From dac44b2c4fe0e1a65eef1361fada921947cea9e5 Mon Sep 17 00:00:00 2001 From: David Chang Date: Thu, 21 Mar 2024 15:03:21 +0800 Subject: [PATCH] ver Mar21st Windows multi_app tasks --- .../185f29bd-5da0-40a6-b69c-ba7f4e0324ef.json | 96 +++++++ .../1f18aa87-af6f-41ef-9853-cdb8f32ebdea.json | 85 ++++++ .../26660ad1-6ebb-4f59-8cba-a8432dfe8d38.json | 68 +++++ .../2c1ebcd7-9c6d-4c9a-afad-900e381ecd5e.json | 127 +++++++++ .../3a93cae4-ad3e-403e-8c12-65303b271818.json | 162 ++++++++++++ .../46407397-a7d5-4c6b-92c6-dbe038b1457b.json | 4 +- .../6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a.json | 72 ++++++ .../74d5859f-ed66-4d3e-aa0e-93d7a592ce41.json | 30 +-- .../8e116af7-7db7-4e35-a68b-b0939c066c78.json | 241 ++++++++++++++++++ .../a82b78bb-7fde-4cb3-94a4-035baf10bcf0.json | 91 +++++++ .../b5062e3e-641c-4e3a-907b-ac864d2e7652.json | 38 +-- .../c867c42d-a52d-4a24-8ae3-f75d256b5618.json | 4 +- .../d1acdb87-bb67-4f30-84aa-990e56a09c92.json | 128 ++++++++++ .../deec51c9-3b1e-4b9e-993c-4776f20e8bb2.json | 101 ++++++++ .../e2392362-125e-4f76-a2ee-524b183a3412.json | 121 +++++++++ .../eb303e01-261e-4972-8c07-c9b4e7a4922a.json | 82 ++++++ .../f918266a-b3e0-4914-865d-4faa564f1aef.json | 4 +- .../6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a.json | 122 ++++----- .../e2392362-125e-4f76-a2ee-524b183a3412.json | 4 +- 19 files changed, 1466 insertions(+), 114 deletions(-) create mode 100644 evaluation_examples/examples/Windows/multi_app/185f29bd-5da0-40a6-b69c-ba7f4e0324ef.json create mode 100644 evaluation_examples/examples/Windows/multi_app/1f18aa87-af6f-41ef-9853-cdb8f32ebdea.json create mode 100644 evaluation_examples/examples/Windows/multi_app/26660ad1-6ebb-4f59-8cba-a8432dfe8d38.json create mode 100644 evaluation_examples/examples/Windows/multi_app/2c1ebcd7-9c6d-4c9a-afad-900e381ecd5e.json create mode 100644 evaluation_examples/examples/Windows/multi_app/3a93cae4-ad3e-403e-8c12-65303b271818.json create mode 100644 evaluation_examples/examples/Windows/multi_app/6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a.json create mode 100644 evaluation_examples/examples/Windows/multi_app/8e116af7-7db7-4e35-a68b-b0939c066c78.json create mode 100644 evaluation_examples/examples/Windows/multi_app/a82b78bb-7fde-4cb3-94a4-035baf10bcf0.json create mode 100644 evaluation_examples/examples/Windows/multi_app/d1acdb87-bb67-4f30-84aa-990e56a09c92.json create mode 100644 evaluation_examples/examples/Windows/multi_app/deec51c9-3b1e-4b9e-993c-4776f20e8bb2.json create mode 100644 evaluation_examples/examples/Windows/multi_app/e2392362-125e-4f76-a2ee-524b183a3412.json create mode 100644 evaluation_examples/examples/Windows/multi_app/eb303e01-261e-4972-8c07-c9b4e7a4922a.json diff --git a/evaluation_examples/examples/Windows/multi_app/185f29bd-5da0-40a6-b69c-ba7f4e0324ef.json b/evaluation_examples/examples/Windows/multi_app/185f29bd-5da0-40a6-b69c-ba7f4e0324ef.json new file mode 100644 index 0000000..203734a --- /dev/null +++ b/evaluation_examples/examples/Windows/multi_app/185f29bd-5da0-40a6-b69c-ba7f4e0324ef.json @@ -0,0 +1,96 @@ +{ + "id": "185f29bd-5da0-40a6-b69c-ba7f4e0324ef", + "snapshot": "libreoffice_calc", + "instruction": "Transfer the data from our 'Employee Performance Evaluation Summary' Excel sheet into our standardized PDF evaluation forms. Each employee's evaluation data should be accurately filled into the designated fields of the PDF form. It's crucial that the final PDF documents retain a uniform and professional look, ready for distribution to our staff or for filing purposes. Furthermore, please ensure that each PDF file is named according to the employee's name as it appears in the Excel document. This will greatly streamline our evaluation process and enhance our efficiency in managing employee performance records. Oh, use \"√\" as mark on characters.", + "source": "authors", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "path": "C:\\Users\\chenj\\Desktop\\Employee Performance Evaluation Summary.xlsx", + "url": "https://drive.google.com/uc?id=1uOzi66bzO_WUnoS4Oqsodrd7_YPLatEk&export=download" + }, + { + "path": "C:\\Users\\chenj\\Desktop\\review_template.pdf", + "url": "https://drive.google.com/uc?id=1YJ4RPGFUuS48tBh31gBerA16JSMw498w&export=download" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "C:\\Users\\chenj\\Desktop\\Employee Performance Evaluation Summary.xlsx" + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 2 + } + }, + { + "type": "open", + "parameters": { + "path": "C:\\Users\\chenj\\Desktop\\review_template.pdf" + } + } + ], + "trajectory": "trajectories/185f29bd-5da0-40a6-b69c-ba7f4e0324ef", + "related_apps": [ + "libreoffice_calc", + "os", + "pdf" + ], + "evaluator": { + "func": "compare_pdfs", + "result": { + "type": "cloud_file", + "path": [ + "https://drive.google.com/uc?id=1kZM90nA1krRmV9ug5_BBe8VlrZRVLiLK&export=download", + "https://drive.google.com/uc?id=1zyLzYYThwyit9ciXpfNfPFlBomolOauY&export=download", + "https://drive.google.com/uc?id=1gMT7JBftuymajMAO5rwksORpeVq3uGmH&export=download", + "https://drive.google.com/uc?id=1x0DdtUSZyBifl1tGIWlWKn255WusJeR4&export=download", + "https://drive.google.com/uc?id=1UAcG32WO8XCXElcanjGwbSpJwFuyOkts&export=download", + "https://drive.google.com/uc?id=1PRgryg7Y5evKnDG2LPtAttVp9qAf5VyZ&export=download", + "https://drive.google.com/uc?id=1JxEDriCS2W7BQLdkIgxu_WFCRa9ib4D7&export=download" + ], + "dest": [ + "Alex Lee_Gold.pdf", + "David Wilson_Gold.pdf", + "Emily Johnson_Gold.pdf", + "John Doe_Gold.pdf", + "Linda Green_Gold.pdf", + "Michael Brown_Gold.pdf", + "Sophia Carter_Gold.pdf" + ], + "multi": true, + "gives": [0,1,2,3,4,5,6] + }, + "expected": { + "type": "vm_file", + "path": [ + "C:\\Users\\chenj\\Desktop\\Alex Lee.pdf", + "C:\\Users\\chenj\\Desktop\\David Wilson.pdf", + "C:\\Users\\chenj\\Desktop\\Emily Johnson.pdf", + "C:\\Users\\chenj\\Desktop\\John Doe.pdf", + "C:\\Users\\chenj\\Desktop\\Linda Green.pdf", + "C:\\Users\\chenj\\Desktop\\Michael Brown.pdf", + "C:\\Users\\chenj\\Desktop\\Sophia Carter.pdf" + ], + "dest": [ + "Alex Lee.pdf", + "David Wilson.pdf", + "Emily Johnson.pdf", + "John Doe.pdf", + "Linda Green.pdf", + "Michael Brown.pdf", + "Sophia Carter.pdf" + ], + "multi": true, + "gives": [0,1,2,3,4,5,6] + } + } +} diff --git a/evaluation_examples/examples/Windows/multi_app/1f18aa87-af6f-41ef-9853-cdb8f32ebdea.json b/evaluation_examples/examples/Windows/multi_app/1f18aa87-af6f-41ef-9853-cdb8f32ebdea.json new file mode 100644 index 0000000..6e49735 --- /dev/null +++ b/evaluation_examples/examples/Windows/multi_app/1f18aa87-af6f-41ef-9853-cdb8f32ebdea.json @@ -0,0 +1,85 @@ + }, + { + "path": "C:\\Users\\chenj\\Desktop\\Grammer test 3.docx", + "url": "https://drive.google.com/uc?id=1QgyQWVOcAJuPaSlrywb9nuFiQDySsTb2&export=download" + }, + { + "path": "C:\\Users\\chenj\\Desktop\\Answer.docx", + "url": "https://drive.google.com/uc?id=1BC2DuWJuZggmf6fXl6Ys9xQMZzU6a1br&export=download" + }, + { + "path": "C:\\Users\\chenj\\Desktop\\Grammar rules PDF\\irregularrules02.pdf", + "url": "https://drive.google.com/uc?id=1Eln9ehX6y6Df2-S_Hp7Ao1teKRu6I1Tg&export=download" + }, + { + "path": "C:\\Users\\chenj\\Desktop\\Grammar rules PDF\\irregularrules01.pdf", + "url": "https://drive.google.com/uc?id=1krdEEdNWvTwMKZU14QtI_xc2lCFVeVcl&export=download" + }, + { + "path": "C:\\Users\\chenj\\Desktop\\Grammar rules PDF\\fragrules.pdf", + "url": "https://drive.google.com/uc?id=1IXyI2KeiXsuh6XV2LelcmhZ2PDh_dBQf&export=download" + }, + { + "path": "C:\\Users\\chenj\\Desktop\\Grammar rules PDF\\csfsrules.pdf", + "url": "https://drive.google.com/uc?id=1ernwGGrjhYNoHVNAevdb2qNKQ0I5n3RP&export=download" + }, + { + "path": "C:\\Users\\chenj\\Desktop\\Public Lecture Teaching Plan.docx", + "url": "https://drive.google.com/uc?id=1ywfVFTEbiSkypZpzLjLmq_ppSbQIC8s8&export=download" + }, + { + "path": "C:\\Users\\chenj\\Desktop\\Course Timetable.xlsx", + "url": "https://drive.google.com/uc?id=1NGtahknRq_kXsXlw0tRQ1_CZp9SljoVg&export=download" + } + ] + } + } + ], + "trajectory": "trajectories/1f18aa87-af6f-41ef-9853-cdb8f32ebdea", + "related_apps": [ + "os", + "libreoffice_writer" + ], + "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "Answer - Word", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5); pyautogui.press(\"enter\");" + ] + } + } + ], + "func": "compare_docx_files", + "expected": { + "type": "cloud_file", + "path": "https://drive.google.com/uc?id=1TOMGWC3OFuP6yEGQuRJMEFWdg2NcBPSs&export=download", + "dest": "Answer gold.docx" + }, + "result": { + "type": "vm_file", + "path": "C:\\Users\\chenj\\Desktop\\Answer.docx", + "dest": "Answer.docx" + }, + "options": { + "ignore_case": true, + "ignore_blanks": true + } + } +} diff --git a/evaluation_examples/examples/Windows/multi_app/26660ad1-6ebb-4f59-8cba-a8432dfe8d38.json b/evaluation_examples/examples/Windows/multi_app/26660ad1-6ebb-4f59-8cba-a8432dfe8d38.json new file mode 100644 index 0000000..8e5b3d7 --- /dev/null +++ b/evaluation_examples/examples/Windows/multi_app/26660ad1-6ebb-4f59-8cba-a8432dfe8d38.json @@ -0,0 +1,68 @@ +{ + "id": "26660ad1-6ebb-4f59-8cba-a8432dfe8d38", + "snapshot": "multiapps", + "instruction": "I want to test the quality of the network environment my laptop is currently in. Please measure my network situation through speedtest.net, export the measurement results, and save them to Documents\\Test\\Speed (if the dir does not exist, create it).", + "source": "https://www.speedtest.net/", + "config": [ + { + "type": "launch", + "parameters": { + "command": [ + "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe", + "--remote-debugging-port=1337" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "ncat.exe", "-k", "-l", "0.0.0.0", "9222", + "--sh-exec", "ncat.exe 127.0.0.1 1337" + ] + } + }, + { + "type": "chrome_open_tabs", + "parameters": { + "urls_to_open": [ + "https://www.speedtest.net/" + ] + } + }, + { + "type": "activate_window", + "parameters": { + "window_name": "Google Chrome" + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; time.sleep(0.5);" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps":[ + "os", + "browser" + ], + "evaluator":{ + "func": "compare_time_in_speedtest_results", + "result":{ + "type": "vm_file", + "path": "C:\\Users\\chenj\\Documents\\Test\\Speed\\Speedtest Results Export-.csv", + "dest": "Speedtest Results Export-.csv", + "time_suffix": true + }, + "expected":{ + "type": "time_diff_range", + "diff_range_in_minutes": "60" + } + } +} diff --git a/evaluation_examples/examples/Windows/multi_app/2c1ebcd7-9c6d-4c9a-afad-900e381ecd5e.json b/evaluation_examples/examples/Windows/multi_app/2c1ebcd7-9c6d-4c9a-afad-900e381ecd5e.json new file mode 100644 index 0000000..a4afcf9 --- /dev/null +++ b/evaluation_examples/examples/Windows/multi_app/2c1ebcd7-9c6d-4c9a-afad-900e381ecd5e.json @@ -0,0 +1,127 @@ +{ + "id": "2c1ebcd7-9c6d-4c9a-afad-900e381ecd5e", + "snapshot": "libreoffice_calc", + "instruction": "Could you please take a moment to review the 'case study' file located within the 'student work' folder? I'm particularly interested in ensuring that the references section at the end of the document adheres to the APA 7th edition formatting guidelines. Making the necessary adjustments if it turns out that the current formatting does not align with APA 7 standards or exists some errors.", + "source": "authors", + "config": [ + { + "type": "command", + "parameters": { + "command": "mkdir \"C:\\Users\\chenj\\Desktop\\students work\" \"C:\\Users\\chenj\\Desktop\\Lec powerpoint\" \"C:\\Users\\chenj\\Desktop\\Grammar test\" \"C:\\Users\\chenj\\Desktop\\Grammar rules PDF\" C:\\Users\\chenj\\Desktop\\FDI", + "shell": true + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "path": "C:\\Users\\chenj\\Desktop\\students work\\Zheng He .docx", + "url": "https://drive.google.com/uc?id=1wI4141LAthnY5m6qcCUaGgDooe4wiTgz&export=download" + }, + { + "path": "C:\\Users\\chenj\\Desktop\\students work\\The literature reviews of weekly readings.docx", + "url": "https://drive.google.com/uc?id=18zoZCNtP-wTkxXp2FhH3O_NdLZKVMPIr&export=download" + }, + { + "path": "C:\\Users\\chenj\\Desktop\\students work\\The British Justice System.docx", + "url": "https://drive.google.com/uc?id=1z3YHSN4CvC5kN1AwTWB_-plRS4p5GAch&export=download" + }, + { + "path": "C:\\Users\\chenj\\Desktop\\students work\\quiz2.docx", + "url": "https://drive.google.com/uc?id=1R5Bii_kvnv_fZVXV-6DMt6Hgq-1gXMo1&export=download" + }, + { + "path": "C:\\Users\\chenj\\Desktop\\students work\\quiz.docx", + "url": "https://drive.google.com/uc?id=1PvlGMVX7YkricrjoPRe0e5VQlHeozRPD&export=download" + }, + { + "path": "C:\\Users\\chenj\\Desktop\\students work\\Q1&2&3.docx", + "url": "https://drive.google.com/uc?id=1kLQ3lnba6p9lqikHhKDdbqrYagHnZWU_&export=download" + }, + { + "path": "C:\\Users\\chenj\\Desktop\\students work\\Photo Ethics in Journalism.docx", + "url": "https://drive.google.com/uc?id=1V6nG6HP_9Kb5KBCRTpaGsRTdPxnJSmRm&export=download" + }, + { + "path": "C:\\Users\\chenj\\Desktop\\students work\\cassie.docx", + "url": "https://drive.google.com/uc?id=1cW9TGJy56vossXxDsdnutPyCbR70af7M&export=download" + }, + { + "path": "C:\\Users\\chenj\\Desktop\\students work\\case study.docx", + "url": "https://drive.google.com/uc?id=11GzpoZvp4qnL2ukXdpbhH-a3zOIHhtDx&export=download" + }, + { + "path": "C:\\Users\\chenj\\Desktop\\Grammar rules PDF\\irregularrules02.pdf", + "url": "https://drive.google.com/uc?id=1Eln9ehX6y6Df2-S_Hp7Ao1teKRu6I1Tg&export=download" + }, + { + "path": "C:\\Users\\chenj\\Desktop\\Grammar rules PDF\\irregularrules01.pdf", + "url": "https://drive.google.com/uc?id=1krdEEdNWvTwMKZU14QtI_xc2lCFVeVcl&export=download" + }, + { + "path": "C:\\Users\\chenj\\Desktop\\Grammar rules PDF\\fragrules.pdf", + "url": "https://drive.google.com/uc?id=1IXyI2KeiXsuh6XV2LelcmhZ2PDh_dBQf&export=download" + }, + { + "path": "C:\\Users\\chenj\\Desktop\\Grammar rules PDF\\csfsrules.pdf", + "url": "https://drive.google.com/uc?id=1ernwGGrjhYNoHVNAevdb2qNKQ0I5n3RP&export=download" + }, + { + "path": "C:\\Users\\chenj\\Desktop\\Public Lecture Teaching Plan.docx", + "url": "https://drive.google.com/uc?id=1ywfVFTEbiSkypZpzLjLmq_ppSbQIC8s8&export=download" + }, + { + "path": "C:\\Users\\chenj\\Desktop\\Course Timetable.xlsx", + "url": "https://drive.google.com/uc?id=1NGtahknRq_kXsXlw0tRQ1_CZp9SljoVg&export=download" + } + ] + } + } + ], + "trajectory": "trajectories/2c1ebcd7-9c6d-4c9a-afad-900e381ecd5e", + "related_apps": [ + ], + "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "case study - Word", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5); pyautogui.press(\"enter\");" + ] + } + } + ], + "func": "compare_references", + "expected": { + "type": "cloud_file", + "path": "https://drive.google.com/uc?id=1325Qfch0JaJ_wJ20ICxMoHeW8KLpK8v0&export=download", + "dest": "case study gold.docx" + }, + "result": { + "type": "vm_file", + "path": "C:\\Users\\chenj\\Desktop\\students work\\case study.docx", + "dest": "case study.docx" + }, + "options": { + "content_only": true, + "reference_base_result": 0.92 + } + } +} diff --git a/evaluation_examples/examples/Windows/multi_app/3a93cae4-ad3e-403e-8c12-65303b271818.json b/evaluation_examples/examples/Windows/multi_app/3a93cae4-ad3e-403e-8c12-65303b271818.json new file mode 100644 index 0000000..1b77444 --- /dev/null +++ b/evaluation_examples/examples/Windows/multi_app/3a93cae4-ad3e-403e-8c12-65303b271818.json @@ -0,0 +1,162 @@ +{ + "id": "3a93cae4-ad3e-403e-8c12-65303b271818", + "snapshot": "libreoffice_calc", + "instruction": "Could you please add a two-hour lecture slot to my weekly course timetable, scheduled for every Wednesday at 12 PM? It seems I accidentally omitted that when setting up my schedule. I'd appreciate you taking care of that for me. Thanks!", + "source": "authors", + "config": [ + { + "type": "command", + "parameters": { + "command": "mkdir \"C:\\Users\\chenj\\Desktop\\students work\" \"C:\\Users\\chenj\\Desktop\\Lec powerpoint\" \"C:\\Users\\chenj\\Desktop\\Grammar test\" \"C:\\Users\\chenj\\Desktop\\Grammar rules PDF\" C:\\Users\\chenj\\Desktop\\FDI", + "shell": true + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "path": "C:\\Users\\chenj\\Desktop\\students work\\Zheng He .docx", + "url": "https://drive.google.com/uc?id=1wI4141LAthnY5m6qcCUaGgDooe4wiTgz&export=download" + }, + { + "path": "C:\\Users\\chenj\\Desktop\\students work\\cassie.docx", + "url": "https://drive.google.com/uc?id=1cW9TGJy56vossXxDsdnutPyCbR70af7M&export=download" + }, + { + "path": "C:\\Users\\chenj\\Desktop\\students work\\case study.docx", + "url": "https://drive.google.com/uc?id=11GzpoZvp4qnL2ukXdpbhH-a3zOIHhtDx&export=download" + }, + { + "path": "C:\\Users\\chenj\\Desktop\\Grammar rules PDF\\irregularrules02.pdf", + "url": "https://drive.google.com/uc?id=1Eln9ehX6y6Df2-S_Hp7Ao1teKRu6I1Tg&export=download" + }, + { + "path": "C:\\Users\\chenj\\Desktop\\Grammar rules PDF\\irregularrules01.pdf", + "url": "https://drive.google.com/uc?id=1krdEEdNWvTwMKZU14QtI_xc2lCFVeVcl&export=download" + }, + { + "path": "C:\\Users\\chenj\\Desktop\\Grammar rules PDF\\fragrules.pdf", + "url": "https://drive.google.com/uc?id=1IXyI2KeiXsuh6XV2LelcmhZ2PDh_dBQf&export=download" + }, + { + "path": "C:\\Users\\chenj\\Desktop\\Grammar rules PDF\\csfsrules.pdf", + "url": "https://drive.google.com/uc?id=1ernwGGrjhYNoHVNAevdb2qNKQ0I5n3RP&export=download" + }, + { + "path": "C:\\Users\\chenj\\Desktop\\Public Lecture Teaching Plan.docx", + "url": "https://drive.google.com/uc?id=1ywfVFTEbiSkypZpzLjLmq_ppSbQIC8s8&export=download" + }, + { + "path": "C:\\Users\\chenj\\Desktop\\Course Timetable.xlsx", + "url": "https://drive.google.com/uc?id=1NGtahknRq_kXsXlw0tRQ1_CZp9SljoVg&export=download" + } + ] + } + } + ], + "trajectory": "trajectories/3a93cae4-ad3e-403e-8c12-65303b271818", + "related_apps": [ + "os", + "libreoffice_calc" + ], + "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "Course Timetable - Excel", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5); pyautogui.press(\"enter\");" + ] + } + } + ], + "func": [ + "compare_table", + "compare_table", + "compare_table" + ], + "result": [ + { + "type": "vm_file", + "path": "C:\\Users\\chenj\\Desktop\\Course Timetable.xlsx", + "dest": "Course Timetable.xlsx" + }, + { + "type": "vm_file", + "path": "C:\\Users\\chenj\\Desktop\\Course Timetable.xlsx", + "dest": "Course Timetable.xlsx" + }, + { + "type": "vm_file", + "path": "C:\\Users\\chenj\\Desktop\\Course Timetable.xlsx", + "dest": "Course Timetable.xlsx" + } + ], + "expected": [ + { + "type": "cloud_file", + "path": "https://drive.google.com/uc?id=1VMOon8byWuoCW2Uk5etGMJLMzAfwFVyB&export=download", + "dest": "Course Timetable gold.xlsx" + }, + { + "type": "cloud_file", + "path": "https://drive.google.com/uc?id=1jAThiIqILZ5t-RFPHVniSvAL8ZJO1H3P&export=download", + "dest": "Course Timetable gold 2.xlsx" + }, + { + "type": "cloud_file", + "path": "https://drive.google.com/uc?id=1U0THDtPCgsw-Rb0N9fjF8DeOepPeUajP&export=download", + "dest": "Course Timetable gold 3.xlsx" + } + ], + "options": [ + { + "rules": [ + { + "type": "sheet_data", + "sheet_idx0": "RNSheet1", + "sheet_idx1": "ENSheet1", + "ignore_case": true + } + ] + }, + { + "rules": [ + { + "type": "sheet_data", + "sheet_idx0": "RNSheet1", + "sheet_idx1": "ENSheet1", + "ignore_case": true + } + ] + }, + { + "rules": [ + { + "type": "sheet_data", + "sheet_idx0": "RNSheet1", + "sheet_idx1": "ENSheet1", + "ignore_case": true + } + ] + } + ], + "conj": "or" + } +} diff --git a/evaluation_examples/examples/Windows/multi_app/46407397-a7d5-4c6b-92c6-dbe038b1457b.json b/evaluation_examples/examples/Windows/multi_app/46407397-a7d5-4c6b-92c6-dbe038b1457b.json index 2106e9e..ff087fe 100644 --- a/evaluation_examples/examples/Windows/multi_app/46407397-a7d5-4c6b-92c6-dbe038b1457b.json +++ b/evaluation_examples/examples/Windows/multi_app/46407397-a7d5-4c6b-92c6-dbe038b1457b.json @@ -74,14 +74,14 @@ { "type": "execute", "parameters": { - "command": "rm -r C:\\Users\\chenj\\AppData\\Roaming\\Thunderbird", + "command": "rd /s /q C:\\Users\\chenj\\AppData\\Roaming\\Thunderbird", "shell": true } }, { "type": "execute", "parameters": { - "command": "mv C:\\Users\\chenj\\Thunderbird C:\\Users\\chenj\\AppData\\Roaming\\Thunderbird", + "command": "move C:\\Users\\chenj\\Thunderbird C:\\Users\\chenj\\AppData\\Roaming\\Thunderbird", "shell": true } }, diff --git a/evaluation_examples/examples/Windows/multi_app/6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a.json b/evaluation_examples/examples/Windows/multi_app/6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a.json new file mode 100644 index 0000000..7268754 --- /dev/null +++ b/evaluation_examples/examples/Windows/multi_app/6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a.json @@ -0,0 +1,72 @@ +{ + "id": "6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a", + "snapshot": "multiapps", + "instruction": "I now want to count the meeting cities of the three machine learning conferences in the past ten years from 2013 to 2019(including 2013 and 2019). I have listed the names and years of the conferences in excel. Please fill in the vacant locations.", + "source": "author", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=19wUxTQeoKr6ihJWJ_9cu2tzKQH0cnxWH", + "path": "C:\\Users\\chenj\\Desktop\\ConferenceCity.xlsx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "C:\\Users\\chenj\\Desktop\\ConferenceCity.xlsx" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "calc", "chrome", "os" + ], + "evaluator": { + "postconfig":[ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1ZcITkIOs2Z86S5L6MShSohFs3_xVfeCP", + "path": "C:\\Users\\chenj\\Desktop\\ConferenceCity_Gold.xlsx" + } + ] + } + }, + { + "type": "activate_window", + "parameters": { + "window_name": "ConferenceCity - Excel" + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5); pyautogui.press(\"enter\");" + ] + } + } + ], + "func": "compare_conference_city_in_order", + "expected": { + "type": "rule", + "rules":{ + "expected": ["Scottsdale","Atlanta","Lake Tahoe","Banff","Beijing",["Montreal", "Montréal"],"San Diego","Lille",["Montreal", "Montréal"],"San Juan",["New York", "New York City", "NYC"],"Barcelona","Toulon","Sydney","Long Beach","Vancouver","Stockholm",["Montreal", "Montréal"],"New Orleans","Long Beach","Vancouver"] + } + }, + "result": { + "type": "vm_file", + "path": "C:\\Users\\chenj\\Desktop\\ConferenceCity.xlsx", + "dest": "ConferenceCity.xlsx" + } + } +} diff --git a/evaluation_examples/examples/Windows/multi_app/74d5859f-ed66-4d3e-aa0e-93d7a592ce41.json b/evaluation_examples/examples/Windows/multi_app/74d5859f-ed66-4d3e-aa0e-93d7a592ce41.json index abe1425..51a0ae4 100644 --- a/evaluation_examples/examples/Windows/multi_app/74d5859f-ed66-4d3e-aa0e-93d7a592ce41.json +++ b/evaluation_examples/examples/Windows/multi_app/74d5859f-ed66-4d3e-aa0e-93d7a592ce41.json @@ -1,14 +1,14 @@ { "id": "74d5859f-ed66-4d3e-aa0e-93d7a592ce41", "snapshot": "chrome", - "instruction": "Help me to set up an initial web extension project with help of the web tool, tagging it \"happy-extension v0.0.1\". Leave description blank for now. Include a background script and browser action, while other features are not required. Remember to unzip the auto-generated folder into \"~/Projects\".", + "instruction": "Help me to set up an initial web extension project with help of the web tool, tagging it \"happy-extension v0.0.1\". Leave description blank for now. Include a background script and browser action, while other features are not required. Remember to unzip the auto-generated folder into \"Documents\\Projects\".", "source": "authors", "config": [ { "type": "launch", "parameters": { "command": [ - "google-chrome", + "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe", "--remote-debugging-port=1337" ] } @@ -17,9 +17,8 @@ "type": "launch", "parameters": { "command": [ - "socat", - "tcp-listen:9222,fork", - "tcp:localhost:1337" + "ncat.exe", "-k", "-l", "0.0.0.0", "9222", + "--sh-exec", "ncat.exe 127.0.0.1 1337" ] } }, @@ -34,19 +33,16 @@ { "type": "execute", "parameters": { - "command": [ - "mkdir", - "-p", - "/home/user/Projects" - ] + "command": "mkdir C:\\Users\\chenj\\Documents\\Projects", + "shell": "true" } }, { "type": "launch", "parameters": { "command": [ - "nautilus", - "/home/user/Projects" + "explorer.exe", + "C:\\Users\\chenj\\Documents\\Projects" ] } } @@ -67,27 +63,27 @@ "result": [ { "type": "vm_file", - "path": "/home/user/Projects/happy-extension/manifest.json", + "path": "C:\\Users\\chenj\\Documents\\Projects\\happy-extension\\manifest.json", "dest": "manifest.json" }, { "type": "vm_file", - "path": "/home/user/Projects/happy-extension/background_script.js", + "path": "C:\\Users\\chenj\\Documents\\Projects\\happy-extension\\background_script.js", "dest": "background_script.js" }, { "type": "vm_file", - "path": "/home/user/Projects/happy-extension/browserAction/index.html", + "path": "C:\\Users\\chenj\\Documents\\Projects\\happy-extension\\browserAction\\index.html", "dest": "index.html" }, { "type": "vm_file", - "path": "/home/user/Projects/happy-extension/browserAction/style.css", + "path": "C:\\Users\\chenj\\Documents\\Projects\\happy-extension\\browserAction\\style.css", "dest": "style.css" }, { "type": "vm_file", - "path": "/home/user/Projects/happy-extension/browserAction/script.js", + "path": "C:\\Users\\chenj\\Documents\\Projects\\happy-extension\\browserAction\\script.js", "dest": "script.js" } ], diff --git a/evaluation_examples/examples/Windows/multi_app/8e116af7-7db7-4e35-a68b-b0939c066c78.json b/evaluation_examples/examples/Windows/multi_app/8e116af7-7db7-4e35-a68b-b0939c066c78.json new file mode 100644 index 0000000..8dba229 --- /dev/null +++ b/evaluation_examples/examples/Windows/multi_app/8e116af7-7db7-4e35-a68b-b0939c066c78.json @@ -0,0 +1,241 @@ +{ + "id": "8e116af7-7db7-4e35-a68b-b0939c066c78", + "snapshot": "libreoffice_calc", + "instruction": "Please update my bookkeeping sheet with the recent transactions from the provided folder, detailing my expenses over the past few days.", + "source": "authors", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "path": "C:\\Users\\chenj\\Desktop\\my_bookkeeping.xlsx", + "url": "https://drive.google.com/uc?id=1QOSpTZPFzFZeC0tng4Gfws544LFln836&export=download" + }, + { + "path": "C:\\Users\\chenj\\Desktop\\receipt_0.jpeg", + "url": "https://drive.google.com/uc?id=1b0BRc-BzXObVCUEonJfRbDsrgxZugj3U&export=download" + }, + { + "path": "C:\\Users\\chenj\\Desktop\\receipt_1.jpg", + "url": "https://drive.google.com/uc?id=1S-JBDqwEf7Z_JXDItK_F4BOHgScTjlyN&export=download" + }, + { + "path": "C:\\Users\\chenj\\Desktop\\receipt_2.jpg", + "url": "https://drive.google.com/uc?id=1Ys2abZi9_0y8sxuj2vCbC0OhjC6YdrC-&export=download" + }, + { + "path": "C:\\Users\\chenj\\Desktop\\receipt_3.pdf", + "url": "https://drive.google.com/uc?id=1sKvBbGDpmUkv891xTqX7w5dtEvchQahd&export=download" + }, + { + "path": "C:\\Users\\chenj\\Desktop\\receipt_4.jpg", + "url": "https://drive.google.com/uc?id=1kW7xH5bc2jRaKGDKHDrgSehTrPgkyzkc&export=download" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "C:\\Users\\chenj\\Desktop\\my_bookkeeping.xlsx" + } + } + ], + "trajectory": "trajectories/8e116af7-7db7-4e35-a68b-b0939c066c78", + "related_apps": [ + "libreoffice_calc", + "os", + "image", + "pdf" + ], + "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "my_bookkeeping - Excel", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5); pyautogui.press(\"enter\");" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 1.0 + } + } + ], + "func": "compare_table", + "result": { + "type": "vm_file", + "path": "C:\\Users\\chenj\\Desktop\\my_bookkeeping.xlsx", + "dest": "my_bookkeeping.xlsx" + }, + "expected": { + "type": "cloud_file", + "path": "https://drive.google.com/uc?id=1ygEDdVlkf2ZyqMxJ_ktqo9G_g--rc6co&export=download", + "dest": "my_bookkeeping_gold.xlsx" + }, + "options": { + "rules": [ + { + "type": "sheet_fuzzy", + "sheet_idx0": "RNSheet1", + "sheet_idx1": "ENSheet1", + "rules": [ + { + "range": [ + "A1:A8", + "B1:B8", + "C1:C8", + "D1:D8", + "E1:E8" + ], + "type": "exact_match" + } + ] + }, + { + "type": "sheet_fuzzy", + "sheet_idx0": "RNSheet1", + "sheet_idx1": "ENSheet1", + "rules": [ + { + "range": [ + "C9:C13" + ], + "type": "exact_match", + "ignore_case": true + } + ] + }, + { + "type": "check_cell", + "sheet_idx": 0, + "coordinate": "D9", + "props": { + "value": { + "method": "approx:0.1", + "ref": -186.93 + } + } + }, + { + "type": "check_cell", + "sheet_idx": 0, + "coordinate": "D10", + "props": { + "value": { + "method": "approx:0.1", + "ref": -3670 + } + } + }, + { + "type": "check_cell", + "sheet_idx": 0, + "coordinate": "D11", + "props": { + "value": { + "method": "approx:0.1", + "ref": -5.7 + } + } + }, + { + "type": "check_cell", + "sheet_idx": 0, + "coordinate": "D12", + "props": { + "value": { + "method": "approx:0.1", + "ref": -154.06 + } + } + }, + { + "type": "check_cell", + "sheet_idx": 0, + "coordinate": "D13", + "props": { + "value": { + "method": "approx:0.1", + "ref": -8.1 + } + } + }, + { + "type": "check_cell", + "sheet_idx": 0, + "coordinate": "E9", + "props": { + "value": { + "method": "approx:0.1", + "ref": 603.07 + } + } + }, + { + "type": "check_cell", + "sheet_idx": 0, + "coordinate": "E10", + "props": { + "value": { + "method": "approx:0.1", + "ref": -3066.93 + } + } + }, + { + "type": "check_cell", + "sheet_idx": 0, + "coordinate": "E11", + "props": { + "value": { + "method": "approx:0.1", + "ref": -3072.63 + } + } + }, + { + "type": "check_cell", + "sheet_idx": 0, + "coordinate": "E12", + "props": { + "value": { + "method": "approx:0.1", + "ref": -3226.69 + } + } + }, + { + "type": "check_cell", + "sheet_idx": 0, + "coordinate": "E13", + "props": { + "value": { + "method": "approx:0.1", + "ref": -3234.79 + } + } + } + ] + } + } +} diff --git a/evaluation_examples/examples/Windows/multi_app/a82b78bb-7fde-4cb3-94a4-035baf10bcf0.json b/evaluation_examples/examples/Windows/multi_app/a82b78bb-7fde-4cb3-94a4-035baf10bcf0.json new file mode 100644 index 0000000..86ff424 --- /dev/null +++ b/evaluation_examples/examples/Windows/multi_app/a82b78bb-7fde-4cb3-94a4-035baf10bcf0.json @@ -0,0 +1,91 @@ +{ + "id": "a82b78bb-7fde-4cb3-94a4-035baf10bcf0", + "snapshot": "libreoffice_calc", + "instruction": "I'm really enjoying this paper. Could you please locate the personal webpages of the initial author and the last three authors? Please include them in a browser bookmark folder titled 'Liked Authors.'", + "source": "authors", + "config": [ + { + "type": "launch", + "parameters": { + "command": [ + "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe", + "--remote-debugging-port=1337" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "ncat.exe", "-k", "-l", "0.0.0.0", "9222", + "--sh-exec", "ncat.exe 127.0.0.1 1337" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 2 + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1IlJ8kU5MlR6OqJHchsSUJzLCmcrG-8N7&export=download&authuser=0&confirm=t&uuid=d2a1810f-edea-4bfd-9d79-e668b9f11876&at=APZUnTVv_eqtC86YzkEU8_jIhC9W:1709522229162", + "path": "C:\\Users\\chenj\\Desktop\\2206.08853.pdf" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "C:\\Users\\chenj\\Desktop\\2206.08853.pdf" + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 2 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('f11'); time.sleep(0.5); pyautogui.click(960, 540); time.sleep(0.5); pyautogui.scroll(-20)" + ] + } + } + ], + "trajectory": "trajectories/a82b78bb-7fde-4cb3-94a4-035baf10bcf0", + "related_apps": [ + "chrome", + "pdf" + ], + "evaluator": { + "func": "is_expected_bookmarks", + "result": { + "type": "bookmarks" + }, + "expected": { + "type": "rule", + "rules": { + "type": "liked_authors_websites_urls", + "names": [ + "Liked Authors" + ], + "urls": [ + ["https://jimfan.me/", "https://research.nvidia.com/person/linxi-jim-fan"], + ["https://research.nvidia.com/person/de-an-huang", "https://ai.stanford.edu/~dahuang/"], + ["https://yukezhu.me/", "https://www.cs.utexas.edu/people/faculty-researchers/yuke-zhu", "https://experts.utexas.edu/yuke_zhu", "https://research.nvidia.com/person/yuke-zhu"], + ["http://tensorlab.cms.caltech.edu/users/anima/", "https://www.eas.caltech.edu/people/anima"] + ] + } + } + } +} diff --git a/evaluation_examples/examples/Windows/multi_app/b5062e3e-641c-4e3a-907b-ac864d2e7652.json b/evaluation_examples/examples/Windows/multi_app/b5062e3e-641c-4e3a-907b-ac864d2e7652.json index c869428..b4ae4ae 100644 --- a/evaluation_examples/examples/Windows/multi_app/b5062e3e-641c-4e3a-907b-ac864d2e7652.json +++ b/evaluation_examples/examples/Windows/multi_app/b5062e3e-641c-4e3a-907b-ac864d2e7652.json @@ -1,17 +1,14 @@ { "id": "b5062e3e-641c-4e3a-907b-ac864d2e7652", "snapshot": "libreoffice_calc", - "instruction": "Please help me to extract the name, e-mail, and affiliation of the first author from each paper in the folder and organize them in an Excel table. Include headers for each field. Sort the authors by their full names alphabetically and save the table as \"~/authors.xlsx\".", + "instruction": "Please help me to extract the name, e-mail, and affiliation of the first author from each paper in the folder and organize them in an Excel table. Include headers for each field. Sort the authors by their full names alphabetically and save the table as \"Documents\\authors.xlsx\".", "source": "authors", "config": [ { "type": "command", "parameters": { - "command": [ - "mkdir", - "-p", - "/home/user/Documents/Papers" - ] + "command": "mkdir C:\\Users\\chenj\\Documents\\Papers", + "shell": true } }, { @@ -19,19 +16,19 @@ "parameters": { "files": [ { - "path": "/home/user/Documents/Papers/zhang_appagent.pdf", + "path": "C:\\Users\\chenj\\Documents\\Papers\\zhang_appagent.pdf", "url": "https://arxiv.org/pdf/2312.13771.pdf" }, { - "path": "/home/user/Documents/Papers/niu_screenagent.pdf", + "path": "C:\\Users\\chenj\\Documents\\Papers\\niu_screenagent.pdf", "url": "https://arxiv.org/pdf/2402.07945.pdf" }, { - "path": "/home/user/Documents/Papers/koh_visualwebarena.pdf", + "path": "C:\\Users\\chenj\\Documents\\Papers\\koh_visualwebarena.pdf", "url": "https://arxiv.org/pdf/2401.13649.pdf" }, { - "path": "/home/user/Documents/Papers/deng_mind2web.pdf", + "path": "C:\\Users\\chenj\\Documents\\Papers\\deng_mind2web.pdf", "url": "https://papers.nips.cc/paper_files/paper/2023/file/5950bf290a1570ea401bf98882128160-Paper-Datasets_and_Benchmarks.pdf" } ] @@ -41,8 +38,8 @@ "type": "launch", "parameters": { "command": [ - "nautilus", - "/home/user/Documents/Papers" + "explorer.exe", + "C:\\Users\\chenj\\Documents\\Papers" ] } } @@ -53,25 +50,10 @@ "os" ], "evaluator": { - "postconfig": [ - { - "type": "execute", - "parameters": { - "command": [ - "libreoffice", - "--convert-to", - "csv:Text - txt - csv (StarCalc):44,34,UTF-8,,,,false,true,true,false,false,1", - "--outdir", - "/home/user", - "/home/user/authors.xlsx" - ] - } - } - ], "func": "compare_table", "result": { "type": "vm_file", - "path": "/home/user/authors.xlsx", + "path": "C:\\Users\\chenj\\authors.xlsx", "dest": "authors.xlsx" }, "expected": { diff --git a/evaluation_examples/examples/Windows/multi_app/c867c42d-a52d-4a24-8ae3-f75d256b5618.json b/evaluation_examples/examples/Windows/multi_app/c867c42d-a52d-4a24-8ae3-f75d256b5618.json index 72d43a0..0fd69f8 100644 --- a/evaluation_examples/examples/Windows/multi_app/c867c42d-a52d-4a24-8ae3-f75d256b5618.json +++ b/evaluation_examples/examples/Windows/multi_app/c867c42d-a52d-4a24-8ae3-f75d256b5618.json @@ -35,14 +35,14 @@ { "type": "execute", "parameters": { - "command": "rm -r C:\\Users\\chenj\\AppData\\Roaming\\Thunderbird", + "command": "rd /s /q C:\\Users\\chenj\\AppData\\Roaming\\Thunderbird", "shell": true } }, { "type": "execute", "parameters": { - "command": "mv C:\\Users\\chenj\\Thunderbird C:\\Users\\chenj\\AppData\\Roaming\\Thunderbird", + "command": "move C:\\Users\\chenj\\Thunderbird C:\\Users\\chenj\\AppData\\Roaming\\Thunderbird", "shell": true } }, diff --git a/evaluation_examples/examples/Windows/multi_app/d1acdb87-bb67-4f30-84aa-990e56a09c92.json b/evaluation_examples/examples/Windows/multi_app/d1acdb87-bb67-4f30-84aa-990e56a09c92.json new file mode 100644 index 0000000..0a671aa --- /dev/null +++ b/evaluation_examples/examples/Windows/multi_app/d1acdb87-bb67-4f30-84aa-990e56a09c92.json @@ -0,0 +1,128 @@ +{ + "id": "d1acdb87-bb67-4f30-84aa-990e56a09c92", + "snapshot": "libreoffice_calc", + "instruction": "Hello! I'm eagerly planning a culinary adventure to Hong Kong and have curated a list of must-visit restaurants that I've been longing to explore. However, I could use some assistance in compiling a few essential details about these establishments. Would you be so kind as to help me out? It would be fantastic if you could search for these restaurants on Google Maps. I'm particularly interested in obtaining their addresses, any available websites, and contact phone numbers. If you could gather this information and input it into my form file, I would be immensely grateful. Many thanks in advance!", + "source": "authors", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "path": "C:\\Users\\chenj\\Desktop\\restaurants.txt", + "url": "https://drive.google.com/uc?id=1IehFLJPZcFv8Ujk31ExbyGLji9AylmmJ&export=download" + }, + { + "path": "C:\\Users\\chenj\\Desktop\\MUST_VISIT.xlsx", + "url": "https://drive.google.com/uc?id=1fXmjvZcwkIcckMIAXi3Hv_JAbVWpgs_l&export=download" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "C:\\Users\\chenj\\Desktop\\MUST_VISIT.xlsx" + } + }, + { + "type": "open", + "parameters": { + "path": "C:\\Users\\chenj\\Desktop\\restaurants.txt" + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 5 + } + }, + { + "type": "activate_window", + "parameters": { + "window_name": "restaurants.txt" + } + } + ], + "trajectory": "trajectories/d1acdb87-bb67-4f30-84aa-990e56a09c92", + "related_apps": [ + "os", + "chrome", + "libreoffice_calc" + ], + "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "MUST_VISIT - Excel", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5); pyautogui.press(\"enter\");" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 1.0 + } + } + ], + "func": "compare_table", + "result": { + "type": "vm_file", + "path": "C:\\Users\\chenj\\Desktop\\MUST_VISIT.xlsx", + "dest": "MUST_VISIT.xlsx" + }, + "expected": { + "type": "cloud_file", + "path": "https://drive.google.com/uc?id=1MV6jBvRbbYwPqeFTd_nX40xzyltNhphl&export=download", + "dest": "MUST_VISIT-gt.xlsx" + }, + "options": { + "rules": [ + { + "type": "sheet_fuzzy", + "sheet_idx0": "RNSheet1", + "sheet_idx1": "ENSheet1", + "rules": [ + { + "range": ["A1:A6", "D1:D6"], + "type": "exact_match" + }, + { + "range": ["B1:B6"], + "type": "fuzzy_match", + "threshold": 85, + "normalization": [ + ["Rd", "Road"], + ["St", "Street"] + ], + "ignore_case": true + }, + { + "range": ["C1:C6"], + "type": "includes", + "trim_leadings": "+ ", + "ignore_chars": " ()-" + } + ] + } + ] + } + } +} diff --git a/evaluation_examples/examples/Windows/multi_app/deec51c9-3b1e-4b9e-993c-4776f20e8bb2.json b/evaluation_examples/examples/Windows/multi_app/deec51c9-3b1e-4b9e-993c-4776f20e8bb2.json new file mode 100644 index 0000000..22de11c --- /dev/null +++ b/evaluation_examples/examples/Windows/multi_app/deec51c9-3b1e-4b9e-993c-4776f20e8bb2.json @@ -0,0 +1,101 @@ +{ + "id": "deec51c9-3b1e-4b9e-993c-4776f20e8bb2", + "snapshot": "libreoffice_calc", + "instruction": "Find a paper list of all the new foundation language models issued on 11st Oct. 2023 via arxiv daily, and organize it into the sheet I opened.", + "source": "authors", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "path": "C:\\Users\\chenj\\Desktop\\New Large Language Models.xlsx", + "url": "https://drive.google.com/uc?id=1NJFAUDzatd5TbBqXeCy3-ok4BWj-xayT&export=download" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "C:\\Users\\chenj\\Desktop\\New Large Language Models.xlsx" + } + } + ], + "trajectory": "trajectories/deec51c9-3b1e-4b9e-993c-4776f20e8bb2", + "related_apps": [ + "libreoffice_calc", + "chrome", + "os" + ], + "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "New Large Language Models - Excel", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5); pyautogui.press(\"enter\");" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 1.0 + } + } + ], + "func": "compare_table", + "result": { + "type": "vm_file", + "path": "C:\\Users\\chenj\\Desktop\\New Large Language Models.xlsx", + "dest": "New Large Language Models.xlsx" + }, + "expected": { + "type": "cloud_file", + "path": "https://drive.google.com/uc?id=1BHOyjFo72b74YKWTqPMaoNvCzICkos-G&export=download", + "dest": "New Large Language Models Gold.xlsx" + }, + "options": { + "rules": [ + { + "type": "sheet_fuzzy", + "sheet_idx0": "RNSheet1", + "sheet_idx1": "ENSheet1", + "rules": [ + { + "range": [ + "B2:B5", + "C2:C5" + ], + "type": "exact_match" + }, + { + "range": [ + "A2:A5" + ], + "type": "fuzzy_match", + "threshold": 90, + "ignore_case": true + } + ] + } + ] + } + } +} diff --git a/evaluation_examples/examples/Windows/multi_app/e2392362-125e-4f76-a2ee-524b183a3412.json b/evaluation_examples/examples/Windows/multi_app/e2392362-125e-4f76-a2ee-524b183a3412.json new file mode 100644 index 0000000..a770c64 --- /dev/null +++ b/evaluation_examples/examples/Windows/multi_app/e2392362-125e-4f76-a2ee-524b183a3412.json @@ -0,0 +1,121 @@ +{ + "id": "e2392362-125e-4f76-a2ee-524b183a3412", + "snapshot": "chrome", + "instruction": "I recently started using the famous personal academic homepage template from academicpages.github.io to build my own personal homepage, and I have cloned it to my local Documents\\Code\\Website folder. According to an online tutorial, I can configure my name and contact information in the _config.yaml file. However, I am not familiar with the YAML file format. Please help me find the sections related to the name and contact information in this file and change them to \"Test Account\" and \"Test@gmail.com\".", + "source": "authors", + "config": [ + { + "type": "command", + "parameters": { + "command": "mkdir C:\\Users\\chenj\\Documents\\Code\\Website", + "shell": true + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "path": "C:\\Users\\chenj\\.tmp.7z", + "url": "https://drive.google.com/uc?id=1LYc6rBSuCNBtTQIg-m9zP6KmlEB_Zfdo&export=download" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "C:\\Program Files\\7-Zip\\7z.exe", + "x", "-oC:\\Users\\chenj\\Documents\\Code\\Website", + "C:\\Users\\chenj\\.tmp.7z" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe", + "--remote-debugging-port=1337" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "ncat.exe", "-k", "-l", "0.0.0.0", "9222", + "--sh-exec", "ncat.exe 127.0.0.1 1337" + ] + } + }, + { + "type": "chrome_open_tabs", + "parameters": { + "urls_to_open": [ + "https://academicpages.github.io/" + ] + } + } + ], + "trajectory": "trajectories/e2392362-125e-4f76-a2ee-524b183a3412", + "related_apps": [ + "chrome", + "os", + "vscode" + ], + "evaluator": { + "postconfig": [ + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);" + ] + } + } + ], + "func": "check_json", + "options": { + "is_yaml": true + }, + "expected": { + "type": "rule", + "rules": { + "expect": [ + { + "key": [ + "name" + ], + "method": "eq", + "ref": "Test Account" + }, + { + "key": [ + "author", + "name" + ], + "method": "eq", + "ref": "Test Account" + }, + { + "key": [ + "author", + "email" + ], + "method": "eq", + "ref": "Test@gmail.com" + } + ] + } + }, + "result": { + "type": "vm_file", + "path": "C:\\Users\\chenj\\Documents\\Code\\Website\\academicpages.github.io\\_config.yml", + "dest": "_config.yaml" + } + } +} diff --git a/evaluation_examples/examples/Windows/multi_app/eb303e01-261e-4972-8c07-c9b4e7a4922a.json b/evaluation_examples/examples/Windows/multi_app/eb303e01-261e-4972-8c07-c9b4e7a4922a.json new file mode 100644 index 0000000..801293e --- /dev/null +++ b/evaluation_examples/examples/Windows/multi_app/eb303e01-261e-4972-8c07-c9b4e7a4922a.json @@ -0,0 +1,82 @@ +{ + "id": "eb303e01-261e-4972-8c07-c9b4e7a4922a", + "snapshot": "libreoffice_impress", + "instruction": "Tomorrow, I'm scheduled to deliver a talk, and my PowerPoint slides and speaking notes are saved on the desktop. Help me insert my planned remarks for each slide into the \"note\" section of the PowerPoint as a reminder. I've completed this task for some slides; assist me in completing the remaining part.", + "source": "authors", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1MdgN8ECxzLgHgjq8eKqrNQt3MPDjnKwa&export=download&authuser=0&confirm=t&uuid=ed5c37db-c565-4ca1-bbd1-bbdba13e9306&at=APZUnTUNi8YTLlZqMZ0r--bBpBEG:1709449877819", + "path": "C:\\Users\\chenj\\Desktop\\lecture1-2021-with-ink.pptx" + }, + { + "url": "https://drive.usercontent.google.com/download?id=1FkPOcsWpsjUXSUld1NblwyVzcsE19uIe&export=download&authuser=0&confirm=t&uuid=27501bc0-732b-4ff7-abf4-a52427aea264&at=APZUnTWleaafIVF2iZkiuHo0vQ66:1709449873140", + "path": "C:\\Users\\chenj\\Desktop\\notes.docx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "C:\\Users\\chenj\\Desktop\\lecture1-2021-with-ink.pptx" + } + } + ], + "trajectory": "trajectories/eb303e01-261e-4972-8c07-c9b4e7a4922a", + "related_apps": [ + "libreoffice_impress", + "libreoffice_writer" + ], + "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "lecture1-2021-with-ink - PowerPoint", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5); pyautogui.press(\"enter\");" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + } + ], + "func": "compare_pptx_files", + "expected": { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=18orj_0q6N4w7ijADOJeU5ZkDDw-RdFUl&export=download&authuser=0&confirm=t&uuid=c05d2bce-bccb-4504-8fe4-7c409788d727&at=APZUnTVlCicnIm0cMdJ9FrZg4MSN:1709453015475", + "dest": "lecture1-2021-with-ink_Gold.pptx" + }, + "result": { + "type": "vm_file", + "path": "C:\\Users\\chenj\\Desktop\\lecture1-2021-with-ink.pptx", + "dest": "lecture1-2021-with-ink.pptx" + }, + "options": { + "examine_shape": false, + "examine_bullets": false + } + } +} diff --git a/evaluation_examples/examples/Windows/multi_app/f918266a-b3e0-4914-865d-4faa564f1aef.json b/evaluation_examples/examples/Windows/multi_app/f918266a-b3e0-4914-865d-4faa564f1aef.json index 09c8983..2336d7a 100644 --- a/evaluation_examples/examples/Windows/multi_app/f918266a-b3e0-4914-865d-4faa564f1aef.json +++ b/evaluation_examples/examples/Windows/multi_app/f918266a-b3e0-4914-865d-4faa564f1aef.json @@ -27,7 +27,7 @@ { "type": "execute", "parameters": { - "command": "rm C:\\Users\\chenj\\Desktop\\calculator.zip", + "command": "del C:\\Users\\chenj\\Desktop\\calculator.zip", "shell": true } } @@ -46,7 +46,7 @@ }, "result": { "type": "vm_file", - "path": "/home/user/Desktop/log.txt", + "path": "C:\\Users\\chenj\\Desktop\\log.txt", "dest": "log.txt" } } diff --git a/evaluation_examples/examples/multi_apps/6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a.json b/evaluation_examples/examples/multi_apps/6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a.json index 414aab4..cb7d65e 100644 --- a/evaluation_examples/examples/multi_apps/6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a.json +++ b/evaluation_examples/examples/multi_apps/6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a.json @@ -1,50 +1,50 @@ { - "id": "6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a", - "snapshot": "multiapps", - "instruction": "I now want to count the meeting cities of the three machine learning conferences in the past ten years from 2013 to 2019(including 2013 and 2019). I have listed the names and years of the conferences in excel. Please fill in the vacant locations.", - "source": "author", - "config": [ - { - "type": "download", - "parameters": { - "files": [ - { - "url": "https://drive.google.com/uc?export=download&id=19wUxTQeoKr6ihJWJ_9cu2tzKQH0cnxWH", - "path": "/home/user/Desktop/ConferenceCity.xlsx" - } - ] - } - }, - { - "type": "open", - "parameters": { - "path": "/home/user/Desktop/ConferenceCity.xlsx" - } - } - ], - "trajectory": "trajectories/", - "related_apps": [ - "calc", "chrome", "os" - ], - "evaluator": { - "postconfig":[ - { - "type": "download", - "parameters": { - "files": [ - { - "url": "https://drive.google.com/uc?export=download&id=1ZcITkIOs2Z86S5L6MShSohFs3_xVfeCP", - "path": "/home/user/Desktop/ConferenceCity_Gold.xlsx" - } - ] - } - }, - { - "type": "activate_window", - "parameters": { - "window_name": "ConferenceCity.xlsx - LibreOffice Calc" - } - }, + "id": "6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a", + "snapshot": "multiapps", + "instruction": "I now want to count the meeting cities of the three machine learning conferences in the past ten years from 2013 to 2019(including 2013 and 2019). I have listed the names and years of the conferences in excel. Please fill in the vacant locations.", + "source": "author", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=19wUxTQeoKr6ihJWJ_9cu2tzKQH0cnxWH", + "path": "/home/user/Desktop/ConferenceCity.xlsx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/Desktop/ConferenceCity.xlsx" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "calc", "chrome", "os" + ], + "evaluator": { + "postconfig":[ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1ZcITkIOs2Z86S5L6MShSohFs3_xVfeCP", + "path": "/home/user/Desktop/ConferenceCity_Gold.xlsx" + } + ] + } + }, + { + "type": "activate_window", + "parameters": { + "window_name": "ConferenceCity.xlsx - LibreOffice Calc" + } + }, { "type": "execute", "parameters": { @@ -55,18 +55,18 @@ ] } } - ], - "func": "compare_conference_city_in_order", - "expected": { - "type": "rule", - "rules":{ - "expected": ["Scottsdale","Atlanta","Lake Tahoe","Banff","Beijing",["Montreal", "Montréal"],"San Diego","Lille",["Montreal", "Montréal"],"San Juan",["New York", "New York City", "NYC"],"Barcelona","Toulon","Sydney","Long Beach","Vancouver","Stockholm",["Montreal", "Montréal"],"New Orleans","Long Beach","Vancouver"] - } - }, - "result": { - "type": "vm_file", - "path": "/home/user/Desktop/ConferenceCity.xlsx", - "dest": "ConferenceCity.xlsx" - } - } - } \ No newline at end of file + ], + "func": "compare_conference_city_in_order", + "expected": { + "type": "rule", + "rules":{ + "expected": ["Scottsdale","Atlanta","Lake Tahoe","Banff","Beijing",["Montreal", "Montréal"],"San Diego","Lille",["Montreal", "Montréal"],"San Juan",["New York", "New York City", "NYC"],"Barcelona","Toulon","Sydney","Long Beach","Vancouver","Stockholm",["Montreal", "Montréal"],"New Orleans","Long Beach","Vancouver"] + } + }, + "result": { + "type": "vm_file", + "path": "/home/user/Desktop/ConferenceCity.xlsx", + "dest": "ConferenceCity.xlsx" + } + } +} diff --git a/evaluation_examples/examples/multi_apps/e2392362-125e-4f76-a2ee-524b183a3412.json b/evaluation_examples/examples/multi_apps/e2392362-125e-4f76-a2ee-524b183a3412.json index b591cfd..eb55e68 100644 --- a/evaluation_examples/examples/multi_apps/e2392362-125e-4f76-a2ee-524b183a3412.json +++ b/evaluation_examples/examples/multi_apps/e2392362-125e-4f76-a2ee-524b183a3412.json @@ -31,7 +31,7 @@ "command": [ "tar", "-xJvf", - ".tmp.tar.xz", + "/home/user/.tmp.tar.xz", "-C", "/home/user/Code/Website/" ] @@ -124,4 +124,4 @@ "dest": "_config.yaml" } } -} \ No newline at end of file +}