diff --git a/evaluation_examples/examples/multi_apps/68a25bd4-59c7-4f4d-975e-da0c8509c848.json b/evaluation_examples/examples/multi_apps/68a25bd4-59c7-4f4d-975e-da0c8509c848.json new file mode 100644 index 0000000..d76f77d --- /dev/null +++ b/evaluation_examples/examples/multi_apps/68a25bd4-59c7-4f4d-975e-da0c8509c848.json @@ -0,0 +1,76 @@ +{ + "id": "68a25bd4-59c7-4f4d-975e-da0c8509c848", + "snapshot": "libreoffice_calc", + "instruction": "I have collected papers and books with link in this calc file. Help me download the pdf file of the first paper as \"paper01.pdf\" to /home/user.", + "source": "", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1g2lhX7CtuT6kc2Bmss0WU5zT1eXnnbEU&export=download&authuser=0&confirm=t&uuid=6c4ebeda-cab3-4b2b-b11a-d725bc72b2ce&at=APZUnTXhm8ukSDB7bI438hBvbdjp:1709201055394", + "path": "/home/user/Desktop/rsc-ebook-collection-2023.xlsx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/Desktop/rsc-ebook-collection-2023.xlsx" + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "google-chrome", + "--remote-debugging-port=1337" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "socat", + "tcp-listen:9222,fork", + "tcp:localhost:1337" + ] + } + }, + { + "type": "chrome_open_tabs", + "parameters": { + "urls_to_open": [ + "https://www.qatarairways.com/en-hk/homepage.html" + ] + } + }, + { + "type": "activate_window", + "parameters": { + "window_name": "Google Chrome" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "libreoffice_calc", + "chrome" + ], + "evaluator": { + "func": "compare_pdfs", + "expected": { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=1kAp7ulAR_h4snny212yg1xyR1cMy3H2Q&export=download&authuser=0&confirm=t&uuid=40dd202b-dc6a-4b4c-b541-b2f09e16ce6f&at=APZUnTWl-vx9Y40otSzf2SBxLXcN:1709201363719", + "dest": "paper01.pdf" + }, + "result": { + "type": "vm_file", + "path": "/home/user/paper01.pdf", + "dest": "paper01.pdf" + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/869de13e-bef9-4b91-ba51-f6708c40b096.json b/evaluation_examples/examples/multi_apps/869de13e-bef9-4b91-ba51-f6708c40b096.json new file mode 100644 index 0000000..1edc6a0 --- /dev/null +++ b/evaluation_examples/examples/multi_apps/869de13e-bef9-4b91-ba51-f6708c40b096.json @@ -0,0 +1,84 @@ +{ + "id": "869de13e-bef9-4b91-ba51-f6708c40b096", + "snapshot": "os", + "instruction": "Can you organize my desktop by identifying schoolwork-related files, coding projects, and random downloads, ensuring no file is misplaced? Specifically, schoolwork files should end up in the 'Work' folder, coding projects in 'Projects', and everything else in 'Miscellaneous'. Use content analysis for files without clear extensions or names.", + "source": "", + "config": [ + { + "type": "command", + "parameters": { + "command": [ + "mkdir", + "-p", + "/home/user/Desktop/Work" + ] + } + }, + { + "type": "command", + "parameters": { + "command": [ + "mkdir", + "-p", + "/home/user/Desktop/Projects" + ] + } + }, + { + "type": "command", + "parameters": { + "command": [ + "mkdir", + "-p", + "/home/user/Desktop/Miscellaneous" + ] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1akdsiRVdq6CUtT-FX8Dpf8ruPTq6DcFn&export=download&authuser=0&confirm=t&uuid=ce2fa96a-454e-43d9-bbe3-98553b7eed0d&at=APZUnTVw_YQ1URTvP34vrmKcw0b4:1705222451052", + "path": "/home/user/project/main.py" + }, + { + "url": "https://drive.usercontent.google.com/download?id=1BkwtqtAzv_K2CrTbJZ0HbMHBffzdD9vc&export=download&authuser=0&confirm=t&uuid=28f77090-deef-49a1-b156-91317881e75e&at=APZUnTXuaR6i_3t3Prslk535GaO5:1705222457290", + "path": "/home/user/project/README.md" + }, + { + "url": "https://drive.usercontent.google.com/download?id=1ea_zF2tbcXOB8w9neBV-U5xI2nnPzIw_&export=download&authuser=0&confirm=t&uuid=9cf8c5bb-a880-475c-b80b-967a0c4fbea4&at=APZUnTUdjIj80F3Mbgi72eZDTZLO:1705222462443", + "path": "/home/user/project/.vscode/settings.json" + } + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "os", + "libreoffice_calc", + "libreoffice_writer", + "libreoffice_impress" + ], + "evaluator": { + "func": "is_extension_installed", + "result": { + "type": "vm_command_line", + "command": [ + "ls", + "/home/user/project/", + "|", + "grep", + "project.code-workspace" + ] + }, + "expected": { + "type": "rule", + "rules": { + "type": "contain", + "expected": "project.code-workspace" + } + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/c7c1e4c3-9e92-4eba-a4b8-689953975ea4.json b/evaluation_examples/examples/multi_apps/c7c1e4c3-9e92-4eba-a4b8-689953975ea4.json new file mode 100644 index 0000000..91b59c1 --- /dev/null +++ b/evaluation_examples/examples/multi_apps/c7c1e4c3-9e92-4eba-a4b8-689953975ea4.json @@ -0,0 +1,110 @@ +{ + "id": "c7c1e4c3-9e92-4eba-a4b8-689953975ea4", + "snapshot": "libreoffice_calc", + "instruction": "I am collecting the contact information of some scholars and have their homepage links listed here. Help me fill in their email address in the form.", + "source": "", + "config": [ + { + "type": "launch", + "parameters": { + "command": [ + "google-chrome", + "--remote-debugging-port=1337" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "socat", + "tcp-listen:9222,fork", + "tcp:localhost:1337" + ] + } + }, + { + "type": "chrome_open_tabs", + "parameters": { + "urls_to_open": [ + "https://www.qatarairways.com/en-hk/homepage.html" + ] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1ZAia9SFnujBlmxCr5G_np6GtHp_Xzvc-&export=download&authuser=0&confirm=t&uuid=40c8cfef-cffb-416a-9e97-43cfbb50f17c&at=APZUnTUKoioAWjNtx7fqYRvf_Pbz:1709208000844", + "path": "/home/user/Desktop/Professor_Contact.xlsx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/Desktop/Professor_Contact.xlsx" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "libreoffice_calc", + "chrome" + ], + "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "Professor_Contact.xlsx - LibreOffice Calc", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + } + ], + "func": "compare_table", + "expected": { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=1npesHVfOx8foPhjb9zYqsxT8_pvDcnII&export=download&authuser=0&confirm=t&uuid=396bb464-1308-4220-b795-ed442ff96303&at=APZUnTVTJMG5GeDCJTratyJZExCb:1709207996808", + "dest": "Professor_Contact_Gold.xlsx" + }, + "result": { + "type": "vm_file", + "path": "/home/user/Desktop/Professor_Contact.xlsx", + "dest": "Professor_Contact.xlsx" + }, + "options": { + "rules": [ + { + "type": "sheet_data", + "sheet_idx0": 0, + "sheet_idx1": "EI0" + } + ] + } + } +} \ No newline at end of file