diff --git a/evaluation_examples/examples/multi_apps/bc2b57f3-686d-4ec9-87ce-edf850b7e442.json b/evaluation_examples/examples/multi_apps/bc2b57f3-686d-4ec9-87ce-edf850b7e442.json new file mode 100644 index 0000000..8e1a33a --- /dev/null +++ b/evaluation_examples/examples/multi_apps/bc2b57f3-686d-4ec9-87ce-edf850b7e442.json @@ -0,0 +1,125 @@ +{ + "id": "bc2b57f3-686d-4ec9-87ce-edf850b7e442", + "snapshot": "libreoffice_calc", + "instruction": "The requirements of my data analysis assignment are listed in \"reminder.docx\" on the desktop. Help me modify my assignment \"asm.xlsx\" saved on the desktop accordingly.", + "source": "authors", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1ZEX26Ns6BWyUKe4FKlGUeKO7G7xSN14x&export=download&authuser=0&confirm=t&uuid=68789dfd-5725-4f2c-9653-7988d2d3b1ff&at=APZUnTWwT4sNxjJwmmYhG-X8uMsb:1709372377572", + "path": "/home/user/Desktop/workbook-with-sample-database.xlsx" + }, + { + "url": "https://drive.usercontent.google.com/download?id=1jlVlK1Ps2JCxT9UiREjABEt8HlhkMb52&export=download&authuser=0&confirm=t&uuid=2fd4d607-c67c-4600-8f42-d7ed90b2dc4f&at=APZUnTVkTeiVpplx8qx3w2ek1j6f:1709373115860", + "path": "/home/user/Desktop/reminder.docx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/Desktop/workbook-with-sample-database.xlsx" + } + } + ], + "trajectory": "trajectories/bc2b57f3-686d-4ec9-87ce-edf850b7e442", + "related_apps": [ + "libreoffice_calc", + "libreoffice_writer" + ], + "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "workbook-with-sample-database.xlsx - LibreOffice Calc", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 5 + } + } + ], + "func": "compare_table", + "result": { + "type": "vm_file", + "path": "/home/user/Desktop/workbook-with-sample-database.xlsx", + "dest": "workbook-with-sample-database.xlsx" + }, + "expected": { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=1wtSpWp1rWUaRZUElqZ1yT0NOiCv4etZY&export=download&authuser=0&confirm=t&uuid=b99c4600-37a0-407e-9c57-7152e09ed24d&at=APZUnTWV6WzfnqnDDnk6QXn3Shu-:1709371237340", + "dest": "workbook-with-sample-database_Gold.xlsx" + }, + "options": { + "rules": [ + { + "type": "sheet_name" + }, + { + "type": "sheet_data", + "sheet_idx0": "RI0", + "sheet_idx1": "EI0" + }, + { + "type": "sheet_data", + "sheet_idx0": "RI1", + "sheet_idx1": "EI1" + }, + { + "type": "sheet_data", + "sheet_idx0": "RI2", + "sheet_idx1": "EI2" + }, + { + "type": "sheet_data", + "sheet_idx0": "RI3", + "sheet_idx1": "EI3" + }, + { + "type": "sheet_data", + "sheet_idx0": "RI4", + "sheet_idx1": "EI4" + }, + { + "type": "sheet_data", + "sheet_idx0": "RI5", + "sheet_idx1": "EI5" + }, + { + "type": "sheet_data", + "sheet_idx0": "RI6", + "sheet_idx1": "EI6" + }, + { + "type": "sheet_data", + "sheet_idx0": "RI7", + "sheet_idx1": "EI7" + } + ] + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/da52d699-e8d2-4dc5-9191-a2199e0b6a9b.json b/evaluation_examples/examples/multi_apps/da52d699-e8d2-4dc5-9191-a2199e0b6a9b.json new file mode 100644 index 0000000..ad5e978 --- /dev/null +++ b/evaluation_examples/examples/multi_apps/da52d699-e8d2-4dc5-9191-a2199e0b6a9b.json @@ -0,0 +1,73 @@ +{ + "id": "da52d699-e8d2-4dc5-9191-a2199e0b6a9b", + "snapshot": "libreoffice_calc", + "instruction": "The spreadsheet on Desktop contains a list of books I read in the year 2022. What is the title of the book that I read the slowest, using the rate of words per day? I have an empty writer file 'book_list_result.docx' on Desktop. Open it and write the title down there.", + "source": "GAIA", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1JGZNCShtmpu7A8Z8lkjc8hdFEAMXZVvh&export=download&authuser=0&confirm=t&uuid=67063da6-2a72-4ed2-92b2-ade508439ce4&at=APZUnTUgS17YjX-D0oSvALwnPosB:1709368886960", + "path": "/home/user/Desktop/2023_validation_Book_Reading_Rate.xlsx" + }, + { + "url": "https://drive.usercontent.google.com/download?id=1iySmK8zvTzgmERH7KQuESP05NBsMunhV&export=download&authuser=0&confirm=t&uuid=130f6cee-0f9a-4f2e-a84d-89a3b302f350&at=APZUnTXugQOTOApe1_zxUbafo2Sp:1709369519349", + "path": "/home/user/Desktop/book_list_result.docx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/Desktop/2023_validation_Book_Reading_Rate.xlsx" + } + } + ], + "trajectory": "trajectories/da52d699-e8d2-4dc5-9191-a2199e0b6a9b", + "related_apps": [ + "libreoffice_calc", + "chrome", + "libreoffice_writer" + ], + "evaluator": { + "func": "compare_docx_files", + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "book_list_result.docx - LibreOffice Writer", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); " + ] + } + } + ], + "expected": { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=1rpvOlHZO0AqC85od8pJtx8YcDPljcejN&export=download&authuser=0&confirm=t&uuid=24a3a5e3-a188-4a41-ad01-a4709dc1c0b6&at=APZUnTWx56rr8-iTuXkfV5poOK-I:1709369145446", + "dest": "book_list_result_Gold.docx" + }, + "result": { + "type": "vm_file", + "path": "/home/user/Desktop/book_list_result.docx", + "dest": "book_list_result.docx" + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/f918266a-b3e0-4914-865d-4faa564f1aef.json b/evaluation_examples/examples/multi_apps/f918266a-b3e0-4914-865d-4faa564f1aef.json new file mode 100644 index 0000000..ea3fbf8 --- /dev/null +++ b/evaluation_examples/examples/multi_apps/f918266a-b3e0-4914-865d-4faa564f1aef.json @@ -0,0 +1,47 @@ +{ + "id": "f918266a-b3e0-4914-865d-4faa564f1aef", + "snapshot": "vscode", + "instruction": "Help me get the output of Python code 'calculator.py' on the desktop. Save it as 'log.txt' at the same place with the python file.", + "source": "GAIA", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1l09TnSiXo-qOK2UazcIdrT_M6JwTfzq7&export=download&authuser=0&confirm=t&uuid=b9c4fc67-2fdd-4dc5-a84f-9ce0b2031c91&at=APZUnTUSK7Codrqfu6J_Lvy5AlT3:1709366036288", + "path": "/home/user/Desktop/calculator.zip" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "/bin/bash", + "-c", + "unzip /home/user/Desktop/calculator.zip -d /home/user/Desktop/ && rm -rf /home/user/Desktop/calculator.zip" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "vscode", + "os" + ], + "evaluator": { + "func": "compare_text_file", + "expected": { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=1-14AgA1nHNL22VD_3QtRzWaMjIBa3RvJ&export=download&authuser=0&confirm=t&uuid=91e2d5bc-a7f0-4abc-9aed-aa8b4956fb45&at=APZUnTU2TksSVngTw6WRJv8wpmoU:1709367652463", + "dest": "log.txt" + }, + "result": { + "type": "vm_file", + "path": "/home/user/Desktop/log.txt", + "dest": "log.txt" + } + } +} \ No newline at end of file