diff --git a/evaluation_examples/examples/libreoffice_writer/0810415c-bde4-4443-9047-d5f70165a697.json b/evaluation_examples/examples/libreoffice_writer/0810415c-bde4-4443-9047-d5f70165a697.json index 0fe3557..b4adf35 100644 --- a/evaluation_examples/examples/libreoffice_writer/0810415c-bde4-4443-9047-d5f70165a697.json +++ b/evaluation_examples/examples/libreoffice_writer/0810415c-bde4-4443-9047-d5f70165a697.json @@ -64,4 +64,4 @@ "dest": "Novels_Intro_Packet.docx" } } -} +} \ No newline at end of file diff --git a/evaluation_examples/examples/libreoffice_writer/6ada715d-3aae-4a32-a6a7-429b2e43fb93.json b/evaluation_examples/examples/libreoffice_writer/6ada715d-3aae-4a32-a6a7-429b2e43fb93.json index f7a3b88..1c1852a 100644 --- a/evaluation_examples/examples/libreoffice_writer/6ada715d-3aae-4a32-a6a7-429b2e43fb93.json +++ b/evaluation_examples/examples/libreoffice_writer/6ada715d-3aae-4a32-a6a7-429b2e43fb93.json @@ -68,7 +68,7 @@ "command": [ "python", "-c", - "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); " + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter');" ] } } diff --git a/evaluation_examples/examples/libreoffice_writer/e528b65e-1107-4b8c-8988-490e4fece599.json b/evaluation_examples/examples/libreoffice_writer/e528b65e-1107-4b8c-8988-490e4fece599.json index a224e68..73571d9 100644 --- a/evaluation_examples/examples/libreoffice_writer/e528b65e-1107-4b8c-8988-490e4fece599.json +++ b/evaluation_examples/examples/libreoffice_writer/e528b65e-1107-4b8c-8988-490e4fece599.json @@ -1,7 +1,7 @@ { "id": "e528b65e-1107-4b8c-8988-490e4fece599", "snapshot": "libreoffice_writer", - "instruction": "Capitalize the first letter of all words.", + "instruction": "Please help me make the first letter of each word to uppercase.", "source": "https://www.youtube.com/watch?v=l25Evu4ohKg", "config": [ { diff --git a/evaluation_examples/examples/vs_code/30167b0c-976b-491b-a9e5-7747607c6a3c.json b/evaluation_examples/examples/vs_code/30167b0c-976b-491b-a9e5-7747607c6a3c.json new file mode 100644 index 0000000..c685c05 --- /dev/null +++ b/evaluation_examples/examples/vs_code/30167b0c-976b-491b-a9e5-7747607c6a3c.json @@ -0,0 +1,62 @@ +{ + "id": "30167b0c-976b-491b-a9e5-7747607c6a3c", + "snapshot": "vscode", + "instruction": "Please help me find the file that contains \"import openai\" in the current VS Code project and open that file.", + "source": "https://download.microsoft.com/download/8/A/4/8A48E46A-C355-4E5C-8417-E6ACD8A207D4/VisualStudioCode-TipsAndTricks-Vol.1.pdf", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://codeload.github.com/onlyphantom/llm-python/zip/refs/heads/main", + "path": "/home/user/Downloads/llm-python-main.zip" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "/bin/bash", + "-c", + "cd /home/user/Downloads && unzip -q llm-python-main.zip" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "code", + "/home/user/Downloads/llm-python-main/" + ] + } + }, + { + "type": "activate_window", + "parameters": { + "window_name": "Visual Studio Code" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "vscode" + ], + "evaluator": { + "func": "check_json_settings", + "expected": { + "type": "rule", + "rules": { + "expect": {} + } + }, + "result": { + "type": "vm_file", + "path": "/home/user/.config/Code/User/settings.json", + "dest": "settings.json" + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/vs_code/5e2d93d8-8ad0-4435-b150-1692aacaa994.json b/evaluation_examples/examples/vs_code/5e2d93d8-8ad0-4435-b150-1692aacaa994.json index 2be4512..3554735 100644 --- a/evaluation_examples/examples/vs_code/5e2d93d8-8ad0-4435-b150-1692aacaa994.json +++ b/evaluation_examples/examples/vs_code/5e2d93d8-8ad0-4435-b150-1692aacaa994.json @@ -4,14 +4,6 @@ "instruction": "Please help me save current project as workspace \"project\" at \"/home/user/\".", "source": "https://www.youtube.com/watch?v=B-s71n0dHUk", "config": [ - { - "type": "launch", - "parameters": { - "command": [ - "code" - ] - } - }, { "type": "command", "parameters": { @@ -41,6 +33,15 @@ ] } }, + { + "type": "launch", + "parameters": { + "command": [ + "code", + "/home/user/project/" + ] + } + }, { "type": "activate_window", "parameters": { @@ -58,6 +59,7 @@ "type": "vm_command_line", "command": [ "ls", + "/home/user/project/", "|", "grep", "project.code-workspace" @@ -71,4 +73,4 @@ } } } -} +} \ No newline at end of file diff --git a/evaluation_examples/examples/vs_code/70745df8-f2f5-42bd-8074-fbc10334fcc5.json b/evaluation_examples/examples/vs_code/70745df8-f2f5-42bd-8074-fbc10334fcc5.json index 8526836..864a07c 100644 --- a/evaluation_examples/examples/vs_code/70745df8-f2f5-42bd-8074-fbc10334fcc5.json +++ b/evaluation_examples/examples/vs_code/70745df8-f2f5-42bd-8074-fbc10334fcc5.json @@ -4,11 +4,33 @@ "instruction": "Please help me open the autosave feature of VS Code and delay AutoSave operations for 500 milliseconds in the VS Code setting.", "source": "https://download.microsoft.com/download/8/A/4/8A48E46A-C355-4E5C-8417-E6ACD8A207D4/VisualStudioCode-TipsAndTricks-Vol.1.pdf", "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://storage.googleapis.com/drive-bulk-export-anonymous/20240207T164539.876Z/4133399871716478688/90e8000b-16d6-481a-a8eb-de0f4613b6c2/1/c7b78392-6016-4d48-9c04-1110303c47c1?authuser", + "path": "/home/user/Downloads/HW-8-main-20240207T164539Z-001.zip" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "/bin/bash", + "-c", + "cd /home/user/Downloads && unzip -q HW-8-main-20240207T164539Z-001.zip" + ] + } + }, { "type": "launch", "parameters": { "command": [ - "code" + "code", + "/home/user/Downloads/HW-8-main/" ] } }, @@ -40,4 +62,4 @@ "dest": "settings.json" } } -} +} \ No newline at end of file diff --git a/evaluation_examples/examples/vs_code/ae506c68-352c-4094-9caa-ee9d42052317.json b/evaluation_examples/examples/vs_code/ae506c68-352c-4094-9caa-ee9d42052317.json index 55394d9..3d59daf 100644 --- a/evaluation_examples/examples/vs_code/ae506c68-352c-4094-9caa-ee9d42052317.json +++ b/evaluation_examples/examples/vs_code/ae506c68-352c-4094-9caa-ee9d42052317.json @@ -36,4 +36,4 @@ "dest": "history.txt" } } -} +} \ No newline at end of file diff --git a/evaluation_examples/examples/vs_code/c6bf789c-ba3a-4209-971d-b63abf0ab733.json b/evaluation_examples/examples/vs_code/c6bf789c-ba3a-4209-971d-b63abf0ab733.json index d1163e8..af52359 100644 --- a/evaluation_examples/examples/vs_code/c6bf789c-ba3a-4209-971d-b63abf0ab733.json +++ b/evaluation_examples/examples/vs_code/c6bf789c-ba3a-4209-971d-b63abf0ab733.json @@ -4,11 +4,33 @@ "instruction": "Please help me modify VS Code setting to hide all \"__pycache__\" folders in the explorer view.", "source": "https://download.microsoft.com/download/8/A/4/8A48E46A-C355-4E5C-8417-E6ACD8A207D4/VisualStudioCode-TipsAndTricks-Vol.1.pdf", "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://storage.googleapis.com/drive-bulk-export-anonymous/20240207T140008.374Z/4133399871716478688/c9171ab5-0655-4c8e-aac8-e2cf6705e9f3/1/01132d9e-2db6-4984-a1ae-e1ff9150367b?authuser", + "path": "/home/user/Downloads/hw_python_oop-master.zip" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "/bin/bash", + "-c", + "cd /home/user/Downloads && unzip -q hw_python_oop-master.zip" + ] + } + }, { "type": "launch", "parameters": { "command": [ - "code" + "code", + "/home/user/Downloads/hw_python_oop-master" ] } }, @@ -30,7 +52,7 @@ "rules": { "expect": { "files.exclude": { - "**/__pycache__": true + "**/__pycache__": true } } } @@ -41,4 +63,4 @@ "dest": "settings.json" } } -} +} \ No newline at end of file diff --git a/evaluation_examples/examples/vs_code/ec71221e-ac43-46f9-89b8-ee7d80f7e1c5.json b/evaluation_examples/examples/vs_code/ec71221e-ac43-46f9-89b8-ee7d80f7e1c5.json index 88aa820..11d1627 100644 --- a/evaluation_examples/examples/vs_code/ec71221e-ac43-46f9-89b8-ee7d80f7e1c5.json +++ b/evaluation_examples/examples/vs_code/ec71221e-ac43-46f9-89b8-ee7d80f7e1c5.json @@ -18,7 +18,10 @@ { "type": "launch", "parameters": { - "command": ["code", "/home/user/Desktop/test.py"] + "command": [ + "code", + "/home/user/Desktop/test.py" + ] } }, { @@ -33,6 +36,36 @@ "vscode" ], "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "test.py - Visual Studio Code" + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5); " + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + } + ], "func": "compare_text_file", "expected": { "type": "cloud_file", @@ -45,4 +78,4 @@ "dest": "test.py" } } -} +} \ No newline at end of file