diff --git a/evaluation_examples/examples/vs_code/0512bb38-d531-4acf-9e7e-0add90816068.json b/evaluation_examples/examples/vs_code/0512bb38-d531-4acf-9e7e-0add90816068.json new file mode 100644 index 0000000..eba98ba --- /dev/null +++ b/evaluation_examples/examples/vs_code/0512bb38-d531-4acf-9e7e-0add90816068.json @@ -0,0 +1,57 @@ +{ + "id": "0512bb38-d531-4acf-9e7e-0add90816068", + "snapshot": "vscode", + "instruction": "Please help me install an extension in VS Code from a local VSIX file \"/home/user/test.vsix\".", + "source": "https://download.microsoft.com/download/8/A/4/8A48E46A-C355-4E5C-8417-E6ACD8A207D4/VisualStudioCode-TipsAndTricks-Vol.1.pdf", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1fu_JH3rQk93MHFzUcXJDKC68X_yXRNwJ&export=download&authuser=0&confirm=t&uuid=efb1a587-645a-4ee8-b8af-cfba2c6473b6&at=APZUnTWqz5t2nKbHZczIMW7aAMsz:1706668206536", + "path": "/home/user/test.vsix" + } + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "code" + ] + } + }, + { + "type": "activate_window", + "parameters": { + "window_name": "Visual Studio Code" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "vscode" + ], + "evaluator": { + "func": "is_extension_installed", + "result": { + "type": "vm_command_line", + "command": [ + "code", + "--list-extensions", + "|", + "grep", + "undefined_publisher.test" + ] + }, + "expected": { + "type": "rule", + "rules": { + "type": "contain", + "expected": "undefined_publisher.test" + } + } + } +} diff --git a/evaluation_examples/examples/vs_code/0ed39f63-6049-43d4-ba4d-5fa2fe04a951.json b/evaluation_examples/examples/vs_code/0ed39f63-6049-43d4-ba4d-5fa2fe04a951.json index c695041..6a96040 100644 --- a/evaluation_examples/examples/vs_code/0ed39f63-6049-43d4-ba4d-5fa2fe04a951.json +++ b/evaluation_examples/examples/vs_code/0ed39f63-6049-43d4-ba4d-5fa2fe04a951.json @@ -28,7 +28,7 @@ } } ], - "trajectory": "trajectories/0ed39f63-6049-43d4-ba4d-5fa2fe04a951", + "trajectory": "trajectories/", "related_apps": [ "vscode" ], diff --git a/evaluation_examples/examples/vs_code/276cc624-87ea-4f08-ab93-f770e3790175.json b/evaluation_examples/examples/vs_code/276cc624-87ea-4f08-ab93-f770e3790175.json index 7001729..bcbff41 100644 --- a/evaluation_examples/examples/vs_code/276cc624-87ea-4f08-ab93-f770e3790175.json +++ b/evaluation_examples/examples/vs_code/276cc624-87ea-4f08-ab93-f770e3790175.json @@ -28,7 +28,9 @@ "expected": { "type": "rule", "rules": { - "expect": {"editor.rulers": [50]} + "expect": { + "editor.wordWrapColumn": 50 + } } }, "result": { diff --git a/evaluation_examples/examples/vs_code/57242fad-77ca-454f-b71b-f187181a9f23.json b/evaluation_examples/examples/vs_code/57242fad-77ca-454f-b71b-f187181a9f23.json new file mode 100644 index 0000000..6b16da9 --- /dev/null +++ b/evaluation_examples/examples/vs_code/57242fad-77ca-454f-b71b-f187181a9f23.json @@ -0,0 +1,46 @@ +{ + "id": "57242fad-77ca-454f-b71b-f187181a9f23", + "snapshot": "vscode", + "instruction": "Please help me create a new python file named \"test.py\" via VS Code and save it at \"/home/user/Desktop\".", + "source": "https://download.microsoft.com/download/8/A/4/8A48E46A-C355-4E5C-8417-E6ACD8A207D4/VisualStudioCode-TipsAndTricks-Vol.1.pdf", + "config": [ + { + "type": "launch", + "parameters": { + "command": [ + "code" + ] + } + }, + { + "type": "activate_window", + "parameters": { + "window_name": "Visual Studio Code" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "vscode" + ], + "evaluator": { + "func": "is_extension_installed", + "result": { + "type": "vm_command_line", + "command": [ + "ls", + "/home/user/Desktop", + "|", + "grep", + "test.py" + ] + }, + "expected": { + "type": "rule", + "rules": { + "type": "contain", + "expected": "test.py" + } + } + } +} diff --git a/evaluation_examples/examples/vs_code/5e2d93d8-8ad0-4435-b150-1692aacaa994.json b/evaluation_examples/examples/vs_code/5e2d93d8-8ad0-4435-b150-1692aacaa994.json new file mode 100644 index 0000000..2be4512 --- /dev/null +++ b/evaluation_examples/examples/vs_code/5e2d93d8-8ad0-4435-b150-1692aacaa994.json @@ -0,0 +1,74 @@ +{ + "id": "5e2d93d8-8ad0-4435-b150-1692aacaa994", + "snapshot": "vscode", + "instruction": "Please help me save current project as workspace \"project\" at \"/home/user/\".", + "source": "https://www.youtube.com/watch?v=B-s71n0dHUk", + "config": [ + { + "type": "launch", + "parameters": { + "command": [ + "code" + ] + } + }, + { + "type": "command", + "parameters": { + "command": [ + "mkdir", + "-p", + "/home/user/project/.vscode" + ] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1akdsiRVdq6CUtT-FX8Dpf8ruPTq6DcFn&export=download&authuser=0&confirm=t&uuid=ce2fa96a-454e-43d9-bbe3-98553b7eed0d&at=APZUnTVw_YQ1URTvP34vrmKcw0b4:1705222451052", + "path": "/home/user/project/main.py" + }, + { + "url": "https://drive.usercontent.google.com/download?id=1BkwtqtAzv_K2CrTbJZ0HbMHBffzdD9vc&export=download&authuser=0&confirm=t&uuid=28f77090-deef-49a1-b156-91317881e75e&at=APZUnTXuaR6i_3t3Prslk535GaO5:1705222457290", + "path": "/home/user/project/README.md" + }, + { + "url": "https://drive.usercontent.google.com/download?id=1ea_zF2tbcXOB8w9neBV-U5xI2nnPzIw_&export=download&authuser=0&confirm=t&uuid=9cf8c5bb-a880-475c-b80b-967a0c4fbea4&at=APZUnTUdjIj80F3Mbgi72eZDTZLO:1705222462443", + "path": "/home/user/project/.vscode/settings.json" + } + ] + } + }, + { + "type": "activate_window", + "parameters": { + "window_name": "Visual Studio Code" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "vscode" + ], + "evaluator": { + "func": "is_extension_installed", + "result": { + "type": "vm_command_line", + "command": [ + "ls", + "|", + "grep", + "project.code-workspace" + ] + }, + "expected": { + "type": "rule", + "rules": { + "type": "contain", + "expected": "project.code-workspace" + } + } + } +} diff --git a/evaluation_examples/examples/vs_code/6ed0a554-cbee-4b44-84ea-fd6c042f4fe1.json b/evaluation_examples/examples/vs_code/6ed0a554-cbee-4b44-84ea-fd6c042f4fe1.json new file mode 100644 index 0000000..1d5f59e --- /dev/null +++ b/evaluation_examples/examples/vs_code/6ed0a554-cbee-4b44-84ea-fd6c042f4fe1.json @@ -0,0 +1,85 @@ +{ + "id": "6ed0a554-cbee-4b44-84ea-fd6c042f4fe1", + "snapshot": "vscode", + "instruction": "Please help me add folder \"/home/user/data1\" and folder \"/home/user/data2\" to the current workspace.", + "source": "https://www.youtube.com/watch?v=B-s71n0dHUk", + "config": [ + { + "type": "command", + "parameters": { + "command": [ + "mkdir", + "-p", + "/home/user/project" + ] + } + }, + { + "type": "command", + "parameters": { + "command": [ + "mkdir", + "-p", + "/home/user/data1" + ] + } + }, + { + "type": "command", + "parameters": { + "command": [ + "mkdir", + "-p", + "/home/user/data2" + ] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1_CzdGGcA70VZnXWbFkxnMCS_HSl2B16A&export=download&authuser=0&confirm=t&uuid=6d08572f-04c6-42ca-8d02-b6adf3f0a271&at=APZUnTU3hL_yPkvSKokYA-Mywj1Z:1706666120779", + "path": "/home/user/project.code-workspace" + } + ] + } + }, + { + "type": "launch", + "parameters": { + "command": ["code", "/home/user/project.code-workspace"] + } + }, + { + "type": "activate_window", + "parameters": { + "window_name": "Visual Studio Code" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "vscode" + ], + "evaluator": { + "func": "check_json_settings", + "expected": { + "type": "rule", + "rules": { + "expect": { + "folders": [ + {"path": "project"}, + {"path": "data1"}, + {"path": "data2"} + ] + } + } + }, + "result": { + "type": "vm_file", + "path": "/home/user/project.code-workspace", + "dest": "project.code-workspace" + } + } +} diff --git a/evaluation_examples/examples/vs_code/70745df8-f2f5-42bd-8074-fbc10334fcc5.json b/evaluation_examples/examples/vs_code/70745df8-f2f5-42bd-8074-fbc10334fcc5.json new file mode 100644 index 0000000..8526836 --- /dev/null +++ b/evaluation_examples/examples/vs_code/70745df8-f2f5-42bd-8074-fbc10334fcc5.json @@ -0,0 +1,43 @@ +{ + "id": "70745df8-f2f5-42bd-8074-fbc10334fcc5", + "snapshot": "vscode", + "instruction": "Please help me open the autosave feature of VS Code and delay AutoSave operations for 500 milliseconds in the VS Code setting.", + "source": "https://download.microsoft.com/download/8/A/4/8A48E46A-C355-4E5C-8417-E6ACD8A207D4/VisualStudioCode-TipsAndTricks-Vol.1.pdf", + "config": [ + { + "type": "launch", + "parameters": { + "command": [ + "code" + ] + } + }, + { + "type": "activate_window", + "parameters": { + "window_name": "Visual Studio Code" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "vscode" + ], + "evaluator": { + "func": "check_json_settings", + "expected": { + "type": "rule", + "rules": { + "expect": { + "files.autoSave": "afterDelay", + "files.autoSaveDelay": 500 + } + } + }, + "result": { + "type": "vm_file", + "path": "/home/user/.config/Code/User/settings.json", + "dest": "settings.json" + } + } +} diff --git a/evaluation_examples/examples/vs_code/c6bf789c-ba3a-4209-971d-b63abf0ab733.json b/evaluation_examples/examples/vs_code/c6bf789c-ba3a-4209-971d-b63abf0ab733.json new file mode 100644 index 0000000..d1163e8 --- /dev/null +++ b/evaluation_examples/examples/vs_code/c6bf789c-ba3a-4209-971d-b63abf0ab733.json @@ -0,0 +1,44 @@ +{ + "id": "c6bf789c-ba3a-4209-971d-b63abf0ab733", + "snapshot": "vscode", + "instruction": "Please help me modify VS Code setting to hide all \"__pycache__\" folders in the explorer view.", + "source": "https://download.microsoft.com/download/8/A/4/8A48E46A-C355-4E5C-8417-E6ACD8A207D4/VisualStudioCode-TipsAndTricks-Vol.1.pdf", + "config": [ + { + "type": "launch", + "parameters": { + "command": [ + "code" + ] + } + }, + { + "type": "activate_window", + "parameters": { + "window_name": "Visual Studio Code" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "vscode" + ], + "evaluator": { + "func": "check_json_settings", + "expected": { + "type": "rule", + "rules": { + "expect": { + "files.exclude": { + "**/__pycache__": true + } + } + } + }, + "result": { + "type": "vm_file", + "path": "/home/user/.config/Code/User/settings.json", + "dest": "settings.json" + } + } +} diff --git a/evaluation_examples/examples/vs_code/e2b5e914-ffe1-44d2-8e92-58f8c5d92bb2.json b/evaluation_examples/examples/vs_code/e2b5e914-ffe1-44d2-8e92-58f8c5d92bb2.json index 756ddc3..e8fa810 100644 --- a/evaluation_examples/examples/vs_code/e2b5e914-ffe1-44d2-8e92-58f8c5d92bb2.json +++ b/evaluation_examples/examples/vs_code/e2b5e914-ffe1-44d2-8e92-58f8c5d92bb2.json @@ -4,6 +4,16 @@ "instruction": "Please modify VS Code's settings to disable error reporting for Python missing imports.", "source": "https://superuser.com/questions/1386061/how-to-suppress-some-python-errors-warnings-in-vs-code", "config": [ + { + "type": "command", + "parameters": { + "command": [ + "code", + "--install-extension", + "ms-python.python" + ] + } + }, { "type": "launch", "parameters": { diff --git a/evaluation_examples/examples/vs_code/ec71221e-ac43-46f9-89b8-ee7d80f7e1c5.json b/evaluation_examples/examples/vs_code/ec71221e-ac43-46f9-89b8-ee7d80f7e1c5.json new file mode 100644 index 0000000..88aa820 --- /dev/null +++ b/evaluation_examples/examples/vs_code/ec71221e-ac43-46f9-89b8-ee7d80f7e1c5.json @@ -0,0 +1,48 @@ +{ + "id": "ec71221e-ac43-46f9-89b8-ee7d80f7e1c5", + "snapshot": "vscode", + "instruction": "Please help me increase the indent of line 2 to line 10 by one tab.", + "source": "https://stackoverflow.com/questions/47903209/how-to-shift-a-block-of-code-left-right-by-one-space-in-vscode", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1k4rv_meWQyYfNMHTWipJnvh3uA5LWnWa&export=download&authuser=0&confirm=t&uuid=e923fec7-a55c-4266-8d40-be017ac54c7d&at=APZUnTVDyVA7V72IC3r2vrjwY0xz:1706666734010", + "path": "/home/user/Desktop/test.py" + } + ] + } + }, + { + "type": "launch", + "parameters": { + "command": ["code", "/home/user/Desktop/test.py"] + } + }, + { + "type": "activate_window", + "parameters": { + "window_name": "Visual Studio Code" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "vscode" + ], + "evaluator": { + "func": "compare_text_file", + "expected": { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=1fNZ4r-G58QVo5yFjS1jTqxFJAXXNyf1G&export=download&authuser=0&confirm=t&uuid=c577a187-1225-4d98-98da-d3b33b9e6c82&at=APZUnTXSUy1dHiC2Oavo5g-wRJe8:1706666819711", + "dest": "test_gold.py" + }, + "result": { + "type": "vm_file", + "path": "/home/user/Desktop/test.py", + "dest": "test.py" + } + } +}