From 980f7290eb523603e62d776e217dba12382c0379 Mon Sep 17 00:00:00 2001 From: Siheng Zhao <1730625285@qq.com> Date: Sat, 20 Jan 2024 19:44:52 +0800 Subject: [PATCH] add vscode examples --- desktop_env/evaluators/metrics/vscode.py | 39 +++++++++++++++ .../276cc624-87ea-4f08-ab93-f770e3790175.json | 40 ++++++++++++++++ .../3486f395-ad68-459c-8c39-ea07de934dd4.json | 12 ----- .../4e60007a-f5be-4bfc-9723-c39affa0a6d3.json | 46 ++++++++++++++++++ .../515630d2-9b30-430c-b06a-e86b0143f7fb.json | 12 ----- .../59ed65c7-e9a6-43db-833f-76d6730c0004.json | 2 +- .../6f7546b0-52f3-4938-9213-52f35454d314.json | 12 ----- .../90f6eeeb-f3c2-4c98-873c-e77d78a45578.json | 12 ----- .../930fdb3b-11a8-46fe-9bac-577332e2640e.json | 45 ++++++++++++++++++ .../9439a27b-18ae-42d8-9778-5f68f891805e.json | 40 ++++++++++++++++ .../9d425400-e9b2-4424-9a4b-d4c7abac4140.json | 40 ++++++++++++++++ .../ae506c68-352c-4094-9caa-ee9d42052317.json | 39 +++++++++++++++ .../e2b5e914-ffe1-44d2-8e92-58f8c5d92bb2.json | 42 +++++++++++++++++ .../ea98c5d7-3cf9-4f9b-8ad3-366b58e0fcae.json | 47 +++++++++++++++++++ 14 files changed, 379 insertions(+), 49 deletions(-) create mode 100644 evaluation_examples/examples/vs_code/276cc624-87ea-4f08-ab93-f770e3790175.json delete mode 100644 evaluation_examples/examples/vs_code/3486f395-ad68-459c-8c39-ea07de934dd4.json create mode 100644 evaluation_examples/examples/vs_code/4e60007a-f5be-4bfc-9723-c39affa0a6d3.json delete mode 100644 evaluation_examples/examples/vs_code/515630d2-9b30-430c-b06a-e86b0143f7fb.json delete mode 100644 evaluation_examples/examples/vs_code/6f7546b0-52f3-4938-9213-52f35454d314.json delete mode 100644 evaluation_examples/examples/vs_code/90f6eeeb-f3c2-4c98-873c-e77d78a45578.json create mode 100644 evaluation_examples/examples/vs_code/930fdb3b-11a8-46fe-9bac-577332e2640e.json create mode 100644 evaluation_examples/examples/vs_code/9439a27b-18ae-42d8-9778-5f68f891805e.json create mode 100644 evaluation_examples/examples/vs_code/9d425400-e9b2-4424-9a4b-d4c7abac4140.json create mode 100644 evaluation_examples/examples/vs_code/ae506c68-352c-4094-9caa-ee9d42052317.json create mode 100644 evaluation_examples/examples/vs_code/e2b5e914-ffe1-44d2-8e92-58f8c5d92bb2.json create mode 100644 evaluation_examples/examples/vs_code/ea98c5d7-3cf9-4f9b-8ad3-366b58e0fcae.json diff --git a/desktop_env/evaluators/metrics/vscode.py b/desktop_env/evaluators/metrics/vscode.py index ac98d72..55f6407 100644 --- a/desktop_env/evaluators/metrics/vscode.py +++ b/desktop_env/evaluators/metrics/vscode.py @@ -1,5 +1,44 @@ from typing import Dict +import json +def check_json_keybindings(actual: str, expected: str, **options) -> float: + """ + Args: + actual (str): path to result text file + expected (str): expected dict{} + + Return: + float: the score + """ + + with open(actual) as f: + data = json.load(f) + + if expected in data: + return 1.0 + else: + return 0.0 + +def check_json_settings(actual: str, expected: str, **options) -> float: + """ + Args: + actual (str): path to result text file + expected (str): expected dict{} + + Return: + float: the score + """ + + with open(actual) as f: + data = json.load(f) + + expect = set(expected.items()) + json = set(data.items()) + + if expect.issubset(json): + return 1.0 + else: + return 0.0 def compare_text_file(actual: str, expected: str, **options) -> float: """ diff --git a/evaluation_examples/examples/vs_code/276cc624-87ea-4f08-ab93-f770e3790175.json b/evaluation_examples/examples/vs_code/276cc624-87ea-4f08-ab93-f770e3790175.json new file mode 100644 index 0000000..629dbe7 --- /dev/null +++ b/evaluation_examples/examples/vs_code/276cc624-87ea-4f08-ab93-f770e3790175.json @@ -0,0 +1,40 @@ +{ + "id": "276cc624-87ea-4f08-ab93-f770e3790175", + "snapshot": "vscode", + "instruction": "Could you help me set the line length to 50 characters for current user in VS Code?", + "source": "https://www.quora.com/unanswered/How-do-you-set-the-line-length-in-Visual-Studio-Code", + "config": [ + { + "type": "launch", + "parameters": { + "command": [ + "code" + ] + } + }, + { + "type": "activate_window", + "parameters": { + "window_name": "Visual Studio Code" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "vscode" + ], + "evaluator": { + "func": "check_json_settings", + "expected": { + "type": "rule", + "rules": { + "expect": {"editor.rulers": [50]} + } + }, + "result": { + "type": "vm_file", + "path": "/home/user/.config/Code/User/settings.json", + "dest": "settings.json" + } + } +} diff --git a/evaluation_examples/examples/vs_code/3486f395-ad68-459c-8c39-ea07de934dd4.json b/evaluation_examples/examples/vs_code/3486f395-ad68-459c-8c39-ea07de934dd4.json deleted file mode 100644 index d99df8c..0000000 --- a/evaluation_examples/examples/vs_code/3486f395-ad68-459c-8c39-ea07de934dd4.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "id": "3486f395-ad68-459c-8c39-ea07de934dd4", - "snapshot": "vscode", - "instruction": "Find me the keyboard shortcut of toggling integrated terminal. ", - "source": "https://www.youtube.com/watch?v=VqCgcpAypFQ", - "config": [], - "trajectory": "trajectories/3486f395-ad68-459c-8c39-ea07de934dd4", - "related_apps": [ - "vscode" - ], - "evaluator": "evaluation_dir" -} diff --git a/evaluation_examples/examples/vs_code/4e60007a-f5be-4bfc-9723-c39affa0a6d3.json b/evaluation_examples/examples/vs_code/4e60007a-f5be-4bfc-9723-c39affa0a6d3.json new file mode 100644 index 0000000..b36a7c8 --- /dev/null +++ b/evaluation_examples/examples/vs_code/4e60007a-f5be-4bfc-9723-c39affa0a6d3.json @@ -0,0 +1,46 @@ +{ + "id": "4e60007a-f5be-4bfc-9723-c39affa0a6d3", + "snapshot": "vscode", + "instruction": "Install autoDocstring extension.", + "source": "https://campbell-muscle-lab.github.io/howtos_Python/pages/documentation/best_practices/vscode_docstring_extension/vscode_docstring_extension.html#:~:text=Type%2C%20Ctrl%20%2B%20Shift%20%2B%20P,select%20the%20NumPy%20docstring%20format.", + "config": [ + { + "type": "launch", + "parameters": { + "command": [ + "code" + ] + } + }, + { + "type": "activate_window", + "parameters": { + "window_name": "Visual Studio Code" + } + } + ], + "trajectory": "trajectories/4e60007a-f5be-4bfc-9723-c39affa0a6d3", + "related_apps": [ + "vscode" + ], + "evaluator": { + "func": "is_extension_installed", + "result": { + "type": "vm_command_line", + "command": [ + "code", + "--list-extensions", + "|", + "grep", + "njpwerner.autodocstring" + ] + }, + "expected": { + "type": "rule", + "rules": { + "type": "contain", + "expected": "njpwerner.autodocstring" + } + } + } +} diff --git a/evaluation_examples/examples/vs_code/515630d2-9b30-430c-b06a-e86b0143f7fb.json b/evaluation_examples/examples/vs_code/515630d2-9b30-430c-b06a-e86b0143f7fb.json deleted file mode 100644 index 8f8bb5c..0000000 --- a/evaluation_examples/examples/vs_code/515630d2-9b30-430c-b06a-e86b0143f7fb.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "id": "515630d2-9b30-430c-b06a-e86b0143f7fb", - "snapshot": "vscode", - "instruction": "Help me enable automatically run code in VS code", - "source": "https://www.quora.com/How-do-I-automatically-run-code-in-Visual-Studio-Code", - "config": [], - "trajectory": "trajectories/", - "related_apps": [ - "vscode" - ], - "evaluator": "evaluation_dir" -} diff --git a/evaluation_examples/examples/vs_code/59ed65c7-e9a6-43db-833f-76d6730c0004.json b/evaluation_examples/examples/vs_code/59ed65c7-e9a6-43db-833f-76d6730c0004.json index 956138d..86250c7 100644 --- a/evaluation_examples/examples/vs_code/59ed65c7-e9a6-43db-833f-76d6730c0004.json +++ b/evaluation_examples/examples/vs_code/59ed65c7-e9a6-43db-833f-76d6730c0004.json @@ -37,7 +37,7 @@ "expected": { "type": "rule", "rules": { - "expect": "100" + "expect": "1" } }, "result": { diff --git a/evaluation_examples/examples/vs_code/6f7546b0-52f3-4938-9213-52f35454d314.json b/evaluation_examples/examples/vs_code/6f7546b0-52f3-4938-9213-52f35454d314.json deleted file mode 100644 index 1dcb21e..0000000 --- a/evaluation_examples/examples/vs_code/6f7546b0-52f3-4938-9213-52f35454d314.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "id": "6f7546b0-52f3-4938-9213-52f35454d314", - "snapshot": "vscode", - "instruction": "Help me ask chatGPT to generate html and css code for a scroll bar?", - "source": "https://www.tiktok.com/@akramovdev/video/7243349980897922306", - "config": [], - "trajectory": "trajectories/", - "related_apps": [ - "vscode" - ], - "evaluator": "evaluation_dir" -} diff --git a/evaluation_examples/examples/vs_code/90f6eeeb-f3c2-4c98-873c-e77d78a45578.json b/evaluation_examples/examples/vs_code/90f6eeeb-f3c2-4c98-873c-e77d78a45578.json deleted file mode 100644 index 955efa2..0000000 --- a/evaluation_examples/examples/vs_code/90f6eeeb-f3c2-4c98-873c-e77d78a45578.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "id": "90f6eeeb-f3c2-4c98-873c-e77d78a45578", - "snapshot": "vscode", - "instruction": "Help me sync extensions and settings across all profiles.", - "source": "https://stackoverflow.com/questions/75866801/how-do-i-sync-extensions-and-their-settings-between-vs-code-profiles", - "config": [], - "trajectory": "trajectories/", - "related_apps": [ - "vscode" - ], - "evaluator": "evaluation_dir" -} diff --git a/evaluation_examples/examples/vs_code/930fdb3b-11a8-46fe-9bac-577332e2640e.json b/evaluation_examples/examples/vs_code/930fdb3b-11a8-46fe-9bac-577332e2640e.json new file mode 100644 index 0000000..a66ecdd --- /dev/null +++ b/evaluation_examples/examples/vs_code/930fdb3b-11a8-46fe-9bac-577332e2640e.json @@ -0,0 +1,45 @@ +{ + "id": "930fdb3b-11a8-46fe-9bac-577332e2640e", + "snapshot": "vscode", + "instruction": "I want to create a shortcut to shift my focus cursor from terminal to Editor in VS Code. Please help me create this shortcut to be 'ctrl+j'.", + "source": "https://superuser.com/questions/1270103/how-to-switch-the-cursor-between-terminal-and-code-in-vscode", + "config": [ + { + "type": "launch", + "parameters": { + "command": [ + "code" + ] + } + }, + { + "type": "activate_window", + "parameters": { + "window_name": "Visual Studio Code" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "vscode" + ], + "evaluator": { + "func": "check_json_keybindings", + "expected": { + "type": "rule", + "rules": { + "expect": + { + "key": "ctrl+j", + "command": "workbench.action.focusActiveEditorGroup", + "when": "terminalFocus" + } + } + }, + "result": { + "type": "vm_file", + "path": "/home/user/.config/Code/User/keybindings.json", + "dest": "keybindings.json" + } + } +} diff --git a/evaluation_examples/examples/vs_code/9439a27b-18ae-42d8-9778-5f68f891805e.json b/evaluation_examples/examples/vs_code/9439a27b-18ae-42d8-9778-5f68f891805e.json new file mode 100644 index 0000000..2f848b2 --- /dev/null +++ b/evaluation_examples/examples/vs_code/9439a27b-18ae-42d8-9778-5f68f891805e.json @@ -0,0 +1,40 @@ +{ + "id": "9439a27b-18ae-42d8-9778-5f68f891805e", + "snapshot": "vscode", + "instruction": "I want to keep my cursor focus in debug console when debugging in VS Code, instead of focusing back to Editor. So please help me modify the setting of VS Code accordingly.", + "source": "https://stackoverflow.com/questions/75832474/how-to-keep-cursor-in-debug-console-when-debugging-in-visual-studio-code", + "config": [ + { + "type": "launch", + "parameters": { + "command": [ + "code" + ] + } + }, + { + "type": "activate_window", + "parameters": { + "window_name": "Visual Studio Code" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "vscode" + ], + "evaluator": { + "func": "check_json_settings", + "expected": { + "type": "rule", + "rules": { + "expect": {"debug.focusEditorOnBreak": false} + } + }, + "result": { + "type": "vm_file", + "path": "/home/user/.config/Code/User/settings.json", + "dest": "settings.json" + } + } +} diff --git a/evaluation_examples/examples/vs_code/9d425400-e9b2-4424-9a4b-d4c7abac4140.json b/evaluation_examples/examples/vs_code/9d425400-e9b2-4424-9a4b-d4c7abac4140.json new file mode 100644 index 0000000..080bdf1 --- /dev/null +++ b/evaluation_examples/examples/vs_code/9d425400-e9b2-4424-9a4b-d4c7abac4140.json @@ -0,0 +1,40 @@ +{ + "id": "9d425400-e9b2-4424-9a4b-d4c7abac4140", + "snapshot": "vscode", + "instruction": "I want to make tabs wrapped over multiple lines when exceeding available space, please help modify the setting of VS Code.", + "source": "https://superuser.com/questions/1466771/is-there-a-way-to-make-editor-tabs-stack-in-vs-code", + "config": [ + { + "type": "launch", + "parameters": { + "command": [ + "code" + ] + } + }, + { + "type": "activate_window", + "parameters": { + "window_name": "Visual Studio Code" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "vscode" + ], + "evaluator": { + "func": "check_json_settings", + "expected": { + "type": "rule", + "rules": { + "expect": {"workbench.editor.wrapTabs": true} + } + }, + "result": { + "type": "vm_file", + "path": "/home/user/.config/Code/User/settings.json", + "dest": "settings.json" + } + } +} diff --git a/evaluation_examples/examples/vs_code/ae506c68-352c-4094-9caa-ee9d42052317.json b/evaluation_examples/examples/vs_code/ae506c68-352c-4094-9caa-ee9d42052317.json new file mode 100644 index 0000000..6bac7ce --- /dev/null +++ b/evaluation_examples/examples/vs_code/ae506c68-352c-4094-9caa-ee9d42052317.json @@ -0,0 +1,39 @@ +{ + "id": "ae506c68-352c-4094-9caa-ee9d42052317", + "snapshot": "vscode", + "instruction": "Could you store the full terminal history of my VS Code terminal into '/home/user/Desktop/history.txt'?", + "source": "", + "config": [ + { + "type": "launch", + "parameters": { + "command": [ + "code" + ] + } + }, + { + "type": "activate_window", + "parameters": { + "window_name": "Visual Studio Code" + } + } + ], + "trajectory": "trajectories/ae506c68-352c-4094-9caa-ee9d42052317", + "related_apps": [ + "vscode" + ], + "evaluator": { + "func": "compare_text_file", + "expected": { + "type": "cloud_file", + "path": "", + "dest": "gold_history.txt" + }, + "result": { + "type": "vm_file", + "path": "Desktop/history.txt", + "dest": "history.txt" + } + } +} diff --git a/evaluation_examples/examples/vs_code/e2b5e914-ffe1-44d2-8e92-58f8c5d92bb2.json b/evaluation_examples/examples/vs_code/e2b5e914-ffe1-44d2-8e92-58f8c5d92bb2.json new file mode 100644 index 0000000..306871a --- /dev/null +++ b/evaluation_examples/examples/vs_code/e2b5e914-ffe1-44d2-8e92-58f8c5d92bb2.json @@ -0,0 +1,42 @@ +{ + "id": "e2b5e914-ffe1-44d2-8e92-58f8c5d92bb2", + "snapshot": "vscode", + "instruction": "I want to disable the missing imports reporting of python error, please modify the setting of VS Code for me.", + "source": "https://superuser.com/questions/1386061/how-to-suppress-some-python-errors-warnings-in-vs-code", + "config": [ + { + "type": "launch", + "parameters": { + "command": [ + "code" + ] + } + }, + { + "type": "activate_window", + "parameters": { + "window_name": "Visual Studio Code" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "vscode" + ], + "evaluator": { + "func": "check_json_settings", + "expected": { + "type": "rule", + "rules": { + "expect": { + "python.analysis.diagnosticSeverityOverrides": {"reportMissingImports": "none"} + } + } + }, + "result": { + "type": "vm_file", + "path": "/home/user/.config/Code/User/settings.json", + "dest": "settings.json" + } + } +} diff --git a/evaluation_examples/examples/vs_code/ea98c5d7-3cf9-4f9b-8ad3-366b58e0fcae.json b/evaluation_examples/examples/vs_code/ea98c5d7-3cf9-4f9b-8ad3-366b58e0fcae.json new file mode 100644 index 0000000..f2667e9 --- /dev/null +++ b/evaluation_examples/examples/vs_code/ea98c5d7-3cf9-4f9b-8ad3-366b58e0fcae.json @@ -0,0 +1,47 @@ +{ + "id": "ea98c5d7-3cf9-4f9b-8ad3-366b58e0fcae", + "snapshot": "vscode", + "instruction": "I want to remove the shortcut 'cmd+f' for Tree view Find (Explorer search) in VS Code explorer view due to shortcut conflict. Can you help me remove this shortcut?", + "source": ["https://superuser.com/questions/1748097/vs-code-disable-tree-view-find-explorer-search", + "https://superuser.com/questions/1417361/how-to-disable-file-filtering-in-vs-code-sidebar-explorer?rq=1" + ], + "config": [ + { + "type": "launch", + "parameters": { + "command": [ + "code" + ] + } + }, + { + "type": "activate_window", + "parameters": { + "window_name": "Visual Studio Code" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "vscode" + ], + "evaluator": { + "func": "check_json_keybindings", + "expected": { + "type": "rule", + "rules": { + "expect": + { + "key": "cmd+f", + "command": "-list.find", + "when": "listFocus && listSupportsFind" + } + } + }, + "result": { + "type": "vm_file", + "path": "/home/user/.config/Code/User/keybindings.json", + "dest": "keybindings.json" + } + } +}