lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add vscode examples

Browse Source
This commit is contained in:
Siheng Zhao
2024-01-20 19:44:52 +08:00
parent 09f3e776ae
commit 980f7290eb
14 changed files with 379 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
evaluation_examples/examples/vs_code/276cc624-87ea-4f08-ab93-f770e3790175.json Normal file
View File

@@ -0,0 +1,40 @@
{
"id": "276cc624-87ea-4f08-ab93-f770e3790175",
"snapshot": "vscode",
"instruction": "Could you help me set the line length to 50 characters for current user in VS Code?",
"source": "https://www.quora.com/unanswered/How-do-you-set-the-line-length-in-Visual-Studio-Code",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"code"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Visual Studio Code"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"vscode"
],
"evaluator": {
"func": "check_json_settings",
"expected": {
"type": "rule",
"rules": {
"expect": {"editor.rulers": [50]}
}
},
"result": {
"type": "vm_file",
"path": "/home/user/.config/Code/User/settings.json",
"dest": "settings.json"
}
}
}

12
evaluation_examples/examples/vs_code/3486f395-ad68-459c-8c39-ea07de934dd4.json
View File

@@ -1,12 +0,0 @@
{
"id": "3486f395-ad68-459c-8c39-ea07de934dd4",
"snapshot": "vscode",
"instruction": "Find me the keyboard shortcut of toggling integrated terminal. ",
"source": "https://www.youtube.com/watch?v=VqCgcpAypFQ",
"config": [],
"trajectory": "trajectories/3486f395-ad68-459c-8c39-ea07de934dd4",
"related_apps": [
"vscode"
],
"evaluator": "evaluation_dir"
}

46
evaluation_examples/examples/vs_code/4e60007a-f5be-4bfc-9723-c39affa0a6d3.json Normal file
View File

@@ -0,0 +1,46 @@
{
"id": "4e60007a-f5be-4bfc-9723-c39affa0a6d3",
"snapshot": "vscode",
"instruction": "Install autoDocstring extension.",
"source": "https://campbell-muscle-lab.github.io/howtos_Python/pages/documentation/best_practices/vscode_docstring_extension/vscode_docstring_extension.html#:~:text=Type%2C%20Ctrl%20%2B%20Shift%20%2B%20P,select%20the%20NumPy%20docstring%20format.",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"code"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Visual Studio Code"
}
}
],
"trajectory": "trajectories/4e60007a-f5be-4bfc-9723-c39affa0a6d3",
"related_apps": [
"vscode"
],
"evaluator": {
"func": "is_extension_installed",
"result": {
"type": "vm_command_line",
"command": [
"code",
"--list-extensions",
"|",
"grep",
"njpwerner.autodocstring"
]
},
"expected": {
"type": "rule",
"rules": {
"type": "contain",
"expected": "njpwerner.autodocstring"
}
}
}
}

12
evaluation_examples/examples/vs_code/515630d2-9b30-430c-b06a-e86b0143f7fb.json
View File

@@ -1,12 +0,0 @@
{
"id": "515630d2-9b30-430c-b06a-e86b0143f7fb",
"snapshot": "vscode",
"instruction": "Help me enable automatically run code in VS code",
"source": "https://www.quora.com/How-do-I-automatically-run-code-in-Visual-Studio-Code",
"config": [],
"trajectory": "trajectories/",
"related_apps": [
"vscode"
],
"evaluator": "evaluation_dir"
}

2
evaluation_examples/examples/vs_code/59ed65c7-e9a6-43db-833f-76d6730c0004.json
View File

@@ -37,7 +37,7 @@
"expected": {
"type": "rule",
"rules": {
"expect": "100"
"expect": "1"
}
},
"result": {

12
evaluation_examples/examples/vs_code/6f7546b0-52f3-4938-9213-52f35454d314.json
View File

@@ -1,12 +0,0 @@
{
"id": "6f7546b0-52f3-4938-9213-52f35454d314",
"snapshot": "vscode",
"instruction": "Help me ask chatGPT to generate html and css code for a scroll bar?",
"source": "https://www.tiktok.com/@akramovdev/video/7243349980897922306",
"config": [],
"trajectory": "trajectories/",
"related_apps": [
"vscode"
],
"evaluator": "evaluation_dir"
}

12
evaluation_examples/examples/vs_code/90f6eeeb-f3c2-4c98-873c-e77d78a45578.json
View File

@@ -1,12 +0,0 @@
{
"id": "90f6eeeb-f3c2-4c98-873c-e77d78a45578",
"snapshot": "vscode",
"instruction": "Help me sync extensions and settings across all profiles.",
"source": "https://stackoverflow.com/questions/75866801/how-do-i-sync-extensions-and-their-settings-between-vs-code-profiles",
"config": [],
"trajectory": "trajectories/",
"related_apps": [
"vscode"
],
"evaluator": "evaluation_dir"
}

45
evaluation_examples/examples/vs_code/930fdb3b-11a8-46fe-9bac-577332e2640e.json Normal file
View File

@@ -0,0 +1,45 @@
{
"id": "930fdb3b-11a8-46fe-9bac-577332e2640e",
"snapshot": "vscode",
"instruction": "I want to create a shortcut to shift my focus cursor from terminal to Editor in VS Code. Please help me create this shortcut to be 'ctrl+j'.",
"source": "https://superuser.com/questions/1270103/how-to-switch-the-cursor-between-terminal-and-code-in-vscode",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"code"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Visual Studio Code"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"vscode"
],
"evaluator": {
"func": "check_json_keybindings",
"expected": {
"type": "rule",
"rules": {
"expect":
{
"key": "ctrl+j",
"command": "workbench.action.focusActiveEditorGroup",
"when": "terminalFocus"
}
}
},
"result": {
"type": "vm_file",
"path": "/home/user/.config/Code/User/keybindings.json",
"dest": "keybindings.json"
}
}
}

40
evaluation_examples/examples/vs_code/9439a27b-18ae-42d8-9778-5f68f891805e.json Normal file
View File

@@ -0,0 +1,40 @@
{
"id": "9439a27b-18ae-42d8-9778-5f68f891805e",
"snapshot": "vscode",
"instruction": "I want to keep my cursor focus in debug console when debugging in VS Code, instead of focusing back to Editor. So please help me modify the setting of VS Code accordingly.",
"source": "https://stackoverflow.com/questions/75832474/how-to-keep-cursor-in-debug-console-when-debugging-in-visual-studio-code",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"code"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Visual Studio Code"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"vscode"
],
"evaluator": {
"func": "check_json_settings",
"expected": {
"type": "rule",
"rules": {
"expect": {"debug.focusEditorOnBreak": false}
}
},
"result": {
"type": "vm_file",
"path": "/home/user/.config/Code/User/settings.json",
"dest": "settings.json"
}
}
}

40
evaluation_examples/examples/vs_code/9d425400-e9b2-4424-9a4b-d4c7abac4140.json Normal file
View File

@@ -0,0 +1,40 @@
{
"id": "9d425400-e9b2-4424-9a4b-d4c7abac4140",
"snapshot": "vscode",
"instruction": "I want to make tabs wrapped over multiple lines when exceeding available space, please help modify the setting of VS Code.",
"source": "https://superuser.com/questions/1466771/is-there-a-way-to-make-editor-tabs-stack-in-vs-code",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"code"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Visual Studio Code"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"vscode"
],
"evaluator": {
"func": "check_json_settings",
"expected": {
"type": "rule",
"rules": {
"expect": {"workbench.editor.wrapTabs": true}
}
},
"result": {
"type": "vm_file",
"path": "/home/user/.config/Code/User/settings.json",
"dest": "settings.json"
}
}
}

39
evaluation_examples/examples/vs_code/ae506c68-352c-4094-9caa-ee9d42052317.json Normal file
View File

@@ -0,0 +1,39 @@
{
"id": "ae506c68-352c-4094-9caa-ee9d42052317",
"snapshot": "vscode",
"instruction": "Could you store the full terminal history of my VS Code terminal into '/home/user/Desktop/history.txt'?",
"source": "",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"code"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Visual Studio Code"
}
}
],
"trajectory": "trajectories/ae506c68-352c-4094-9caa-ee9d42052317",
"related_apps": [
"vscode"
],
"evaluator": {
"func": "compare_text_file",
"expected": {
"type": "cloud_file",
"path": "",
"dest": "gold_history.txt"
},
"result": {
"type": "vm_file",
"path": "Desktop/history.txt",
"dest": "history.txt"
}
}
}

42
evaluation_examples/examples/vs_code/e2b5e914-ffe1-44d2-8e92-58f8c5d92bb2.json Normal file
View File

@@ -0,0 +1,42 @@
{
"id": "e2b5e914-ffe1-44d2-8e92-58f8c5d92bb2",
"snapshot": "vscode",
"instruction": "I want to disable the missing imports reporting of python error, please modify the setting of VS Code for me.",
"source": "https://superuser.com/questions/1386061/how-to-suppress-some-python-errors-warnings-in-vs-code",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"code"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Visual Studio Code"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"vscode"
],
"evaluator": {
"func": "check_json_settings",
"expected": {
"type": "rule",
"rules": {
"expect": {
"python.analysis.diagnosticSeverityOverrides": {"reportMissingImports": "none"}
}
}
},
"result": {
"type": "vm_file",
"path": "/home/user/.config/Code/User/settings.json",
"dest": "settings.json"
}
}
}

47
evaluation_examples/examples/vs_code/ea98c5d7-3cf9-4f9b-8ad3-366b58e0fcae.json Normal file
View File

@@ -0,0 +1,47 @@
{
"id": "ea98c5d7-3cf9-4f9b-8ad3-366b58e0fcae",
"snapshot": "vscode",
"instruction": "I want to remove the shortcut 'cmd+f' for Tree view Find (Explorer search) in VS Code explorer view due to shortcut conflict. Can you help me remove this shortcut?",
"source": ["https://superuser.com/questions/1748097/vs-code-disable-tree-view-find-explorer-search",
"https://superuser.com/questions/1417361/how-to-disable-file-filtering-in-vs-code-sidebar-explorer?rq=1"
],
"config": [
{
"type": "launch",
"parameters": {
"command": [
"code"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Visual Studio Code"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"vscode"
],
"evaluator": {
"func": "check_json_keybindings",
"expected": {
"type": "rule",
"rules": {
"expect":
{
"key": "cmd+f",
"command": "-list.find",
"when": "listFocus && listSupportsFind"
}
}
},
"result": {
"type": "vm_file",
"path": "/home/user/.config/Code/User/keybindings.json",
"dest": "keybindings.json"
}
}
}