lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add vsc examples and fix prev issues

Browse Source
This commit is contained in:
Siheng Zhao
2024-01-31 10:37:00 +08:00
parent cb7643713e
commit cffed446fc
10 changed files with 411 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
evaluation_examples/examples/vs_code/0512bb38-d531-4acf-9e7e-0add90816068.json Normal file
View File

@@ -0,0 +1,57 @@
{
"id": "0512bb38-d531-4acf-9e7e-0add90816068",
"snapshot": "vscode",
"instruction": "Please help me install an extension in VS Code from a local VSIX file \"/home/user/test.vsix\".",
"source": "https://download.microsoft.com/download/8/A/4/8A48E46A-C355-4E5C-8417-E6ACD8A207D4/VisualStudioCode-TipsAndTricks-Vol.1.pdf",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1fu_JH3rQk93MHFzUcXJDKC68X_yXRNwJ&export=download&authuser=0&confirm=t&uuid=efb1a587-645a-4ee8-b8af-cfba2c6473b6&at=APZUnTWqz5t2nKbHZczIMW7aAMsz:1706668206536",
"path": "/home/user/test.vsix"
}
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"code"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Visual Studio Code"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"vscode"
],
"evaluator": {
"func": "is_extension_installed",
"result": {
"type": "vm_command_line",
"command": [
"code",
"--list-extensions",
"|",
"grep",
"undefined_publisher.test"
]
},
"expected": {
"type": "rule",
"rules": {
"type": "contain",
"expected": "undefined_publisher.test"
}
}
}
}

2
evaluation_examples/examples/vs_code/0ed39f63-6049-43d4-ba4d-5fa2fe04a951.json
View File

@@ -28,7 +28,7 @@
}
}
],
"trajectory": "trajectories/0ed39f63-6049-43d4-ba4d-5fa2fe04a951",
"trajectory": "trajectories/",
"related_apps": [
"vscode"
],

4
evaluation_examples/examples/vs_code/276cc624-87ea-4f08-ab93-f770e3790175.json
View File

@@ -28,7 +28,9 @@
"expected": {
"type": "rule",
"rules": {
"expect": {"editor.rulers": [50]}
"expect": {
"editor.wordWrapColumn": 50
}
}
},
"result": {

46
evaluation_examples/examples/vs_code/57242fad-77ca-454f-b71b-f187181a9f23.json Normal file
View File

@@ -0,0 +1,46 @@
{
"id": "57242fad-77ca-454f-b71b-f187181a9f23",
"snapshot": "vscode",
"instruction": "Please help me create a new python file named \"test.py\" via VS Code and save it at \"/home/user/Desktop\".",
"source": "https://download.microsoft.com/download/8/A/4/8A48E46A-C355-4E5C-8417-E6ACD8A207D4/VisualStudioCode-TipsAndTricks-Vol.1.pdf",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"code"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Visual Studio Code"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"vscode"
],
"evaluator": {
"func": "is_extension_installed",
"result": {
"type": "vm_command_line",
"command": [
"ls",
"/home/user/Desktop",
"|",
"grep",
"test.py"
]
},
"expected": {
"type": "rule",
"rules": {
"type": "contain",
"expected": "test.py"
}
}
}
}

74
evaluation_examples/examples/vs_code/5e2d93d8-8ad0-4435-b150-1692aacaa994.json Normal file
View File

@@ -0,0 +1,74 @@
{
"id": "5e2d93d8-8ad0-4435-b150-1692aacaa994",
"snapshot": "vscode",
"instruction": "Please help me save current project as workspace \"project\" at \"/home/user/\".",
"source": "https://www.youtube.com/watch?v=B-s71n0dHUk",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"code"
]
}
},
{
"type": "command",
"parameters": {
"command": [
"mkdir",
"-p",
"/home/user/project/.vscode"
]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1akdsiRVdq6CUtT-FX8Dpf8ruPTq6DcFn&export=download&authuser=0&confirm=t&uuid=ce2fa96a-454e-43d9-bbe3-98553b7eed0d&at=APZUnTVw_YQ1URTvP34vrmKcw0b4:1705222451052",
"path": "/home/user/project/main.py"
},
{
"url": "https://drive.usercontent.google.com/download?id=1BkwtqtAzv_K2CrTbJZ0HbMHBffzdD9vc&export=download&authuser=0&confirm=t&uuid=28f77090-deef-49a1-b156-91317881e75e&at=APZUnTXuaR6i_3t3Prslk535GaO5:1705222457290",
"path": "/home/user/project/README.md"
},
{
"url": "https://drive.usercontent.google.com/download?id=1ea_zF2tbcXOB8w9neBV-U5xI2nnPzIw_&export=download&authuser=0&confirm=t&uuid=9cf8c5bb-a880-475c-b80b-967a0c4fbea4&at=APZUnTUdjIj80F3Mbgi72eZDTZLO:1705222462443",
"path": "/home/user/project/.vscode/settings.json"
}
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Visual Studio Code"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"vscode"
],
"evaluator": {
"func": "is_extension_installed",
"result": {
"type": "vm_command_line",
"command": [
"ls",
"|",
"grep",
"project.code-workspace"
]
},
"expected": {
"type": "rule",
"rules": {
"type": "contain",
"expected": "project.code-workspace"
}
}
}
}

85
evaluation_examples/examples/vs_code/6ed0a554-cbee-4b44-84ea-fd6c042f4fe1.json Normal file
View File

@@ -0,0 +1,85 @@
{
"id": "6ed0a554-cbee-4b44-84ea-fd6c042f4fe1",
"snapshot": "vscode",
"instruction": "Please help me add folder \"/home/user/data1\" and folder \"/home/user/data2\" to the current workspace.",
"source": "https://www.youtube.com/watch?v=B-s71n0dHUk",
"config": [
{
"type": "command",
"parameters": {
"command": [
"mkdir",
"-p",
"/home/user/project"
]
}
},
{
"type": "command",
"parameters": {
"command": [
"mkdir",
"-p",
"/home/user/data1"
]
}
},
{
"type": "command",
"parameters": {
"command": [
"mkdir",
"-p",
"/home/user/data2"
]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1_CzdGGcA70VZnXWbFkxnMCS_HSl2B16A&export=download&authuser=0&confirm=t&uuid=6d08572f-04c6-42ca-8d02-b6adf3f0a271&at=APZUnTU3hL_yPkvSKokYA-Mywj1Z:1706666120779",
"path": "/home/user/project.code-workspace"
}
]
}
},
{
"type": "launch",
"parameters": {
"command": ["code", "/home/user/project.code-workspace"]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Visual Studio Code"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"vscode"
],
"evaluator": {
"func": "check_json_settings",
"expected": {
"type": "rule",
"rules": {
"expect": {
"folders": [
{"path": "project"},
{"path": "data1"},
{"path": "data2"}
]
}
}
},
"result": {
"type": "vm_file",
"path": "/home/user/project.code-workspace",
"dest": "project.code-workspace"
}
}
}

43
evaluation_examples/examples/vs_code/70745df8-f2f5-42bd-8074-fbc10334fcc5.json Normal file
View File

@@ -0,0 +1,43 @@
{
"id": "70745df8-f2f5-42bd-8074-fbc10334fcc5",
"snapshot": "vscode",
"instruction": "Please help me open the autosave feature of VS Code and delay AutoSave operations for 500 milliseconds in the VS Code setting.",
"source": "https://download.microsoft.com/download/8/A/4/8A48E46A-C355-4E5C-8417-E6ACD8A207D4/VisualStudioCode-TipsAndTricks-Vol.1.pdf",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"code"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Visual Studio Code"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"vscode"
],
"evaluator": {
"func": "check_json_settings",
"expected": {
"type": "rule",
"rules": {
"expect": {
"files.autoSave": "afterDelay",
"files.autoSaveDelay": 500
}
}
},
"result": {
"type": "vm_file",
"path": "/home/user/.config/Code/User/settings.json",
"dest": "settings.json"
}
}
}

44
evaluation_examples/examples/vs_code/c6bf789c-ba3a-4209-971d-b63abf0ab733.json Normal file
View File

@@ -0,0 +1,44 @@
{
"id": "c6bf789c-ba3a-4209-971d-b63abf0ab733",
"snapshot": "vscode",
"instruction": "Please help me modify VS Code setting to hide all \"__pycache__\" folders in the explorer view.",
"source": "https://download.microsoft.com/download/8/A/4/8A48E46A-C355-4E5C-8417-E6ACD8A207D4/VisualStudioCode-TipsAndTricks-Vol.1.pdf",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"code"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Visual Studio Code"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"vscode"
],
"evaluator": {
"func": "check_json_settings",
"expected": {
"type": "rule",
"rules": {
"expect": {
"files.exclude": {
"**/__pycache__": true
}
}
}
},
"result": {
"type": "vm_file",
"path": "/home/user/.config/Code/User/settings.json",
"dest": "settings.json"
}
}
}

10
evaluation_examples/examples/vs_code/e2b5e914-ffe1-44d2-8e92-58f8c5d92bb2.json
View File

@@ -4,6 +4,16 @@
"instruction": "Please modify VS Code's settings to disable error reporting for Python missing imports.",
"source": "https://superuser.com/questions/1386061/how-to-suppress-some-python-errors-warnings-in-vs-code",
"config": [
{
"type": "command",
"parameters": {
"command": [
"code",
"--install-extension",
"ms-python.python"
]
}
},
{
"type": "launch",
"parameters": {

48
evaluation_examples/examples/vs_code/ec71221e-ac43-46f9-89b8-ee7d80f7e1c5.json Normal file
View File

@@ -0,0 +1,48 @@
{
"id": "ec71221e-ac43-46f9-89b8-ee7d80f7e1c5",
"snapshot": "vscode",
"instruction": "Please help me increase the indent of line 2 to line 10 by one tab.",
"source": "https://stackoverflow.com/questions/47903209/how-to-shift-a-block-of-code-left-right-by-one-space-in-vscode",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1k4rv_meWQyYfNMHTWipJnvh3uA5LWnWa&export=download&authuser=0&confirm=t&uuid=e923fec7-a55c-4266-8d40-be017ac54c7d&at=APZUnTVDyVA7V72IC3r2vrjwY0xz:1706666734010",
"path": "/home/user/Desktop/test.py"
}
]
}
},
{
"type": "launch",
"parameters": {
"command": ["code", "/home/user/Desktop/test.py"]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Visual Studio Code"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"vscode"
],
"evaluator": {
"func": "compare_text_file",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1fNZ4r-G58QVo5yFjS1jTqxFJAXXNyf1G&export=download&authuser=0&confirm=t&uuid=c577a187-1225-4d98-98da-d3b33b9e6c82&at=APZUnTXSUy1dHiC2Oavo5g-wRJe8:1706666819711",
"dest": "test_gold.py"
},
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/test.py",
"dest": "test.py"
}
}
}