lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Finish loading the vscode examples v1; Improve on the infra: Add accessibility tree into the observation; Add activate window function, etc

Browse Source
This commit is contained in:
Timothyxxx
2024-01-14 18:30:49 +08:00
parent 2228f346a9
commit d52b692ee5
16 changed files with 368 additions and 157 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
evaluation_examples/examples/vs_code/0ed39f63-6049-43d4-ba4d-5fa2fe04a951.json
View File

@@ -1,7 +1,7 @@
{
"id": "0ed39f63-6049-43d4-ba4d-5fa2fe04a951",
"snapshot": "vscode",
"instruction": "Could you help me find and replace \"text\" with \"test\" in this file?",
"instruction": "Please change all the places that say \"text\" to \"test\" in this document for me.",
"source": "https://www.quora.com/How-do-you-find-and-replace-text-in-Visual-Studio-Code",
"config": [
{
@@ -20,6 +20,12 @@
"parameters": {
"command": ["code", "Desktop/vscode_replace_text.txt"]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Visual Studio Code"
}
}
],
"trajectory": "trajectories/0ed39f63-6049-43d4-ba4d-5fa2fe04a951",

46
evaluation_examples/examples/vs_code/53ad5833-3455-407b-bbc6-45b4c79ab8fb.json
View File

@@ -1,19 +1,50 @@
{
"id": "53ad5833-3455-407b-bbc6-45b4c79ab8fb",
"snapshot": "vscode",
"instruction": "Could you help me open the project at /home/user/project?",
"instruction": "I'd like the \"project\" in the \"user\" folder under \"home\" to be opened with VS Code, please.",
"source": "https://www.youtube.com/watch?v=VqCgcpAypFQ",
"config": [
{
"type": "launch",
"parameters": {
"command": ["code"]
"command": [
"code"
]
}
},
{
"type": "command",
"parameters": {
"command": ["mkdir", "-p", "/home/user/project"]
"command": [
"mkdir",
"-p",
"/home/user/project/.vscode"
]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1akdsiRVdq6CUtT-FX8Dpf8ruPTq6DcFn&export=download&authuser=0&confirm=t&uuid=ce2fa96a-454e-43d9-bbe3-98553b7eed0d&at=APZUnTVw_YQ1URTvP34vrmKcw0b4:1705222451052",
"path": "/home/user/project/main.py"
},
{
"url": "https://drive.usercontent.google.com/download?id=1BkwtqtAzv_K2CrTbJZ0HbMHBffzdD9vc&export=download&authuser=0&confirm=t&uuid=28f77090-deef-49a1-b156-91317881e75e&at=APZUnTXuaR6i_3t3Prslk535GaO5:1705222457290",
"path": "/home/user/project/README.md"
},
{
"url": "https://drive.usercontent.google.com/download?id=1ea_zF2tbcXOB8w9neBV-U5xI2nnPzIw_&export=download&authuser=0&confirm=t&uuid=9cf8c5bb-a880-475c-b80b-967a0c4fbea4&at=APZUnTUdjIj80F3Mbgi72eZDTZLO:1705222462443",
"path": "/home/user/project/.vscode/settings.json"
}
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Visual Studio Code"
}
}
],
@@ -22,6 +53,14 @@
"vscode"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "Visual Studio Code"
}
}
],
"func": "compare_config",
"expected": {
"type": "rule",
@@ -31,6 +70,7 @@
},
"result": {
"type": "vscode_config",
"vscode_extension_command": "OpenProject",
"path": "OpenProject.txt",
"dest": "OpenProject.txt"
}

11
evaluation_examples/examples/vs_code/59ed65c7-e9a6-43db-833f-76d6730c0004.json
View File

@@ -16,9 +16,15 @@
}
},
{
"type": "open",
"type": "launch",
"parameters": {
"path": "Desktop/main.py"
"command": ["code", "Desktop/main.py"]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Visual Studio Code"
}
}
],
@@ -36,6 +42,7 @@
},
"result": {
"type": "vscode_config",
"vscode_extension_command": "GetBreakPoint",
"path": "GetBreakPoint.txt",
"dest": "GetBreakPoint.txt"
}

24
evaluation_examples/examples/vs_code/982d12a5-beab-424f-8d38-d2a48429e511.json
View File

@@ -3,7 +3,22 @@
"snapshot": "vscode",
"instruction": "Could you help me change the color theme to Dark?",
"source": "https://www.youtube.com/watch?v=ORrELERGIHs",
"config": [],
"config": [
{
"type": "launch",
"parameters": {
"command": [
"code"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Visual Studio Code"
}
}
],
"trajectory": "trajectories/982d12a5-beab-424f-8d38-d2a48429e511",
"related_apps": [
"vscode"
@@ -11,11 +26,14 @@
"evaluator": {
"func": "compare_config",
"expected": {
"type": "string",
"string": "2"
"type": "rule",
"rules": {
"expect": "2"
}
},
"result": {
"type": "vscode_config",
"vscode_extension_command": "GetColorTheme",
"path": "GetColorTheme.txt",
"dest": "GetColorTheme.txt"
}

40
evaluation_examples/examples/vs_code/eabc805a-bfcf-4460-b250-ac92135819f6.json
View File

@@ -3,20 +3,44 @@
"snapshot": "vscode",
"instruction": "Help me install the extension Python.",
"source": "https://www.youtube.com/watch?v=VqCgcpAypFQ",
"config": [],
"config": [
{
"type": "launch",
"parameters": {
"command": [
"code"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Visual Studio Code"
}
}
],
"trajectory": "trajectories/eabc805a-bfcf-4460-b250-ac92135819f6",
"related_apps": [
"vscode"
],
"evaluator": {
"func": "compare_answer",
"expected": {
"type": "string",
"string": "ms-python.python\n"
},
"func": "is_extension_installed",
"result": {
"type": "command_line",
"command": "code --list-extensions | grep ms-python.python"
"type": "vm_command_line",
"command": [
"code",
"--list-extensions",
"|",
"grep",
"ms-python.python"
]
},
"expected": {
"type": "rule",
"rules": {
"type": "contain",
"expected": "ms-python.python"
}
}
}
}