diff --git a/desktop_env/evaluators/metrics/vscode.py b/desktop_env/evaluators/metrics/vscode.py index 41710ae..b641d43 100644 --- a/desktop_env/evaluators/metrics/vscode.py +++ b/desktop_env/evaluators/metrics/vscode.py @@ -33,7 +33,7 @@ def check_json_keybindings(actual: str, expected: str, **options) -> float: break else: return 0.0 - expected = expected['expect'] + expected = expected['expected'] if expected in data: return 1.0 else: @@ -55,7 +55,7 @@ def check_json_settings(actual: str, expected: str, **options) -> float: with open(actual, 'r') as f: data = json.load(f) - expect = expected['expect'] + expect = expected['expected'] data_copy = copy.deepcopy(data) data_copy.update(expect) if data == data_copy: @@ -93,7 +93,7 @@ def compare_config(actual: str, rules: Dict, **options) -> float: with open(actual) as f1: actual_text = f1.read() - if actual_text == rules['expect']: + if actual_text == rules['expected']: return 1.0 return 0.0 @@ -110,7 +110,7 @@ def compare_answer(actual: str, rules: Dict, **options) -> float: if not actual: return 0. - if actual == rules['expect']: + if actual == rules['expected']: return 1.0 # TODO: can use text embedding to get non-zero return diff --git a/evaluation_examples/examples/vs_code/276cc624-87ea-4f08-ab93-f770e3790175.json b/evaluation_examples/examples/vs_code/276cc624-87ea-4f08-ab93-f770e3790175.json index bcbff41..b69d76a 100644 --- a/evaluation_examples/examples/vs_code/276cc624-87ea-4f08-ab93-f770e3790175.json +++ b/evaluation_examples/examples/vs_code/276cc624-87ea-4f08-ab93-f770e3790175.json @@ -28,7 +28,7 @@ "expected": { "type": "rule", "rules": { - "expect": { + "expected": { "editor.wordWrapColumn": 50 } } diff --git a/evaluation_examples/examples/vs_code/30167b0c-976b-491b-a9e5-7747607c6a3c.json b/evaluation_examples/examples/vs_code/30167b0c-976b-491b-a9e5-7747607c6a3c.json index c685c05..4b2bd69 100644 --- a/evaluation_examples/examples/vs_code/30167b0c-976b-491b-a9e5-7747607c6a3c.json +++ b/evaluation_examples/examples/vs_code/30167b0c-976b-491b-a9e5-7747607c6a3c.json @@ -50,7 +50,7 @@ "expected": { "type": "rule", "rules": { - "expect": {} + "expected": {} } }, "result": { diff --git a/evaluation_examples/examples/vs_code/53ad5833-3455-407b-bbc6-45b4c79ab8fb.json b/evaluation_examples/examples/vs_code/53ad5833-3455-407b-bbc6-45b4c79ab8fb.json index b597a03..14d217a 100644 --- a/evaluation_examples/examples/vs_code/53ad5833-3455-407b-bbc6-45b4c79ab8fb.json +++ b/evaluation_examples/examples/vs_code/53ad5833-3455-407b-bbc6-45b4c79ab8fb.json @@ -86,7 +86,7 @@ "expected": { "type": "rule", "rules": { - "expect": "project" + "expected": "project" } }, "result": { diff --git a/evaluation_examples/examples/vs_code/6ed0a554-cbee-4b44-84ea-fd6c042f4fe1.json b/evaluation_examples/examples/vs_code/6ed0a554-cbee-4b44-84ea-fd6c042f4fe1.json index 1d5f59e..7e1f03d 100644 --- a/evaluation_examples/examples/vs_code/6ed0a554-cbee-4b44-84ea-fd6c042f4fe1.json +++ b/evaluation_examples/examples/vs_code/6ed0a554-cbee-4b44-84ea-fd6c042f4fe1.json @@ -67,7 +67,7 @@ "expected": { "type": "rule", "rules": { - "expect": { + "expected": { "folders": [ {"path": "project"}, {"path": "data1"}, diff --git a/evaluation_examples/examples/vs_code/70745df8-f2f5-42bd-8074-fbc10334fcc5.json b/evaluation_examples/examples/vs_code/70745df8-f2f5-42bd-8074-fbc10334fcc5.json index 864a07c..51a59ad 100644 --- a/evaluation_examples/examples/vs_code/70745df8-f2f5-42bd-8074-fbc10334fcc5.json +++ b/evaluation_examples/examples/vs_code/70745df8-f2f5-42bd-8074-fbc10334fcc5.json @@ -50,7 +50,7 @@ "expected": { "type": "rule", "rules": { - "expect": { + "expected": { "files.autoSave": "afterDelay", "files.autoSaveDelay": 500 } diff --git a/evaluation_examples/examples/vs_code/930fdb3b-11a8-46fe-9bac-577332e2640e.json b/evaluation_examples/examples/vs_code/930fdb3b-11a8-46fe-9bac-577332e2640e.json index c6a61b5..e2aa73e 100644 --- a/evaluation_examples/examples/vs_code/930fdb3b-11a8-46fe-9bac-577332e2640e.json +++ b/evaluation_examples/examples/vs_code/930fdb3b-11a8-46fe-9bac-577332e2640e.json @@ -28,7 +28,7 @@ "expected": { "type": "rule", "rules": { - "expect": + "expected": { "key": "ctrl+j", "command": "workbench.action.focusActiveEditorGroup", diff --git a/evaluation_examples/examples/vs_code/9439a27b-18ae-42d8-9778-5f68f891805e.json b/evaluation_examples/examples/vs_code/9439a27b-18ae-42d8-9778-5f68f891805e.json index e0d9425..d91b3fb 100644 --- a/evaluation_examples/examples/vs_code/9439a27b-18ae-42d8-9778-5f68f891805e.json +++ b/evaluation_examples/examples/vs_code/9439a27b-18ae-42d8-9778-5f68f891805e.json @@ -28,7 +28,7 @@ "expected": { "type": "rule", "rules": { - "expect": {"debug.focusEditorOnBreak": false} + "expected": {"debug.focusEditorOnBreak": false} } }, "result": { diff --git a/evaluation_examples/examples/vs_code/982d12a5-beab-424f-8d38-d2a48429e511.json b/evaluation_examples/examples/vs_code/982d12a5-beab-424f-8d38-d2a48429e511.json index e1dc8c7..0403a74 100644 --- a/evaluation_examples/examples/vs_code/982d12a5-beab-424f-8d38-d2a48429e511.json +++ b/evaluation_examples/examples/vs_code/982d12a5-beab-424f-8d38-d2a48429e511.json @@ -1,9 +1,16 @@ { "id": "982d12a5-beab-424f-8d38-d2a48429e511", "snapshot": "vscode", - "instruction": "Please help me change the color theme of VS Code to Dark.", + "instruction": "Please help me change the color theme of VS Code to Visual Studio Dark.", "source": "https://www.youtube.com/watch?v=ORrELERGIHs", "config": [ + { + "type": "command", + "parameters": { + "command": "SETTINGS_PATH=\"$HOME/.config/Code/User/settings.json\"; [ ! -f \"$SETTINGS_PATH\" ] && echo '{\"workbench.colorTheme\": \"Red\"}' > \"$SETTINGS_PATH\" || (grep -q \"workbench.colorTheme\" \"$SETTINGS_PATH\" && sed -i 's/\"workbench.colorTheme\": \"[^\"]*\"/\"workbench.colorTheme\": \"Red\"/' \"$SETTINGS_PATH\" || jq '. + {\"workbench.colorTheme\": \"Red\"}' \"$SETTINGS_PATH\" > temp.json && mv temp.json \"$SETTINGS_PATH\")", + "shell": true + } + }, { "type": "launch", "parameters": { @@ -28,14 +35,13 @@ "expected": { "type": "rule", "rules": { - "expect": "2" + "expected": "{\"workbench.colorTheme\": \"Visual Studio Dark\"}\n" } }, "result": { - "type": "vscode_config", - "vscode_extension_command": "GetColorTheme", - "path": "/home/user/GetColorTheme.txt", - "dest": "GetColorTheme.txt" + "type": "vm_file", + "path": "/home/user/.config/Code/User/settings.json", + "dest": "settings.json" } } } diff --git a/evaluation_examples/examples/vs_code/9d425400-e9b2-4424-9a4b-d4c7abac4140.json b/evaluation_examples/examples/vs_code/9d425400-e9b2-4424-9a4b-d4c7abac4140.json index aa12ec8..1de9d06 100644 --- a/evaluation_examples/examples/vs_code/9d425400-e9b2-4424-9a4b-d4c7abac4140.json +++ b/evaluation_examples/examples/vs_code/9d425400-e9b2-4424-9a4b-d4c7abac4140.json @@ -28,7 +28,7 @@ "expected": { "type": "rule", "rules": { - "expect": {"workbench.editor.wrapTabs": true} + "expected": {"workbench.editor.wrapTabs": true} } }, "result": { diff --git a/evaluation_examples/examples/vs_code/c6bf789c-ba3a-4209-971d-b63abf0ab733.json b/evaluation_examples/examples/vs_code/c6bf789c-ba3a-4209-971d-b63abf0ab733.json index af52359..3c176f3 100644 --- a/evaluation_examples/examples/vs_code/c6bf789c-ba3a-4209-971d-b63abf0ab733.json +++ b/evaluation_examples/examples/vs_code/c6bf789c-ba3a-4209-971d-b63abf0ab733.json @@ -50,7 +50,7 @@ "expected": { "type": "rule", "rules": { - "expect": { + "expected": { "files.exclude": { "**/__pycache__": true } diff --git a/evaluation_examples/examples/vs_code/e2b5e914-ffe1-44d2-8e92-58f8c5d92bb2.json b/evaluation_examples/examples/vs_code/e2b5e914-ffe1-44d2-8e92-58f8c5d92bb2.json index e8fa810..2cbff94 100644 --- a/evaluation_examples/examples/vs_code/e2b5e914-ffe1-44d2-8e92-58f8c5d92bb2.json +++ b/evaluation_examples/examples/vs_code/e2b5e914-ffe1-44d2-8e92-58f8c5d92bb2.json @@ -38,7 +38,7 @@ "expected": { "type": "rule", "rules": { - "expect": { + "expected": { "python.analysis.diagnosticSeverityOverrides": {"reportMissingImports": "none"} } } diff --git a/evaluation_examples/examples/vs_code/ea98c5d7-3cf9-4f9b-8ad3-366b58e0fcae.json b/evaluation_examples/examples/vs_code/ea98c5d7-3cf9-4f9b-8ad3-366b58e0fcae.json index 309cf1e..f337c89 100644 --- a/evaluation_examples/examples/vs_code/ea98c5d7-3cf9-4f9b-8ad3-366b58e0fcae.json +++ b/evaluation_examples/examples/vs_code/ea98c5d7-3cf9-4f9b-8ad3-366b58e0fcae.json @@ -30,7 +30,7 @@ "expected": { "type": "rule", "rules": { - "expect": + "expected": { "key": "ctrl+f", "command": "-list.find",