diff --git a/desktop_env/evaluators/metrics/__init__.py b/desktop_env/evaluators/metrics/__init__.py index b07f71d..59a72d2 100644 --- a/desktop_env/evaluators/metrics/__init__.py +++ b/desktop_env/evaluators/metrics/__init__.py @@ -16,7 +16,7 @@ from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, co from .gimp import increase_saturation, decrease_brightness, check_file_exists, compare_triangle_positions from .general import check_csv, check_accessibility_tree, check_list, run_sqlite3, check_json from .thunderbird import check_thunderbird_prefs, check_thunderbird_filter -from .vscode import compare_text_file, compare_config, compare_answer, is_extension_installed +from .vscode import compare_text_file, compare_config, compare_answer, is_extension_installed, check_json_settings, check_json_keybindings from .impress import check_image_stretch_and_center, check_slide_numbers_color, compare_pptx_files, check_strikethrough, \ check_for_audio, check_formula_shape from .impress import check_slide_orientation_Portrait, contains_mp4_video diff --git a/desktop_env/evaluators/metrics/vscode.py b/desktop_env/evaluators/metrics/vscode.py index 55f6407..14e9df1 100644 --- a/desktop_env/evaluators/metrics/vscode.py +++ b/desktop_env/evaluators/metrics/vscode.py @@ -1,5 +1,5 @@ from typing import Dict -import json +import json, copy def check_json_keybindings(actual: str, expected: str, **options) -> float: """ @@ -10,36 +10,58 @@ def check_json_keybindings(actual: str, expected: str, **options) -> float: Return: float: the score """ + def direct_load_json(fp): + try: + with open(fp, 'r') as f: + data = json.load(f) + return data + except: + return None + + def skip_first_line_load_json(fp): + try: + with open(fp, 'r') as f: + f.readline() + data = json.load(f) + return data + except: + return None - with open(actual) as f: - data = json.load(f) - + for func in [direct_load_json, skip_first_line_load_json]: + data = func(actual) + if data is not None and type(data) == list: + break + else: + return 0.0 + expected = expected['expect'] if expected in data: return 1.0 else: return 0.0 + def check_json_settings(actual: str, expected: str, **options) -> float: """ Args: actual (str): path to result text file - expected (str): expected dict{} + expected (dict): expected dict{}, containing key "expect" Return: float: the score """ - with open(actual) as f: + with open(actual, 'r') as f: data = json.load(f) - - expect = set(expected.items()) - json = set(data.items()) - - if expect.issubset(json): + + expect = expected['expect'] + data_copy = copy.deepcopy(data) + data_copy.update(expect) + if data == data_copy: return 1.0 else: return 0.0 + def compare_text_file(actual: str, expected: str, **options) -> float: """ Args: diff --git a/evaluation_examples/examples/vs_code/0ed39f63-6049-43d4-ba4d-5fa2fe04a951.json b/evaluation_examples/examples/vs_code/0ed39f63-6049-43d4-ba4d-5fa2fe04a951.json index 2c7a72f..be1f5a6 100644 --- a/evaluation_examples/examples/vs_code/0ed39f63-6049-43d4-ba4d-5fa2fe04a951.json +++ b/evaluation_examples/examples/vs_code/0ed39f63-6049-43d4-ba4d-5fa2fe04a951.json @@ -10,7 +10,7 @@ "files": [ { "url": "https://drive.usercontent.google.com/download?id=1I0tp77_4Qwuz_JW0Tluo_DJzScTARkOZ&export=download&authuser=0&confirm=t&uuid=3e090432-df3f-4f68-8c77-f0f06d399d08&at=APZUnTUFLQTdU4MtnLGUnOVAVcxW:1704282082382", - "path": "Desktop/vscode_replace_text.txt" + "path": "/home/user/Desktop/vscode_replace_text.txt" } ] } @@ -18,7 +18,7 @@ { "type": "launch", "parameters": { - "command": ["code", "Desktop/vscode_replace_text.txt"] + "command": ["code", "/home/user/Desktop/vscode_replace_text.txt"] } }, { @@ -41,7 +41,7 @@ }, "result": { "type": "vm_file", - "path": "Desktop/vscode_replace_text.txt", + "path": "/home/user/Desktop/vscode_replace_text.txt", "dest": "vscode_replace_text.txt" } } diff --git a/evaluation_examples/examples/vs_code/53ad5833-3455-407b-bbc6-45b4c79ab8fb.json b/evaluation_examples/examples/vs_code/53ad5833-3455-407b-bbc6-45b4c79ab8fb.json index 11fb7e5..290643b 100644 --- a/evaluation_examples/examples/vs_code/53ad5833-3455-407b-bbc6-45b4c79ab8fb.json +++ b/evaluation_examples/examples/vs_code/53ad5833-3455-407b-bbc6-45b4c79ab8fb.json @@ -71,7 +71,7 @@ "result": { "type": "vscode_config", "vscode_extension_command": "OpenProject", - "path": "OpenProject.txt", + "path": "/home/user/OpenProject.txt", "dest": "OpenProject.txt" } } diff --git a/evaluation_examples/examples/vs_code/982d12a5-beab-424f-8d38-d2a48429e511.json b/evaluation_examples/examples/vs_code/982d12a5-beab-424f-8d38-d2a48429e511.json index 8c0748d..f5144e8 100644 --- a/evaluation_examples/examples/vs_code/982d12a5-beab-424f-8d38-d2a48429e511.json +++ b/evaluation_examples/examples/vs_code/982d12a5-beab-424f-8d38-d2a48429e511.json @@ -34,7 +34,7 @@ "result": { "type": "vscode_config", "vscode_extension_command": "GetColorTheme", - "path": "GetColorTheme.txt", + "path": "/home/user/GetColorTheme.txt", "dest": "GetColorTheme.txt" } } diff --git a/evaluation_examples/examples/vs_code/ea98c5d7-3cf9-4f9b-8ad3-366b58e0fcae.json b/evaluation_examples/examples/vs_code/ea98c5d7-3cf9-4f9b-8ad3-366b58e0fcae.json index f2667e9..e230539 100644 --- a/evaluation_examples/examples/vs_code/ea98c5d7-3cf9-4f9b-8ad3-366b58e0fcae.json +++ b/evaluation_examples/examples/vs_code/ea98c5d7-3cf9-4f9b-8ad3-366b58e0fcae.json @@ -1,7 +1,7 @@ { "id": "ea98c5d7-3cf9-4f9b-8ad3-366b58e0fcae", "snapshot": "vscode", - "instruction": "I want to remove the shortcut 'cmd+f' for Tree view Find (Explorer search) in VS Code explorer view due to shortcut conflict. Can you help me remove this shortcut?", + "instruction": "I want to remove the shortcut 'ctrl+f' for Tree view Find (Explorer search) in VS Code explorer view due to shortcut conflict. Can you help me remove this shortcut?", "source": ["https://superuser.com/questions/1748097/vs-code-disable-tree-view-find-explorer-search", "https://superuser.com/questions/1417361/how-to-disable-file-filtering-in-vs-code-sidebar-explorer?rq=1" ], @@ -32,7 +32,7 @@ "rules": { "expect": { - "key": "cmd+f", + "key": "ctrl+f", "command": "-list.find", "when": "listFocus && listSupportsFind" }