diff --git a/desktop_env/evaluators/metrics/vscode.py b/desktop_env/evaluators/metrics/vscode.py index d207aae..82129f6 100644 --- a/desktop_env/evaluators/metrics/vscode.py +++ b/desktop_env/evaluators/metrics/vscode.py @@ -58,16 +58,20 @@ def check_json_settings(actual: str, expected: str, **options) -> float: if not actual: return 0. - with open(actual, 'r') as f: - data = json.load(f) + try: + with open(actual, 'r') as f: + data = json.load(f) + except Exception as e: + return 0.0 expect = expected['expected'] - data_copy = copy.deepcopy(data) - data_copy.update(expect) - if data == data_copy: - return 1.0 - else: - return 0.0 + + # Check if all expected key-value pairs are in the actual data + for key, value in expect.items(): + if key not in data or data[key] != value: + return 0.0 + + return 1.0 def compare_text_file(actual: str, expected: str, **options) -> float: diff --git a/evaluation_examples/examples/vs_code/276cc624-87ea-4f08-ab93-f770e3790175.json b/evaluation_examples/examples/vs_code/276cc624-87ea-4f08-ab93-f770e3790175.json index d35ba53..bc421a9 100644 --- a/evaluation_examples/examples/vs_code/276cc624-87ea-4f08-ab93-f770e3790175.json +++ b/evaluation_examples/examples/vs_code/276cc624-87ea-4f08-ab93-f770e3790175.json @@ -1,7 +1,7 @@ { "id": "276cc624-87ea-4f08-ab93-f770e3790175", "snapshot": "vscode", - "instruction": "Please help me set the current user's line length to 50 characters in VS Code.", + "instruction": "Please help me set the current user's line length for code wrapping to 50 characters in VS Code.", "source": "https://www.quora.com/unanswered/How-do-you-set-the-line-length-in-Visual-Studio-Code", "config": [ { diff --git a/evaluation_examples/examples/vs_code/7c4cc09e-7a92-40dd-8338-b2286535c4ed.json b/evaluation_examples/examples/vs_code/7c4cc09e-7a92-40dd-8338-b2286535c4ed.json index f25886a..8345b1b 100644 --- a/evaluation_examples/examples/vs_code/7c4cc09e-7a92-40dd-8338-b2286535c4ed.json +++ b/evaluation_examples/examples/vs_code/7c4cc09e-7a92-40dd-8338-b2286535c4ed.json @@ -1,7 +1,7 @@ { "id": "7c4cc09e-7a92-40dd-8338-b2286535c4ed", "snapshot": "vscode", - "instruction": "Please help me change the display language of VS Code to \"Arabic\".", + "instruction": "Please help me change the display language of VS Code to \"Arabic\" without using any extensions.", "source": "", "config": [ {