fix some vscode eval problems

desktop_env/evaluators/metrics/__init__.py

@@ -16,7 +16,7 @@ from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, co
 from .gimp import increase_saturation, decrease_brightness, check_file_exists, compare_triangle_positions
 from .general import check_csv, check_accessibility_tree, check_list, run_sqlite3, check_json
 from .thunderbird import check_thunderbird_prefs, check_thunderbird_filter
-from .vscode import compare_text_file, compare_config, compare_answer, is_extension_installed
+from .vscode import compare_text_file, compare_config, compare_answer, is_extension_installed, check_json_settings, check_json_keybindings
 from .impress import check_image_stretch_and_center, check_slide_numbers_color, compare_pptx_files, check_strikethrough, \
     check_for_audio, check_formula_shape
 from .impress import check_slide_orientation_Portrait, contains_mp4_video

desktop_env/evaluators/metrics/vscode.py

@@ -1,5 +1,5 @@
 from typing import Dict
-import json
+import json, copy
 
 def check_json_keybindings(actual: str, expected: str, **options) -> float:
     """
@@ -10,36 +10,58 @@ def check_json_keybindings(actual: str, expected: str, **options) -> float:
     Return:
         float: the score
     """
+    def direct_load_json(fp):
+        try:
+            with open(fp, 'r') as f:
+                data = json.load(f)
+            return data
+        except:
+            return None
+    
+    def skip_first_line_load_json(fp):
+        try:
+            with open(fp, 'r') as f:
+                f.readline()
+                data = json.load(f)
+            return data
+        except:
+            return None
 
-    with open(actual) as f:
+    for func in [direct_load_json, skip_first_line_load_json]:
-        data = json.load(f)
+        data = func(actual)
-
+        if data is not None and type(data) == list:
+            break
+    else:
+        return 0.0
+    expected = expected['expect']
     if expected in data:
         return 1.0
     else:
         return 0.0
 
+
 def check_json_settings(actual: str, expected: str, **options) -> float:
     """
     Args:
         actual (str): path to result text file
-        expected (str): expected dict{}
+        expected (dict): expected dict{}, containing key "expect"
 
     Return:
         float: the score
     """
 
-    with open(actual) as f:
+    with open(actual, 'r') as f:
         data = json.load(f)
-        
+    
-    expect = set(expected.items())
+    expect = expected['expect']
-    json = set(data.items())
+    data_copy = copy.deepcopy(data)
-
+    data_copy.update(expect)
-    if expect.issubset(json):
+    if data == data_copy:
         return 1.0
     else:
         return 0.0
 
+
 def compare_text_file(actual: str, expected: str, **options) -> float:
     """
     Args:

evaluation_examples/examples/vs_code/0ed39f63-6049-43d4-ba4d-5fa2fe04a951.json

@@ -10,7 +10,7 @@
         "files": [
           {
             "url": "https://drive.usercontent.google.com/download?id=1I0tp77_4Qwuz_JW0Tluo_DJzScTARkOZ&export=download&authuser=0&confirm=t&uuid=3e090432-df3f-4f68-8c77-f0f06d399d08&at=APZUnTUFLQTdU4MtnLGUnOVAVcxW:1704282082382",
-            "path": "Desktop/vscode_replace_text.txt"
+            "path": "/home/user/Desktop/vscode_replace_text.txt"
           }
         ]
       }
@@ -18,7 +18,7 @@
     {
       "type": "launch",
       "parameters": {
-        "command": ["code", "Desktop/vscode_replace_text.txt"]
+        "command": ["code", "/home/user/Desktop/vscode_replace_text.txt"]
       }
     },
     {
@@ -41,7 +41,7 @@
     },
     "result": {
       "type": "vm_file",
-      "path": "Desktop/vscode_replace_text.txt",
+      "path": "/home/user/Desktop/vscode_replace_text.txt",
       "dest": "vscode_replace_text.txt"
     }
   }

evaluation_examples/examples/vs_code/53ad5833-3455-407b-bbc6-45b4c79ab8fb.json

@@ -71,7 +71,7 @@
     "result": {
       "type": "vscode_config",
       "vscode_extension_command": "OpenProject",
-      "path": "OpenProject.txt",
+      "path": "/home/user/OpenProject.txt",
       "dest": "OpenProject.txt"
     }
   }

evaluation_examples/examples/vs_code/982d12a5-beab-424f-8d38-d2a48429e511.json

@@ -34,7 +34,7 @@
     "result": {
       "type": "vscode_config",
       "vscode_extension_command": "GetColorTheme",
-      "path": "GetColorTheme.txt",
+      "path": "/home/user/GetColorTheme.txt",
       "dest": "GetColorTheme.txt"
     }
   }

evaluation_examples/examples/vs_code/ea98c5d7-3cf9-4f9b-8ad3-366b58e0fcae.json

@@ -1,7 +1,7 @@
 {
   "id": "ea98c5d7-3cf9-4f9b-8ad3-366b58e0fcae",
   "snapshot": "vscode",
-  "instruction": "I want to remove the shortcut 'cmd+f' for Tree view Find (Explorer search) in VS Code explorer view due to shortcut conflict. Can you help me remove this shortcut?",
+  "instruction": "I want to remove the shortcut 'ctrl+f' for Tree view Find (Explorer search) in VS Code explorer view due to shortcut conflict. Can you help me remove this shortcut?",
   "source": ["https://superuser.com/questions/1748097/vs-code-disable-tree-view-find-explorer-search",
               "https://superuser.com/questions/1417361/how-to-disable-file-filtering-in-vs-code-sidebar-explorer?rq=1"
             ],
@@ -32,7 +32,7 @@
       "rules": {
         "expect": 
           {
-              "key": "cmd+f",
+              "key": "ctrl+f",
               "command": "-list.find",
               "when": "listFocus && listSupportsFind"
           }