update vscode

desktop_env/evaluators/getters/general.py

@@ -0,0 +1,23 @@
+from typing import Dict
+
+import os
+import requests
+
+
+def get_string(env, config: Dict[str, str]) -> str:
+    """
+    Config:
+        string (str)
+    """
+
+    return config["string"]
+
+def get_command_line(env, config: Dict[str, str]) -> str:
+    """
+    Config:
+        string (str)
+    """
+    
+    f = os.popen(config["command"])
+    
+    return f.read()

desktop_env/evaluators/metrics/vscode.py

@@ -20,5 +20,13 @@ def compare_text_file(actual: str, expected: str, **options) -> float:
         return 1.0
     return 0.0
 
+def compare_answer(actual: str, expected: str, **options) -> float:
+
+    if actual == expected:
+        return 1.0
+    
+    # TODO: can use text embedding to get non-zero return
+    return 0.0
+
 if __name__ == '__main__':
     print(compare_text_file("README.md", "README.md"))

evaluation_examples/examples/vs_code/eabc805a-bfcf-4460-b250-ac92135819f6.json

@@ -9,5 +9,14 @@
     "vscode"
   ],
   "evaluator": {
+    "func": "compare_answer",
+    "expected": {
+      "type": "string",
+      "string": "ms-python.python\n"
+    },
+    "result": {
+      "type": "command_line",
+      "command": "code --list-extensions | grep ms-python.python"
+    }
   }
 }