update vscode

desktop_env/evaluators/getters/general.py

@@ -0,0 +1,23 @@
+from typing import Dict
+
+import os
+import requests
+
+
+def get_string(env, config: Dict[str, str]) -> str:
+    """
+    Config:
+        string (str)
+    """
+
+    return config["string"]
+
+def get_command_line(env, config: Dict[str, str]) -> str:
+    """
+    Config:
+        string (str)
+    """
+    
+    f = os.popen(config["command"])
+    
+    return f.read()

desktop_env/evaluators/metrics/vscode.py

@@ -20,5 +20,13 @@ def compare_text_file(actual: str, expected: str, **options) -> float:
         return 1.0
     return 0.0
 
+def compare_answer(actual: str, expected: str, **options) -> float:
+
+    if actual == expected:
+        return 1.0
+    
+    # TODO: can use text embedding to get non-zero return
+    return 0.0
+
 if __name__ == '__main__':
     print(compare_text_file("README.md", "README.md"))