diff --git a/desktop_env/evaluators/getters/general.py b/desktop_env/evaluators/getters/general.py new file mode 100644 index 0000000..22a20c7 --- /dev/null +++ b/desktop_env/evaluators/getters/general.py @@ -0,0 +1,23 @@ +from typing import Dict + +import os +import requests + + +def get_string(env, config: Dict[str, str]) -> str: + """ + Config: + string (str) + """ + + return config["string"] + +def get_command_line(env, config: Dict[str, str]) -> str: + """ + Config: + string (str) + """ + + f = os.popen(config["command"]) + + return f.read() \ No newline at end of file diff --git a/desktop_env/evaluators/metrics/vscode.py b/desktop_env/evaluators/metrics/vscode.py index 9c5bb3b..9efef3e 100644 --- a/desktop_env/evaluators/metrics/vscode.py +++ b/desktop_env/evaluators/metrics/vscode.py @@ -20,5 +20,13 @@ def compare_text_file(actual: str, expected: str, **options) -> float: return 1.0 return 0.0 +def compare_answer(actual: str, expected: str, **options) -> float: + + if actual == expected: + return 1.0 + + # TODO: can use text embedding to get non-zero return + return 0.0 + if __name__ == '__main__': print(compare_text_file("README.md", "README.md")) \ No newline at end of file diff --git a/evaluation_examples/examples/vs_code/eabc805a-bfcf-4460-b250-ac92135819f6.json b/evaluation_examples/examples/vs_code/eabc805a-bfcf-4460-b250-ac92135819f6.json index ce982b8..bf63054 100644 --- a/evaluation_examples/examples/vs_code/eabc805a-bfcf-4460-b250-ac92135819f6.json +++ b/evaluation_examples/examples/vs_code/eabc805a-bfcf-4460-b250-ac92135819f6.json @@ -9,5 +9,14 @@ "vscode" ], "evaluator": { + "func": "compare_answer", + "expected": { + "type": "string", + "string": "ms-python.python\n" + }, + "result": { + "type": "command_line", + "command": "code --list-extensions | grep ms-python.python" + } } }