diff --git a/desktop_env/evaluators/getters/__init__.py b/desktop_env/evaluators/getters/__init__.py index 7f59d4b..546b8db 100644 --- a/desktop_env/evaluators/getters/__init__.py +++ b/desktop_env/evaluators/getters/__init__.py @@ -3,3 +3,5 @@ from .info import get_vm_screen_size, get_vm_window_size, get_vm_wallpaper from .misc import get_rule, get_accessibility_tree from .vlc import get_vlc_playing_info, get_vlc_config from .chrome import get_default_search_engine, get_bookmarks +from .replay import get_replay +from .vscode import get_vscode_config diff --git a/desktop_env/evaluators/getters/replay.py b/desktop_env/evaluators/getters/replay.py index e69de29..a0d1d62 100644 --- a/desktop_env/evaluators/getters/replay.py +++ b/desktop_env/evaluators/getters/replay.py @@ -0,0 +1,19 @@ +from typing import List, Dict, Any + +def get_replay(env, trajectory: List[Dict[str, Any]]) -> None: + + def parse(action): + if action["type"] == "hotkey": + keys = "', '".join(action["param"]) + return f"pyautogui.hotkey('{keys}')" + + if action["type"] == "typewrite": + text = action["param"] + return f"pyautogui.typewrite('{text}')" + + if action["type"] == "press": + key = action["param"] + return f"pyautogui.press('{key}')" + + for action in trajectory: + env.controller.execute_python_command(parse(action)) \ No newline at end of file diff --git a/desktop_env/evaluators/getters/vscode.py b/desktop_env/evaluators/getters/vscode.py new file mode 100644 index 0000000..7758d17 --- /dev/null +++ b/desktop_env/evaluators/getters/vscode.py @@ -0,0 +1,13 @@ +from typing import Dict +from typing import Any +from replay import get_replay +from file import get_vm_file + +def get_vscode_config(env, config: Dict[str, Any]) -> str: + + get_replay(env, config["trajectory"]) + + return get_vm_file(env, { + "path": config["path"], + "dest": config["dest"] + }) \ No newline at end of file diff --git a/desktop_env/evaluators/metrics/vscode.py b/desktop_env/evaluators/metrics/vscode.py index 9efef3e..a76910b 100644 --- a/desktop_env/evaluators/metrics/vscode.py +++ b/desktop_env/evaluators/metrics/vscode.py @@ -1,11 +1,8 @@ def compare_text_file(actual: str, expected: str, **options) -> float: """ Args: - actual (str): path to result xlsx - expected (str): path to gold xlsx - options (Dict[str, List[str]]): dict like - { - } + actual (str): path to result text file + expected (str): path to gold text file Return: float: the score @@ -20,7 +17,32 @@ def compare_text_file(actual: str, expected: str, **options) -> float: return 1.0 return 0.0 +def compare_config(actual: str, expected: str, **options) -> float: + """ + Args: + actual (str): path to result text file + expected (str): gold string + + Return: + float: the score + """ + + with open(actual) as f1: + actual_text = f1.read() + + if actual_text == expected: + return 1.0 + return 0.0 + def compare_answer(actual: str, expected: str, **options) -> float: + """ + Args: + actual (str): result string + expected (str): gold string + + Return: + float: the score + """ if actual == expected: return 1.0 diff --git a/evaluation_examples/examples/vs_code/53ad5833-3455-407b-bbc6-45b4c79ab8fb.json b/evaluation_examples/examples/vs_code/53ad5833-3455-407b-bbc6-45b4c79ab8fb.json index 893eada..824f604 100644 --- a/evaluation_examples/examples/vs_code/53ad5833-3455-407b-bbc6-45b4c79ab8fb.json +++ b/evaluation_examples/examples/vs_code/53ad5833-3455-407b-bbc6-45b4c79ab8fb.json @@ -15,5 +15,21 @@ "related_apps": [ "vscode" ], - "evaluator": "evaluation_dir" + "evaluator": { + "func": "compare_config", + "expected": { + "type": "string", + "string": "project" + }, + "result": { + "type": "vscode_config", + "trajectory": [ + {"type": "hotkey", "param": ["command", "shift", "p"]}, + {"type": "typewrite", "param": "OpenProject"}, + {"type": "press", "param": "enter"} + ], + "path": "OpenProject.txt", + "dest": "OpenProject.txt" + } + } }