implement action replay for vscode and gimp evaluation

desktop_env/evaluators/getters/__init__.py

@@ -3,3 +3,5 @@ from .info import get_vm_screen_size, get_vm_window_size, get_vm_wallpaper
 from .misc import get_rule, get_accessibility_tree
 from .vlc import get_vlc_playing_info, get_vlc_config
 from .chrome import get_default_search_engine, get_bookmarks
+from .replay import get_replay
+from .vscode import get_vscode_config

desktop_env/evaluators/getters/replay.py

@@ -0,0 +1,19 @@
+from typing import List, Dict, Any
+
+def get_replay(env, trajectory: List[Dict[str, Any]]) -> None:
+
+    def parse(action):
+        if action["type"] == "hotkey":
+            keys = "', '".join(action["param"])
+            return f"pyautogui.hotkey('{keys}')"
+
+        if action["type"] == "typewrite":
+            text = action["param"]
+            return f"pyautogui.typewrite('{text}')"
+
+        if action["type"] == "press":
+            key = action["param"]
+            return f"pyautogui.press('{key}')"
+
+    for action in trajectory:
+        env.controller.execute_python_command(parse(action))

desktop_env/evaluators/getters/vscode.py

@@ -0,0 +1,13 @@
+from typing import Dict
+from typing import Any
+from replay import get_replay
+from file import get_vm_file
+
+def get_vscode_config(env, config: Dict[str, Any]) -> str:
+
+    get_replay(env, config["trajectory"])
+
+    return get_vm_file(env, {
+            "path": config["path"],
+            "dest": config["dest"]
+        })

desktop_env/evaluators/metrics/vscode.py

@@ -1,11 +1,8 @@
 def compare_text_file(actual: str, expected: str, **options) -> float:
     """
     Args:
-        actual (str): path to result xlsx
+        actual (str): path to result text file
-        expected (str): path to gold xlsx
+        expected (str): path to gold text file
-        options (Dict[str, List[str]]): dict like
-          {
-          }
 
     Return:
         float: the score
@@ -20,7 +17,32 @@ def compare_text_file(actual: str, expected: str, **options) -> float:
         return 1.0
     return 0.0
 
+def compare_config(actual: str, expected: str, **options) -> float:
+    """
+    Args:
+        actual (str): path to result text file
+        expected (str): gold string
+
+    Return:
+        float: the score
+    """
+    
+    with open(actual) as f1:
+        actual_text = f1.read()
+
+    if actual_text == expected:
+        return 1.0
+    return 0.0
+
 def compare_answer(actual: str, expected: str, **options) -> float:
+    """
+    Args:
+        actual (str): result string
+        expected (str): gold string
+
+    Return:
+        float: the score
+    """
 
     if actual == expected:
         return 1.0

evaluation_examples/examples/vs_code/53ad5833-3455-407b-bbc6-45b4c79ab8fb.json

@@ -15,5 +15,21 @@
   "related_apps": [
     "vscode"
   ],
-  "evaluator": "evaluation_dir"
+  "evaluator": {
+    "func": "compare_config",
+    "expected": {
+      "type": "string",
+      "string": "project"
+    },
+    "result": {
+      "type": "vscode_config",
+      "trajectory": [
+        {"type": "hotkey", "param": ["command", "shift", "p"]},
+        {"type": "typewrite", "param": "OpenProject"},
+        {"type": "press", "param": "enter"}
+      ],
+      "path": "OpenProject.txt",
+      "dest": "OpenProject.txt"
+    }
+  }
 }