Finish loading the vscode examples v1; Improve on the infra: Add accessibility tree into the observation; Add activate window function, etc

desktop_env/evaluators/getters/__init__.py

@@ -1,7 +1,9 @@
+from .chrome import get_default_search_engine, get_cookie_data, get_bookmarks, get_open_tabs_info, get_pdf_from_url, \
+    get_shortcuts_on_desktop
 from .file import get_cloud_file, get_vm_file, get_cache_file
+from .general import get_vm_command_line
 from .info import get_vm_screen_size, get_vm_window_size, get_vm_wallpaper
 from .misc import get_rule, get_accessibility_tree
-from .vlc import get_vlc_playing_info, get_vlc_config
-from .chrome import get_default_search_engine, get_cookie_data, get_bookmarks, get_open_tabs_info, get_pdf_from_url, get_shortcuts_on_desktop
 from .replay import get_replay
+from .vlc import get_vlc_playing_info, get_vlc_config
 from .vscode import get_vscode_config

desktop_env/evaluators/getters/general.py

@@ -1,22 +1,19 @@
-import os
+import logging
 from typing import Dict
+import requests
+
+logger = logging.getLogger("desktopenv.getters.general")
 
 
-def get_string(env, config: Dict[str, str]) -> str:
-    """
-    Config:
-        string (str)
-    """
+def get_vm_command_line(env, config: Dict[str, str]):
+    vm_ip = env.vm_ip
+    port = 5000
+    command = config["command"]
 
-    return config["string"]
+    response = requests.post(f"http://{vm_ip}:{port}/execute", json={"command": command})
 
-
-def get_command_line(env, config: Dict[str, str]) -> str:
-    """
-    Config:
-        string (str)
-    """
-
-    f = os.popen(config["command"])
-
-    return f.read()
+    if response.status_code == 200:
+        return response.json()["output"]
+    else:
+        logger.error("Failed to get vm command line. Status code: %d", response.status_code)
+        return None

desktop_env/evaluators/getters/replay.py

@@ -2,6 +2,7 @@ from typing import List, Dict, Any
 
 
 def get_replay(env, trajectory: List[Dict[str, Any]]) -> None:
+    # fixme: need to be combined with the accessibility tree to activate the selection of the target window
     def parse(action):
         if action["type"] == "hotkey":
             keys = "', '".join(action["param"])

desktop_env/evaluators/getters/vscode.py

@@ -1,14 +1,30 @@
-from typing import Any
-from typing import Dict
+import logging
+from typing import Any, Dict
 
 from .file import get_vm_file
 from .replay import get_replay
 
+logger = logging.getLogger("desktopenv.getters.vscode")
+
 
 def get_vscode_config(env, config: Dict[str, Any]) -> str:
-    trajectory = [{"type": "hotkey", "param": ["command", "shift", "p"]},
-                  {"type": "typewrite", "param": "OpenProject"},
-                  {"type": "press", "param": "enter"}]
+    os_type = env.vm_platform
+    vscode_extension_command = config["vscode_extension_command"]
+
+    # fixme: depends on how we config and install the vscode in virtual machine, need to be aligned and double-checked
+
+    if os_type == "MacOS":
+        trajectory = [
+            {"type": "hotkey", "param": ["command", "shift", "p"]},
+            {"type": "typewrite", "param": vscode_extension_command},
+            {"type": "press", "param": "enter"}
+        ]
+    else:
+        trajectory = [
+            {"type": "hotkey", "param": ["ctrl", "shift", "p"]},
+            {"type": "typewrite", "param": vscode_extension_command},
+            {"type": "press", "param": "enter"}
+        ]
 
     get_replay(env, trajectory)
 

desktop_env/evaluators/metrics/__init__.py

@@ -13,4 +13,4 @@ from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, co
 from .gimp import increase_saturation, decrease_brightness, check_file_exists, compare_triangle_positions
 from .general import check_csv, check_accessibility_tree, check_list, run_sqlite3
 from .thunderbird import check_thunderbird_prefs, check_thunderbird_filter
-from .vscode import compare_text_file, compare_config, compare_answer
+from .vscode import compare_text_file, compare_config, compare_answer, is_extension_installed

desktop_env/evaluators/metrics/vscode.py

@@ -10,6 +10,8 @@ def compare_text_file(actual: str, expected: str, **options) -> float:
     Return:
         float: the score
     """
+    if not actual:
+        return 0.
 
     with open(actual) as f1:
         actual_text = f1.read()
@@ -22,6 +24,9 @@ def compare_text_file(actual: str, expected: str, **options) -> float:
 
 
 def compare_config(actual: str, rules: Dict, **options) -> float:
+    if not actual:
+        return 0.
+
     with open(actual) as f1:
         actual_text = f1.read()
 
@@ -39,9 +44,24 @@ def compare_answer(actual: str, rules: Dict, **options) -> float:
     Return:
         float: the score
     """
+    if not actual:
+        return 0.
 
     if actual == rules['expect']:
         return 1.0
 
     # TODO: can use text embedding to get non-zero return
     return 0.0
+
+
+def is_extension_installed(actual: str, rules: Dict, **options):
+    if rules['type'] == 'contain':
+        if rules['expected'] in actual:
+            return 1.0
+        return 0.0
+    elif rules['type'] == 'not_contain':
+        if rules['expected'] not in actual:
+            return 1.0
+        return 0.0
+    else:
+        raise NotImplementedError