Finish loading the vscode examples v1; Improve on the infra: Add accessibility tree into the observation; Add activate window function, etc

desktop_env/evaluators/getters/__init__.py

@@ -1,7 +1,9 @@
+from .chrome import get_default_search_engine, get_cookie_data, get_bookmarks, get_open_tabs_info, get_pdf_from_url, \
+    get_shortcuts_on_desktop
 from .file import get_cloud_file, get_vm_file, get_cache_file
+from .general import get_vm_command_line
 from .info import get_vm_screen_size, get_vm_window_size, get_vm_wallpaper
 from .misc import get_rule, get_accessibility_tree
-from .vlc import get_vlc_playing_info, get_vlc_config
-from .chrome import get_default_search_engine, get_cookie_data, get_bookmarks, get_open_tabs_info, get_pdf_from_url, get_shortcuts_on_desktop
 from .replay import get_replay
+from .vlc import get_vlc_playing_info, get_vlc_config
 from .vscode import get_vscode_config

desktop_env/evaluators/getters/general.py

@@ -1,22 +1,19 @@
-import os
+import logging
 from typing import Dict
+import requests
+
+logger = logging.getLogger("desktopenv.getters.general")
 
 
-def get_string(env, config: Dict[str, str]) -> str:
-    """
-    Config:
-        string (str)
-    """
+def get_vm_command_line(env, config: Dict[str, str]):
+    vm_ip = env.vm_ip
+    port = 5000
+    command = config["command"]
 
-    return config["string"]
+    response = requests.post(f"http://{vm_ip}:{port}/execute", json={"command": command})
 
-
-def get_command_line(env, config: Dict[str, str]) -> str:
-    """
-    Config:
-        string (str)
-    """
-
-    f = os.popen(config["command"])
-
-    return f.read()
+    if response.status_code == 200:
+        return response.json()["output"]
+    else:
+        logger.error("Failed to get vm command line. Status code: %d", response.status_code)
+        return None

desktop_env/evaluators/getters/replay.py

@@ -2,6 +2,7 @@ from typing import List, Dict, Any
 
 
 def get_replay(env, trajectory: List[Dict[str, Any]]) -> None:
+    # fixme: need to be combined with the accessibility tree to activate the selection of the target window
     def parse(action):
         if action["type"] == "hotkey":
             keys = "', '".join(action["param"])

desktop_env/evaluators/getters/vscode.py

@@ -1,14 +1,30 @@
-from typing import Any
-from typing import Dict
+import logging
+from typing import Any, Dict
 
 from .file import get_vm_file
 from .replay import get_replay
 
+logger = logging.getLogger("desktopenv.getters.vscode")
+
 
 def get_vscode_config(env, config: Dict[str, Any]) -> str:
-    trajectory = [{"type": "hotkey", "param": ["command", "shift", "p"]},
-                  {"type": "typewrite", "param": "OpenProject"},
-                  {"type": "press", "param": "enter"}]
+    os_type = env.vm_platform
+    vscode_extension_command = config["vscode_extension_command"]
+
+    # fixme: depends on how we config and install the vscode in virtual machine, need to be aligned and double-checked
+
+    if os_type == "MacOS":
+        trajectory = [
+            {"type": "hotkey", "param": ["command", "shift", "p"]},
+            {"type": "typewrite", "param": vscode_extension_command},
+            {"type": "press", "param": "enter"}
+        ]
+    else:
+        trajectory = [
+            {"type": "hotkey", "param": ["ctrl", "shift", "p"]},
+            {"type": "typewrite", "param": vscode_extension_command},
+            {"type": "press", "param": "enter"}
+        ]
 
     get_replay(env, trajectory)