Finish loading the vscode examples v1; Improve on the infra: Add accessibility tree into the observation; Add activate window function, etc

desktop_env/controllers/setup.py

@@ -1,18 +1,18 @@
 import json
-import time
+import logging
 import os.path
+import time
 import traceback
 import uuid
-
-from typing import Dict, List
 from typing import Any, Union, Optional
+from typing import Dict, List
 
 import requests
 from playwright.sync_api import sync_playwright
 from requests_toolbelt.multipart.encoder import MultipartEncoder
+
 from desktop_env.evaluators.metrics.utils import compare_urls
 
-import logging
 logger = logging.getLogger("desktopenv.setup")
 
 
@@ -58,31 +58,31 @@ class SetupController:
         # can add other setup steps
 
     # ZDY_COMMENT: merged with launch
-    #def _command_setup(self, command: str):
-        #"""
-        #Directly send a command into the virtual machine os for setting up.
-        #"""
-        #payload = json.dumps({"command": command})
-        #headers = {
-            #'Content-Type': 'application/json'
-        #}
-        #timeout = 5
-        #timout_whitelist = ["vlc"]
-#
-        #try:
-#
-            #response = requests.post(self.http_server + "/execute", headers=headers, data=payload, timeout=timeout)
-            #if response.status_code == 200:
-                #print("Command executed successfully:", response.text)
-            #else:
-                #print("Failed to execute command. Status code:", response.status_code)
-        #except requests.exceptions.Timeout as e:
-            #if command in timout_whitelist:
-                #print("Command executed successfully:", command)
-            #else:
-                #print("An error occurred while trying to execute the command:", e)
-        #except requests.exceptions.RequestException as e:
-            #print("An error occurred while trying to execute the command:", e)
+    # def _command_setup(self, command: str):
+    # """
+    # Directly send a command into the virtual machine os for setting up.
+    # """
+    # payload = json.dumps({"command": command})
+    # headers = {
+    # 'Content-Type': 'application/json'
+    # }
+    # timeout = 5
+    # timout_whitelist = ["vlc"]
+    #
+    # try:
+    #
+    # response = requests.post(self.http_server + "/execute", headers=headers, data=payload, timeout=timeout)
+    # if response.status_code == 200:
+    # print("Command executed successfully:", response.text)
+    # else:
+    # print("Failed to execute command. Status code:", response.status_code)
+    # except requests.exceptions.Timeout as e:
+    # if command in timout_whitelist:
+    # print("Command executed successfully:", command)
+    # else:
+    # print("An error occurred while trying to execute the command:", e)
+    # except requests.exceptions.RequestException as e:
+    # print("An error occurred while trying to execute the command:", e)
 
     def _download_setup(self, files: List[Dict[str, str]]):
         """
@@ -225,9 +225,14 @@ class SetupController:
         except requests.exceptions.RequestException as e:
             logger.error("An error occurred while trying to send the request: %s", e)
 
-    def _execute_setup( self, command: List[str]
-                      , stdout: str = "", stderr: str = ""
-                      , shell: bool = False, until: Optional[Dict[str, Any]] = None):
+    def _execute_setup(
+            self,
+            command: List[str],
+            stdout: str = "",
+            stderr: str = "",
+            shell: bool = False,
+            until: Optional[Dict[str, Any]] = None
+    ):
         if not command:
             raise Exception("Empty comman to launch.")
 
@@ -249,10 +254,10 @@ class SetupController:
                     if stderr:
                         with open(os.path.join(self.cache_dir, stderr), "w") as f:
                             f.write(results["error"])
-                    logger.info( "Command executed successfully: %s -> %s"
-                               , " ".join(command)
-                               , response.text
-                               )
+                    logger.info("Command executed successfully: %s -> %s"
+                                , " ".join(command)
+                                , response.text
+                                )
                 else:
                     logger.error("Failed to launch application. Status code: %s", response.text)
                     results = None
@@ -264,13 +269,13 @@ class SetupController:
                 results = None
                 nb_failings += 1
 
-            if len(until)==0:
+            if len(until) == 0:
                 terminates = True
             elif results is not None:
-                terminates = "returncode" in until and results["returncode"]==until["returncode"]\
-                          or "stdout" in until and until["stdout"] in results["output"]\
-                          or "stderr" in until and until["stderr"] in results["error"]
-            terminates = terminates or nb_failings>=5
+                terminates = "returncode" in until and results["returncode"] == until["returncode"] \
+                             or "stdout" in until and until["stdout"] in results["output"] \
+                             or "stderr" in until and until["stderr"] in results["error"]
+            terminates = terminates or nb_failings >= 5
             if not terminates:
                 time.sleep(0.3)
 
@@ -293,6 +298,25 @@ class SetupController:
         # TODO
         raise NotImplementedError()
 
+    def _activate_window_setup(self, window_name: str):
+        if not window_name:
+            raise Exception(f"Setup Open - Invalid path ({window_name}).")
+
+        payload = json.dumps({"window_name": window_name})
+        headers = {
+            'Content-Type': 'application/json'
+        }
+
+        # send request to server to open file
+        try:
+            response = requests.post(self.http_server + "/setup" + "/activate_window", headers=headers, data=payload)
+            if response.status_code == 200:
+                logger.info("Command executed successfully: %s", response.text)
+            else:
+                logger.error(f"Failed to activate window {window_name}. Status code: %s", response.text)
+        except requests.exceptions.RequestException as e:
+            logger.error("An error occurred while trying to send the request: %s", e)
+
     # Chrome setup
     def _chrome_open_tabs_setup(self, urls_to_open: List[str]):
         host = self.vm_ip