1. fix quote and \ characters in execute_command ; 2. add terminal output text as extra observation ; 3. move get_vm_*() to reset()

desktop_env/envs/desktop_env.py

@@ -83,9 +83,9 @@ class DesktopEnv(gym.Env):
         self.controller = PythonController(vm_ip=self.vm_ip)
         self.setup_controller = SetupController(vm_ip=self.vm_ip, cache_dir=self.cache_dir)
 
-        # Meta info of the VM
-        self.vm_platform = self.controller.get_vm_platform()
-        self.vm_screen_size = self.controller.get_vm_screen_size()
+        # Meta info of the VM, move to the reset() function
+        self.vm_platform: str = "" # self.controller.get_vm_platform()
+        self.vm_screen_size = None # self.controller.get_vm_screen_size()
 
         # mode: human or machine
         assert action_space in ["computer_13", "pyautogui"]
@@ -188,6 +188,9 @@ class DesktopEnv(gym.Env):
         _execute_command(["vmrun", "-T", "ws", "revertToSnapshot", self.path_to_vm, self.snapshot_path])
         time.sleep(5)
 
+        self.vm_platform = self.controller.get_vm_platform()
+        self.vm_screen_size = self.controller.get_vm_screen_size()
+
         logger.info("Starting emulator...")
         self._start_emulator()
         logger.info("Emulator started.")
@@ -217,6 +220,7 @@ class DesktopEnv(gym.Env):
         time.sleep(pause)
         observation = {
             "screenshot": self._get_obs(),
+            "terminal": self.controller.get_terminal_output(),
             "instruction": self.instruction
         }
         reward = 0  # todo: Define reward calculation for each example