add key down key up

desktop_env/controllers/keyboard.py

@@ -12,6 +12,15 @@ class AbstractKeyboardController(ABC):
     @abstractmethod
     def key(self, key: str):
         raise NotImplementedError
+    
+    @abstractmethod
+    def key_down(self, key: str):
+        raise NotImplementedError
+
+    @abstractmethod
+    def key_up(self, key: str):
+        raise NotImplementedError
+
 
 class XDoToolKeyboardController(AbstractKeyboardController, XDoToolController):
     def __init__(self, ssh_connection: Connection):
@@ -23,6 +32,12 @@ class XDoToolKeyboardController(AbstractKeyboardController, XDoToolController):
     def key(self, key: str):
         self._execute_xdotool_command(f"key {key}")
 
+    def key_down(self, key: str):
+        self._execute_xdotool_command(f"keydown {key}")
+
+    def key_up(self, key: str):
+        self._execute_xdotool_command(f"keyup {key}")
+
 class PythonKeyboardController(AbstractKeyboardController, PythonController):
     def __init__(self, http_server: str):
         super().__init__(http_server=http_server)
@@ -32,4 +47,10 @@ class PythonKeyboardController(AbstractKeyboardController, PythonController):
         self._execute_python_command(self.command.format(command=f"keyboard.write('{text}')"))
 
     def key(self, key: str):
-        self._execute_python_command(self.command.format(command=f"keyboard.press_and_release('{key}')"))
+        self._execute_python_command(self.command.format(command=f"keyboard.press_and_release('{key}')"))
+
+    def key_down(self, key: str):
+        self._execute_python_command(self.command.format(command=f"keyboard.press('{key}')"))
+
+    def key_up(self, key: str):
+        self._execute_python_command(self.command.format(command=f"keyboard.release('{key}')"))

desktop_env/envs/desktop_env.py

@@ -18,7 +18,9 @@ class Action(Enum):
     MOUSE_UP = 2
     MOUSE_MOVE = 3
     KEY = 4
-    TYPE = 5
+    KEY_DOWN = 5
+    KEY_UP = 6
+    TYPE = 7
 
 VM_TYPE = Literal['ubuntu', 'windows']
 
@@ -163,6 +165,12 @@ class DesktopEnv(gym.Env):
         elif action_type == Action.KEY:
             key_sequence = ''.join(map(chr, action['key']))  # Convert integer array to string
             self.keyboard_controller.key(key_sequence)
+        elif action_type == Action.KEY_DOWN:
+            key_sequence = ''.join(map(chr, action['key']))  # Convert integer array to string
+            self.keyboard_controller.key_down(key_sequence)
+        elif action_type == Action.KEY_UP:
+            key_sequence = ''.join(map(chr, action['key']))  # Convert integer array to string
+            self.keyboard_controller.key_up(key_sequence)
         elif action_type == Action.TYPE:
             text = ''.join(map(chr, action['text']))  # Convert integer array to string
             self.keyboard_controller.type(text)