diff --git a/desktop_env/controllers/keyboard.py b/desktop_env/controllers/keyboard.py index 7cb79f8..5178491 100644 --- a/desktop_env/controllers/keyboard.py +++ b/desktop_env/controllers/keyboard.py @@ -12,6 +12,15 @@ class AbstractKeyboardController(ABC): @abstractmethod def key(self, key: str): raise NotImplementedError + + @abstractmethod + def key_down(self, key: str): + raise NotImplementedError + + @abstractmethod + def key_up(self, key: str): + raise NotImplementedError + class XDoToolKeyboardController(AbstractKeyboardController, XDoToolController): def __init__(self, ssh_connection: Connection): @@ -23,6 +32,12 @@ class XDoToolKeyboardController(AbstractKeyboardController, XDoToolController): def key(self, key: str): self._execute_xdotool_command(f"key {key}") + def key_down(self, key: str): + self._execute_xdotool_command(f"keydown {key}") + + def key_up(self, key: str): + self._execute_xdotool_command(f"keyup {key}") + class PythonKeyboardController(AbstractKeyboardController, PythonController): def __init__(self, http_server: str): super().__init__(http_server=http_server) @@ -32,4 +47,10 @@ class PythonKeyboardController(AbstractKeyboardController, PythonController): self._execute_python_command(self.command.format(command=f"keyboard.write('{text}')")) def key(self, key: str): - self._execute_python_command(self.command.format(command=f"keyboard.press_and_release('{key}')")) \ No newline at end of file + self._execute_python_command(self.command.format(command=f"keyboard.press_and_release('{key}')")) + + def key_down(self, key: str): + self._execute_python_command(self.command.format(command=f"keyboard.press('{key}')")) + + def key_up(self, key: str): + self._execute_python_command(self.command.format(command=f"keyboard.release('{key}')")) diff --git a/desktop_env/envs/desktop_env.py b/desktop_env/envs/desktop_env.py index 501c135..98fa586 100644 --- a/desktop_env/envs/desktop_env.py +++ b/desktop_env/envs/desktop_env.py @@ -18,7 +18,9 @@ class Action(Enum): MOUSE_UP = 2 MOUSE_MOVE = 3 KEY = 4 - TYPE = 5 + KEY_DOWN = 5 + KEY_UP = 6 + TYPE = 7 VM_TYPE = Literal['ubuntu', 'windows'] @@ -163,6 +165,12 @@ class DesktopEnv(gym.Env): elif action_type == Action.KEY: key_sequence = ''.join(map(chr, action['key'])) # Convert integer array to string self.keyboard_controller.key(key_sequence) + elif action_type == Action.KEY_DOWN: + key_sequence = ''.join(map(chr, action['key'])) # Convert integer array to string + self.keyboard_controller.key_down(key_sequence) + elif action_type == Action.KEY_UP: + key_sequence = ''.join(map(chr, action['key'])) # Convert integer array to string + self.keyboard_controller.key_up(key_sequence) elif action_type == Action.TYPE: text = ''.join(map(chr, action['text'])) # Convert integer array to string self.keyboard_controller.type(text)