mouse and keyboard controllers for windows and linux

desktop_env/controllers/keyboard.py

@@ -1,9 +1,10 @@
 from abc import ABC, abstractmethod
 from fabric import Connection
 
-from xdotool import XDoToolController
+from .xdotool import XDoToolController
+from .python import PythonController
 
-class AbstractMouseController(ABC):
+class AbstractKeyboardController(ABC):
     @abstractmethod
     def type(self, text: str):
         raise NotImplementedError
@@ -12,7 +13,7 @@ class AbstractMouseController(ABC):
     def key(self, key: str):
         raise NotImplementedError
 
-class XDoToolKeyboardController(AbstractMouseController, XDoToolController):
+class XDoToolKeyboardController(AbstractKeyboardController, XDoToolController):
     def __init__(self, ssh_connection: Connection):
         super().__init__(ssh_connection=ssh_connection)
 
@@ -22,16 +23,13 @@ class XDoToolKeyboardController(AbstractMouseController, XDoToolController):
     def key(self, key: str):
         self._execute_xdotool_command(f"key {key}")
 
-class PythonKeyboardController(AbstractMouseController):
-    def __init__(self, ssh_connection: Connection):
-        self.ssh_connection = ssh_connection
-    
-    def _execute_python_command(self, command: list[str]) -> None:
-        result = self.ssh_connection.run(f"sudo python3 -c 'import keyboard; keyboard.{command}'", hide=True)
-        return result.stdout.strip()
+class PythonKeyboardController(AbstractKeyboardController, PythonController):
+    def __init__(self, http_server: str):
+        super().__init__(http_server=http_server)
+        self.command = "python -c \"import keyboard; {command}\""
 
     def type(self, text: str):
-        self._execute_python_command(f"write({text})")
+        self._execute_python_command(self.command.format(command=f"keyboard.write('{text}')"))
 
     def key(self, key: str):
-        self._execute_python_command(f"press_and_release({key})")
+        self._execute_python_command(self.command.format(command=f"keyboard.press_and_release('{key}')"))

desktop_env/controllers/mouse.py

@@ -1,37 +1,144 @@
+from enum import Enum
+
 from abc import ABC, abstractmethod
 from fabric import Connection
 
-from xdotool import XDoToolController
+from .xdotool import XDoToolController
+from .python import PythonController
+class MouseClick(Enum):
+    LEFT = 1
+    MIDDLE = 2
+    RIGHT = 3
+    WHEEL_UP = 4
+    WHEEL_DOWN = 5
 
 class AbstractMouseController(ABC):
     @abstractmethod
-    def type(self, text: str):
+    def mouse_move(self, x: int, y: int):
         raise NotImplementedError
 
     @abstractmethod
-    def key(self, key: str):
+    def left_down(self):
+        raise NotImplementedError
+    
+    @abstractmethod
+    def left_up(self):
+        raise NotImplementedError
+    
+    @abstractmethod
+    def left_click(self):
         raise NotImplementedError
 
-class XDoToolKeyboardController(AbstractMouseController, XDoToolController):
+    @abstractmethod
+    def middle_down(self):
+        raise NotImplementedError
+    
+    @abstractmethod
+    def middle_up(self):
+        raise NotImplementedError
+    
+    @abstractmethod
+    def middle_click(self):
+        raise NotImplementedError
+    
+    @abstractmethod
+    def right_down(self):
+        raise NotImplementedError
+    
+    @abstractmethod
+    def right_up(self):
+        raise NotImplementedError
+    
+    @abstractmethod
+    def right_click(self):
+        raise NotImplementedError
+    
+    @abstractmethod
+    def scroll_up(self):
+        raise NotImplementedError
+    
+    @abstractmethod
+    def scroll_down(self):
+        raise NotImplementedError
+
+
+class XDoToolMouseController(AbstractMouseController, XDoToolController):
     def __init__(self, ssh_connection: Connection):
         super().__init__(ssh_connection=ssh_connection)
 
-    def type(self, text: str):
-        self._execute_xdotool_command(f"type {text}")
+    def mouse_move(self, x: int, y: int):
+        self._execute_xdotool_command(f"mousemove {x} {y}")
 
-    def key(self, key: str):
-        self._execute_xdotool_command(f"key {key}")
-
-class PythonKeyboardController(AbstractMouseController):
-    def __init__(self, ssh_connection: Connection):
-        self.ssh_connection = ssh_connection
+    def left_down(self):
+        self._execute_xdotool_command(f"mousedown 1")
     
-    def _execute_python_command(self, command: list[str]) -> None:
-        result = self.ssh_connection.run(f"sudo python3 -c 'import keyboard; keyboard.{command}'", hide=True)
-        return result.stdout.strip()
+    def left_up(self):
+        self._execute_xdotool_command(f"mouseup 1")
 
-    def type(self, text: str):
-        self._execute_python_command(f"write({text})")
+    def left_click(self):
+        self._execute_xdotool_command(f"click 1")
 
-    def key(self, key: str):
-        self._execute_python_command(f"press_and_release({key})")
+    def middle_down(self):
+        self._execute_xdotool_command(f"mousedown 2")
+    
+    def middle_up(self):
+        self._execute_xdotool_command(f"mouseup 2")
+
+    def middle_click(self):
+        self._execute_xdotool_command(f"click 2")
+    
+    def right_down(self):
+        self._execute_xdotool_command(f"mousedown 3")
+    
+    def right_up(self):
+        self._execute_xdotool_command(f"mouseup 3")
+    
+    def right_click(self):
+        self._execute_xdotool_command(f"click 3")
+    
+    def scroll_up(self):
+        self._execute_xdotool_command(f"click 4")
+    
+    def scroll_down(self):
+        self._execute_xdotool_command(f"click 5")
+
+class PythonMouseController(AbstractMouseController, PythonController):
+    def __init__(self, http_server: str):
+        super().__init__(http_server=http_server)
+        self.command = "python -c \"import mouse; {command}\""
+
+    def mouse_move(self, x: int, y: int):
+        self._execute_python_command(self.command.format(command=f"mouse.move({x}, {y})"))
+
+    def left_down(self):
+        self._execute_python_command(self.command.format(command="mouse.press(button='left')"))
+    
+    def left_up(self):
+        self._execute_python_command(self.command.format(command="mouse.release(button='left')"))
+
+    def left_click(self):
+        self._execute_python_command(self.command.format(command="mouse.click(button='left')"))
+
+    def middle_down(self):
+        self._execute_python_command(self.command.format(command="mouse.press(button='middle')"))
+    
+    def middle_up(self):
+        self._execute_python_command(self.command.format(command="mouse.release(button='middle')"))
+
+    def middle_click(self):
+        self._execute_python_command(self.command.format(command="mouse.click(button='middle')"))
+    
+    def right_down(self):
+        self._execute_python_command(self.command.format(command="mouse.press(button='right')"))
+    
+    def right_up(self):
+        self._execute_python_command(self.command.format(command="mouse.release(button='right')"))
+    
+    def right_click(self):
+        self._execute_python_command(self.command.format(command="mouse.click(button='right')"))
+    
+    def scroll_up(self):
+        self._execute_python_command(self.command.format(command="mouse.wheel(10)"))
+    
+    def scroll_down(self):
+        self._execute_python_command(self.command.format(command="mouse.wheel(-10)"))

desktop_env/controllers/python.py

@@ -0,0 +1,34 @@
+import requests
+import json
+
+class PythonController:
+    def __init__(self, http_server: str):
+        self.http_server = http_server
+    
+    def _execute_python_command(self, command: str) -> None:
+        payload = json.dumps({
+            "command": command
+        })
+        headers = {
+            'Content-Type': 'application/json'
+        }
+        
+        try:
+            response = requests.post(self.http_server + "/execute", headers=headers, data=payload)
+            if response.status_code == 200:
+                print("Command executed successfully:", response.text)
+            else:
+                print("Failed to execute command. Status code:", response.status_code)
+        except requests.exceptions.RequestException as e:
+            print("An error occurred while trying to execute the command:", e)
+
+# example usage
+if __name__ == '__main__':
+    # replace with your actual server URL of the vm
+    server_url = "http://192.168.7.129:5000"  
+    controller = PythonController(server_url)
+
+    # example commands    
+    python_command = "python -c \"import keyboard; keyboard.write('hello world')\""
+    python_command = "python -c \"import mouse; mouse.move(100,100);mouse.right_click()\""
+    controller._execute_python_command(python_command)