add more actions

controller.py

@@ -1,7 +1,19 @@
+from enum import Enum
+
 import numpy as np
 from PIL import Image
 
-from simulator import EmulatorSimulator
+from simulator import EmulatorSimulator, MouseClick
+
+
+class Action(Enum):
+    CLICK = "click"
+    MOUSE_DOWN = "mousedown"
+    MOUSE_UP = "mouseup"
+    MOUSE_MOVE = "mousemove"
+    KEY = "key"
+    TYPE = "type"
+
 
 class Controller:
     def __init__(self, vm_name: str, username: str, password: str, host: str) -> None:
@@ -13,5 +25,17 @@ class Controller:
         with Image.open(image_path) as img:
             return np.array(img)
     
-    def step(self) -> None:
+    def step(self, action: Action, **action_args) -> None:
-        self.simulator.keyboard_type("hello word")
+        if action == Action.CLICK:
+            print(action_args)
+            self.simulator.click(**action_args)
+        elif action == Action.MOUSE_DOWN:
+            self.simulator.mousedown(**action_args)
+        elif action == Action.MOUSE_UP:
+            self.simulator.mouseup(**action_args)
+        elif action == Action.MOUSE_MOVE:
+            self.simulator.mouse_move(**action_args)
+        elif action == Action.KEY:
+            self.simulator.key(**action_args)
+        elif action == Action.TYPE:
+            self.simulator.type(**action_args)

main.py

@@ -1,4 +1,4 @@
-from controller import Controller
+from controller import Controller, Action, MouseClick
 
 controller = Controller(vm_name="KUbuntu-23.10", username="username", password="password", host="192.168.56.101")
 
@@ -6,4 +6,9 @@ input("enter to continue")
 img = controller.get_state()
 print(img)
 input("enter to continue")
-controller.step()
+controller.step(action=Action.MOUSE_MOVE, x=100, y=100)
+input("enter to continue")
+controller.step(action=Action.CLICK, click=MouseClick.LEFT)
+input("enter to continue")
+controller.step(action=Action.TYPE, text="hello world")
+

simulator.py

@@ -1,6 +1,17 @@
+from enum import Enum
+
 import subprocess
 from fabric import Connection
 
+
+class MouseClick(Enum):
+    LEFT = 1
+    MIDDLE = 2
+    RIGHT = 3
+    WHEEL_UP = 4
+    WHEEL_DOWN = 5
+
+
 class EmulatorSimulator:
     def __init__(self, vm_name: str, username: str, password: str, host:str, snapshot_name: str = "snapshot"):
         self.vm_name = vm_name
@@ -34,16 +45,22 @@ class EmulatorSimulator:
     def save_state(self) -> None:
         self._execute_command(["VBoxManage", "snapshot", self.vm_name, "take", self.snapshot_name])
 
-    def send_click(self, click: int) -> None:
+    def click(self, click: MouseClick) -> None:
-        self._execute_xdotool_command(f"click {click}")
+        self._execute_xdotool_command(f"click {click.value}")
 
-    def send_key(self) -> None:
+    def mousedown(self, click: MouseClick) -> None:
-        pass
+        self._execute_xdotool_command(f"mousedown {click}")
+    
+    def mouseup(self, click: MouseClick) -> None:
+        self._execute_xdotool_command(f"mouseup {click}")
 
     def mouse_move(self, x: int, y: int) -> None:
         self._execute_xdotool_command(f"mousemove {x} {y}")
 
-    def keyboard_type(self, text: str) -> None:
+    def key(self, key: str) -> None:
+        self._execute_xdotool_command(f"key {key}")
+
+    def type(self, text: str) -> None:
         self._execute_xdotool_command(f"type {text}")
 
     def get_screenshot(self) -> str: