Merge branch 'main' into zdy

desktop_env/controllers/python.py

@@ -1,11 +1,15 @@
 import json
+import logging
+import random
 from typing import Any, Dict, Optional
+
 import requests
+
 from desktop_env.envs.actions import KEYBOARD_KEYS
 
-import logging
 logger = logging.getLogger("desktopenv.pycontroller")
 
+
 class PythonController:
     def __init__(self, http_server: str, pkgs_prefix: str = "python -c \"import pyautogui; {command}\""):
         self.http_server = http_server
@@ -71,6 +75,10 @@ class PythonController:
 
         action_type = action["action_type"]
         parameters = action["parameters"] if "parameters" in action else {}
+        move_mode = random.choice(
+            ["pyautogui.easeInQuad", "pyautogui.easeOutQuad", "pyautogui.easeInOutQuad", "pyautogui.easeInBounce",
+             "pyautogui.easeInElastic"])
+        duration = random.uniform(0.5, 1)
 
         if action_type == "MOVE_TO":
             if parameters == {} or None:
@@ -78,7 +86,7 @@ class PythonController:
             elif "x" in parameters and "y" in parameters:
                 x = parameters["x"]
                 y = parameters["y"]
-                self.execute_python_command(f"pyautogui.moveTo({x}, {y})")
+                self.execute_python_command(f"pyautogui.moveTo({x}, {y}, {duration}, {move_mode})")
             else:
                 raise Exception(f"Unknown parameters: {parameters}")
 
@@ -220,6 +228,27 @@ class PythonController:
         else:
             raise Exception(f"Unknown action type: {action_type}")
 
+    def get_vm_screen_size(self):
+        """
+        Gets the size of the vm screen.
+        """
+        response = requests.post(self.http_server + "/screen_size")
+        if response.status_code == 200:
+            return response.json()
+        else:
+            logger.error("Failed to get screen size. Status code: %d", response.status_code)
+            return None
+
+    def get_vm_window_size(self, app_class_name: str):
+        """
+        Gets the size of the vm app window.
+        """
+        response = requests.post(self.http_server + "/window_size", data={"app_class_name": app_class_name})
+        if response.status_code == 200:
+            return response.json()
+        else:
+            logger.error("Failed to get window size. Status code: %d", response.status_code)
+            return None
 
     def get_vlc_status(self, host='localhost', port=8080, password='password'):
         url = f'http://{host}:{port}/requests/status.xml'
@@ -227,8 +256,7 @@ class PythonController:
         response = requests.get(url, auth=('', password))
 
         if response.status_code == 200:
-            print("File downloaded successfully")
             return response.content
         else:
-            print("Failed to get vlc status. Status code:", response.status_code)
+            logger.error("Failed to get vlc status. Status code: %d", response.status_code)
             return None