Initialize VLC getters and metrics, fix some bugs in infra logic, needs to be refactored later on

desktop_env/controllers/python.py

@@ -80,7 +80,8 @@ class PythonController:
                 y = parameters["y"]
                 if "num_clicks" in parameters:
                     num_clicks = parameters["num_clicks"]
-                    self.execute_python_command(f"pyautogui.click(button='{button}', x={x}, y={y}, clicks={num_clicks})")
+                    self.execute_python_command(
+                        f"pyautogui.click(button='{button}', x={x}, y={y}, clicks={num_clicks})")
                 else:
                     self.execute_python_command(f"pyautogui.click(button='{button}', x={x}, y={y})")
             elif "button" in parameters and "x" not in parameters and "y" not in parameters:
@@ -143,7 +144,8 @@ class PythonController:
             if "x" in parameters and "y" in parameters:
                 x = parameters["x"]
                 y = parameters["y"]
-                self.execute_python_command(f"pyautogui.dragTo({x}, {y}, duration=1.0, button='left', mouseDownUp=True)")
+                self.execute_python_command(
+                    f"pyautogui.dragTo({x}, {y}, duration=1.0, button='left', mouseDownUp=True)")
 
         elif action_type == "SCROLL":
             # todo: check if it is related to the operating system, as https://github.com/TheDuckAI/DuckTrack/blob/main/ducktrack/playback.py pointed out
@@ -206,3 +208,16 @@ class PythonController:
 
         else:
             raise Exception(f"Unknown action type: {action_type}")
+
+
+    def get_vlc_status(self, host='localhost', port=8080, password='password'):
+        url = f'http://{host}:{port}/requests/status.xml'
+
+        response = requests.get(url, auth=('', password))
+
+        if response.status_code == 200:
+            print("File downloaded successfully")
+            return response.content
+        else:
+            print("Failed to get vlc status. Status code:", response.status_code)
+            return None

desktop_env/controllers/setup.py

@@ -10,12 +10,11 @@ from typing import Any
 
 
 class SetupController:
-    def __init__( self
-                , http_server: str
-                , cache_dir: str
-                ):
-        self.http_server = http_server + "/setup"
+    def __init__(self, http_server: str, cache_dir: str):
+        self.http_server: str = http_server
+        self.http_server_setup_root = http_server + "/setup"
         self.cache_dir: str = cache_dir
+
     def reset_cache_dir(self, cache_dir: str):
         self.cache_dir = cache_dir
 
@@ -48,6 +47,32 @@ class SetupController:
         # self._open_setup(config)
         # can add other setup steps
 
+    def _command_setup(self, command: str):
+        """
+        Directly send a command into the virtual machine os for setting up.
+        """
+        payload = json.dumps({"command": command})
+        headers = {
+            'Content-Type': 'application/json'
+        }
+        timeout = 5
+        timout_whitelist = ["vlc"]
+
+        try:
+
+            response = requests.post(self.http_server + "/execute", headers=headers, data=payload, timeout=timeout)
+            if response.status_code == 200:
+                print("Command executed successfully:", response.text)
+            else:
+                print("Failed to execute command. Status code:", response.status_code)
+        except requests.exceptions.Timeout as e:
+            if command in timout_whitelist:
+                print("Command executed successfully:", command)
+            else:
+                print("An error occurred while trying to execute the command:", e)
+        except requests.exceptions.RequestException as e:
+            print("An error occurred while trying to execute the command:", e)
+
     def _download_setup(self, files: List[Dict[str, str]]):
         """
         Args:
@@ -66,12 +91,9 @@ class SetupController:
         for f in files:
             url: str = f["url"]
             path: str = f["path"]
-            cache_path: str = os.path.join( self.cache_dir
-                                          , "{:}_{:}".format(
-                                              uuid.uuid5(uuid.NAMESPACE_URL, url)
-                                            , os.path.basename(path)
-                                            )
-                                          )
+            cache_path: str = os.path.join(self.cache_dir, "{:}_{:}".format(
+                uuid.uuid5(uuid.NAMESPACE_URL, url),
+                os.path.basename(path)))
 
             if not url or not path:
                 raise Exception(f"Setup Download - Invalid URL ({url}) or path ({path}).")
@@ -97,21 +119,21 @@ class SetupController:
                 if not downloaded:
                     raise requests.RequestException(f"Failed to download {url}. No retries left. Error: {e}")
 
-            #payload = json.dumps({"url": url, "path": path})
-            #headers = {
-                #'Content-Type': 'application/json'
-            #}
+            # payload = json.dumps({"url": url, "path": path})
+            # headers = {
+            # 'Content-Type': 'application/json'
+            # }
 
-            form = MultipartEncoder( { "file_path": path
-                                     , "file_data": (os.path.basename(path), open(cache_path, "rb"))
-                                     }
-                                   )
+            form = MultipartEncoder({
+                "file_path": path,
+                "file_data": (os.path.basename(path), open(cache_path, "rb"))
+            })
             headers = {"Content-Type": form.content_type}
             print(form.content_type)
 
             # send request to server to upload file
             try:
-                response = requests.post(self.http_server + "/upload", headers=headers, data=form)
+                response = requests.post(self.http_server_setup_root + "/upload", headers=headers, data=form)
                 if response.status_code == 200:
                     print("Command executed successfully:", response.text)
                 else:
@@ -136,7 +158,7 @@ class SetupController:
 
         # send request to server to change wallpaper
         try:
-            response = requests.post(self.http_server + "/change_wallpaper", headers=headers, data=payload)
+            response = requests.post(self.http_server_setup_root + "/change_wallpaper", headers=headers, data=payload)
             if response.status_code == 200:
                 print("Command executed successfully:", response.text)
             else:
@@ -163,7 +185,7 @@ class SetupController:
 
         # send request to server to open file
         try:
-            response = requests.post(self.http_server + "/open_file", headers=headers, data=payload)
+            response = requests.post(self.http_server_setup_root + "/open_file", headers=headers, data=payload)
             if response.status_code == 200:
                 print("Command executed successfully:", response.text)
             else:

desktop_env/envs/desktop_env.py

@@ -85,7 +85,8 @@ class DesktopEnv(gym.Env):
         # Initialize emulator and controller
         print("Initializing...")
         self._start_emulator()
-        self.host = f"http://{self._get_vm_ip()}:5000"
+        self.vm_ip = self._get_vm_ip()
+        self.host = f"http://{self.vm_ip}:5000"
         self.controller = PythonController(http_server=self.host)
         self.setup_controller = SetupController(http_server=self.host, cache_dir=self.cache_dir)
 

desktop_env/evaluators/getters/__init__.py

@@ -1,2 +1,3 @@
 from .file import get_cloud_file, get_vm_file
 from .misc import get_rule
+from .vlc import get_vlc_playing_info

desktop_env/evaluators/getters/file.py

@@ -3,6 +3,7 @@ from typing import Dict
 import os
 import requests
 
+
 def get_cloud_file(env, config: Dict[str, str]) -> str:
     """
     Config:
@@ -25,6 +26,7 @@ def get_cloud_file(env, config: Dict[str, str]) -> str:
 
     return _path
 
+
 def get_vm_file(env, config: Dict[str, str]) -> str:
     """
     Config:
@@ -33,12 +35,9 @@ def get_vm_file(env, config: Dict[str, str]) -> str:
     """
 
     _path = os.path.join(env.cache_dir, config["dest"])
-    if os.path.exists(_path):
-        return _path
 
     file = env.controller.get_file(config["path"])
     with open(_path, "wb") as f:
         f.write(file)
 
     return _path
-

desktop_env/evaluators/metrics/__init__.py

@@ -2,4 +2,5 @@ from .table import compare_table
 from .table import check_sheet_list, check_xlsx_freeze, check_zoom
 from .docs import find_default_font, contains_page_break, compare_docx_files, compare_docx_tables, compare_line_spacing, compare_insert_equation
 from .docs import compare_font_names, compare_subscript_contains, has_page_numbers_in_footers
-from .docs import is_first_line_centered, check_file_exists, compare_contains_image
+from .docs import is_first_line_centered, check_file_exists, compare_contains_image
+from .vlc import is_vlc_playing

evaluation_examples/examples/vlc/59f21cfb-0120-4326-b255-a5b827b38967.json

@@ -1,12 +1,42 @@
 {
   "id": "59f21cfb-0120-4326-b255-a5b827b38967",
   "snapshot": "base_setup",
-  "instruction": "Could you help me play the file at FILE_PATH?",
+  "instruction": "Play the music video on my desktop",
   "source": "https://docs.videolan.me/vlc-user/desktop/3.0/en/basic/media.html#playing-a-file",
-  "config": [],
+  "config": [
+    {
+      "type": "download",
+      "parameters": {
+        "files": [
+          {
+            "url": "https://drive.usercontent.google.com/download?id=14-vhVMVw53e0l-MDVBFbngFAE1jMqvgm&export=download&authuser=0&confirm=t&uuid=d31607ed-0075-4fe5-b68c-b24b6eec356e&at=APZUnTV0Wy0672VFGrQChgHmd1Ba:1704337791613",
+            "path": "Desktop/Rick Astley - Never Gonna Give You Up (Official Music Video).mp4"
+          }
+        ]
+      }
+    },
+    {
+      "type": "command",
+      "parameters": {
+        "command": "vlc"
+      }
+    }
+  ],
   "trajectory": "trajectories/",
   "related_apps": [
     "vlc"
   ],
-  "evaluator": "evaluation_dir"
+  "evaluator": {
+    "func": "is_vlc_playing",
+    "expected": {
+      "type": "rule",
+      "rules": {
+        "file_path": "Desktop/Rick Astley - Never Gonna Give You Up (Official Music Video).mp4"
+      }
+    },
+    "result": {
+      "type": "vlc_playing_info",
+      "dest": "status.xml"
+    }
+  }
 }