Refactor examples; Start to load examples into benchmark; vlc initialization

desktop_env/__init__.py

@@ -0,0 +1 @@
+

desktop_env/controllers/setup.py

@@ -4,6 +4,7 @@ import json
 from typing import Dict, List
 from typing import Any
 
+
 class SetupController:
     def __init__(self, http_server: str):
         self.http_server = http_server + "/setup"
@@ -31,10 +32,10 @@ class SetupController:
             assert hasattr(self, setup_function)
             getattr(self, setup_function)(**parameters)
 
-        #self._download_setup(config)
-        #self._change_wallpaper(config)
+        # self._download_setup(config)
+        # self._change_wallpaper(config)
         # self._tidy_desktop(config) todo: implement this
-        #self._open_setup(config)
+        # self._open_setup(config)
         # can add other setup steps
 
     def _download_setup(self, files: List[Dict[str, str]]):
@@ -47,11 +48,11 @@ class SetupController:
               }
         """
 
-        #if not config:
-            #return
-        #if not 'download' in config:
-            #return
-        #for url, path in config['download']:
+        # if not config:
+        # return
+        # if not 'download' in config:
+        # return
+        # for url, path in config['download']:
         for f in files:
             url: str = f["url"]
             path: str = f["path"]
@@ -75,12 +76,12 @@ class SetupController:
                 print("An error occurred while trying to send the request:", e)
 
     def _change_wallpaper_setup(self, path: str):
-        #if not config:
-            #return
-        #if not 'wallpaper' in config:
-            #return
+        # if not config:
+        # return
+        # if not 'wallpaper' in config:
+        # return
 
-        #path = config['wallpaper']
+        # path = config['wallpaper']
         if not path:
             raise Exception(f"Setup Wallpaper - Invalid path ({path}).")
 
@@ -103,11 +104,11 @@ class SetupController:
         raise NotImplementedError
 
     def _open_setup(self, path: str):
-        #if not config:
-            #return
-        #if not 'open' in config:
-            #return
-        #for path in config['open']:
+        # if not config:
+        # return
+        # if not 'open' in config:
+        # return
+        # for path in config['open']:
         if not path:
             raise Exception(f"Setup Open - Invalid path ({path}).")
 

desktop_env/envs/desktop_env.py

@@ -3,23 +3,24 @@ from __future__ import annotations
 import os
 import subprocess
 import time
-#import uuid
-#import platform
+# import uuid
+# import platform
 from typing import List, Dict
 from typing import Callable, Any, Optional
 import tempfile
 
 import gymnasium as gym
-#import requests
+# import requests
 
 from desktop_env.controllers.python import PythonController
 from desktop_env.controllers.setup import SetupController
-#from desktop_env.evaluators import eval_funcs
+# from desktop_env.evaluators import eval_funcs
 from desktop_env.evaluators import metrics, getters
 
 Metric = Callable[[Any, Any], float]
 Getter = Callable[[gym.Env, Dict[str, Any]], Any]
 
+
 def _execute_command(command: List[str]) -> None:
     if command[:4] == ["vmrun", "-T", "ws", "start"]:
         p = subprocess.Popen(command)
@@ -92,7 +93,7 @@ class DesktopEnv(gym.Env):
         self.expected_getter: Getter = getattr(getters, "get_{:}".format(self.evaluator["expected"]["type"]))
 
         # episodic stuffs, like tmp dir and counters
-        self.tmp_dir: str = self.tmp_dir_base # just an init value, updated during reset
+        self.tmp_dir: str = self.tmp_dir_base  # just an init value, updated during reset
         self._traj_no: int = -1
         self._step_no: int = 0
 
@@ -102,7 +103,7 @@ class DesktopEnv(gym.Env):
                 output = subprocess.check_output("vmrun -T ws list", shell=True, stderr=subprocess.STDOUT)
                 output = output.decode()
                 output: List[str] = output.splitlines()
-                #if self.path_to_vm.lstrip("~/") in output:
+                # if self.path_to_vm.lstrip("~/") in output:
                 if self.path_to_vm in output:
                     print("VM is running.")
                     break
@@ -130,9 +131,9 @@ class DesktopEnv(gym.Env):
         _execute_command(["vmrun", "-T", "ws" "snapshot", self.path_to_vm, self.snapshot_path])
 
     def _get_screenshot(self):
-        #random_uuid = str(uuid.uuid4())
-        #os.makedirs(os.path.join("tmp", random_uuid), exist_ok=True)
-        #image_path = os.path.join("tmp", random_uuid, "screenshot.png")
+        # random_uuid = str(uuid.uuid4())
+        # os.makedirs(os.path.join("tmp", random_uuid), exist_ok=True)
+        # image_path = os.path.join("tmp", random_uuid, "screenshot.png")
         image_path: str = os.path.join(self.tmp_dir, "screenshots", "{:d}.png".format(self._step_no))
 
         # Get the screenshot and save to the image_path
@@ -168,9 +169,10 @@ class DesktopEnv(gym.Env):
         self._step_no = 0
 
         print("Setup new temp dir...")
-        self.tmp_dir = tempfile.mkdtemp( prefix="{:d}.{:}.".format(self._traj_no, self.task_id)
-                                       , dir=self.tmp_dir_base
-                                       )
+        self.tmp_dir = tempfile.mkdtemp(
+            prefix="{:d}.{:}.".format(self._traj_no, self.task_id),
+            dir=self.tmp_dir_base
+        )
         os.makedirs(os.path.join(self.tmp_dir, "screenshots"))
 
         print("Reverting to snapshot to {}...".format(self.snapshot_path))
@@ -218,14 +220,14 @@ class DesktopEnv(gym.Env):
         """
 
         # todo: make this more flexible by refactoring
-        #eval_func = eval_funcs[self.evaluator["func"]]
-        #eval_func_vars = {}
-#
-        #for var_name, file_info in self.evaluator["paths"].items():
-            #path = copy_file_to_local(file_info)
-            #eval_func_vars[var_name] = path
-#
-        #return eval_func(**eval_func_vars)
+        # eval_func = eval_funcs[self.evaluator["func"]]
+        # eval_func_vars = {}
+        #
+        # for var_name, file_info in self.evaluator["paths"].items():
+        # path = copy_file_to_local(file_info)
+        # eval_func_vars[var_name] = path
+        #
+        # return eval_func(**eval_func_vars)
 
         result = self.result_getter(self, self.evaluator["result"])
         expected = self.expected_getter(self, self.evaluator["expected"])

desktop_env/server/main.py

@@ -4,9 +4,9 @@ import platform
 import subprocess
 import requests
 
-#import Xlib.display
+# import Xlib.display
 import pyautogui
-#from PIL import ImageGrab, Image
+# from PIL import ImageGrab, Image
 from PIL import Image
 from flask import Flask, request, jsonify, send_file
 
@@ -47,7 +47,6 @@ def capture_screen_with_cursor():
     # Ensure the screenshots directory exists
     os.makedirs(os.path.dirname(file_path), exist_ok=True)
 
-
     # fixme: This is a temporary fix for the cursor not being captured on Windows and Linux
     if user_platform == "Windows" or user_platform == "Linux":
         def _download_image(url, path):
@@ -181,10 +180,10 @@ def open_file():
         return f"File not found: {path}", 404
 
     try:
-        if platform.system()=="Windows":
+        if platform.system() == "Windows":
             os.startfile(path)
         else:
-            open_cmd: str = "open" if platform.system()=="Darwin" else "xdg-open"
+            open_cmd: str = "open" if platform.system() == "Darwin" else "xdg-open"
             subprocess.Popen([open_cmd, str(path)])
         return "File opened successfully"
     except Exception as e:

evaluation_examples/examples/libreoffice_calc/f9584479-3d0d-4c79-affa-9ad7afdd8850.json

@@ -10,7 +10,7 @@
 				"files": [
 					{
 						"url": "https://drive.usercontent.google.com/download?id=1rwhniaClEkF8XFzdfaNUA6GmAiy4syMZ&export=download&authuser=0&confirm=t&uuid=6fdd5b04-85f4-45e1-ad74-368f8f2a82ab&at=APZUnTUP-JxPxLfNls6jXWghblQ5:1701766091851",
-						"path": "C:\\Users\\tianbaox\\Desktop\\Quarterly_Product_Sales_by_Zone.xlsx"
+						"path": "Desktop/Quarterly_Product_Sales_by_Zone.xlsx"
 					}
 				]
 			}
@@ -18,7 +18,7 @@
 		{
 			"type": "open",
 			"parameters": {
-				"path": "C:\\Users\\tianbaox\\Desktop\\Quarterly_Product_Sales_by_Zone.xlsx"
+				"path": "Desktop/Quarterly_Product_Sales_by_Zone.xlsx"
 			}
 		}
   ],
@@ -35,7 +35,7 @@
 		},
 		"result": {
 			"type": "vm_file",
-			"path": "C:\\Users\\tianbaox\\Desktop\\Quarterly_Product_Sales_by_Zone.xlsx",
+			"path": "Desktop/Quarterly_Product_Sales_by_Zone.xlsx",
 			"dest": "Quarterly_Product_Sales_by_Zone.xlsx"
 		}
   }

main.py

@@ -7,14 +7,15 @@ def human_agent():
     Runs the Gym environment with human input.
     """
 
-    with open("evaluation_examples/examples/f9584479-3d0d-4c79-affa-9ad7afdd8850.json", "r") as f:
+    with open("evaluation_examples/examples/libreoffice_calc/f9584479-3d0d-4c79-affa-9ad7afdd8850.json", "r") as f:
         example = json.load(f)
-    example["snapshot"] = "base_setup"
+    example["snapshot"] = "base_setup2"
 
-    env = DesktopEnv( path_to_vm=r"C:\Users\tianbaox\Documents\Virtual Machines\Ubuntu\Ubuntu.vmx"
-                    , action_space="computer_13"
-                    , task_config=example
-                    )
+    env = DesktopEnv(
+        path_to_vm=r"C:\Users\tianbaox\Documents\Virtual Machines\Ubuntu\Ubuntu.vmx",
+        action_space="computer_13",
+        task_config=example
+    )
 
     # reset the environment to certain snapshot
     observation = env.reset()