Merge branch 'zdy'

.gitignore

@@ -160,4 +160,9 @@ instructor-embedding
 # frontend cache
 frontend/node_modules/
 frontend/.next/
-frontend/.idea
+frontend/.idea
+
+tags
+snapshots
+branch_flag
+branch-config

desktop_env/controllers/setup.py

@@ -1,6 +1,8 @@
 import requests
 import json
 
+from typing import Dict, List
+from typing import Any
 
 class SetupController:
     def __init__(self, http_server: str):
@@ -14,18 +16,42 @@ class SetupController:
             ...
         }
         """
-        self._download_setup(config)
+
-        self._change_wallpaper(config)
+        for cfg in config:
+            config_type: str = cfg["type"]
+            parameters: Dict[str, Any] = cfg["parameters"]
+
+            # Assumes all the setup the functions should follow this name
+            # protocol
+            setup_function: str = "_{:}_setup".format(config_type)
+            assert hasattr(self, setup_function)
+            getattr(self, setup_function)(**parameters)
+
+        #self._download_setup(config)
+        #self._change_wallpaper(config)
         # self._tidy_desktop(config) todo: implement this
-        self._open_setup(config)
+        #self._open_setup(config)
         # can add other setup steps
 
-    def _download_setup(self, config):
+    def _download_setup(self, files: List[Dict[str, str]]):
-        if not config:
+        """
-            return
+        Args:
-        if not 'download' in config:
+            files (List[Dict[str, str]]): files to download. lisf of dict like
-            return
+              {
-        for url, path in config['download']:
+                "url": str, the url to download
+                "path": str, the path on the VM to store the downloaded file
+              }
+        """
+
+        #if not config:
+            #return
+        #if not 'download' in config:
+            #return
+        #for url, path in config['download']:
+        for f in files:
+            url: str = f["url"]
+            path: str = f["path"]
+
             if not url or not path:
                 raise Exception(f"Setup Download - Invalid URL ({url}) or path ({path}).")
 
@@ -44,12 +70,13 @@ class SetupController:
             except requests.exceptions.RequestException as e:
                 print("An error occurred while trying to send the request:", e)
 
-    def _change_wallpaper(self, config):
+    def _change_wallpaper_setup(self, path: str):
-        if not config:
+        #if not config:
-            return
+            #return
-        if not 'wallpaper' in config:
+        #if not 'wallpaper' in config:
-            return
+            #return
-        path = config['wallpaper']
+
+        #path = config['wallpaper']
         if not path:
             raise Exception(f"Setup Wallpaper - Invalid path ({path}).")
 
@@ -68,29 +95,29 @@ class SetupController:
         except requests.exceptions.RequestException as e:
             print("An error occurred while trying to send the request:", e)
 
-    def _tidy_desktop(self, config):
+    def _tidy_desktop_setup(self, **config):
         raise NotImplementedError
 
-    def _open_setup(self, config):
+    def _open_setup(self, path: str):
-        if not config:
+        #if not config:
-            return
+            #return
-        if not 'open' in config:
+        #if not 'open' in config:
-            return
+            #return
-        for path in config['open']:
+        #for path in config['open']:
-            if not path:
+        if not path:
-                raise Exception(f"Setup Open - Invalid path ({path}).")
+            raise Exception(f"Setup Open - Invalid path ({path}).")
 
-            payload = json.dumps({"path": path})
+        payload = json.dumps({"path": path})
-            headers = {
+        headers = {
-                'Content-Type': 'application/json'
+            'Content-Type': 'application/json'
-            }
+        }
 
-            # send request to server to open file
+        # send request to server to open file
-            try:
+        try:
-                response = requests.post(self.http_server + "/open_file", headers=headers, data=payload)
+            response = requests.post(self.http_server + "/open_file", headers=headers, data=payload)
-                if response.status_code == 200:
+            if response.status_code == 200:
-                    print("Command executed successfully:", response.text)
+                print("Command executed successfully:", response.text)
-                else:
+            else:
-                    print("Failed to open file. Status code:", response.text)
+                print("Failed to open file. Status code:", response.text)
-            except requests.exceptions.RequestException as e:
+        except requests.exceptions.RequestException as e:
-                print("An error occurred while trying to send the request:", e)
+            print("An error occurred while trying to send the request:", e)

desktop_env/envs/desktop_env.py

@@ -60,7 +60,7 @@ class DesktopEnv(gym.Env):
     def _start_emulator(self):
         while True:
             try:
-                output = subprocess.check_output(f"vmrun -T ws list", shell=True, stderr=subprocess.STDOUT)
+                output = subprocess.check_output("vmrun -T ws list", shell=True, stderr=subprocess.STDOUT)
                 output = output.decode()
                 if self.path_to_vm.lstrip("~/") in output:
                     print("VM is running.")

desktop_env/server/main.py

@@ -4,9 +4,10 @@ import platform
 import subprocess
 import requests
 
-import Xlib.display
+#import Xlib.display
 import pyautogui
-from PIL import ImageGrab, Image
+#from PIL import ImageGrab, Image
+from PIL import Image
 from flask import Flask, request, jsonify, send_file
 
 app = Flask(__name__)
@@ -180,7 +181,11 @@ def open_file():
         return f"File not found: {path}", 404
 
     try:
-        os.startfile(path)
+        if platform.system()=="Windows":
+            os.startfile(path)
+        else:
+            open_cmd: str = "open" if platform.system()=="Darwin" else "xdg-open"
+            subprocess.Popen([open_cmd, str(path)])
         return "File opened successfully"
     except Exception as e:
         return f"Failed to open {path}. Error: {e}", 500

evaluation_examples/examples/f9584479-3d0d-4c79-affa-9ad7afdd8850.json

@@ -3,17 +3,25 @@
   "snapshot": "libreoffice_calc",
   "instruction": "Fill the missing row and column which show the total value",
   "source": "https://youtube.com/shorts/feldd-Pn48c?si=9xJiem2uAHm6Jshb",
-  "config": {
+  "config": [
-    "download": [
+		{
-      [
+			"type": "download",
-        "https://drive.usercontent.google.com/download?id=1rwhniaClEkF8XFzdfaNUA6GmAiy4syMZ&export=download&authuser=0&confirm=t&uuid=6fdd5b04-85f4-45e1-ad74-368f8f2a82ab&at=APZUnTUP-JxPxLfNls6jXWghblQ5:1701766091851",
+			"parameters": {
-        "C:\\Users\\tianbaox\\Desktop\\Quarterly_Product_Sales_by_Zone.xlsx"
+				"files": [
-      ]
+					{
-    ],
+						"url": "https://drive.usercontent.google.com/download?id=1rwhniaClEkF8XFzdfaNUA6GmAiy4syMZ&export=download&authuser=0&confirm=t&uuid=6fdd5b04-85f4-45e1-ad74-368f8f2a82ab&at=APZUnTUP-JxPxLfNls6jXWghblQ5:1701766091851",
-    "open": [
+						"path": "C:\\Users\\tianbaox\\Desktop\\Quarterly_Product_Sales_by_Zone.xlsx"
-      "C:\\Users\\tianbaox\\Desktop\\Quarterly_Product_Sales_by_Zone.xlsx"
+					}
-    ]
+				]
-  },
+			}
+		},
+		{
+			"type": "open",
+			"parameters": {
+				"path": "C:\\Users\\tianbaox\\Desktop\\Quarterly_Product_Sales_by_Zone.xlsx"
+			}
+		}
+  ],
   "trajectory": "trajectories/f9584479-3d0d-4c79-affa-9ad7afdd8850",
   "related_apps": [
     "libreoffice calc"

main.py

@@ -7,19 +7,16 @@ def human_agent():
     Runs the Gym environment with human input.
     """
 
-    with open("evaluation_examples/examples/37608790-6147-45d0-9f20-1137bb35703d.json", "r") as f:
+    with open("evaluation_examples/examples/f9584479-3d0d-4c79-affa-9ad7afdd8850.json", "r") as f:
         example = json.load(f)
 
-    env = DesktopEnv(
+    env = DesktopEnv( path_to_vm=r"C:\Users\tianbaox\Documents\Virtual Machines\Ubuntu\Ubuntu.vmx"
-        # path_to_vm=r"""C:\Users\tianbaox\Downloads\Windows 10 x64\Windows 10 x64.vmx""",
+                    , action_space="computer_13"
-        path_to_vm=r"""C:\Users\tianbaox\Documents\Virtual Machines\Ubuntu\Ubuntu.vmx""",
+                    , snapshot_path="base_setup"
-        #  path_to_vm="/home/yuri/vmware/Ubuntu 64-bit/Ubuntu 64-bit.vmx",
+                    , instruction=example["instruction"]
-        action_space="computer_13",
+                    , config=example["config"]
-        snapshot_path="base_setup",
+                    , evaluator=example["evaluator"]
-        instruction=example["instruction"],
+                    )
-        # config=example["config"],
-        # evaluator=example["evaluator"]
-    )
 
     # reset the environment to certain snapshot
     observation = env.reset()