diff --git a/desktop_env/__init__.py b/desktop_env/__init__.py index e69de29..8b13789 100644 --- a/desktop_env/__init__.py +++ b/desktop_env/__init__.py @@ -0,0 +1 @@ + diff --git a/desktop_env/controllers/setup.py b/desktop_env/controllers/setup.py index b42b2ff..4ef71d6 100644 --- a/desktop_env/controllers/setup.py +++ b/desktop_env/controllers/setup.py @@ -4,6 +4,7 @@ import json from typing import Dict, List from typing import Any + class SetupController: def __init__(self, http_server: str): self.http_server = http_server + "/setup" @@ -31,10 +32,10 @@ class SetupController: assert hasattr(self, setup_function) getattr(self, setup_function)(**parameters) - #self._download_setup(config) - #self._change_wallpaper(config) + # self._download_setup(config) + # self._change_wallpaper(config) # self._tidy_desktop(config) todo: implement this - #self._open_setup(config) + # self._open_setup(config) # can add other setup steps def _download_setup(self, files: List[Dict[str, str]]): @@ -47,11 +48,11 @@ class SetupController: } """ - #if not config: - #return - #if not 'download' in config: - #return - #for url, path in config['download']: + # if not config: + # return + # if not 'download' in config: + # return + # for url, path in config['download']: for f in files: url: str = f["url"] path: str = f["path"] @@ -75,12 +76,12 @@ class SetupController: print("An error occurred while trying to send the request:", e) def _change_wallpaper_setup(self, path: str): - #if not config: - #return - #if not 'wallpaper' in config: - #return + # if not config: + # return + # if not 'wallpaper' in config: + # return - #path = config['wallpaper'] + # path = config['wallpaper'] if not path: raise Exception(f"Setup Wallpaper - Invalid path ({path}).") @@ -103,11 +104,11 @@ class SetupController: raise NotImplementedError def _open_setup(self, path: str): - #if not config: - #return - #if not 'open' in config: - #return - #for path in config['open']: + # if not config: + # return + # if not 'open' in config: + # return + # for path in config['open']: if not path: raise Exception(f"Setup Open - Invalid path ({path}).") diff --git a/desktop_env/envs/desktop_env.py b/desktop_env/envs/desktop_env.py index 712ade8..4afcbf1 100644 --- a/desktop_env/envs/desktop_env.py +++ b/desktop_env/envs/desktop_env.py @@ -3,23 +3,24 @@ from __future__ import annotations import os import subprocess import time -#import uuid -#import platform +# import uuid +# import platform from typing import List, Dict from typing import Callable, Any, Optional import tempfile import gymnasium as gym -#import requests +# import requests from desktop_env.controllers.python import PythonController from desktop_env.controllers.setup import SetupController -#from desktop_env.evaluators import eval_funcs +# from desktop_env.evaluators import eval_funcs from desktop_env.evaluators import metrics, getters Metric = Callable[[Any, Any], float] Getter = Callable[[gym.Env, Dict[str, Any]], Any] + def _execute_command(command: List[str]) -> None: if command[:4] == ["vmrun", "-T", "ws", "start"]: p = subprocess.Popen(command) @@ -92,7 +93,7 @@ class DesktopEnv(gym.Env): self.expected_getter: Getter = getattr(getters, "get_{:}".format(self.evaluator["expected"]["type"])) # episodic stuffs, like tmp dir and counters - self.tmp_dir: str = self.tmp_dir_base # just an init value, updated during reset + self.tmp_dir: str = self.tmp_dir_base # just an init value, updated during reset self._traj_no: int = -1 self._step_no: int = 0 @@ -102,7 +103,7 @@ class DesktopEnv(gym.Env): output = subprocess.check_output("vmrun -T ws list", shell=True, stderr=subprocess.STDOUT) output = output.decode() output: List[str] = output.splitlines() - #if self.path_to_vm.lstrip("~/") in output: + # if self.path_to_vm.lstrip("~/") in output: if self.path_to_vm in output: print("VM is running.") break @@ -130,9 +131,9 @@ class DesktopEnv(gym.Env): _execute_command(["vmrun", "-T", "ws" "snapshot", self.path_to_vm, self.snapshot_path]) def _get_screenshot(self): - #random_uuid = str(uuid.uuid4()) - #os.makedirs(os.path.join("tmp", random_uuid), exist_ok=True) - #image_path = os.path.join("tmp", random_uuid, "screenshot.png") + # random_uuid = str(uuid.uuid4()) + # os.makedirs(os.path.join("tmp", random_uuid), exist_ok=True) + # image_path = os.path.join("tmp", random_uuid, "screenshot.png") image_path: str = os.path.join(self.tmp_dir, "screenshots", "{:d}.png".format(self._step_no)) # Get the screenshot and save to the image_path @@ -168,9 +169,10 @@ class DesktopEnv(gym.Env): self._step_no = 0 print("Setup new temp dir...") - self.tmp_dir = tempfile.mkdtemp( prefix="{:d}.{:}.".format(self._traj_no, self.task_id) - , dir=self.tmp_dir_base - ) + self.tmp_dir = tempfile.mkdtemp( + prefix="{:d}.{:}.".format(self._traj_no, self.task_id), + dir=self.tmp_dir_base + ) os.makedirs(os.path.join(self.tmp_dir, "screenshots")) print("Reverting to snapshot to {}...".format(self.snapshot_path)) @@ -218,14 +220,14 @@ class DesktopEnv(gym.Env): """ # todo: make this more flexible by refactoring - #eval_func = eval_funcs[self.evaluator["func"]] - #eval_func_vars = {} -# - #for var_name, file_info in self.evaluator["paths"].items(): - #path = copy_file_to_local(file_info) - #eval_func_vars[var_name] = path -# - #return eval_func(**eval_func_vars) + # eval_func = eval_funcs[self.evaluator["func"]] + # eval_func_vars = {} + # + # for var_name, file_info in self.evaluator["paths"].items(): + # path = copy_file_to_local(file_info) + # eval_func_vars[var_name] = path + # + # return eval_func(**eval_func_vars) result = self.result_getter(self, self.evaluator["result"]) expected = self.expected_getter(self, self.evaluator["expected"]) diff --git a/desktop_env/server/main.py b/desktop_env/server/main.py index 9f3c5a4..c4ab500 100644 --- a/desktop_env/server/main.py +++ b/desktop_env/server/main.py @@ -4,9 +4,9 @@ import platform import subprocess import requests -#import Xlib.display +# import Xlib.display import pyautogui -#from PIL import ImageGrab, Image +# from PIL import ImageGrab, Image from PIL import Image from flask import Flask, request, jsonify, send_file @@ -47,7 +47,6 @@ def capture_screen_with_cursor(): # Ensure the screenshots directory exists os.makedirs(os.path.dirname(file_path), exist_ok=True) - # fixme: This is a temporary fix for the cursor not being captured on Windows and Linux if user_platform == "Windows" or user_platform == "Linux": def _download_image(url, path): @@ -181,10 +180,10 @@ def open_file(): return f"File not found: {path}", 404 try: - if platform.system()=="Windows": + if platform.system() == "Windows": os.startfile(path) else: - open_cmd: str = "open" if platform.system()=="Darwin" else "xdg-open" + open_cmd: str = "open" if platform.system() == "Darwin" else "xdg-open" subprocess.Popen([open_cmd, str(path)]) return "File opened successfully" except Exception as e: diff --git a/evaluation_examples/examples/0bf05a7d-b28b-44d2-955a-50b41e24012a.json b/evaluation_examples/examples/libreoffice_calc/0bf05a7d-b28b-44d2-955a-50b41e24012a.json similarity index 100% rename from evaluation_examples/examples/0bf05a7d-b28b-44d2-955a-50b41e24012a.json rename to evaluation_examples/examples/libreoffice_calc/0bf05a7d-b28b-44d2-955a-50b41e24012a.json diff --git a/evaluation_examples/examples/2bd59342-0664-4ccb-ba87-79379096cc08.json b/evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json similarity index 100% rename from evaluation_examples/examples/2bd59342-0664-4ccb-ba87-79379096cc08.json rename to evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json diff --git a/evaluation_examples/examples/37608790-6147-45d0-9f20-1137bb35703d.json b/evaluation_examples/examples/libreoffice_calc/37608790-6147-45d0-9f20-1137bb35703d.json similarity index 100% rename from evaluation_examples/examples/37608790-6147-45d0-9f20-1137bb35703d.json rename to evaluation_examples/examples/libreoffice_calc/37608790-6147-45d0-9f20-1137bb35703d.json diff --git a/evaluation_examples/examples/7a4e4bc8-922c-4c84-865c-25ba34136be1.json b/evaluation_examples/examples/libreoffice_calc/7a4e4bc8-922c-4c84-865c-25ba34136be1.json similarity index 100% rename from evaluation_examples/examples/7a4e4bc8-922c-4c84-865c-25ba34136be1.json rename to evaluation_examples/examples/libreoffice_calc/7a4e4bc8-922c-4c84-865c-25ba34136be1.json diff --git a/evaluation_examples/examples/7b802dad-6e0f-4204-9815-d4e3f57627d8.json b/evaluation_examples/examples/libreoffice_calc/7b802dad-6e0f-4204-9815-d4e3f57627d8.json similarity index 100% rename from evaluation_examples/examples/7b802dad-6e0f-4204-9815-d4e3f57627d8.json rename to evaluation_examples/examples/libreoffice_calc/7b802dad-6e0f-4204-9815-d4e3f57627d8.json diff --git a/evaluation_examples/examples/7efeb4b1-3d19-4762-b163-63328d66303b.json b/evaluation_examples/examples/libreoffice_calc/7efeb4b1-3d19-4762-b163-63328d66303b.json similarity index 100% rename from evaluation_examples/examples/7efeb4b1-3d19-4762-b163-63328d66303b.json rename to evaluation_examples/examples/libreoffice_calc/7efeb4b1-3d19-4762-b163-63328d66303b.json diff --git a/evaluation_examples/examples/a9f325aa-8c05-4e4f-8341-9e4358565f4f.json b/evaluation_examples/examples/libreoffice_calc/a9f325aa-8c05-4e4f-8341-9e4358565f4f.json similarity index 100% rename from evaluation_examples/examples/a9f325aa-8c05-4e4f-8341-9e4358565f4f.json rename to evaluation_examples/examples/libreoffice_calc/a9f325aa-8c05-4e4f-8341-9e4358565f4f.json diff --git a/evaluation_examples/examples/d681960f-7bc3-4286-9913-a8812ba3261a.json b/evaluation_examples/examples/libreoffice_calc/d681960f-7bc3-4286-9913-a8812ba3261a.json similarity index 100% rename from evaluation_examples/examples/d681960f-7bc3-4286-9913-a8812ba3261a.json rename to evaluation_examples/examples/libreoffice_calc/d681960f-7bc3-4286-9913-a8812ba3261a.json diff --git a/evaluation_examples/examples/eb03d19a-b88d-4de4-8a64-ca0ac66f426b.json b/evaluation_examples/examples/libreoffice_calc/eb03d19a-b88d-4de4-8a64-ca0ac66f426b.json similarity index 100% rename from evaluation_examples/examples/eb03d19a-b88d-4de4-8a64-ca0ac66f426b.json rename to evaluation_examples/examples/libreoffice_calc/eb03d19a-b88d-4de4-8a64-ca0ac66f426b.json diff --git a/evaluation_examples/examples/ecb0df7a-4e8d-4a03-b162-053391d3afaf.json b/evaluation_examples/examples/libreoffice_calc/ecb0df7a-4e8d-4a03-b162-053391d3afaf.json similarity index 100% rename from evaluation_examples/examples/ecb0df7a-4e8d-4a03-b162-053391d3afaf.json rename to evaluation_examples/examples/libreoffice_calc/ecb0df7a-4e8d-4a03-b162-053391d3afaf.json diff --git a/evaluation_examples/examples/f9584479-3d0d-4c79-affa-9ad7afdd8850.json b/evaluation_examples/examples/libreoffice_calc/f9584479-3d0d-4c79-affa-9ad7afdd8850.json similarity index 83% rename from evaluation_examples/examples/f9584479-3d0d-4c79-affa-9ad7afdd8850.json rename to evaluation_examples/examples/libreoffice_calc/f9584479-3d0d-4c79-affa-9ad7afdd8850.json index 61adf0f..6a2094e 100644 --- a/evaluation_examples/examples/f9584479-3d0d-4c79-affa-9ad7afdd8850.json +++ b/evaluation_examples/examples/libreoffice_calc/f9584479-3d0d-4c79-affa-9ad7afdd8850.json @@ -10,7 +10,7 @@ "files": [ { "url": "https://drive.usercontent.google.com/download?id=1rwhniaClEkF8XFzdfaNUA6GmAiy4syMZ&export=download&authuser=0&confirm=t&uuid=6fdd5b04-85f4-45e1-ad74-368f8f2a82ab&at=APZUnTUP-JxPxLfNls6jXWghblQ5:1701766091851", - "path": "C:\\Users\\tianbaox\\Desktop\\Quarterly_Product_Sales_by_Zone.xlsx" + "path": "Desktop/Quarterly_Product_Sales_by_Zone.xlsx" } ] } @@ -18,7 +18,7 @@ { "type": "open", "parameters": { - "path": "C:\\Users\\tianbaox\\Desktop\\Quarterly_Product_Sales_by_Zone.xlsx" + "path": "Desktop/Quarterly_Product_Sales_by_Zone.xlsx" } } ], @@ -35,7 +35,7 @@ }, "result": { "type": "vm_file", - "path": "C:\\Users\\tianbaox\\Desktop\\Quarterly_Product_Sales_by_Zone.xlsx", + "path": "Desktop/Quarterly_Product_Sales_by_Zone.xlsx", "dest": "Quarterly_Product_Sales_by_Zone.xlsx" } } diff --git a/main.py b/main.py index 0613713..c900ed7 100644 --- a/main.py +++ b/main.py @@ -7,14 +7,15 @@ def human_agent(): Runs the Gym environment with human input. """ - with open("evaluation_examples/examples/f9584479-3d0d-4c79-affa-9ad7afdd8850.json", "r") as f: + with open("evaluation_examples/examples/libreoffice_calc/f9584479-3d0d-4c79-affa-9ad7afdd8850.json", "r") as f: example = json.load(f) - example["snapshot"] = "base_setup" + example["snapshot"] = "base_setup2" - env = DesktopEnv( path_to_vm=r"C:\Users\tianbaox\Documents\Virtual Machines\Ubuntu\Ubuntu.vmx" - , action_space="computer_13" - , task_config=example - ) + env = DesktopEnv( + path_to_vm=r"C:\Users\tianbaox\Documents\Virtual Machines\Ubuntu\Ubuntu.vmx", + action_space="computer_13", + task_config=example + ) # reset the environment to certain snapshot observation = env.reset()