From 857f73f61d6fdbb011510474d8e070c635517b68 Mon Sep 17 00:00:00 2001 From: David Chang Date: Thu, 23 Nov 2023 22:25:32 +0800 Subject: [PATCH 1/4] ver Nov23rd trying to set up the environment --- .gitignore | 4 +++- desktop_env/envs/desktop_env.py | 12 +++++++----- main.py | 17 ++++++++++------- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index b3945ef..591df91 100644 --- a/.gitignore +++ b/.gitignore @@ -160,4 +160,6 @@ instructor-embedding # frontend cache frontend/node_modules/ frontend/.next/ -frontend/.idea \ No newline at end of file +frontend/.idea + +tags diff --git a/desktop_env/envs/desktop_env.py b/desktop_env/envs/desktop_env.py index 8b51c5b..3293440 100644 --- a/desktop_env/envs/desktop_env.py +++ b/desktop_env/envs/desktop_env.py @@ -74,7 +74,7 @@ class DesktopEnv(gym.Env): def _wait_for_emulator_load(self): while True: try: - output = subprocess.check_output(f"vmrun -T ws list", shell=True, stderr=subprocess.STDOUT) + output = subprocess.check_output("vmrun -T ws list", shell=True, stderr=subprocess.STDOUT) output = output.decode() if self.path_to_vm.lstrip("~/") in output: print("VM is running.") @@ -127,16 +127,18 @@ class DesktopEnv(gym.Env): return image_path def _get_obs(self): + print("OBS 1") screenshot_image_path = self._get_screenshot() + print("OBS 2") with Image.open(screenshot_image_path) as img: return np.array(img) def reset(self): - input() - self._execute_command(["vmrun", "-T", "ws", "revertToSnapshot", self.path_to_vm, self.snapshot_path]) - input() + input("Reset #1 PE") + #self._execute_command(["vmrun", "-T", "ws", "revertToSnapshot", self.path_to_vm, self.snapshot_path]) + input("Revert to snapshot #2 PE") self._start_emulator() - input() + input("Started emulator #3 PE") self._wait_for_emulator_load() observation = self._get_obs() diff --git a/main.py b/main.py index 2cbcd18..a18cbc3 100644 --- a/main.py +++ b/main.py @@ -1,4 +1,4 @@ -from pprint import pprint +#from pprint import pprint from desktop_env.envs.desktop_env import DesktopEnv, Action, MouseClick def get_human_action(): @@ -38,13 +38,16 @@ def human_agent(): """ Runs the Gym environment with human input. """ - env = DesktopEnv(path_to_vm="/home/yuri/vmware/Windows 10 x64/Windows 10 x64.vmx", + #env = DesktopEnv( path_to_vm="/home/yuri/vmware/Windows 10 x64/Windows 10 x64.vmx" # path_to_vm="/home/yuri/vmware/Ubuntu 64-bit/Ubuntu 64-bit.vmx", - username="user", - password="password", - # host="192.168.7.128", - host="http://192.168.7.129:5000", - vm_os="windows") + env = DesktopEnv( path_to_vm="/home/david/vmware/KUbuntu 64-bit/KUbuntu 64-bit.vmx" + , username="david" + , password="123456" + , host="192.168.174.129" + #host="http://192.168.7.129:5000", + #vm_os="windows") + , vm_os="ubuntu" + ) observation = env.reset() done = False From b37df609b9957f6379e90bf2c3f9de00bf1f7803 Mon Sep 17 00:00:00 2001 From: David Chang Date: Wed, 20 Dec 2023 20:14:39 +0800 Subject: [PATCH 2/4] ver Dec20th some adaptations to make in run on my device --- .gitignore | 1 + branch-config/.gitignore | 1 + branch-config/autodelete | 1 + branch-config/destpath | 1 + branch-config/filelist | 4 ++++ branch-config/ignorelist | 0 branch-config/includelist | 0 branch-config/symlist | 0 branch_flag | 1 + desktop_env/envs/desktop_env.py | 2 +- desktop_env/server/main.py | 2 +- .../examples/f9584479-3d0d-4c79-affa-9ad7afdd8850.json | 10 +++++----- main.py | 9 +++++---- 13 files changed, 21 insertions(+), 11 deletions(-) create mode 100644 branch-config/.gitignore create mode 100644 branch-config/autodelete create mode 100644 branch-config/destpath create mode 100644 branch-config/filelist create mode 100644 branch-config/ignorelist create mode 100644 branch-config/includelist create mode 100644 branch-config/symlist create mode 100644 branch_flag diff --git a/.gitignore b/.gitignore index 591df91..184e64e 100644 --- a/.gitignore +++ b/.gitignore @@ -163,3 +163,4 @@ frontend/.next/ frontend/.idea tags +snapshots diff --git a/branch-config/.gitignore b/branch-config/.gitignore new file mode 100644 index 0000000..60c9e47 --- /dev/null +++ b/branch-config/.gitignore @@ -0,0 +1 @@ +comment diff --git a/branch-config/autodelete b/branch-config/autodelete new file mode 100644 index 0000000..573541a --- /dev/null +++ b/branch-config/autodelete @@ -0,0 +1 @@ +0 diff --git a/branch-config/destpath b/branch-config/destpath new file mode 100644 index 0000000..bd01d04 --- /dev/null +++ b/branch-config/destpath @@ -0,0 +1 @@ +snapshots diff --git a/branch-config/filelist b/branch-config/filelist new file mode 100644 index 0000000..eb3b3a1 --- /dev/null +++ b/branch-config/filelist @@ -0,0 +1,4 @@ +desktop_env +main.py +requirements.txt +evaluation_examples diff --git a/branch-config/ignorelist b/branch-config/ignorelist new file mode 100644 index 0000000..e69de29 diff --git a/branch-config/includelist b/branch-config/includelist new file mode 100644 index 0000000..e69de29 diff --git a/branch-config/symlist b/branch-config/symlist new file mode 100644 index 0000000..e69de29 diff --git a/branch_flag b/branch_flag new file mode 100644 index 0000000..9daeafb --- /dev/null +++ b/branch_flag @@ -0,0 +1 @@ +test diff --git a/desktop_env/envs/desktop_env.py b/desktop_env/envs/desktop_env.py index 34270c8..93f81b3 100644 --- a/desktop_env/envs/desktop_env.py +++ b/desktop_env/envs/desktop_env.py @@ -154,7 +154,7 @@ class DesktopEnv(gym.Env): _path = os.path.join("tmp", random_uuid, "tmp.xlsx") if _file_info["type"] == "cloud_file": url = _file_info["path"] - response = requests.get(url, stream=True) + response = requests.get(url, stream=True, verify=False) response.raise_for_status() with open(_path, 'wb') as f: diff --git a/desktop_env/server/main.py b/desktop_env/server/main.py index 245054e..e1cb27e 100644 --- a/desktop_env/server/main.py +++ b/desktop_env/server/main.py @@ -151,7 +151,7 @@ def download_file(): max_retries = 3 for i in range(max_retries): try: - response = requests.get(url, stream=True) + response = requests.get(url, stream=True, verify=False) response.raise_for_status() with open(path, 'wb') as f: diff --git a/evaluation_examples/examples/f9584479-3d0d-4c79-affa-9ad7afdd8850.json b/evaluation_examples/examples/f9584479-3d0d-4c79-affa-9ad7afdd8850.json index 5a01d56..ae47d10 100644 --- a/evaluation_examples/examples/f9584479-3d0d-4c79-affa-9ad7afdd8850.json +++ b/evaluation_examples/examples/f9584479-3d0d-4c79-affa-9ad7afdd8850.json @@ -6,12 +6,12 @@ "config": { "download": [ [ - "https://drive.usercontent.google.com/download?id=1rwhniaClEkF8XFzdfaNUA6GmAiy4syMZ&export=download&authuser=0&confirm=t&uuid=6fdd5b04-85f4-45e1-ad74-368f8f2a82ab&at=APZUnTUP-JxPxLfNls6jXWghblQ5:1701766091851", - "C:\\Users\\tianbaox\\Desktop\\Quarterly_Product_Sales_by_Zone.xlsx" + "http://101.43.24.67/s/DbaHsQpPA7dxrA8/download/Quarterly_Product_Sales_by_Zone.xlsx", + "/home/david/Quarterly_Product_Sales_by_Zone.xlsx" ] ], "open": [ - "C:\\Users\\tianbaox\\Desktop\\Quarterly_Product_Sales_by_Zone.xlsx" + "/home/david/Quarterly_Product_Sales_by_Zone.xlsx" ] }, "trajectory": "trajectories/f9584479-3d0d-4c79-affa-9ad7afdd8850", @@ -23,11 +23,11 @@ "paths": { "expected": { "type": "cloud_file", - "path": "https://drive.usercontent.google.com/download?id=17f1wZuJPvUEc5at_Fy3c18VFdOk0x7xz&export=download&authuser=0&confirm=t&uuid=6d2edffd-0ce0-426e-9820-8af25b4667f3&at=APZUnTVh7JS85dwZBaV2hytWQgDK:1702361510956" + "path": "http://101.43.24.67/s/BAfFwa3689XTYoo/download/Quarterly_Product_Sales_by_Zone_gold.xlsx" }, "actual": { "type": "vm_file", - "path": "C:\\Users\\tianbaox\\Desktop\\Quarterly_Product_Sales_by_Zone.xlsx" + "path": "/home/david/Quarterly_Product_Sales_by_Zone.xlsx" } } } diff --git a/main.py b/main.py index b7ed019..a116d59 100644 --- a/main.py +++ b/main.py @@ -7,17 +7,18 @@ def human_agent(): Runs the Gym environment with human input. """ - with open("evaluation_examples/examples/37608790-6147-45d0-9f20-1137bb35703d.json", "r") as f: + with open("evaluation_examples/examples/f9584479-3d0d-4c79-affa-9ad7afdd8850.json", "r") as f: example = json.load(f) #env = DesktopEnv( path_to_vm="/home/yuri/vmware/Windows 10 x64/Windows 10 x64.vmx" # path_to_vm="/home/yuri/vmware/Ubuntu 64-bit/Ubuntu 64-bit.vmx", env = DesktopEnv( path_to_vm="/home/david/vmware/KUbuntu 64-bit/KUbuntu 64-bit.vmx" , action_space="computer_13" - , snapshot_path="base_setup" + #, snapshot_path="base_setup" + , snapshot_path="Init5" , instruction=example["instruction"] - #, config=example["config"] - #, evaluator=example["evaluator"] + , config=example["config"] + , evaluator=example["evaluator"] ) # reset the environment to certain snapshot From f4949c3e901b4b6acce53bd4fbaf303298771cae Mon Sep 17 00:00:00 2001 From: David Chang Date: Wed, 20 Dec 2023 20:55:47 +0800 Subject: [PATCH 3/4] ver Dec20th fixed bug caused by windows-only os.startfile --- desktop_env/server/main.py | 11 ++++++++--- main.py | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/desktop_env/server/main.py b/desktop_env/server/main.py index e1cb27e..1e88787 100644 --- a/desktop_env/server/main.py +++ b/desktop_env/server/main.py @@ -4,9 +4,10 @@ import platform import subprocess import requests -import Xlib.display +#import Xlib.display import pyautogui -from PIL import ImageGrab, Image +#from PIL import ImageGrab, Image +from PIL import Image from flask import Flask, request, jsonify, send_file app = Flask(__name__) @@ -180,7 +181,11 @@ def open_file(): return f"File not found: {path}", 404 try: - os.startfile(path) + if platform.system()=="Windows": + os.startfile(path) + else: + open_cmd: str = "open" if platform.system()=="Darwin" else "xdg-open" + subprocess.Popen([open_cmd, str(path)]) return "File opened successfully" except Exception as e: return f"Failed to open {path}. Error: {e}", 500 diff --git a/main.py b/main.py index a116d59..a96bd08 100644 --- a/main.py +++ b/main.py @@ -15,7 +15,7 @@ def human_agent(): env = DesktopEnv( path_to_vm="/home/david/vmware/KUbuntu 64-bit/KUbuntu 64-bit.vmx" , action_space="computer_13" #, snapshot_path="base_setup" - , snapshot_path="Init5" + , snapshot_path="Init6" , instruction=example["instruction"] , config=example["config"] , evaluator=example["evaluator"] From 4a643abc31ad237801b51feddb3f6a59b6836a7e Mon Sep 17 00:00:00 2001 From: David Chang Date: Thu, 21 Dec 2023 10:30:23 +0800 Subject: [PATCH 4/4] ver Dec21st updated setup configs from dict-style to list-style to support more flexible setup steps --- desktop_env/controllers/setup.py | 101 +++++++++++------- .../f9584479-3d0d-4c79-affa-9ad7afdd8850.json | 30 ++++-- 2 files changed, 83 insertions(+), 48 deletions(-) diff --git a/desktop_env/controllers/setup.py b/desktop_env/controllers/setup.py index 1b3e581..a550389 100644 --- a/desktop_env/controllers/setup.py +++ b/desktop_env/controllers/setup.py @@ -1,6 +1,8 @@ import requests import json +from typing import Dict, List +from typing import Any class SetupController: def __init__(self, http_server: str): @@ -14,18 +16,42 @@ class SetupController: ... } """ - self._download_setup(config) - self._change_wallpaper(config) + + for cfg in config: + config_type: str = cfg["type"] + parameters: Dict[str, Any] = cfg["parameters"] + + # Assumes all the setup the functions should follow this name + # protocol + setup_function: str = "_{:}_setup".format(config_type) + assert hasattr(self, setup_function) + getattr(self, setup_function)(**parameters) + + #self._download_setup(config) + #self._change_wallpaper(config) # self._tidy_desktop(config) todo: implement this - self._open_setup(config) + #self._open_setup(config) # can add other setup steps - def _download_setup(self, config): - if not config: - return - if not 'download' in config: - return - for url, path in config['download']: + def _download_setup(self, files: List[Dict[str, str]]): + """ + Args: + files (List[Dict[str, str]]): files to download. lisf of dict like + { + "url": str, the url to download + "path": str, the path on the VM to store the downloaded file + } + """ + + #if not config: + #return + #if not 'download' in config: + #return + #for url, path in config['download']: + for f in files: + url: str = f["url"] + path: str = f["path"] + if not url or not path: raise Exception(f"Setup Download - Invalid URL ({url}) or path ({path}).") @@ -44,12 +70,13 @@ class SetupController: except requests.exceptions.RequestException as e: print("An error occurred while trying to send the request:", e) - def _change_wallpaper(self, config): - if not config: - return - if not 'wallpaper' in config: - return - path = config['wallpaper'] + def _change_wallpaper_setup(self, path: str): + #if not config: + #return + #if not 'wallpaper' in config: + #return + + #path = config['wallpaper'] if not path: raise Exception(f"Setup Wallpaper - Invalid path ({path}).") @@ -68,29 +95,29 @@ class SetupController: except requests.exceptions.RequestException as e: print("An error occurred while trying to send the request:", e) - def _tidy_desktop(self, config): + def _tidy_desktop_setup(self, **config): raise NotImplementedError - def _open_setup(self, config): - if not config: - return - if not 'open' in config: - return - for path in config['open']: - if not path: - raise Exception(f"Setup Open - Invalid path ({path}).") + def _open_setup(self, path: str): + #if not config: + #return + #if not 'open' in config: + #return + #for path in config['open']: + if not path: + raise Exception(f"Setup Open - Invalid path ({path}).") - payload = json.dumps({"path": path}) - headers = { - 'Content-Type': 'application/json' - } + payload = json.dumps({"path": path}) + headers = { + 'Content-Type': 'application/json' + } - # send request to server to open file - try: - response = requests.post(self.http_server + "/open_file", headers=headers, data=payload) - if response.status_code == 200: - print("Command executed successfully:", response.text) - else: - print("Failed to open file. Status code:", response.text) - except requests.exceptions.RequestException as e: - print("An error occurred while trying to send the request:", e) + # send request to server to open file + try: + response = requests.post(self.http_server + "/open_file", headers=headers, data=payload) + if response.status_code == 200: + print("Command executed successfully:", response.text) + else: + print("Failed to open file. Status code:", response.text) + except requests.exceptions.RequestException as e: + print("An error occurred while trying to send the request:", e) diff --git a/evaluation_examples/examples/f9584479-3d0d-4c79-affa-9ad7afdd8850.json b/evaluation_examples/examples/f9584479-3d0d-4c79-affa-9ad7afdd8850.json index ae47d10..3581f21 100644 --- a/evaluation_examples/examples/f9584479-3d0d-4c79-affa-9ad7afdd8850.json +++ b/evaluation_examples/examples/f9584479-3d0d-4c79-affa-9ad7afdd8850.json @@ -3,17 +3,25 @@ "snapshot": "libreoffice_calc", "instruction": "Fill the missing row and column which show the total value", "source": "https://youtube.com/shorts/feldd-Pn48c?si=9xJiem2uAHm6Jshb", - "config": { - "download": [ - [ - "http://101.43.24.67/s/DbaHsQpPA7dxrA8/download/Quarterly_Product_Sales_by_Zone.xlsx", - "/home/david/Quarterly_Product_Sales_by_Zone.xlsx" - ] - ], - "open": [ - "/home/david/Quarterly_Product_Sales_by_Zone.xlsx" - ] - }, + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "http://101.43.24.67/s/DbaHsQpPA7dxrA8/download/Quarterly_Product_Sales_by_Zone.xlsx", + "path": "/home/david/Quarterly_Product_Sales_by_Zone.xlsx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/david/Quarterly_Product_Sales_by_Zone.xlsx" + } + } + ], "trajectory": "trajectories/f9584479-3d0d-4c79-affa-9ad7afdd8850", "related_apps": [ "libreoffice calc"