From 2163a08a0db38e1a60bed7c3b45c9d7203c831d9 Mon Sep 17 00:00:00 2001 From: David Chang Date: Fri, 22 Dec 2023 19:03:54 +0800 Subject: [PATCH 1/4] ver Dec22ndv4 recorded action history --- desktop_env/envs/desktop_env.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/desktop_env/envs/desktop_env.py b/desktop_env/envs/desktop_env.py index 712ade8..d1a8026 100644 --- a/desktop_env/envs/desktop_env.py +++ b/desktop_env/envs/desktop_env.py @@ -91,10 +91,12 @@ class DesktopEnv(gym.Env): self.result_getter: Getter = getattr(getters, "get_{:}".format(self.evaluator["result"]["type"])) self.expected_getter: Getter = getattr(getters, "get_{:}".format(self.evaluator["expected"]["type"])) - # episodic stuffs, like tmp dir and counters + # episodic stuffs, like tmp dir and counters, will be updated or reset + # when calling self.reset() self.tmp_dir: str = self.tmp_dir_base # just an init value, updated during reset self._traj_no: int = -1 self._step_no: int = 0 + self.action_history: List[Dict[str, any]] = [] def _start_emulator(self): while True: @@ -166,6 +168,7 @@ class DesktopEnv(gym.Env): print("Setting counters...") self._traj_no += 1 self._step_no = 0 + self.action_history.clear() print("Setup new temp dir...") self.tmp_dir = tempfile.mkdtemp( prefix="{:d}.{:}.".format(self._traj_no, self.task_id) @@ -200,6 +203,7 @@ class DesktopEnv(gym.Env): elif self.action_space == "pyautogui": # the set of all possible python commands insides `pyautogui` self.controller.execute_python_command(action) + self.action_history.append(action) # todo: maybe for the better here we need to add a logic to wait until the rendering is done time.sleep(pause) From 82e3353f65c25ae23595cff050ff4633db2d094a Mon Sep 17 00:00:00 2001 From: David Chang Date: Mon, 25 Dec 2023 14:40:30 +0800 Subject: [PATCH 2/4] ver Dec25th added cache and upload function for setup --- .gitignore | 1 + desktop_env/controllers/setup.py | 60 ++++++++++++++++--- desktop_env/envs/desktop_env.py | 26 ++++---- desktop_env/server/main.py | 10 ++++ .../f9584479-3d0d-4c79-affa-9ad7afdd8850.json | 54 ++++++++--------- main.py | 2 +- requirements.txt | 3 +- 7 files changed, 107 insertions(+), 49 deletions(-) diff --git a/.gitignore b/.gitignore index 184e64e..490eebb 100644 --- a/.gitignore +++ b/.gitignore @@ -163,4 +163,5 @@ frontend/.next/ frontend/.idea tags +tags-opts snapshots diff --git a/desktop_env/controllers/setup.py b/desktop_env/controllers/setup.py index b42b2ff..26cb474 100644 --- a/desktop_env/controllers/setup.py +++ b/desktop_env/controllers/setup.py @@ -1,12 +1,22 @@ import requests import json +from requests_toolbelt.multipart.encoder import MultipartEncoder + +import uuid +import os.path from typing import Dict, List from typing import Any class SetupController: - def __init__(self, http_server: str): + def __init__( self + , http_server: str + , cache_dir: str + ): self.http_server = http_server + "/setup" + self.cache_dir: str = cache_dir + def reset_cache_dir(self, cache_dir: str): + self.cache_dir = cache_dir def setup(self, config: List[Dict[str, Any]]): """ @@ -55,22 +65,56 @@ class SetupController: for f in files: url: str = f["url"] path: str = f["path"] + cache_path: str = os.path.join( self.cache_dir + , "{:}_{:}".format( + uuid.uuid5(uuid.NAMESPACE_URL, url) + , os.path.basename(path) + ) + ) if not url or not path: raise Exception(f"Setup Download - Invalid URL ({url}) or path ({path}).") - payload = json.dumps({"url": url, "path": path}) - headers = { - 'Content-Type': 'application/json' - } + if not os.path.exists(cache_path): + max_retries = 3 + downloaded = False + for i in range(max_retries): + try: + response = requests.get(url, stream=True, verify=False) + response.raise_for_status() - # send request to server to download file + with open(cache_path, 'wb') as f: + for chunk in response.iter_content(chunk_size=8192): + if chunk: + f.write(chunk) + print("File downloaded successfully") + downloaded = True + break + + except requests.RequestException as e: + print(f"Failed to download {url}. Retrying... ({max_retries - i - 1} attempts left)") + if not downloaded: + raise requests.RequestException(f"Failed to download {url}. No retries left. Error: {e}") + + #payload = json.dumps({"url": url, "path": path}) + #headers = { + #'Content-Type': 'application/json' + #} + + form = MultipartEncoder( { "file_path": path + , "file_data": (os.path.basename(path), open(cache_path, "rb")) + } + ) + headers = {"Content-Type": form.content_type} + print(form.content_type) + + # send request to server to upload file try: - response = requests.post(self.http_server + "/download_file", headers=headers, data=payload) + response = requests.post(self.http_server + "/upload", headers=headers, data=form) if response.status_code == 200: print("Command executed successfully:", response.text) else: - print("Failed to download file. Status code:", response.text) + print("Failed to upload file. Status code:", response.text) except requests.exceptions.RequestException as e: print("An error occurred while trying to send the request:", e) diff --git a/desktop_env/envs/desktop_env.py b/desktop_env/envs/desktop_env.py index d1a8026..eea834a 100644 --- a/desktop_env/envs/desktop_env.py +++ b/desktop_env/envs/desktop_env.py @@ -66,18 +66,6 @@ class DesktopEnv(gym.Env): self.tmp_dir_base: str = tmp_dir self.cache_dir_base: str = cache_dir - # Initialize emulator and controller - print("Initializing...") - self._start_emulator() - self.host = f"http://{self._get_vm_ip()}:5000" - self.controller = PythonController(http_server=self.host) - self.setup_controller = SetupController(http_server=self.host) - - # mode: human or machine - assert action_space in ["computer_13", "pyautogui"] - self.action_space = action_space - # todo: define the action space and the observation space as gym did, or extend theirs - # task-aware stuffs self.snapshot_path = task_config["snapshot"] # todo: handling the logic of snapshot directory self.task_id: str = task_config["id"] @@ -91,6 +79,18 @@ class DesktopEnv(gym.Env): self.result_getter: Getter = getattr(getters, "get_{:}".format(self.evaluator["result"]["type"])) self.expected_getter: Getter = getattr(getters, "get_{:}".format(self.evaluator["expected"]["type"])) + # Initialize emulator and controller + print("Initializing...") + self._start_emulator() + self.host = f"http://{self._get_vm_ip()}:5000" + self.controller = PythonController(http_server=self.host) + self.setup_controller = SetupController(http_server=self.host, cache_dir=self.cache_dir) + + # mode: human or machine + assert action_space in ["computer_13", "pyautogui"] + self.action_space = action_space + # todo: define the action space and the observation space as gym did, or extend theirs + # episodic stuffs, like tmp dir and counters, will be updated or reset # when calling self.reset() self.tmp_dir: str = self.tmp_dir_base # just an init value, updated during reset @@ -165,6 +165,8 @@ class DesktopEnv(gym.Env): self.result_getter: Getter = getattr(getters, "get_{:}".format(self.evaluator["result"]["type"])) self.expected_getter: Getter = getattr(getters, "get_{:}".format(self.evaluator["expected"]["type"])) + self.setup_controller.reset_cache_dir(self.cache_dir) + print("Setting counters...") self._traj_no += 1 self._step_no = 0 diff --git a/desktop_env/server/main.py b/desktop_env/server/main.py index 1e88787..67dc015 100644 --- a/desktop_env/server/main.py +++ b/desktop_env/server/main.py @@ -96,6 +96,16 @@ def get_file(): # If the file is not found, return a 404 error return jsonify({"error": "File not found"}), 404 +@app.route("/setup/upload", methods=["POST"]) +def upload_file(): + # Retrieve filename from the POST request + if 'file_path' in request.form and 'file_data' in request.files: + file_path = request.form['file_path'] + file = request.files["file_data"] + file.save(file_path) + return "File Uploaded" + else: + return jsonify({"error": "file_path and file_data are required"}), 400 @app.route('/platform', methods=['GET']) def get_platform(): diff --git a/evaluation_examples/examples/f9584479-3d0d-4c79-affa-9ad7afdd8850.json b/evaluation_examples/examples/f9584479-3d0d-4c79-affa-9ad7afdd8850.json index 539c4aa..60decfa 100644 --- a/evaluation_examples/examples/f9584479-3d0d-4c79-affa-9ad7afdd8850.json +++ b/evaluation_examples/examples/f9584479-3d0d-4c79-affa-9ad7afdd8850.json @@ -4,23 +4,23 @@ "instruction": "Fill the missing row and column which show the total value", "source": "https://youtube.com/shorts/feldd-Pn48c?si=9xJiem2uAHm6Jshb", "config": [ - { - "type": "download", - "parameters": { - "files": [ - { - "url": "http://101.43.24.67/s/DbaHsQpPA7dxrA8/download/Quarterly_Product_Sales_by_Zone.xlsx", - "path": "/home/david/Quarterly_Product_Sales_by_Zone.xlsx" - } - ] - } - }, - { - "type": "open", - "parameters": { - "path": "/home/david/Quarterly_Product_Sales_by_Zone.xlsx" - } - } + { + "type": "download", + "parameters": { + "files": [ + { + "url": "http://101.43.24.67/s/DbaHsQpPA7dxrA8/download/Quarterly_Product_Sales_by_Zone.xlsx", + "path": "/home/david/Quarterly_Product_Sales_by_Zone.xlsx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/david/Quarterly_Product_Sales_by_Zone.xlsx" + } + } ], "trajectory": "trajectories/f9584479-3d0d-4c79-affa-9ad7afdd8850", "related_apps": [ @@ -28,15 +28,15 @@ ], "evaluator": { "func": "compare_table", - "expected": { - "type": "cloud_file", - "path": "http://101.43.24.67/s/BAfFwa3689XTYoo/download/Quarterly_Product_Sales_by_Zone_gold.xlsx", - "dest": "Quarterly_Product_Sales_by_Zone_gold.xlsx" - }, - "result": { - "type": "vm_file", - "path": "/home/david/Quarterly_Product_Sales_by_Zone.xlsx", - "dest": "Quarterly_Product_Sales_by_Zone.xlsx" - } + "expected": { + "type": "cloud_file", + "path": "http://101.43.24.67/s/BAfFwa3689XTYoo/download/Quarterly_Product_Sales_by_Zone_gold.xlsx", + "dest": "Quarterly_Product_Sales_by_Zone_gold.xlsx" + }, + "result": { + "type": "vm_file", + "path": "/home/david/Quarterly_Product_Sales_by_Zone.xlsx", + "dest": "Quarterly_Product_Sales_by_Zone.xlsx" + } } } diff --git a/main.py b/main.py index 911b34f..147a137 100644 --- a/main.py +++ b/main.py @@ -9,7 +9,7 @@ def human_agent(): with open("evaluation_examples/examples/f9584479-3d0d-4c79-affa-9ad7afdd8850.json", "r") as f: example = json.load(f) - example["snapshot"] = "Init6" + example["snapshot"] = "Snapshot 10" #env = DesktopEnv( path_to_vm="/home/yuri/vmware/Windows 10 x64/Windows 10 x64.vmx" # path_to_vm="/home/yuri/vmware/Ubuntu 64-bit/Ubuntu 64-bit.vmx", diff --git a/requirements.txt b/requirements.txt index 0a27eff..f3907ea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,4 +13,5 @@ pyautogui~=0.9.54 psutil~=5.9.6 tqdm~=4.65.0 pandas~=2.0.3 -flask~=3.0.0 \ No newline at end of file +flask~=3.0.0 +requests-toolbelt~=1.0.0 From ba77c276e67a241682ee2a7c6ab35d5d08d1e4d0 Mon Sep 17 00:00:00 2001 From: David Chang Date: Mon, 25 Dec 2023 19:50:19 +0800 Subject: [PATCH 3/4] ver Dec25thv2 implemented functions to load sparklines from xlsx --- desktop_env/evaluators/metrics/__init__.py | 2 +- desktop_env/evaluators/metrics/table.py | 72 +++++++++++++++++-- .../2bd59342-0664-4ccb-ba87-79379096cc08.json | 38 +++++++--- .../37608790-6147-45d0-9f20-1137bb35703d.json | 50 +++++++------ requirements.txt | 3 + 5 files changed, 127 insertions(+), 38 deletions(-) diff --git a/desktop_env/evaluators/metrics/__init__.py b/desktop_env/evaluators/metrics/__init__.py index 740c9e2..f8dfefd 100644 --- a/desktop_env/evaluators/metrics/__init__.py +++ b/desktop_env/evaluators/metrics/__init__.py @@ -1 +1 @@ -from .table import compare_table +from .table import compare_table, compare_with_sparklines diff --git a/desktop_env/evaluators/metrics/table.py b/desktop_env/evaluators/metrics/table.py index bd49297..d984c0f 100644 --- a/desktop_env/evaluators/metrics/table.py +++ b/desktop_env/evaluators/metrics/table.py @@ -1,14 +1,74 @@ -def compare_table(expected, actual): - import pandas as pd +import pandas as pd +import zipfile +import lxml.etree +import lxml.cssselect +from lxml.etree import _Element +import xmltodict +#import pylightxl + +from typing import Dict, List +#from typing import Any + +def compare_table(actual, expected): df1 = pd.read_excel(expected) df2 = pd.read_excel(actual) # Compare the DataFrames return 1 if df1.equals(df2) else 0 +_xlsx_namespaces = [ ("x14", "http://schemas.microsoft.com/office/spreadsheetml/2009/9/main") + , ("xm", "http://schemas.microsoft.com/office/excel/2006/main") + ] +_xlsx_ns_mapping = dict(_xlsx_namespaces) +_xlsx_ns_imapping = dict(map(lambda itm: (itm[1], itm[0]), _xlsx_namespaces)) +_sparklines_selector = lxml.cssselect.CSSSelector("x14|sparkline", namespaces=_xlsx_ns_mapping) +#print(_sparklines_selector.css) +def _load_sparklines(xlsx_file: str) -> Dict[str, str]: + """ + This function modifies data_frame in-place + + Args: + xlsx_file (str): path to xlsx + + Returns: + List[Dict[str, str]]: sparkline definitions in form of + { + "F3": "Sheet1!C3:E3" + } + """ + + # read xlsx + with zipfile.ZipFile(xlsx_file, "r") as z_f: + with z_f.open("xl/worksheets/sheet1.xml") as f: + sheet1: _Element = lxml.etree.fromstring(f.read()) + sparklines: List[_Element] = _sparklines_selector(sheet1) + + sparklines_dict: Dict[str, str] = {} + for sp_l in sparklines: + sparkline_xml: str = lxml.etree.tostring(sp_l, encoding="unicode") + sparkline: Dict[str, Dict[str, str]] = xmltodict.parse( sparkline_xml + , process_namespaces=True + , namespaces=_xlsx_ns_imapping + ) + sparklines_dict[sparkline["x14:sparkline"]["xm:sqref"]] = sparkline["x14:sparkline"]["xm:f"] + return sparklines_dict + +def compare_with_sparklines(actual: str, expected: str) -> float: + df1 = pd.read_excel(actual) + df2 = pd.read_excel(expected) + normal_content_metric: bool = df1.equals(df2) + + sp1 = _load_sparklines(actual) + sp2 = _load_sparklines(expected) + sparkline_metric: bool = sp1 == sp2 + + return float(normal_content_metric and sparkline_metric) if __name__ == '__main__': - path1 = "" - path2 = "" - - print(compare_table(path1, path2)) + #path1 = "" + #path2 = "" + #print(compare_table(path1, path2)) + + path1 = "../../../../../任务数据/LibreOffice Calc/OrderId_Month_Chart_gold.xlsx" + path2 = "../../../../../任务数据/LibreOffice Calc/OrderId_Month_Chart.xlsx" + print(compare_with_sparklines(path1, path2)) diff --git a/evaluation_examples/examples/2bd59342-0664-4ccb-ba87-79379096cc08.json b/evaluation_examples/examples/2bd59342-0664-4ccb-ba87-79379096cc08.json index 98d773a..284f162 100644 --- a/evaluation_examples/examples/2bd59342-0664-4ccb-ba87-79379096cc08.json +++ b/evaluation_examples/examples/2bd59342-0664-4ccb-ba87-79379096cc08.json @@ -3,20 +3,38 @@ "snapshot": "libreoffice_calc", "instruction": "Make sparkline chart line by line", "source": "https://www.youtube.com/shorts/L3Z-F1QTQFY", - "config": { - "download": [ - [ - "", - "C:\\Users\\tianbaox\\Desktop\\OrderId_Month_Chart.xlsx" - ] - ], - "open": [ - "C:\\Users\\tianbaox\\Desktop\\OrderId_Month_Chart.xlsx" + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://101.43.24.67/s/wrEyMi8HsmFjQrZ/download/OrderId_Month_Chart.xlsx", + "path": "/home/david/OrderId_Month_Chart.xlsx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/david/OrderId_Month_Chart.xlsx" ] }, "trajectory": "trajectories/2bd59342-0664-4ccb-ba87-79379096cc08", "related_apps": [ "libreoffice calc" ], - "evaluator": "evaluation_dir" + "evaluator": { + "expected": { + "type": "cloud_file", + "path": "https://101.43.24.67/s/t7pgJxNoAGFQWEM/download/OrderId_Month_Chart_gold.xlsx", + "dest": "OrderId_Month_Chart_gold.xlsx" + }, + "result": { + "type": "vm_file", + "path": "/home/david/OrderId_Month_Chart.xlsx", + "dest": "OrderId_Month_Chart.xlsx" + } + } } diff --git a/evaluation_examples/examples/37608790-6147-45d0-9f20-1137bb35703d.json b/evaluation_examples/examples/37608790-6147-45d0-9f20-1137bb35703d.json index e25e698..b34683a 100644 --- a/evaluation_examples/examples/37608790-6147-45d0-9f20-1137bb35703d.json +++ b/evaluation_examples/examples/37608790-6147-45d0-9f20-1137bb35703d.json @@ -3,32 +3,40 @@ "snapshot": "libreoffice_calc", "instruction": "Help me fill the columns of First Name, Last Name and Rank", "source": "https://www.youtube.com/shorts/uzPo_CPCHH8", - "config": { - "download": [ - [ - "https://drive.usercontent.google.com/download?id=1wDqap5cBfxnlqTNrZG61k_wDWTujl6AU&export=download&authuser=0&confirm=t&uuid=fd183b89-76b7-4dc5-880e-1045ed769562&at=APZUnTWp9RMafMg0xohhBWazN3YD:1701785710674", - "C:\\Users\\tianbaox\\Desktop\\Employee_Roles_and_Ranks.xlsx" - ] - ], - "open": [ - "C:\\Users\\tianbaox\\Desktop\\Employee_Roles_and_Ranks.xlsx" - ] - }, + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://101.43.24.67/s/FBip5fXoR4KEJaa", + "path": "/home/david/Employee_Roles_and_Ranks.xlsx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/david/Employee_Roles_and_Ranks.xlsx" + } + } + ], "trajectory": "trajectories/37608790-6147-45d0-9f20-1137bb35703d", "related_apps": [ "libreoffice calc" ], "evaluator": { - "func": "compare_table(expected, actual)", - "paths": { - "expected": { - "type": "cloud_file", - "path": "https://drive.usercontent.google.com/download?id=1dxpiUqP_CVvQp5tddxlwO3Cp1BqJ-ZDE&export=download&authuser=0&confirm=t&uuid=ccd204c7-07ce-4fdf-a5d4-a7e4f37b9ce6&at=APZUnTVBs7TgrVrDXpkiU8S7WbQo:1702360836747" - }, - "actual": { - "type": "vm_file", - "path": "C:\\Users\\tianbaox\\Desktop\\Employee_Roles_and_Ranks.xlsx" - } + "func": "compare_table", + "expected": { + "type": "cloud_file", + "path": "https://101.43.24.67/s/wr7B4GeotNNoeHD", + "dest": "Employee_Roles_and_Ranks_gold.xlsx" + }, + "result": { + "type": "vm_file", + "path": "/home/david/Employee_Roles_and_Ranks.xlsx", + "dest": "Employee_Roles_and_Ranks.xlsx" } } } diff --git a/requirements.txt b/requirements.txt index f3907ea..28d6d88 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,3 +15,6 @@ tqdm~=4.65.0 pandas~=2.0.3 flask~=3.0.0 requests-toolbelt~=1.0.0 +lxml +cssselect +xmltodict From bc2ef08da7cf8fd6f40721a468228e667c1620a6 Mon Sep 17 00:00:00 2001 From: David Chang Date: Mon, 25 Dec 2023 20:41:32 +0800 Subject: [PATCH 4/4] ver Dec25thv3 added config for new example with metrics under implementation --- .../0cecd4f3-74de-457b-ba94-29ad6b5dafb6.json | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 evaluation_examples/examples/libreoffice_calc/0cecd4f3-74de-457b-ba94-29ad6b5dafb6.json diff --git a/evaluation_examples/examples/libreoffice_calc/0cecd4f3-74de-457b-ba94-29ad6b5dafb6.json b/evaluation_examples/examples/libreoffice_calc/0cecd4f3-74de-457b-ba94-29ad6b5dafb6.json new file mode 100644 index 0000000..0090cb1 --- /dev/null +++ b/evaluation_examples/examples/libreoffice_calc/0cecd4f3-74de-457b-ba94-29ad6b5dafb6.json @@ -0,0 +1,57 @@ +{ + "id": "0cecd4f3-74de-457b-ba94-29ad6b5dafb6", + "snapshot": "libreoffice_calc", + "instruction": "Copy sheet1 and insert it before sheet2", + "source": "https://www.libreofficehelp.com/add-insert-delete-copy-move-rename-a-worksheet-in-libreoffice-calc/", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://101.43.24.67/s/LLE8tmGkpNeGBtZ/download/copy_sheet_insert.xlsx", + "path": "/home/david/copy_sheet_insert.xlsx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/david/copy_sheet_insert.xlsx" + } + } + ], + "trajectory": "trajectories/0cecd4f3-74de-457b-ba94-29ad6b5dafb6", + "related_apps": [ + "libreoffice_calc" + ], + "evaluator": { + "func": "check_sheet_list", + "result": { + "type": "vm_file", + "path": "/home/david/copy_sheet_insert.xlsx", + "dest": "copy_sheet_insert.xlsx" + }, + "expected": { + "type": "rule", + "rules": [ + { + "type": "sheet_name", + "sheet_idx": 0, + "sheet_name": "Sheet1" + }, + { + "type": "sheet_data", + "sheet_idx0": 0, + "sheet_idx1": 1 + }, + { + "type": "sheet_name", + "sheet_idx": 2, + "sheet_name": "Sheet2" + } + ] + } + } +}