diff --git a/.gitignore b/.gitignore index 3fb2a90..cedaca8 100644 --- a/.gitignore +++ b/.gitignore @@ -163,6 +163,7 @@ frontend/.next/ frontend/.idea tags +tags-opts snapshots branch_flag branch-config diff --git a/desktop_env/controllers/setup.py b/desktop_env/controllers/setup.py index 4ef71d6..e28287b 100644 --- a/desktop_env/controllers/setup.py +++ b/desktop_env/controllers/setup.py @@ -1,13 +1,23 @@ import requests import json +from requests_toolbelt.multipart.encoder import MultipartEncoder + +import uuid +import os.path from typing import Dict, List from typing import Any class SetupController: - def __init__(self, http_server: str): + def __init__( self + , http_server: str + , cache_dir: str + ): self.http_server = http_server + "/setup" + self.cache_dir: str = cache_dir + def reset_cache_dir(self, cache_dir: str): + self.cache_dir = cache_dir def setup(self, config: List[Dict[str, Any]]): """ @@ -56,22 +66,56 @@ class SetupController: for f in files: url: str = f["url"] path: str = f["path"] + cache_path: str = os.path.join( self.cache_dir + , "{:}_{:}".format( + uuid.uuid5(uuid.NAMESPACE_URL, url) + , os.path.basename(path) + ) + ) if not url or not path: raise Exception(f"Setup Download - Invalid URL ({url}) or path ({path}).") - payload = json.dumps({"url": url, "path": path}) - headers = { - 'Content-Type': 'application/json' - } + if not os.path.exists(cache_path): + max_retries = 3 + downloaded = False + for i in range(max_retries): + try: + response = requests.get(url, stream=True) + response.raise_for_status() - # send request to server to download file + with open(cache_path, 'wb') as f: + for chunk in response.iter_content(chunk_size=8192): + if chunk: + f.write(chunk) + print("File downloaded successfully") + downloaded = True + break + + except requests.RequestException as e: + print(f"Failed to download {url}. Retrying... ({max_retries - i - 1} attempts left)") + if not downloaded: + raise requests.RequestException(f"Failed to download {url}. No retries left. Error: {e}") + + #payload = json.dumps({"url": url, "path": path}) + #headers = { + #'Content-Type': 'application/json' + #} + + form = MultipartEncoder( { "file_path": path + , "file_data": (os.path.basename(path), open(cache_path, "rb")) + } + ) + headers = {"Content-Type": form.content_type} + print(form.content_type) + + # send request to server to upload file try: - response = requests.post(self.http_server + "/download_file", headers=headers, data=payload) + response = requests.post(self.http_server + "/upload", headers=headers, data=form) if response.status_code == 200: print("Command executed successfully:", response.text) else: - print("Failed to download file. Status code:", response.text) + print("Failed to upload file. Status code:", response.text) except requests.exceptions.RequestException as e: print("An error occurred while trying to send the request:", e) diff --git a/desktop_env/envs/desktop_env.py b/desktop_env/envs/desktop_env.py index 4afcbf1..ec00181 100644 --- a/desktop_env/envs/desktop_env.py +++ b/desktop_env/envs/desktop_env.py @@ -67,18 +67,6 @@ class DesktopEnv(gym.Env): self.tmp_dir_base: str = tmp_dir self.cache_dir_base: str = cache_dir - # Initialize emulator and controller - print("Initializing...") - self._start_emulator() - self.host = f"http://{self._get_vm_ip()}:5000" - self.controller = PythonController(http_server=self.host) - self.setup_controller = SetupController(http_server=self.host) - - # mode: human or machine - assert action_space in ["computer_13", "pyautogui"] - self.action_space = action_space - # todo: define the action space and the observation space as gym did, or extend theirs - # task-aware stuffs self.snapshot_path = task_config["snapshot"] # todo: handling the logic of snapshot directory self.task_id: str = task_config["id"] @@ -92,10 +80,24 @@ class DesktopEnv(gym.Env): self.result_getter: Getter = getattr(getters, "get_{:}".format(self.evaluator["result"]["type"])) self.expected_getter: Getter = getattr(getters, "get_{:}".format(self.evaluator["expected"]["type"])) - # episodic stuffs, like tmp dir and counters - self.tmp_dir: str = self.tmp_dir_base # just an init value, updated during reset + # Initialize emulator and controller + print("Initializing...") + self._start_emulator() + self.host = f"http://{self._get_vm_ip()}:5000" + self.controller = PythonController(http_server=self.host) + self.setup_controller = SetupController(http_server=self.host, cache_dir=self.cache_dir) + + # mode: human or machine + assert action_space in ["computer_13", "pyautogui"] + self.action_space = action_space + # todo: define the action space and the observation space as gym did, or extend theirs + + # episodic stuffs, like tmp dir and counters, will be updated or reset + # when calling self.reset() + self.tmp_dir: str = self.tmp_dir_base # just an init value, updated during reset self._traj_no: int = -1 self._step_no: int = 0 + self.action_history: List[Dict[str, any]] = [] def _start_emulator(self): while True: @@ -164,9 +166,12 @@ class DesktopEnv(gym.Env): self.result_getter: Getter = getattr(getters, "get_{:}".format(self.evaluator["result"]["type"])) self.expected_getter: Getter = getattr(getters, "get_{:}".format(self.evaluator["expected"]["type"])) + self.setup_controller.reset_cache_dir(self.cache_dir) + print("Setting counters...") self._traj_no += 1 self._step_no = 0 + self.action_history.clear() print("Setup new temp dir...") self.tmp_dir = tempfile.mkdtemp( @@ -202,6 +207,7 @@ class DesktopEnv(gym.Env): elif self.action_space == "pyautogui": # the set of all possible python commands insides `pyautogui` self.controller.execute_python_command(action) + self.action_history.append(action) # todo: maybe for the better here we need to add a logic to wait until the rendering is done time.sleep(pause) diff --git a/desktop_env/evaluators/metrics/__init__.py b/desktop_env/evaluators/metrics/__init__.py index 740c9e2..f8dfefd 100644 --- a/desktop_env/evaluators/metrics/__init__.py +++ b/desktop_env/evaluators/metrics/__init__.py @@ -1 +1 @@ -from .table import compare_table +from .table import compare_table, compare_with_sparklines diff --git a/desktop_env/evaluators/metrics/table.py b/desktop_env/evaluators/metrics/table.py index bd49297..d984c0f 100644 --- a/desktop_env/evaluators/metrics/table.py +++ b/desktop_env/evaluators/metrics/table.py @@ -1,14 +1,74 @@ -def compare_table(expected, actual): - import pandas as pd +import pandas as pd +import zipfile +import lxml.etree +import lxml.cssselect +from lxml.etree import _Element +import xmltodict +#import pylightxl + +from typing import Dict, List +#from typing import Any + +def compare_table(actual, expected): df1 = pd.read_excel(expected) df2 = pd.read_excel(actual) # Compare the DataFrames return 1 if df1.equals(df2) else 0 +_xlsx_namespaces = [ ("x14", "http://schemas.microsoft.com/office/spreadsheetml/2009/9/main") + , ("xm", "http://schemas.microsoft.com/office/excel/2006/main") + ] +_xlsx_ns_mapping = dict(_xlsx_namespaces) +_xlsx_ns_imapping = dict(map(lambda itm: (itm[1], itm[0]), _xlsx_namespaces)) +_sparklines_selector = lxml.cssselect.CSSSelector("x14|sparkline", namespaces=_xlsx_ns_mapping) +#print(_sparklines_selector.css) +def _load_sparklines(xlsx_file: str) -> Dict[str, str]: + """ + This function modifies data_frame in-place + + Args: + xlsx_file (str): path to xlsx + + Returns: + List[Dict[str, str]]: sparkline definitions in form of + { + "F3": "Sheet1!C3:E3" + } + """ + + # read xlsx + with zipfile.ZipFile(xlsx_file, "r") as z_f: + with z_f.open("xl/worksheets/sheet1.xml") as f: + sheet1: _Element = lxml.etree.fromstring(f.read()) + sparklines: List[_Element] = _sparklines_selector(sheet1) + + sparklines_dict: Dict[str, str] = {} + for sp_l in sparklines: + sparkline_xml: str = lxml.etree.tostring(sp_l, encoding="unicode") + sparkline: Dict[str, Dict[str, str]] = xmltodict.parse( sparkline_xml + , process_namespaces=True + , namespaces=_xlsx_ns_imapping + ) + sparklines_dict[sparkline["x14:sparkline"]["xm:sqref"]] = sparkline["x14:sparkline"]["xm:f"] + return sparklines_dict + +def compare_with_sparklines(actual: str, expected: str) -> float: + df1 = pd.read_excel(actual) + df2 = pd.read_excel(expected) + normal_content_metric: bool = df1.equals(df2) + + sp1 = _load_sparklines(actual) + sp2 = _load_sparklines(expected) + sparkline_metric: bool = sp1 == sp2 + + return float(normal_content_metric and sparkline_metric) if __name__ == '__main__': - path1 = "" - path2 = "" - - print(compare_table(path1, path2)) + #path1 = "" + #path2 = "" + #print(compare_table(path1, path2)) + + path1 = "../../../../../任务数据/LibreOffice Calc/OrderId_Month_Chart_gold.xlsx" + path2 = "../../../../../任务数据/LibreOffice Calc/OrderId_Month_Chart.xlsx" + print(compare_with_sparklines(path1, path2)) diff --git a/desktop_env/server/main.py b/desktop_env/server/main.py index c4ab500..7f485ea 100644 --- a/desktop_env/server/main.py +++ b/desktop_env/server/main.py @@ -95,6 +95,16 @@ def get_file(): # If the file is not found, return a 404 error return jsonify({"error": "File not found"}), 404 +@app.route("/setup/upload", methods=["POST"]) +def upload_file(): + # Retrieve filename from the POST request + if 'file_path' in request.form and 'file_data' in request.files: + file_path = request.form['file_path'] + file = request.files["file_data"] + file.save(file_path) + return "File Uploaded" + else: + return jsonify({"error": "file_path and file_data are required"}), 400 @app.route('/platform', methods=['GET']) def get_platform(): diff --git a/evaluation_examples/examples/libreoffice_calc/0cecd4f3-74de-457b-ba94-29ad6b5dafb6.json b/evaluation_examples/examples/libreoffice_calc/0cecd4f3-74de-457b-ba94-29ad6b5dafb6.json new file mode 100644 index 0000000..bcde623 --- /dev/null +++ b/evaluation_examples/examples/libreoffice_calc/0cecd4f3-74de-457b-ba94-29ad6b5dafb6.json @@ -0,0 +1,57 @@ +{ + "id": "0cecd4f3-74de-457b-ba94-29ad6b5dafb6", + "snapshot": "libreoffice_calc", + "instruction": "Copy sheet1 and insert it before sheet2", + "source": "https://www.libreofficehelp.com/add-insert-delete-copy-move-rename-a-worksheet-in-libreoffice-calc/", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1ejNXBNOZtn64ugmvXot21pOEjx5xa-I5&export=download&authuser=0&confirm=t&uuid=61aa93e2-03f7-4b28-8e4a-cdff16a642f7&at=APZUnTVgPAHHfXaEjfKau5CDY1_K:1703509323791", + "path": "Desktop/copy_sheet_insert.xlsx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "Desktop/copy_sheet_insert.xlsx" + } + } + ], + "trajectory": "trajectories/0cecd4f3-74de-457b-ba94-29ad6b5dafb6", + "related_apps": [ + "libreoffice_calc" + ], + "evaluator": { + "func": "check_sheet_list", + "result": { + "type": "vm_file", + "path": "Desktop/copy_sheet_insert.xlsx", + "dest": "copy_sheet_insert.xlsx" + }, + "expected": { + "type": "rule", + "rules": [ + { + "type": "sheet_name", + "sheet_idx": 0, + "sheet_name": "Sheet1" + }, + { + "type": "sheet_data", + "sheet_idx0": 0, + "sheet_idx1": 1 + }, + { + "type": "sheet_name", + "sheet_idx": 2, + "sheet_name": "Sheet2" + } + ] + } + } +} diff --git a/evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json b/evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json index 98d773a..51c6892 100644 --- a/evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json +++ b/evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json @@ -3,20 +3,38 @@ "snapshot": "libreoffice_calc", "instruction": "Make sparkline chart line by line", "source": "https://www.youtube.com/shorts/L3Z-F1QTQFY", - "config": { - "download": [ - [ - "", - "C:\\Users\\tianbaox\\Desktop\\OrderId_Month_Chart.xlsx" - ] - ], - "open": [ - "C:\\Users\\tianbaox\\Desktop\\OrderId_Month_Chart.xlsx" + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1uywX5XWMvesnb4-8LPKEzr2HFU7HmoIu&export=download&authuser=0&confirm=t&uuid=267bfe49-a861-4272-ae7c-39c95df35e84&at=APZUnTUbs-FF06hSMv3yWfdXc02l:1703508870351", + "path": "Desktop/OrderId_Month_Chart.xlsx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "Desktop/OrderId_Month_Chart.xlsx" ] }, "trajectory": "trajectories/2bd59342-0664-4ccb-ba87-79379096cc08", "related_apps": [ "libreoffice calc" ], - "evaluator": "evaluation_dir" + "evaluator": { + "expected": { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=1KQJJLVPGtTL_7ArEWvwwbFbJSiA3cgSE&export=download&authuser=0&confirm=t&uuid=6b11c721-caad-439a-b369-4c13c7a485df&at=APZUnTV5-1isKrDKSHV9NeJ6TDeS:1703509054094", + "dest": "OrderId_Month_Chart_gold.xlsx" + }, + "result": { + "type": "vm_file", + "path": "Desktop/OrderId_Month_Chart.xlsx", + "dest": "OrderId_Month_Chart.xlsx" + } + } } diff --git a/evaluation_examples/examples/libreoffice_calc/37608790-6147-45d0-9f20-1137bb35703d.json b/evaluation_examples/examples/libreoffice_calc/37608790-6147-45d0-9f20-1137bb35703d.json index e25e698..a16483e 100644 --- a/evaluation_examples/examples/libreoffice_calc/37608790-6147-45d0-9f20-1137bb35703d.json +++ b/evaluation_examples/examples/libreoffice_calc/37608790-6147-45d0-9f20-1137bb35703d.json @@ -3,32 +3,40 @@ "snapshot": "libreoffice_calc", "instruction": "Help me fill the columns of First Name, Last Name and Rank", "source": "https://www.youtube.com/shorts/uzPo_CPCHH8", - "config": { - "download": [ - [ - "https://drive.usercontent.google.com/download?id=1wDqap5cBfxnlqTNrZG61k_wDWTujl6AU&export=download&authuser=0&confirm=t&uuid=fd183b89-76b7-4dc5-880e-1045ed769562&at=APZUnTWp9RMafMg0xohhBWazN3YD:1701785710674", - "C:\\Users\\tianbaox\\Desktop\\Employee_Roles_and_Ranks.xlsx" - ] - ], - "open": [ - "C:\\Users\\tianbaox\\Desktop\\Employee_Roles_and_Ranks.xlsx" - ] - }, + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "", + "path": "Desktop/Employee_Roles_and_Ranks.xlsx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "Desktop/Employee_Roles_and_Ranks.xlsx" + } + } + ], "trajectory": "trajectories/37608790-6147-45d0-9f20-1137bb35703d", "related_apps": [ "libreoffice calc" ], "evaluator": { - "func": "compare_table(expected, actual)", - "paths": { - "expected": { - "type": "cloud_file", - "path": "https://drive.usercontent.google.com/download?id=1dxpiUqP_CVvQp5tddxlwO3Cp1BqJ-ZDE&export=download&authuser=0&confirm=t&uuid=ccd204c7-07ce-4fdf-a5d4-a7e4f37b9ce6&at=APZUnTVBs7TgrVrDXpkiU8S7WbQo:1702360836747" - }, - "actual": { - "type": "vm_file", - "path": "C:\\Users\\tianbaox\\Desktop\\Employee_Roles_and_Ranks.xlsx" - } + "func": "compare_table", + "expected": { + "type": "cloud_file", + "path": "", + "dest": "Employee_Roles_and_Ranks_gold.xlsx" + }, + "result": { + "type": "vm_file", + "path": "Desktop/Employee_Roles_and_Ranks.xlsx", + "dest": "Employee_Roles_and_Ranks.xlsx" } } } diff --git a/requirements.txt b/requirements.txt index 0a27eff..28d6d88 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,4 +13,8 @@ pyautogui~=0.9.54 psutil~=5.9.6 tqdm~=4.65.0 pandas~=2.0.3 -flask~=3.0.0 \ No newline at end of file +flask~=3.0.0 +requests-toolbelt~=1.0.0 +lxml +cssselect +xmltodict