From 19214f2107610158dd70b313d2c3b7a9c300050e Mon Sep 17 00:00:00 2001 From: David Chang Date: Wed, 17 Jan 2024 22:43:26 +0800 Subject: [PATCH] ver Jan17thv2 updated compare_table with compare the shown value through exported csv --- desktop_env/evaluators/getters/file.py | 92 +++++++++++----- desktop_env/evaluators/metrics/table.py | 44 ++++++-- .../0bf05a7d-b28b-44d2-955a-50b41e24012a.json | 103 ++++++++++++++++-- .../21df9241-f8d7-4509-b7f1-37e501a823f7.json | 46 ++++++-- .../a01fbce3-2793-461f-ab86-43680ccbae25.json | 64 ++++++++--- ...3-2793-461f-ab86-43680ccbae25.json.nosetup | 85 +++++++++++++++ main.py | 4 +- 7 files changed, 371 insertions(+), 67 deletions(-) create mode 100644 evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json.nosetup diff --git a/desktop_env/evaluators/getters/file.py b/desktop_env/evaluators/getters/file.py index 6714b0e..95c0a18 100644 --- a/desktop_env/evaluators/getters/file.py +++ b/desktop_env/evaluators/getters/file.py @@ -1,50 +1,92 @@ import os -from typing import Dict -from typing import Optional +from typing import Dict, List, Set +from typing import Optional, Any, Union import requests -def get_cloud_file(env, config: Dict[str, str]) -> str: +def get_cloud_file(env, config: Dict[str, Any]) -> Union[str, List[str]]: """ Config: - path (str): the url to download from - dest (str): file name of the downloaded file + path (str|List[str]): the url to download from + dest (str|List[str])): file name of the downloaded file + multi (bool) : optional. if path and dest are lists providing + information of multiple files. defaults to False + gives (List[int]): optional. defaults to [0]. which files are directly + returned to the metric. if len==1, str is returned; else, list is + returned. """ - _path = os.path.join(env.cache_dir, config["dest"]) - if os.path.exists(_path): - return _path + if not config.get("multi", False): + paths: List[str] = [config["path"]] + dests: List[str] = [config["dest"]] + else: + paths: List[str] = config["path"] + dests: List[str] = config["dest"] + cache_paths: List[str] = [] - url = config["path"] - response = requests.get(url, stream=True) - response.raise_for_status() + gives: Set[int] = set(config.get("gives", [0])) - with open(_path, 'wb') as f: - for chunk in response.iter_content(chunk_size=8192): - if chunk: - f.write(chunk) + for i, (p, d) in enumerate(zip(paths, dests)): + _path = os.path.join(env.cache_dir, d) + if i in gives: + cache_paths.append(_path) - return _path + if os.path.exists(_path): + #return _path + continue + + url = p + response = requests.get(url, stream=True) + response.raise_for_status() + + with open(_path, 'wb') as f: + for chunk in response.iter_content(chunk_size=8192): + if chunk: + f.write(chunk) + + return cache_paths[0] if len(cache_paths)==1 else cache_paths -def get_vm_file(env, config: Dict[str, str]) -> Optional[str]: +def get_vm_file(env, config: Dict[str, Any]) -> Union[Optional[str], List[Optional[str]]]: """ Config: path (str): absolute path on the VM to fetch dest (str): file name of the downloaded file + multi (bool) : optional. if path and dest are lists providing + information of multiple files. defaults to False + gives (List[int]): optional. defaults to [0]. which files are directly + returned to the metric. if len==1, str is returned; else, list is + returned. """ - _path = os.path.join(env.cache_dir, config["dest"]) + if not config.get("multi", False): + paths: List[str] = [config["path"]] + dests: List[str] = [config["dest"]] + else: + paths: List[str] = config["path"] + dests: List[str] = config["dest"] + cache_paths: List[str] = [] - file = env.controller.get_file(config["path"]) - if file is None: - return None - # raise FileNotFoundError("File not found on VM: {:}".format(config["path"])) - with open(_path, "wb") as f: - f.write(file) + gives: Set[int] = set(config.get("gives", [0])) - return _path + for i, (p, d) in enumerate(zip(paths, dests)): + _path = os.path.join(env.cache_dir, d) + + file = env.controller.get_file(p) + if file is None: + #return None + # raise FileNotFoundError("File not found on VM: {:}".format(config["path"])) + if i in gives: + cache_paths.append(None) + continue + + if i in gives: + cache_paths.append(_path) + with open(_path, "wb") as f: + f.write(file) + + return cache_paths[0] if len(cache_paths)==1 else cache_paths def get_cache_file(env, config: Dict[str, str]) -> str: diff --git a/desktop_env/evaluators/metrics/table.py b/desktop_env/evaluators/metrics/table.py index 03daef5..e09b9f9 100644 --- a/desktop_env/evaluators/metrics/table.py +++ b/desktop_env/evaluators/metrics/table.py @@ -3,6 +3,8 @@ import operator from numbers import Number from typing import Any, Union from typing import Dict, List +import os.path +import itertools import openpyxl import pandas as pd @@ -26,6 +28,7 @@ def compare_table(actual: str, expected: str, **options) -> float: * chart * number_format "chart_props": list of str, giving the converned chart properties + "as_shown": bool, TODO } Return: @@ -35,10 +38,35 @@ def compare_table(actual: str, expected: str, **options) -> float: if actual is None: return 0. - df1 = pd.read_excel(expected) - df2 = pd.read_excel(actual) - metric: bool = df1.equals(df2) - logger.debug("Normal Contents Metric: {:}".format(metric)) + if options.get("as_shown", False): + expected_csv: str = os.path.splitext(expected)[0] + ".csv" + actual_csv: str = os.path.splitext(actual)[0] + ".csv" + + with open(expected_csv) as f: + expected_lines: List[str] = list( itertools.dropwhile( lambda l: len(l)==0 + , map( lambda l: l.strip() + , reversed(f.read().splitlines()) + ) + ) + ) + if options.get("ignore_case", False): + expected_lines = [l.lower() for l in expected_lines] + with open(actual_csv) as f: + actual_lines: List[str] = list( itertools.dropwhile( lambda l: len(l)==0 + , map( lambda l: l.strip() + , reversed(f.read().splitlines()) + ) + ) + ) + if options.get("ignore_case", False): + actual_lines = [l.lower() for l in actual_lines] + metric: bool = expected_lines==actual_lines + logger.debug("Content Metric just as shown: %s", metric) + else: + df1 = pd.read_excel(expected) + df2 = pd.read_excel(actual) + metric: bool = df1.equals(df2) + logger.debug("Normal Content Metric: {:}".format(metric)) features: List[str] = options.get("features", []) for ftr in features: @@ -219,6 +247,8 @@ if __name__ == '__main__': # print(check_zoom(path1, {"relation": "lt", "ref_value": 100})) # print(check_zoom(path2, {"relation": "lt", "ref_value": 100})) - path1 = "../../任务数据/LibreOffice Calc/Padding_Decimals_In_Formular_gold.xlsx" - data_frame: pd.DataFrame = pd.read_excel(path1) - print(data_frame) + path1 = "../../任务数据/LibreOffice Calc/Customers_New_7digit_Id.xlsx" + path2 = "../../任务数据/LibreOffice Calc/Customers_New_7digit_Id_gold.xlsx" + #data_frame: pd.DataFrame = pd.read_excel(path1) + #print(data_frame) + print(compare_table(path1, path2, as_shown=True)) diff --git a/evaluation_examples/examples/libreoffice_calc/0bf05a7d-b28b-44d2-955a-50b41e24012a.json b/evaluation_examples/examples/libreoffice_calc/0bf05a7d-b28b-44d2-955a-50b41e24012a.json index 6f92ac5..524b2d1 100644 --- a/evaluation_examples/examples/libreoffice_calc/0bf05a7d-b28b-44d2-955a-50b41e24012a.json +++ b/evaluation_examples/examples/libreoffice_calc/0bf05a7d-b28b-44d2-955a-50b41e24012a.json @@ -3,20 +3,101 @@ "snapshot": "libreoffice_calc", "instruction": "I would like to pad all the numbers in the 'Old ID' column with zeros in front, to fill them up to seven digits in the 'New 7 Digit ID' column.", "source": "https://www.youtube.com/shorts/FPAQaDTS8VY", - "config": { - "download": [ - [ - "", - "C:\\Users\\tianbaox\\Desktop\\Customers_New_7digit_Id.xlsx" - ] - ], - "open": [ - "C:\\Users\\tianbaox\\Desktop\\Customers_New_7digit_Id.xlsx" - ] + "config": [ + { + "type": "download", + "parameters": { + "file": [ + { + "url": "https://drive.usercontent.google.com/download?id=1DqGy5JRKOuZMRJ8O76d4Cds4WaRyz8V1&export=download&authuser=0&confirm=t&uuid=fa0694d1-2a77-4fd2-89d3-d9b854317823&at=APZUnTU9BxqG7E8tLZ104c0E8BEL:1705501029016", + "path": "/home/user/Customers_New_7digit_Id.xlsx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/Customers_New_7digit_Id.xlsx" + } + } }, "trajectory": "trajectories/0bf05a7d-b28b-44d2-955a-50b41e24012a", "related_apps": [ "libreoffice calc" ], - "evaluator": "evaluation_dir" + "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "Customers_New_7digit_Id.xlsx - LibreOffice Calc", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "libreoffice", + "--convert-to", + "csv:Text - txt - csv (StarCalc):44,34,UTF-8,,,,false,true,true", + "--outdir", + "/home/user", + "/home/user/Customers_New_7digit_Id.xlsx" + ] + } + } + ], + "func": "compare_table", + "result": { + "type": "vm_file", + "path": [ + "/home/user/Customers_New_7digit_Id.xlsx", + "/home/user/Customers_New_7digit_Id.csv" + ], + "dest": [ + "Customers_New_7digit_Id.xlsx", + "Customers_New_7digit_Id.csv" + ], + "multi": true + }, + "expected": { + "type": "cloud_file", + "path": [ + "https://drive.usercontent.google.com/download?id=1zXz5k5A403IR0GE6DRRXRgQZrSIoVFSz&export=download&authuser=0&confirm=t&uuid=ba70b841-969c-4d91-9288-0011aeecf251&at=APZUnTWx3LL9udCgJAh-VMFzzfod:1705501007861", + "https://drive.usercontent.google.com/download?id=1h1GnUpyj92K7FXiHJ1xVaUYW_UYMDLPM&export=download&authuser=0&confirm=t&uuid=98de75bd-ba32-4ceb-97a7-b8d303a8dc96&at=APZUnTUewdfFIsyC3UlSlMcmmRbo:1705500978790" + ], + "dest": [ + "Customers_New_7digit_Id_gold.xlsx", + "Customers_New_7digit_Id_gold.csv" + ], + "multi": true + }, + "options": { + "as_shown": true + } + } } diff --git a/evaluation_examples/examples/libreoffice_calc/21df9241-f8d7-4509-b7f1-37e501a823f7.json b/evaluation_examples/examples/libreoffice_calc/21df9241-f8d7-4509-b7f1-37e501a823f7.json index 302eef0..516dc91 100644 --- a/evaluation_examples/examples/libreoffice_calc/21df9241-f8d7-4509-b7f1-37e501a823f7.json +++ b/evaluation_examples/examples/libreoffice_calc/21df9241-f8d7-4509-b7f1-37e501a823f7.json @@ -50,23 +50,55 @@ "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" ] } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "libreoffice", + "--convert-to", + "csv:Text - txt - csv (StarCalc):44,34,UTF-8,,,,false,true,true", + "--outdir", + "/home/user", + "/home/user/Represent_in_millions_billions.xlsx" + ] + } } ], "func": "compare_table", "result": { "type": "vm_file", - "path": "/home/user/Represent_in_millions_billions.xlsx", - "dest": "Represent_in_millions_billions.xlsx" + "path": [ + "/home/user/Represent_in_millions_billions.xlsx", + "/home/user/Represent_in_millions_billions.csv" + ], + "dest": [ + "Represent_in_millions_billions.xlsx", + "Represent_in_millions_billions.csv" + ], + "multi": true }, "expected": { "type": "cloud_file", - "path": "https://drive.usercontent.google.com/download?id=1Jy6lZexhU5t0eW1GwXJ_csnwIe0Xiy9-&export=download&authuser=0&confirm=t&uuid=601701e7-9eb8-4ce8-83d5-35916094a15d&at=APZUnTW4WE-plIC5MmWTuFu24qLL:1703857882995", - "dest": "Represent_in_millions_billions_gold.xlsx" + "path": [ + "https://drive.usercontent.google.com/download?id=1Jy6lZexhU5t0eW1GwXJ_csnwIe0Xiy9-&export=download&authuser=0&confirm=t&uuid=601701e7-9eb8-4ce8-83d5-35916094a15d&at=APZUnTW4WE-plIC5MmWTuFu24qLL:1703857882995", + "https://drive.usercontent.google.com/download?id=1CjzmJ4vqwPI7DMTR_6a8ytO-W-4xavBS&export=download&authuser=0&confirm=t&uuid=32c3430a-6d3d-47cc-bb5c-e683963b19f7&at=APZUnTWSBLNtkrZTix_BRl9gyaek:1705497493679" + ], + "dest": [ + "Represent_in_millions_billions_gold.xlsx", + "Represent_in_millions_billions_gold.csv" + ], + "multi": true }, "options": { - "features": [ - "number_format" - ] + "as_shown": true, + "ignore_case": true } } } diff --git a/evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json b/evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json index 113e491..50dcc83 100644 --- a/evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json +++ b/evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json @@ -1,7 +1,7 @@ { "id": "a01fbce3-2793-461f-ab86-43680ccbae25", "snapshot": "libreoffice_calc", - "instruction": "Could you help me setting decimal separator as a dot (.)?", + "instruction": "I need to set the decimal separator as a comma (,) for localized data representation and clarity in visualization. Can you assist with this?", "source": "https://superuser.com/questions/1250677/how-to-set-decimal-separator-in-libre-office-calc", "config": [ { @@ -14,6 +14,12 @@ } ] } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/Set_Decimal_Separator_Dot.xlsx" + } } ], "trajectory": "trajectories/a01fbce3-2793-461f-ab86-43680ccbae25", @@ -44,26 +50,54 @@ "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" ] } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "libreoffice", + "--convert-to", + "csv:Text - txt - csv (StarCalc):44,34,UTF-8,,,,false,true,true", + "--outdir", + "/home/user", + "/home/user/Set_Decimal_Separator_Dot.xlsx" + ] + } } ], - "func": "check_libre_locale", + "func": "compare_table", "result": { "type": "vm_file", - "path": "/home/user/.config/libreoffice/4/user/registrymodifications.xcu", - "dest": "registrymodifications.xcu" + "path": [ + "/home/user/Set_Decimal_Separator_Dot.xlsx", + "/home/user/Set_Decimal_Separator_Dot.csv" + ], + "dest": [ + "Set_Decimal_Separator_Dot.xlsx", + "Set_Decimal_Separator_Dot.csv" + ], + "multi": true }, "expected": { - "type": "rule", - "rules": { - "locale_set": [ - "ru-*", - "de-*", - "fr-*", - "pt-*", - "es-*", - "it-*" - ] - } + "type": "cloud_file", + "path": [ + "https://drive.usercontent.google.com/download?id=15O0l5fxVi1JX_12KOLfbxWPHjXPZPon5&export=download&authuser=0&confirm=t&uuid=395e6c57-11a7-4b33-af4c-98ff2390742b&at=APZUnTVKcrUGrjRfBEwT_AD53Cmn:1705497822975", + "https://drive.usercontent.google.com/download?id=1rKDWcovxw4Qtd3RHs7M5p_QqryI0SQO3&export=download&authuser=0&confirm=t&uuid=eb6ffb6d-f7c2-44d8-ad77-db6c0aaf5cc7&at=APZUnTWr2VxrJPiiKVMdFd0IykrR:1705497846507" + ], + "dest": [ + "Set_Decimal_Separator_Dot_gold.xlsx", + "Set_Decimal_Separator_Dot_gold.csv" + ], + "multi": true + }, + "options": { + "as_shown": true } } } diff --git a/evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json.nosetup b/evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json.nosetup new file mode 100644 index 0000000..9844796 --- /dev/null +++ b/evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json.nosetup @@ -0,0 +1,85 @@ +{ + "id": "a01fbce3-2793-461f-ab86-43680ccbae25", + "snapshot": "libreoffice_calc", + "instruction": "I need to set the decimal separator as a comma (,) for localized data representation and clarity in visualization. Can you assist with this?", + "source": "https://superuser.com/questions/1250677/how-to-set-decimal-separator-in-libre-office-calc", + "config": [], + "trajectory": "trajectories/a01fbce3-2793-461f-ab86-43680ccbae25", + "related_apps": [ + "libreoffice_calc" + ], + "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "Set_Decimal_Separator_Dot.xlsx - LibreOffice Calc", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "libreoffice", + "--convert-to", + "csv:Text - txt - csv (StarCalc):44,34,UTF-8,,,,false,true,true", + "--outdir", + "/home/user", + "/home/user/Set_Decimal_Separator_Dot.xlsx" + ] + } + } + ], + "func": "compare_table", + "result": { + "type": "vm_file", + "path": [ + "/home/user/Set_Decimal_Separator_Dot.xlsx", + "/home/user/Set_Decimal_Separator_Dot.csv" + ], + "dest": [ + "Set_Decimal_Separator_Dot.xlsx", + "Set_Decimal_Separator_Dot.csv" + ], + "multi": true + }, + "expected": { + "type": "cloud_file", + "path": [ + "https://drive.usercontent.google.com/download?id=15O0l5fxVi1JX_12KOLfbxWPHjXPZPon5&export=download&authuser=0&confirm=t&uuid=395e6c57-11a7-4b33-af4c-98ff2390742b&at=APZUnTVKcrUGrjRfBEwT_AD53Cmn:1705497822975", + "https://drive.usercontent.google.com/download?id=1rKDWcovxw4Qtd3RHs7M5p_QqryI0SQO3&export=download&authuser=0&confirm=t&uuid=eb6ffb6d-f7c2-44d8-ad77-db6c0aaf5cc7&at=APZUnTWr2VxrJPiiKVMdFd0IykrR:1705497846507" + ], + "dest": [ + "Set_Decimal_Separator_Dot_gold.xlsx", + "Set_Decimal_Separator_Dot_gold.csv" + ], + "multi": true + }, + "options": { + "as_shown": true + } + } +} diff --git a/main.py b/main.py index 17155de..2027736 100644 --- a/main.py +++ b/main.py @@ -46,9 +46,9 @@ def human_agent(): Runs the Gym environment with human input. """ - with open("evaluation_examples/examples/thunderbird/d088f539-cab4-4f9a-ac92-9999fc3a656e.json.nosetup", "r") as f: + with open("evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json.nosetup", "r") as f: example = json.load(f) - example["snapshot"] = "Snapshot 24" + example["snapshot"] = "Snapshot 26" env = DesktopEnv( path_to_vm="../../../../大文件/镜像/Ubuntu-1218/Ubuntu/Ubuntu.vmx" , action_space="computer_13"