From a6b6022ecbe0ef9b081aa70f4bbe4d89ddf8224c Mon Sep 17 00:00:00 2001 From: David Chang Date: Tue, 26 Dec 2023 16:46:50 +0800 Subject: [PATCH] ver Dec26th evaluation metric checking result file according to rules --- desktop_env/evaluators/getters/__init__.py | 1 + desktop_env/evaluators/getters/misc.py | 8 ++ desktop_env/evaluators/metrics/__init__.py | 1 + desktop_env/evaluators/metrics/table.py | 86 +++++++++++++++++-- .../4188d3a4-077d-46b7-9c86-23e1a036f6c1.json | 43 ++++++++++ requirements.txt | 1 + 6 files changed, 134 insertions(+), 6 deletions(-) create mode 100644 desktop_env/evaluators/getters/misc.py create mode 100644 evaluation_examples/examples/libreoffice_calc/4188d3a4-077d-46b7-9c86-23e1a036f6c1.json diff --git a/desktop_env/evaluators/getters/__init__.py b/desktop_env/evaluators/getters/__init__.py index 7d8c3fd..81a23fd 100644 --- a/desktop_env/evaluators/getters/__init__.py +++ b/desktop_env/evaluators/getters/__init__.py @@ -1 +1,2 @@ from .file import get_cloud_file, get_vm_file +from .misc import get_rule diff --git a/desktop_env/evaluators/getters/misc.py b/desktop_env/evaluators/getters/misc.py new file mode 100644 index 0000000..20327b0 --- /dev/null +++ b/desktop_env/evaluators/getters/misc.py @@ -0,0 +1,8 @@ +from typing import TypeVar + +R = TypeVar("Rule") +def get_rule(env, config: R) -> R: + """ + Returns the rule as-is. + """ + return config diff --git a/desktop_env/evaluators/metrics/__init__.py b/desktop_env/evaluators/metrics/__init__.py index f8dfefd..623f948 100644 --- a/desktop_env/evaluators/metrics/__init__.py +++ b/desktop_env/evaluators/metrics/__init__.py @@ -1 +1,2 @@ from .table import compare_table, compare_with_sparklines +from .table import check_sheet_list, check_xlsx_freeze diff --git a/desktop_env/evaluators/metrics/table.py b/desktop_env/evaluators/metrics/table.py index d984c0f..8dfa40b 100644 --- a/desktop_env/evaluators/metrics/table.py +++ b/desktop_env/evaluators/metrics/table.py @@ -5,9 +5,12 @@ import lxml.cssselect from lxml.etree import _Element import xmltodict #import pylightxl +import openpyxl +#from openpyxl import Workbook +from openpyxl.worksheet.worksheet import Worksheet from typing import Dict, List -#from typing import Any +from typing import Any def compare_table(actual, expected): df1 = pd.read_excel(expected) @@ -42,7 +45,7 @@ def _load_sparklines(xlsx_file: str) -> Dict[str, str]: with z_f.open("xl/worksheets/sheet1.xml") as f: sheet1: _Element = lxml.etree.fromstring(f.read()) sparklines: List[_Element] = _sparklines_selector(sheet1) - + sparklines_dict: Dict[str, str] = {} for sp_l in sparklines: sparkline_xml: str = lxml.etree.tostring(sp_l, encoding="unicode") @@ -57,18 +60,89 @@ def compare_with_sparklines(actual: str, expected: str) -> float: df1 = pd.read_excel(actual) df2 = pd.read_excel(expected) normal_content_metric: bool = df1.equals(df2) + print("Normal Contents Metric: {:}".format(normal_content_metric)) sp1 = _load_sparklines(actual) sp2 = _load_sparklines(expected) sparkline_metric: bool = sp1 == sp2 + print("Sparkline Metric: {:}".format(sparkline_metric)) return float(normal_content_metric and sparkline_metric) +def check_sheet_list(result: str, rules: List[Dict[str, Any]]) -> float: + #workbook: Workbook = openpyxl.load_workbook(filename=result) + workbook = pd.ExcelFile(result) + worksheet_names: List[str] = workbook.sheet_names + + passes = True + for r in rules: + if r["type"]=="sheet_name": + expected_name: str = worksheet_names[r["sheet_idx"]] + actual_name: str = r["sheet_name"] + metric: bool = expected_name==actual_name + print("Assertion: {:d}.{:} is {:} - {:}".format(r["sheet_idx"], actual_name, expected_name, metric)) + passes = passes and metric + elif r["type"]=="sheet_data": + if isinstance(r["sheet_idx0"], int): + df1: pd.DataFrame = pd.read_excel(workbook, r["sheet_idx0"]) + else: + file_name: str + sheet_idx: str + file_name, sheet_idx = r["sheet_idx0"].rsplit("@", maxsplit=1) + sheet_idx: int = int(sheet_idx) + df1: pd.DataFrame = pd.read_excel(file_name, sheet_idx) + if isinstance(r["sheet_idx1"], int): + df2: pd.DataFrame = pd.read_excel(workbook, r["sheet_idx1"]) + else: + file_name: str + sheet_idx: str + file_name, sheet_idx = r["sheet_idx1"].rsplit("@", maxsplit=1) + sheet_idx: int = int(sheet_idx) + df2: pd.DataFrame = pd.read_excel(file_name, sheet_idx) + metric: bool = df1.equals(df2) + print("Assertion: {:} == {:} - {:}".format(r["sheet_idx0"], r["sheet_idx1"], metric)) + passes = passes and metric + else: + raise NotImplementedError("Unimplemented sheet check: {:}".format(r["type"])) + + return float(passes) + +def check_xlsx_freeze(result: str, rules: Dict[str, str]) -> float: + worksheet: Worksheet = openpyxl.load_workbook(filename=result).active + return float(worksheet.freeze_panes==rules["position"]) + if __name__ == '__main__': #path1 = "" #path2 = "" #print(compare_table(path1, path2)) - - path1 = "../../../../../任务数据/LibreOffice Calc/OrderId_Month_Chart_gold.xlsx" - path2 = "../../../../../任务数据/LibreOffice Calc/OrderId_Month_Chart.xlsx" - print(compare_with_sparklines(path1, path2)) + + #path1 = "../../../../../任务数据/LibreOffice Calc/OrderId_Month_Chart_gold.xlsx" + #path2 = "../../../../../任务数据/LibreOffice Calc/OrderId_Month_Chart.xlsx" + #print(compare_with_sparklines(path1, path2)) + + #path1 = "../../../../../任务数据/LibreOffice Calc/Freeze_row_column_gold.xlsx" + #path2 = "../../../../../任务数据/LibreOffice Calc/Freeze_row_column.xlsx" + #workbook1: Workbook = openpyxl.load_workbook(filename=path1) + #worksheet1: Worksheet = workbook1.active + #print(worksheet1.freeze_panes) + #workbook2: Workbook = openpyxl.load_workbook(filename=path2) + #worksheet2: Worksheet = workbook2.active + #print(worksheet2.freeze_panes) + #rule = {"position": "C6"} + #print(check_xlsx_freeze(path1, rule)) + + path1 = "../../../../../任务数据/LibreOffice Calc/copy_sheet_insert_gold.xlsx" + rule = [ { "type": "sheet_name" + , "sheet_idx": 0 + , "sheet_name": "Sheet1" + } + , { "type": "sheet_data" + , "sheet_idx0": "../../../../../任务数据/LibreOffice Calc/copy_sheet_insert.xlsx@0" + , "sheet_idx1": 1 + } + , { "type": "sheet_name" + , "sheet_idx": 2 + , "sheet_name": "Sheet2" + } + ] + print(check_sheet_list(path1, rule)) diff --git a/evaluation_examples/examples/libreoffice_calc/4188d3a4-077d-46b7-9c86-23e1a036f6c1.json b/evaluation_examples/examples/libreoffice_calc/4188d3a4-077d-46b7-9c86-23e1a036f6c1.json new file mode 100644 index 0000000..673f9ce --- /dev/null +++ b/evaluation_examples/examples/libreoffice_calc/4188d3a4-077d-46b7-9c86-23e1a036f6c1.json @@ -0,0 +1,43 @@ +{ + "id": "4188d3a4-077d-46b7-9c86-23e1a036f6c1", + "snapshot": "libreoffice_calc", + "instruction": "Help me freeze the range A1:B5 on this sheet", + "source": "https://www.libreofficehelp.com/freeze-unfreeze-rows-columns-ranges-calc/", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://101.43.24.67/s/H7k3zLNaNcMWyLB/download/Freeze_row_column.xlsx", + "path": "/home/david/Freeze_row_column.xlsx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/david/Freeze_row_column.xlsx" + } + } + ], + "trajectory": "trajectories/4188d3a4-077d-46b7-9c86-23e1a036f6c1", + "related_apps": [ + "libreoffice_calc" + ], + "evaluators": { + "func": "check_xlsx_freeze", + "result": { + "type": "vm_file", + "path": "/home/david/Freeze_row_column.xlsx", + "dest": "Freeze_row_column.xlsx" + }, + "expected": { + "type": "rule", + "rules": { + "position": "C6" + } + } + } +} diff --git a/requirements.txt b/requirements.txt index 28d6d88..c5a7b86 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,3 +18,4 @@ requests-toolbelt~=1.0.0 lxml cssselect xmltodict +openpyxl