diff --git a/desktop_env/evaluators/metrics/__init__.py b/desktop_env/evaluators/metrics/__init__.py index b137c87..69d65c5 100644 --- a/desktop_env/evaluators/metrics/__init__.py +++ b/desktop_env/evaluators/metrics/__init__.py @@ -1,3 +1,3 @@ -from .table import compare_table, compare_with_sparklines, compare_with_charts -from .table import check_sheet_list, check_xlsx_freeze -from .docs import find_default_font, contains_page_break, compare_docx_files \ No newline at end of file +from .table import compare_table +from .table import check_sheet_list, check_xlsx_freeze, check_zoom +from .docs import find_default_font, contains_page_break, compare_docx_files diff --git a/desktop_env/evaluators/metrics/table.py b/desktop_env/evaluators/metrics/table.py index 4591b82..f1a3eda 100644 --- a/desktop_env/evaluators/metrics/table.py +++ b/desktop_env/evaluators/metrics/table.py @@ -1,56 +1,72 @@ import pandas as pd import openpyxl +from openpyxl import Workbook from openpyxl.worksheet.worksheet import Worksheet from .utils import load_charts, load_sparklines +import operator from typing import Dict, List -from typing import Any +from typing import Any, Union +from numbers import Number -def compare_table(actual, expected): - df1 = pd.read_excel(expected) - df2 = pd.read_excel(actual) - - # Compare the DataFrames - return 1 if df1.equals(df2) else 0 - - -def compare_with_sparklines(actual: str, expected: str) -> float: - df1 = pd.read_excel(actual) - df2 = pd.read_excel(expected) - normal_content_metric: bool = df1.equals(df2) - print("Normal Contents Metric: {:}".format(normal_content_metric)) - - sp1 = load_sparklines(actual) - sp2 = load_sparklines(expected) - sparkline_metric: bool = sp1 == sp2 - print("Sparkline Metric: {:}".format(sparkline_metric)) - - return float(normal_content_metric and sparkline_metric) - - -def compare_with_charts(actual: str, expected: str, **options) -> float: +def compare_table(actual: str, expected: str, **options) -> float: """ Args: actual (str): path to result xlsx expected (str): path to gold xlsx - options (Dict[str, List[str]]): dict like {"chart_props": list of str} - giving the concerned chart properties + options (Dict[str, List[str]]): dict like + { + "features": list of str for other features, supports: + * sparkline + * chart + * number_format + "chart_props": list of str, giving the converned chart properties + } + + Return: + float: the score """ - df1 = pd.read_excel(actual) - df2 = pd.read_excel(expected) - normal_content_metric: bool = df1.equals(df2) - print("Normal Contents Metric: {:}".format(normal_content_metric)) + df1 = pd.read_excel(expected) + df2 = pd.read_excel(actual) + metric: bool = df1.equals(df2) + print("Normal Contents Metric: {:}".format(metric)) - charts1 = load_charts(actual, **options) - charts2 = load_charts(expected, **options) - chart_metric: bool = charts1 == charts2 - print("Chart Metric: {:}".format(chart_metric)) + features: List[str] = options.get("features", []) + for ftr in features: + workbook1: Workbook = openpyxl.load_workbook(actual) + workbook2: Workbook = openpyxl.load_workbook(expected) - return float(normal_content_metric and chart_metric) + if ftr=="sparkline": + sp1 = load_sparklines(actual) + sp2 = load_sparklines(expected) + new_metric: bool = sp1 == sp2 + print("Sparkline Metric: {:}".format(new_metric)) + elif ftr=="chart": + charts1 = load_charts(workbook1, **options) + charts2 = load_charts(workbook2, **options) + new_metric: bool = charts1 == charts2 + print("Chart Metric: {:}".format(new_metric)) + elif ftr=="number_format": + number_formats1: List[str] = [ c.number_format.lower()\ + for col in workbook1.active.iter_cols()\ + for c in col\ + if c.data_type=="n" + ] + number_formats2: List[str] = [ c.number_format.lower()\ + for col in workbook2.active.iter_cols()\ + for c in col\ + if c.data_type=="n" + ] + new_metric: bool = number_formats1==number_formats2 + print("Number Format Metric: {:}".format(new_metric)) + else: + raise NotImplementedError("Unsupported xlsx feature: {:}".format(ftr)) + metric = metric and new_metric + return float(metric) def check_sheet_list(result: str, rules: List[Dict[str, Any]]) -> float: # workbook: Workbook = openpyxl.load_workbook(filename=result) @@ -90,11 +106,17 @@ def check_sheet_list(result: str, rules: List[Dict[str, Any]]) -> float: return float(passes) - def check_xlsx_freeze(result: str, rules: Dict[str, str]) -> float: worksheet: Worksheet = openpyxl.load_workbook(filename=result).active return float(worksheet.freeze_panes == rules["position"]) +def check_zoom(result: str, rules: Dict[str, Union[str, Number]]) -> float: + worksheet = openpyxl.load_workbook(filename=result).active + zoom_scale: Number = worksheet.sheet_view.zoomScale or 100. + return float( getattr(operator, rules["relation"])( zoom_scale + , rules["ref_value"] + ) + ) if __name__ == '__main__': # path1 = "" @@ -132,6 +154,38 @@ if __name__ == '__main__': # ] # print(check_sheet_list(path1, rule)) - path1 = "../../../../../任务数据/LibreOffice Calc/Create_column_charts_using_statistics_gold.xlsx" - path2 = "../../../../../任务数据/LibreOffice Calc/Create_column_charts_using_statistics_gold2.xlsx" - print(compare_with_charts(path1, path2, chart_props=["type", "direction"])) + #path1 = "../../任务数据/LibreOffice Calc/Create_column_charts_using_statistics_gold.xlsx" + #path2 = "../../任务数据/LibreOffice Calc/Create_column_charts_using_statistics_gold2.xlsx" + #print(compare_table(path1, path2, features=["chart"], chart_props=["type", "direction"])) + + #path1 = "../../任务数据/LibreOffice Calc/Represent_in_millions_billions_gold.xlsx" + #path2 = "../../任务数据/LibreOffice Calc/Represent_in_millions_billions_gold3.xlsx" + #workbook1: Workbook = openpyxl.load_workbook(filename=path1) + #worksheet1: Worksheet = workbook1.active +# + #import itertools + #for col, r in itertools.product( ['A', 'B', 'C'] + #, range(1, 9) + #): + #position: str = "{:}{:d}".format(col, r) + #print(worksheet1[position]) + #print(worksheet1[position].value) + #print(worksheet1[position].number_format) + #print(compare_table(path1, path2, features=["number_format"])) + + path1 = "../../任务数据/LibreOffice Calc/Zoom_Out_Oversized_Cells_gold.xlsx" + path2 = "../../任务数据/LibreOffice Calc/Zoom_Out_Oversized_Cells.xlsx" + #workbook1: Workbook = openpyxl.load_workbook(filename=path1) + #worksheet1: Worksheet = workbook1.active + #print(worksheet1.sheet_view.zoomScale) + #print(type(worksheet1.sheet_view.zoomScale)) +# + #import os + #import os.path + #for wb in filter( lambda f: f.endswith(".xlsx") + #, os.listdir("../../任务数据/LibreOffice Calc/") + #): + #path = os.path.join("../../任务数据/LibreOffice Calc/", wb) + #print(wb, openpyxl.load_workbook(filename=path).active.sheet_view.zoomScale) + print(check_zoom(path1, {"relation": "lt", "ref_value": 100})) + print(check_zoom(path2, {"relation": "lt", "ref_value": 100})) diff --git a/desktop_env/evaluators/metrics/utils.py b/desktop_env/evaluators/metrics/utils.py index 65db158..b826d87 100644 --- a/desktop_env/evaluators/metrics/utils.py +++ b/desktop_env/evaluators/metrics/utils.py @@ -56,10 +56,10 @@ def load_sparklines(xlsx_file: str) -> Dict[str, str]: # type: "scatterChart" | "lineChart" | "barChart" # direction: "bar" (hori) | "col" (vert) # xtitle, ytitle, ztitle: str -def load_charts(xlsx_file: str, **options) -> Dict[str, Any]: +def load_charts(xlsx_file: Workbook, **options) -> Dict[str, Any]: """ Args: - xlsx_file (str): path to xlsx + xlsx_file (Workbook): concerned excel book options (Dict[str, List[str]]): dict like {"chart_props": list of str} giving the concerned chart properties @@ -67,8 +67,8 @@ def load_charts(xlsx_file: str, **options) -> Dict[str, Any]: Dict[str, Any]: information of charts """ - workbook: Workbook = openpyxl.load_workbook(filename=xlsx_file) - worksheet: Worksheet = workbook.active + #workbook: Workbook = openpyxl.load_workbook(filename=xlsx_file) + worksheet: Worksheet = xlsx_file.active charts: List[ChartBase] = worksheet._charts chart_set: Dict[str, Any] = {} diff --git a/evaluation_examples/examples/libreoffice_calc/1334ca3e-f9e3-4db8-9ca7-b4c653be7d17.json b/evaluation_examples/examples/libreoffice_calc/1334ca3e-f9e3-4db8-9ca7-b4c653be7d17.json new file mode 100644 index 0000000..aa131f6 --- /dev/null +++ b/evaluation_examples/examples/libreoffice_calc/1334ca3e-f9e3-4db8-9ca7-b4c653be7d17.json @@ -0,0 +1,44 @@ +{ + "id": "1334ca3e-f9e3-4db8-9ca7-b4c653be7d17", + "snapshot": "libreoffice_calc", + "instruction": "The cells are so big that I can not click on the cell I want, zoom out a little bit.", + "source": "https://techcommunity.microsoft.com/t5/excel/excel-workbook-top-way-too-big-can-t-see-rows-and-columns/m-p/4014694", + "config": [ + { + "type": "download", + "parameters": { + "file": [ + { + "url": "https://drive.usercontent.google.com/download?id=1Wkepf_vic9o7CZFiosZ4jZT_Hy2WbRPZ&export=download&authuser=0&confirm=t&uuid=bc2ce901-a6bb-433f-bcce-cbe42d813f18&at=APZUnTVQcGTcXjwqenmtSH6IMFkM:1703858853235", + "path": "Desktop/Zoom_Out_Oversized_Cells.xlsx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "Desktop/Zoom_Out_Oversized_Cells.xlsx" + } + } + ], + "trajectory": "trajectories/1334ca3e-f9e3-4db8-9ca7-b4c653be7d17", + "related_apps": [ + "libreoffice_calc" + ], + "evaluator": { + "func": "check_zoom", + "result": { + "type": "vm_file", + "path": "Desktop/Zoom_Out_Oversized_Cells.xlsx", + "dest": "Zoom_Out_Oversized_Cells.xlsx" + }, + "expected": { + "type": "rule", + "rules": { + "relation": "lt", + "ref_value": 260 + } + } + } +} diff --git a/evaluation_examples/examples/libreoffice_calc/21df9241-f8d7-4509-b7f1-37e501a823f7.json b/evaluation_examples/examples/libreoffice_calc/21df9241-f8d7-4509-b7f1-37e501a823f7.json new file mode 100644 index 0000000..a54a774 --- /dev/null +++ b/evaluation_examples/examples/libreoffice_calc/21df9241-f8d7-4509-b7f1-37e501a823f7.json @@ -0,0 +1,47 @@ +{ + "id": "21df9241-f8d7-4509-b7f1-37e501a823f7", + "snapshot": "libreoffice_calc", + "instruction": "Change the representation of colum \"Parameter\" and show in Millions (M) and Billions (B). Keep one decimal and place a white space between the digits and the unit.", + "source": "https://www.youtube.com/watch?v=p5C4V_AO1UU", + "config": [ + { + "type": "download", + "parameters": { + "file": [ + { + "url": "https://drive.usercontent.google.com/download?id=16PowrQA4E71xUoJmpXPHy0dr9HBcTRmo&export=download&authuser=0&confirm=t&uuid=9a6265f7-585c-4cf8-b321-3b859aec1e68&at=APZUnTWzzOw85wws0ojXNPsIwnjE:1703858126178", + "path": "Desktop/Represent_in_millions_billions.xlsx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "Desktop/Represent_in_millions_billions.xlsx" + } + } + ], + "trajectory": "trajectories/21df9241-f8d7-4509-b7f1-37e501a823f7", + "related_apps": [ + "libreoffice_calc" + ], + "evaluator": { + "func": "compare_table", + "result": { + "type": "vm_file", + "path": "Desktop/Represent_in_millions_billions.xlsx", + "dest": "Represent_in_millions_billions.xlsx" + }, + "expected": { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=1Jy6lZexhU5t0eW1GwXJ_csnwIe0Xiy9-&export=download&authuser=0&confirm=t&uuid=601701e7-9eb8-4ce8-83d5-35916094a15d&at=APZUnTW4WE-plIC5MmWTuFu24qLL:1703857882995", + "dest": "Represent_in_millions_billions_gold.xlsx" + }, + "options": { + "features": [ + "number_format" + ] + } + } +} diff --git a/evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json b/evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json index d58c4b9..f0b143f 100644 --- a/evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json +++ b/evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json @@ -27,7 +27,7 @@ "libreoffice calc" ], "evaluator": { - "func": "compare_with_sparklines", + "func": "compare_table", "expected": { "type": "cloud_file", "path": "https://drive.usercontent.google.com/download?id=1KQJJLVPGtTL_7ArEWvwwbFbJSiA3cgSE&export=download&authuser=0&confirm=t&uuid=6b11c721-caad-439a-b369-4c13c7a485df&at=APZUnTV5-1isKrDKSHV9NeJ6TDeS:1703509054094", @@ -37,6 +37,10 @@ "type": "vm_file", "path": "Desktop/OrderId_Month_Chart.xlsx", "dest": "OrderId_Month_Chart.xlsx" - } + }, + "options": { + "features": [ + "sparkline" + ] } -} \ No newline at end of file +} diff --git a/evaluation_examples/examples/libreoffice_calc/347ef137-7eeb-4c80-a3bb-0951f26a8aff.json b/evaluation_examples/examples/libreoffice_calc/347ef137-7eeb-4c80-a3bb-0951f26a8aff.json index 38f853b..d359ca2 100644 --- a/evaluation_examples/examples/libreoffice_calc/347ef137-7eeb-4c80-a3bb-0951f26a8aff.json +++ b/evaluation_examples/examples/libreoffice_calc/347ef137-7eeb-4c80-a3bb-0951f26a8aff.json @@ -27,7 +27,7 @@ "libreoffice_calc" ], "evaluator": { - "func": "compare_with_charts", + "func": "compare_table", "result": { "type": "vm_file", "path": "Desktop/Create_column_charts_using_statistics.xlsx", @@ -39,6 +39,9 @@ "dest": "Create_column_charts_using_statistics_gold.xlsx" }, "options": { + "features": [ + "chart" + ], "chart_props": [ "type", "direction" diff --git a/evaluation_examples/examples/libreoffice_calc/f9584479-3d0d-4c79-affa-9ad7afdd8850.json b/evaluation_examples/examples/libreoffice_calc/f9584479-3d0d-4c79-affa-9ad7afdd8850.json index 6a2094e..a9cae16 100644 --- a/evaluation_examples/examples/libreoffice_calc/f9584479-3d0d-4c79-affa-9ad7afdd8850.json +++ b/evaluation_examples/examples/libreoffice_calc/f9584479-3d0d-4c79-affa-9ad7afdd8850.json @@ -21,13 +21,13 @@ "path": "Desktop/Quarterly_Product_Sales_by_Zone.xlsx" } } - ], - "trajectory": "trajectories/f9584479-3d0d-4c79-affa-9ad7afdd8850", - "related_apps": [ - "libreoffice calc" - ], - "evaluator": { - "func": "compare_table", + ], + "trajectory": "trajectories/f9584479-3d0d-4c79-affa-9ad7afdd8850", + "related_apps": [ + "libreoffice calc" + ], + "evaluator": { + "func": "compare_table", "expected": { "type": "cloud_file", "path": "https://drive.usercontent.google.com/download?id=17f1wZuJPvUEc5at_Fy3c18VFdOk0x7xz&export=download&authuser=0&confirm=t&uuid=6d2edffd-0ce0-426e-9820-8af25b4667f3&at=APZUnTVh7JS85dwZBaV2hytWQgDK:1702361510956", @@ -38,5 +38,5 @@ "path": "Desktop/Quarterly_Product_Sales_by_Zone.xlsx", "dest": "Quarterly_Product_Sales_by_Zone.xlsx" } - } + } }