ver Dec26th

evaluation metric checking result file according to rules

desktop_env/evaluators/getters/__init__.py

@@ -1 +1,2 @@
 from .file import get_cloud_file, get_vm_file
+from .misc import get_rule

desktop_env/evaluators/getters/misc.py

@@ -0,0 +1,8 @@
+from typing import TypeVar
+
+R = TypeVar("Rule")
+def get_rule(env, config: R) -> R:
+    """
+    Returns the rule as-is.
+    """
+    return config

desktop_env/evaluators/metrics/__init__.py

@@ -1 +1,2 @@
 from .table import compare_table, compare_with_sparklines
+from .table import check_sheet_list, check_xlsx_freeze

desktop_env/evaluators/metrics/table.py

@@ -5,9 +5,12 @@ import lxml.cssselect
 from lxml.etree import _Element
 import xmltodict
 #import pylightxl
+import openpyxl
+#from openpyxl import Workbook
+from openpyxl.worksheet.worksheet import Worksheet
 
 from typing import Dict, List
-#from typing import Any
+from typing import Any
 
 def compare_table(actual, expected):
     df1 = pd.read_excel(expected)
@@ -42,7 +45,7 @@ def _load_sparklines(xlsx_file: str) -> Dict[str, str]:
         with z_f.open("xl/worksheets/sheet1.xml") as f:
             sheet1: _Element = lxml.etree.fromstring(f.read())
             sparklines: List[_Element] = _sparklines_selector(sheet1)
-            
+
     sparklines_dict: Dict[str, str] = {}
     for sp_l in sparklines:
         sparkline_xml: str = lxml.etree.tostring(sp_l, encoding="unicode")
@@ -57,18 +60,89 @@ def compare_with_sparklines(actual: str, expected: str) -> float:
     df1 = pd.read_excel(actual)
     df2 = pd.read_excel(expected)
     normal_content_metric: bool = df1.equals(df2)
+    print("Normal Contents Metric: {:}".format(normal_content_metric))
 
     sp1 = _load_sparklines(actual)
     sp2 = _load_sparklines(expected)
     sparkline_metric: bool = sp1 == sp2
+    print("Sparkline Metric: {:}".format(sparkline_metric))
 
     return float(normal_content_metric and sparkline_metric)
 
+def check_sheet_list(result: str, rules: List[Dict[str, Any]]) -> float:
+    #workbook: Workbook = openpyxl.load_workbook(filename=result)
+    workbook = pd.ExcelFile(result)
+    worksheet_names: List[str] = workbook.sheet_names
+
+    passes = True
+    for r in rules:
+        if r["type"]=="sheet_name":
+            expected_name: str = worksheet_names[r["sheet_idx"]]
+            actual_name: str = r["sheet_name"]
+            metric: bool = expected_name==actual_name
+            print("Assertion: {:d}.{:} is {:} - {:}".format(r["sheet_idx"], actual_name, expected_name, metric))
+            passes = passes and metric
+        elif r["type"]=="sheet_data":
+            if isinstance(r["sheet_idx0"], int):
+                df1: pd.DataFrame = pd.read_excel(workbook, r["sheet_idx0"])
+            else:
+                file_name: str
+                sheet_idx: str
+                file_name, sheet_idx = r["sheet_idx0"].rsplit("@", maxsplit=1)
+                sheet_idx: int = int(sheet_idx)
+                df1: pd.DataFrame = pd.read_excel(file_name, sheet_idx)
+            if isinstance(r["sheet_idx1"], int):
+                df2: pd.DataFrame = pd.read_excel(workbook, r["sheet_idx1"])
+            else:
+                file_name: str
+                sheet_idx: str
+                file_name, sheet_idx = r["sheet_idx1"].rsplit("@", maxsplit=1)
+                sheet_idx: int = int(sheet_idx)
+                df2: pd.DataFrame = pd.read_excel(file_name, sheet_idx)
+            metric: bool = df1.equals(df2)
+            print("Assertion: {:} == {:} - {:}".format(r["sheet_idx0"], r["sheet_idx1"], metric))
+            passes = passes and metric
+        else:
+            raise NotImplementedError("Unimplemented sheet check: {:}".format(r["type"]))
+
+    return float(passes)
+
+def check_xlsx_freeze(result: str, rules: Dict[str, str]) -> float:
+    worksheet: Worksheet = openpyxl.load_workbook(filename=result).active
+    return float(worksheet.freeze_panes==rules["position"])
+
 if __name__ == '__main__':
     #path1 = ""
     #path2 = ""
     #print(compare_table(path1, path2))
-    
-    path1 = "../../../../../任务数据/LibreOffice Calc/OrderId_Month_Chart_gold.xlsx"
-    path2 = "../../../../../任务数据/LibreOffice Calc/OrderId_Month_Chart.xlsx"
-    print(compare_with_sparklines(path1, path2))
+
+    #path1 = "../../../../../任务数据/LibreOffice Calc/OrderId_Month_Chart_gold.xlsx"
+    #path2 = "../../../../../任务数据/LibreOffice Calc/OrderId_Month_Chart.xlsx"
+    #print(compare_with_sparklines(path1, path2))
+
+    #path1 = "../../../../../任务数据/LibreOffice Calc/Freeze_row_column_gold.xlsx"
+    #path2 = "../../../../../任务数据/LibreOffice Calc/Freeze_row_column.xlsx"
+    #workbook1: Workbook = openpyxl.load_workbook(filename=path1)
+    #worksheet1: Worksheet = workbook1.active
+    #print(worksheet1.freeze_panes)
+    #workbook2: Workbook = openpyxl.load_workbook(filename=path2)
+    #worksheet2: Worksheet = workbook2.active
+    #print(worksheet2.freeze_panes)
+    #rule = {"position": "C6"}
+    #print(check_xlsx_freeze(path1, rule))
+
+    path1 = "../../../../../任务数据/LibreOffice Calc/copy_sheet_insert_gold.xlsx"
+    rule = [ { "type": "sheet_name"
+             , "sheet_idx": 0
+             , "sheet_name": "Sheet1"
+             }
+           , { "type": "sheet_data"
+             , "sheet_idx0": "../../../../../任务数据/LibreOffice Calc/copy_sheet_insert.xlsx@0"
+             , "sheet_idx1": 1
+             }
+           , { "type": "sheet_name"
+             , "sheet_idx": 2
+             , "sheet_name": "Sheet2"
+             }
+           ]
+    print(check_sheet_list(path1, rule))

evaluation_examples/examples/libreoffice_calc/4188d3a4-077d-46b7-9c86-23e1a036f6c1.json

@@ -0,0 +1,43 @@
+{
+	"id": "4188d3a4-077d-46b7-9c86-23e1a036f6c1",
+  "snapshot": "libreoffice_calc",
+	"instruction": "Help me freeze the range A1:B5 on this sheet",
+	"source": "https://www.libreofficehelp.com/freeze-unfreeze-rows-columns-ranges-calc/",
+	"config": [
+		{
+			"type": "download",
+			"parameters": {
+				"files": [
+					{
+						"url": "https://101.43.24.67/s/H7k3zLNaNcMWyLB/download/Freeze_row_column.xlsx",
+						"path": "/home/david/Freeze_row_column.xlsx"
+					}
+				]
+			}
+		},
+		{
+			"type": "open",
+			"parameters": {
+				"path": "/home/david/Freeze_row_column.xlsx"
+			}
+		}
+	],
+	"trajectory": "trajectories/4188d3a4-077d-46b7-9c86-23e1a036f6c1",
+	"related_apps": [
+		"libreoffice_calc"
+	],
+	"evaluators": {
+		"func": "check_xlsx_freeze",
+		"result": {
+			"type": "vm_file",
+			"path": "/home/david/Freeze_row_column.xlsx",
+			"dest": "Freeze_row_column.xlsx"
+		},
+		"expected": {
+			"type": "rule",
+			"rules": {
+					"position": "C6"
+			}
+		}
+	}
+}

requirements.txt

@@ -18,3 +18,4 @@ requests-toolbelt~=1.0.0
 lxml
 cssselect
 xmltodict
+openpyxl