diff --git a/branch-config/filelist b/branch-config/filelist index 0ba2d96..b386813 100644 --- a/branch-config/filelist +++ b/branch-config/filelist @@ -10,3 +10,5 @@ experiment_screenshot.py experiment_screenshot_a11y_tree.py experiment_screenshot_seeact.py experiment_screenshot_som.py + +quick_compare_table.py diff --git a/clear_sheetcopilot_jsons.py b/clear_sheetcopilot_jsons.py new file mode 100644 index 0000000..796a73c --- /dev/null +++ b/clear_sheetcopilot_jsons.py @@ -0,0 +1,31 @@ +#!/usr/bin/python3 + +import json +import shutil +import os +import os.path +from typing import Dict, List +from typing import Any + +path = "evaluation_examples/examples/sheetcopilot" +for f in filter(lambda f: f.endswith(".json"), os.listdir(path)): + with open(os.path.join(path, f)) as f_obj: + config: Dict[str, Any] = json.load(f_obj) + + options: List[Dict[str, Any]] = [] + deletes = False + for opt in config["evaluator"]["options"]["rules"]: + if opt["type"] == "style" and opt["props"] == [ "font_name", "font_color" + , "font_bold", "font_italic" + , "font_underline", "bgcolor" + ]: + deletes = True + continue + else: + options.append(opt) + + if deletes: + config["evaluator"]["options"]["rules"] = options + shutil.move(os.path.join(path, f), os.path.join(path, f + ".old")) + with open(os.path.join(path, f), "w") as f_obj: + json.dump(config, f_obj, indent="\t", ensure_ascii=False) diff --git a/desktop_env/evaluators/metrics/utils.py b/desktop_env/evaluators/metrics/utils.py index ac8b879..8afffb1 100644 --- a/desktop_env/evaluators/metrics/utils.py +++ b/desktop_env/evaluators/metrics/utils.py @@ -107,7 +107,10 @@ def load_charts(xlsx_file: Workbook, sheet_name: str, **options) -> Dict[str, An """ # workbook: Workbook = openpyxl.load_workbook(filename=xlsx_file) - worksheet: Worksheet = xlsx_file[sheet_name] + try: + worksheet: Worksheet = xlsx_file[sheet_name] + except KeyError: + return {} charts: List[ChartBase] = worksheet._charts chart_set: Dict[str, Any] = {} @@ -203,7 +206,10 @@ def load_pivot_tables(xlsx_file: Workbook, sheet_name: str, **options) -> Dict[s } """ - worksheet: Worksheet = xlsx_file[sheet_name] + try: + worksheet: Worksheet = xlsx_file[sheet_name] + except KeyError: + return {} pivots: List[PivotTableDefinition] = worksheet._pivots pivot_set: Dict[str, Any] = {} @@ -354,7 +360,10 @@ def load_xlsx_styles(xlsx_file: Workbook, sheet_name: str, **options) -> Dict[st } """ - worksheet: Worksheet = xlsx_file[sheet_name] + try: + worksheet: Worksheet = xlsx_file[sheet_name] + except KeyError: + return {} style_dict: Dict[str, List[Any]] = {} concerned_styles: List[str] = options.get("props", []) @@ -416,7 +425,10 @@ def load_rows_or_cols(xlsx_file: Workbook, sheet_name: str, **options)\ Dict[Union[int, str], Dict[str, Any]]: row/column information """ - worksheet: Worksheet = xlsx_file[sheet_name] + try: + worksheet: Worksheet = xlsx_file[sheet_name] + except KeyError: + return {} objs: DimensionHolder = getattr(worksheet, "{:}_dimensions".format(options["obj"])) obj_set: Dict[int, Any] = {} @@ -431,7 +443,10 @@ def load_rows_or_cols(xlsx_file: Workbook, sheet_name: str, **options)\ def load_filters(xlsx_file: Workbook, sheet_name: str, **options) -> Dict[str, Any]: # function load_filters {{{ # - worksheet: Worksheet = xlsx_file[sheet_name] + try: + worksheet: Worksheet = xlsx_file[sheet_name] + except KeyError: + return {} filters: AutoFilter = worksheet.auto_filter filter_dict: Dict[str, Any] = {} diff --git a/evaluation_examples/examples/sheetcopilot/035f41ba-6653-43ab-aa63-c86d449d62e5.json b/evaluation_examples/examples/sheetcopilot/035f41ba-6653-43ab-aa63-c86d449d62e5.json index be9f508..a267101 100644 --- a/evaluation_examples/examples/sheetcopilot/035f41ba-6653-43ab-aa63-c86d449d62e5.json +++ b/evaluation_examples/examples/sheetcopilot/035f41ba-6653-43ab-aa63-c86d449d62e5.json @@ -72,24 +72,16 @@ "options": { "rules": [ { - "type": "style", - "sheet_idx0": 1, - "sheet_idx1": "EI1", - "props": [ - "font_name", - "font_color", - "font_bold", - "font_italic", - "font_underline", - "bgcolor" - ] + "type": "sheet_data", + "sheet_idx0": "RNSheet1", + "sheet_idx1": "ENSheet1" }, { "type": "sheet_data", - "sheet_idx0": 1, - "sheet_idx1": "EI1" + "sheet_idx0": "RNSheet2", + "sheet_idx1": "ENSheet2" } ] } } -} \ No newline at end of file +} diff --git a/evaluation_examples/examples/sheetcopilot/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f.json b/evaluation_examples/examples/sheetcopilot/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f.json index 35ece13..89b6269 100644 --- a/evaluation_examples/examples/sheetcopilot/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f.json +++ b/evaluation_examples/examples/sheetcopilot/04d9aeaf-7bed-4024-bedb-e10e6f00eb7f.json @@ -71,25 +71,12 @@ }, "options": { "rules": [ - { - "type": "style", - "sheet_idx0": 0, - "sheet_idx1": "EI0", - "props": [ - "font_name", - "font_color", - "font_bold", - "font_italic", - "font_underline", - "bgcolor" - ] - }, { "type": "sheet_data", - "sheet_idx0": 0, - "sheet_idx1": "EI0" + "sheet_idx0": "RNSheet2", + "sheet_idx1": "ENSheet2" } ] } } -} \ No newline at end of file +} diff --git a/evaluation_examples/examples/sheetcopilot/0acbd372-ca7a-4507-b949-70673120190f.json b/evaluation_examples/examples/sheetcopilot/0acbd372-ca7a-4507-b949-70673120190f.json index 9f4ae32..5888f47 100644 --- a/evaluation_examples/examples/sheetcopilot/0acbd372-ca7a-4507-b949-70673120190f.json +++ b/evaluation_examples/examples/sheetcopilot/0acbd372-ca7a-4507-b949-70673120190f.json @@ -71,19 +71,6 @@ }, "options": { "rules": [ - { - "type": "style", - "sheet_idx0": 0, - "sheet_idx1": "EI0", - "props": [ - "font_name", - "font_color", - "font_bold", - "font_italic", - "font_underline", - "bgcolor" - ] - }, { "type": "sheet_data", "sheet_idx0": 0, diff --git a/evaluation_examples/examples/sheetcopilot/0acbd372-ca7a-4507-b949-70673120190f.json.old b/evaluation_examples/examples/sheetcopilot/0acbd372-ca7a-4507-b949-70673120190f.json.old new file mode 100644 index 0000000..9f4ae32 --- /dev/null +++ b/evaluation_examples/examples/sheetcopilot/0acbd372-ca7a-4507-b949-70673120190f.json.old @@ -0,0 +1,95 @@ +{ + "id": "0acbd372-ca7a-4507-b949-70673120190f", + "snapshot": "libreoffice_calc", + "instruction": "Calculate Net Income in a new column with header \"Net Income\" and format all data as accounting number type.", + "source": "SheetCopilot@121", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?id=1VoEpRbXGxxrxIp-a2iGPqgVPQNdswQB-&export=download", + "path": "/home/user/NetIncome.xlsx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/NetIncome.xlsx" + } + } + ], + "trajectory": "trajectories/0acbd372-ca7a-4507-b949-70673120190f", + "related_apps": [ + "libreoffice_calc" + ], + "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "NetIncome.xlsx - LibreOffice Calc", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + } + ], + "func": "compare_table", + "expected": { + "type": "cloud_file", + "path": "https://drive.google.com/uc?id=1NGZNpzoB3YDdDiWZy4eZtZMwM9Fw1vw2&export=download", + "dest": "1_NetIncome_gt1.xlsx" + }, + "result": { + "type": "vm_file", + "path": "/home/user/NetIncome.xlsx", + "dest": "NetIncome.xlsx" + }, + "options": { + "rules": [ + { + "type": "style", + "sheet_idx0": 0, + "sheet_idx1": "EI0", + "props": [ + "font_name", + "font_color", + "font_bold", + "font_italic", + "font_underline", + "bgcolor" + ] + }, + { + "type": "sheet_data", + "sheet_idx0": 0, + "sheet_idx1": "EI0" + } + ] + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/sheetcopilot/12382c62-0cd1-4bf2-bdc8-1d20bf9b2371.json b/evaluation_examples/examples/sheetcopilot/12382c62-0cd1-4bf2-bdc8-1d20bf9b2371.json index 421da24..b17352c 100644 --- a/evaluation_examples/examples/sheetcopilot/12382c62-0cd1-4bf2-bdc8-1d20bf9b2371.json +++ b/evaluation_examples/examples/sheetcopilot/12382c62-0cd1-4bf2-bdc8-1d20bf9b2371.json @@ -73,8 +73,8 @@ "rules": [ { "type": "chart", - "sheet_idx0": 0, - "sheet_idx1": "EI0", + "sheet_idx0": "RNSheet2", + "sheet_idx1": "ENSheet2", "chart_props": [ "type", "title" @@ -83,4 +83,4 @@ ] } } -} \ No newline at end of file +} diff --git a/evaluation_examples/examples/sheetcopilot/1d17d234-e39d-4ed7-b46f-4417922a4e7c.json b/evaluation_examples/examples/sheetcopilot/1d17d234-e39d-4ed7-b46f-4417922a4e7c.json index ec81c9e..fa158d2 100644 --- a/evaluation_examples/examples/sheetcopilot/1d17d234-e39d-4ed7-b46f-4417922a4e7c.json +++ b/evaluation_examples/examples/sheetcopilot/1d17d234-e39d-4ed7-b46f-4417922a4e7c.json @@ -71,19 +71,6 @@ }, "options": { "rules": [ - { - "type": "style", - "sheet_idx0": 0, - "sheet_idx1": "EI0", - "props": [ - "font_name", - "font_color", - "font_bold", - "font_italic", - "font_underline", - "bgcolor" - ] - }, { "type": "sheet_data", "sheet_idx0": 0, diff --git a/evaluation_examples/examples/sheetcopilot/1d17d234-e39d-4ed7-b46f-4417922a4e7c.json.old b/evaluation_examples/examples/sheetcopilot/1d17d234-e39d-4ed7-b46f-4417922a4e7c.json.old new file mode 100644 index 0000000..ec81c9e --- /dev/null +++ b/evaluation_examples/examples/sheetcopilot/1d17d234-e39d-4ed7-b46f-4417922a4e7c.json.old @@ -0,0 +1,95 @@ +{ + "id": "1d17d234-e39d-4ed7-b46f-4417922a4e7c", + "snapshot": "libreoffice_calc", + "instruction": "Create a new sheet named \"Sheet2\" and merge cells A1:C1 to write the header \"Investment Summary\". Beneath that, merge cells A2:B2 to write \"High Interest Rate\" and merge cells C2:D2 to form \"Low Interest Rate\".", + "source": "SheetCopilot@73", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?id=1qILGOZpTwo2armXyGZM96ao-ydHRpZ_g&export=download", + "path": "/home/user/FutureValue.xlsx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/FutureValue.xlsx" + } + } + ], + "trajectory": "trajectories/1d17d234-e39d-4ed7-b46f-4417922a4e7c", + "related_apps": [ + "libreoffice_calc" + ], + "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "FutureValue.xlsx - LibreOffice Calc", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + } + ], + "func": "compare_table", + "expected": { + "type": "cloud_file", + "path": "https://drive.google.com/uc?id=1KCTx2yEwTlSLa8doQ74ucE-opbCxQCeh&export=download", + "dest": "5_FutureValue_gt1.xlsx" + }, + "result": { + "type": "vm_file", + "path": "/home/user/FutureValue.xlsx", + "dest": "FutureValue.xlsx" + }, + "options": { + "rules": [ + { + "type": "style", + "sheet_idx0": 0, + "sheet_idx1": "EI0", + "props": [ + "font_name", + "font_color", + "font_bold", + "font_italic", + "font_underline", + "bgcolor" + ] + }, + { + "type": "sheet_data", + "sheet_idx0": 0, + "sheet_idx1": "EI0" + } + ] + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/sheetcopilot/21ab7b40-77c2-4ae6-8321-e00d3a086c73.json b/evaluation_examples/examples/sheetcopilot/21ab7b40-77c2-4ae6-8321-e00d3a086c73.json index 3c3427b..0040efa 100644 --- a/evaluation_examples/examples/sheetcopilot/21ab7b40-77c2-4ae6-8321-e00d3a086c73.json +++ b/evaluation_examples/examples/sheetcopilot/21ab7b40-77c2-4ae6-8321-e00d3a086c73.json @@ -71,19 +71,6 @@ }, "options": { "rules": [ - { - "type": "style", - "sheet_idx0": 0, - "sheet_idx1": "EI0", - "props": [ - "font_name", - "font_color", - "font_bold", - "font_italic", - "font_underline", - "bgcolor" - ] - }, { "type": "sheet_data", "sheet_idx0": 0, diff --git a/evaluation_examples/examples/sheetcopilot/21ab7b40-77c2-4ae6-8321-e00d3a086c73.json.old b/evaluation_examples/examples/sheetcopilot/21ab7b40-77c2-4ae6-8321-e00d3a086c73.json.old new file mode 100644 index 0000000..3c3427b --- /dev/null +++ b/evaluation_examples/examples/sheetcopilot/21ab7b40-77c2-4ae6-8321-e00d3a086c73.json.old @@ -0,0 +1,108 @@ +{ + "id": "21ab7b40-77c2-4ae6-8321-e00d3a086c73", + "snapshot": "libreoffice_calc", + "instruction": "Please calculate the period rate for my data in a new column with header \"Period Rate (%)\", convert the results as number type, and highlight the highest result as green.", + "source": "SheetCopilot@124", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?id=1zejyjqheZOyLaYXS2X8ds6QrZCj3P096&export=download", + "path": "/home/user/PeriodRate.xlsx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/PeriodRate.xlsx" + } + } + ], + "trajectory": "trajectories/21ab7b40-77c2-4ae6-8321-e00d3a086c73", + "related_apps": [ + "libreoffice_calc" + ], + "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "PeriodRate.xlsx - LibreOffice Calc", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + } + ], + "func": "compare_table", + "expected": { + "type": "cloud_file", + "path": "https://drive.google.com/uc?id=1aK3pWJ4LZx7aPFu16-MxzL0HzXGcQ2U1&export=download", + "dest": "1_PeriodRate_gt1.xlsx" + }, + "result": { + "type": "vm_file", + "path": "/home/user/PeriodRate.xlsx", + "dest": "PeriodRate.xlsx" + }, + "options": { + "rules": [ + { + "type": "style", + "sheet_idx0": 0, + "sheet_idx1": "EI0", + "props": [ + "font_name", + "font_color", + "font_bold", + "font_italic", + "font_underline", + "bgcolor" + ] + }, + { + "type": "sheet_data", + "sheet_idx0": 0, + "sheet_idx1": "EI0" + }, + { + "type": "style", + "sheet_idx0": 0, + "sheet_idx1": "EI0", + "props": [ + "font_bold", + "font_color", + "bgcolor", + "font_name", + "font_italic", + "font_underline" + ] + } + ] + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/sheetcopilot/30e3e107-1cfb-46ee-a755-2cd080d7ba6a.json b/evaluation_examples/examples/sheetcopilot/30e3e107-1cfb-46ee-a755-2cd080d7ba6a.json index a4cbce9..dd32682 100644 --- a/evaluation_examples/examples/sheetcopilot/30e3e107-1cfb-46ee-a755-2cd080d7ba6a.json +++ b/evaluation_examples/examples/sheetcopilot/30e3e107-1cfb-46ee-a755-2cd080d7ba6a.json @@ -71,19 +71,6 @@ }, "options": { "rules": [ - { - "type": "style", - "sheet_idx0": 0, - "sheet_idx1": "EI0", - "props": [ - "font_name", - "font_color", - "font_bold", - "font_italic", - "font_underline", - "bgcolor" - ] - }, { "type": "sheet_data", "sheet_idx0": 0, diff --git a/evaluation_examples/examples/sheetcopilot/30e3e107-1cfb-46ee-a755-2cd080d7ba6a.json.old b/evaluation_examples/examples/sheetcopilot/30e3e107-1cfb-46ee-a755-2cd080d7ba6a.json.old new file mode 100644 index 0000000..a4cbce9 --- /dev/null +++ b/evaluation_examples/examples/sheetcopilot/30e3e107-1cfb-46ee-a755-2cd080d7ba6a.json.old @@ -0,0 +1,106 @@ +{ + "id": "30e3e107-1cfb-46ee-a755-2cd080d7ba6a", + "snapshot": "libreoffice_calc", + "instruction": "Please create a new sheet. Merge cells A1:C1 in the new sheet and write \"Demographic Profile\" with blue fill and bold white text. Then I want to create three pivot tables to show the percentage of Sex, Civil Status , and Highest Educational Attainment.", + "source": "SheetCopilot@9", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?id=1c9egJM5T3_-FTJd9yFfQfQ2sCEgerzSl&export=download", + "path": "/home/user/DemographicProfile.xlsx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/DemographicProfile.xlsx" + } + } + ], + "trajectory": "trajectories/30e3e107-1cfb-46ee-a755-2cd080d7ba6a", + "related_apps": [ + "libreoffice_calc" + ], + "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "DemographicProfile.xlsx - LibreOffice Calc", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + } + ], + "func": "compare_table", + "expected": { + "type": "cloud_file", + "path": "https://drive.google.com/uc?id=1fiqcYR0w4XXpgPInSR5CGIh2_3-sjQv5&export=download", + "dest": "1_DemographicProfile_gt1.xlsx" + }, + "result": { + "type": "vm_file", + "path": "/home/user/DemographicProfile.xlsx", + "dest": "DemographicProfile.xlsx" + }, + "options": { + "rules": [ + { + "type": "style", + "sheet_idx0": 0, + "sheet_idx1": "EI0", + "props": [ + "font_name", + "font_color", + "font_bold", + "font_italic", + "font_underline", + "bgcolor" + ] + }, + { + "type": "sheet_data", + "sheet_idx0": 0, + "sheet_idx1": "EI0" + }, + { + "type": "pivot_table", + "sheet_idx0": 0, + "sheet_idx1": "EI0", + "pivot_props": [ + "col_fields", + "filter", + "row_fields", + "data_fields" + ] + } + ] + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/sheetcopilot/447b9505-7a2f-4863-9dd1-69395482eb4b.json b/evaluation_examples/examples/sheetcopilot/447b9505-7a2f-4863-9dd1-69395482eb4b.json index 00c61a8..c53bb07 100644 --- a/evaluation_examples/examples/sheetcopilot/447b9505-7a2f-4863-9dd1-69395482eb4b.json +++ b/evaluation_examples/examples/sheetcopilot/447b9505-7a2f-4863-9dd1-69395482eb4b.json @@ -71,19 +71,6 @@ }, "options": { "rules": [ - { - "type": "style", - "sheet_idx0": 0, - "sheet_idx1": "EI0", - "props": [ - "font_name", - "font_color", - "font_bold", - "font_italic", - "font_underline", - "bgcolor" - ] - }, { "type": "sheet_data", "sheet_idx0": 0, diff --git a/evaluation_examples/examples/sheetcopilot/447b9505-7a2f-4863-9dd1-69395482eb4b.json.old b/evaluation_examples/examples/sheetcopilot/447b9505-7a2f-4863-9dd1-69395482eb4b.json.old new file mode 100644 index 0000000..00c61a8 --- /dev/null +++ b/evaluation_examples/examples/sheetcopilot/447b9505-7a2f-4863-9dd1-69395482eb4b.json.old @@ -0,0 +1,95 @@ +{ + "id": "447b9505-7a2f-4863-9dd1-69395482eb4b", + "snapshot": "libreoffice_calc", + "instruction": "Divide the values through column C to P by 1 million and put the results along with the year headers in a new sheet. Set the results type as accounting.", + "source": "SheetCopilot@28", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?id=1v2Hfrd5fE2JG7H_ED6NbtC8G7hd8aOrM&export=download", + "path": "/home/user/EasyGDPBreakdown.xlsx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/EasyGDPBreakdown.xlsx" + } + } + ], + "trajectory": "trajectories/447b9505-7a2f-4863-9dd1-69395482eb4b", + "related_apps": [ + "libreoffice_calc" + ], + "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "EasyGDPBreakdown.xlsx - LibreOffice Calc", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + } + ], + "func": "compare_table", + "expected": { + "type": "cloud_file", + "path": "https://drive.google.com/uc?id=1cx0tSCOEGbsjVWhHJB1W_z15gAReEzIk&export=download", + "dest": "5_EasyGDPBreakdown_gt1.xlsx" + }, + "result": { + "type": "vm_file", + "path": "/home/user/EasyGDPBreakdown.xlsx", + "dest": "EasyGDPBreakdown.xlsx" + }, + "options": { + "rules": [ + { + "type": "style", + "sheet_idx0": 0, + "sheet_idx1": "EI0", + "props": [ + "font_name", + "font_color", + "font_bold", + "font_italic", + "font_underline", + "bgcolor" + ] + }, + { + "type": "sheet_data", + "sheet_idx0": 0, + "sheet_idx1": "EI0" + } + ] + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/sheetcopilot/5780a545-4e20-4230-95b4-cac135ef119f.json b/evaluation_examples/examples/sheetcopilot/5780a545-4e20-4230-95b4-cac135ef119f.json index 881ccc0..0a30df6 100644 --- a/evaluation_examples/examples/sheetcopilot/5780a545-4e20-4230-95b4-cac135ef119f.json +++ b/evaluation_examples/examples/sheetcopilot/5780a545-4e20-4230-95b4-cac135ef119f.json @@ -71,19 +71,6 @@ }, "options": { "rules": [ - { - "type": "style", - "sheet_idx0": 0, - "sheet_idx1": "EI0", - "props": [ - "font_name", - "font_color", - "font_bold", - "font_italic", - "font_underline", - "bgcolor" - ] - }, { "type": "sheet_data", "sheet_idx0": 0, diff --git a/evaluation_examples/examples/sheetcopilot/5780a545-4e20-4230-95b4-cac135ef119f.json.old b/evaluation_examples/examples/sheetcopilot/5780a545-4e20-4230-95b4-cac135ef119f.json.old new file mode 100644 index 0000000..881ccc0 --- /dev/null +++ b/evaluation_examples/examples/sheetcopilot/5780a545-4e20-4230-95b4-cac135ef119f.json.old @@ -0,0 +1,116 @@ +{ + "id": "5780a545-4e20-4230-95b4-cac135ef119f", + "snapshot": "libreoffice_calc", + "instruction": "In a new sheet, merge cells A1:B1 and write bold text \"Summer sales ($)\" with blue fill color and white text color. Then create a pivot table starting in this new sheet for my data in Sheet 1 to calculate the sum of revenue for each promotion. Plot a 3D pie chart illustrating the results. Set the chart title as \"Revenue of Each Promotion ($)\" and move the legends to the bottom. Don't forget to add outside end data labels for the chart.", + "source": "SheetCopilot@53", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?id=1I29YyV7zFdNqbsQw0SWD9pDuLz3oHlge&export=download", + "path": "/home/user/EntireSummerSales.xlsx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/EntireSummerSales.xlsx" + } + } + ], + "trajectory": "trajectories/5780a545-4e20-4230-95b4-cac135ef119f", + "related_apps": [ + "libreoffice_calc" + ], + "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "EntireSummerSales.xlsx - LibreOffice Calc", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + } + ], + "func": "compare_table", + "expected": { + "type": "cloud_file", + "path": "https://drive.google.com/uc?id=1zHi--HRZwHvyMyvSXx62qO6Laiql7zG3&export=download", + "dest": "4_EntireSummerSales_gt1.xlsx" + }, + "result": { + "type": "vm_file", + "path": "/home/user/EntireSummerSales.xlsx", + "dest": "EntireSummerSales.xlsx" + }, + "options": { + "rules": [ + { + "type": "style", + "sheet_idx0": 0, + "sheet_idx1": "EI0", + "props": [ + "font_name", + "font_color", + "font_bold", + "font_italic", + "font_underline", + "bgcolor" + ] + }, + { + "type": "sheet_data", + "sheet_idx0": 0, + "sheet_idx1": "EI0" + }, + { + "type": "chart", + "sheet_idx0": 0, + "sheet_idx1": "EI0", + "chart_props": [ + "type", + "legend", + "title" + ] + }, + { + "type": "pivot_table", + "sheet_idx0": 0, + "sheet_idx1": "EI0", + "pivot_props": [ + "col_fields", + "filter", + "row_fields", + "data_fields" + ] + } + ] + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/sheetcopilot/8fa9072b-ea9b-4679-84c6-420f3fe4c697.json b/evaluation_examples/examples/sheetcopilot/8fa9072b-ea9b-4679-84c6-420f3fe4c697.json index 7d5ad88..b12daaa 100644 --- a/evaluation_examples/examples/sheetcopilot/8fa9072b-ea9b-4679-84c6-420f3fe4c697.json +++ b/evaluation_examples/examples/sheetcopilot/8fa9072b-ea9b-4679-84c6-420f3fe4c697.json @@ -71,19 +71,6 @@ }, "options": { "rules": [ - { - "type": "style", - "sheet_idx0": 0, - "sheet_idx1": "EI0", - "props": [ - "font_name", - "font_color", - "font_bold", - "font_italic", - "font_underline", - "bgcolor" - ] - }, { "type": "sheet_data", "sheet_idx0": 0, diff --git a/evaluation_examples/examples/sheetcopilot/8fa9072b-ea9b-4679-84c6-420f3fe4c697.json.old b/evaluation_examples/examples/sheetcopilot/8fa9072b-ea9b-4679-84c6-420f3fe4c697.json.old new file mode 100644 index 0000000..7d5ad88 --- /dev/null +++ b/evaluation_examples/examples/sheetcopilot/8fa9072b-ea9b-4679-84c6-420f3fe4c697.json.old @@ -0,0 +1,118 @@ +{ + "id": "8fa9072b-ea9b-4679-84c6-420f3fe4c697", + "snapshot": "libreoffice_calc", + "instruction": "In a new sheet, merge cells A1:B1 and write bold text \"Summer sales ($)\" with yellow fill color and white text color. Then create a pivot table starting in this new sheet for my data in Sheet 1 to calculate the sum of revenue for each week. Plot a line chart illustrating the trend of the results with chart title \"Revenue of Each Week\". Set the horizontal axis title as \"Week\" and remove the vertical axis title as well as legend.", + "source": "SheetCopilot@51", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?id=1I29YyV7zFdNqbsQw0SWD9pDuLz3oHlge&export=download", + "path": "/home/user/EntireSummerSales.xlsx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/EntireSummerSales.xlsx" + } + } + ], + "trajectory": "trajectories/8fa9072b-ea9b-4679-84c6-420f3fe4c697", + "related_apps": [ + "libreoffice_calc" + ], + "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "EntireSummerSales.xlsx - LibreOffice Calc", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + } + ], + "func": "compare_table", + "expected": { + "type": "cloud_file", + "path": "https://drive.google.com/uc?id=1VZP8zkroAoDZC4fxDfLR5HcnuOUChuh1&export=download", + "dest": "2_EntireSummerSales_gt1.xlsx" + }, + "result": { + "type": "vm_file", + "path": "/home/user/EntireSummerSales.xlsx", + "dest": "EntireSummerSales.xlsx" + }, + "options": { + "rules": [ + { + "type": "style", + "sheet_idx0": 0, + "sheet_idx1": "EI0", + "props": [ + "font_name", + "font_color", + "font_bold", + "font_italic", + "font_underline", + "bgcolor" + ] + }, + { + "type": "sheet_data", + "sheet_idx0": 0, + "sheet_idx1": "EI0" + }, + { + "type": "chart", + "sheet_idx0": 0, + "sheet_idx1": "EI0", + "chart_props": [ + "xtitle", + "ytitle", + "type", + "legend", + "title" + ] + }, + { + "type": "pivot_table", + "sheet_idx0": 0, + "sheet_idx1": "EI0", + "pivot_props": [ + "col_fields", + "filter", + "row_fields", + "data_fields" + ] + } + ] + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/sheetcopilot/9b534cd8-d497-4ca8-8444-82105b87d6f4.json b/evaluation_examples/examples/sheetcopilot/9b534cd8-d497-4ca8-8444-82105b87d6f4.json index 64368f5..9d09c50 100644 --- a/evaluation_examples/examples/sheetcopilot/9b534cd8-d497-4ca8-8444-82105b87d6f4.json +++ b/evaluation_examples/examples/sheetcopilot/9b534cd8-d497-4ca8-8444-82105b87d6f4.json @@ -71,19 +71,6 @@ }, "options": { "rules": [ - { - "type": "style", - "sheet_idx0": 0, - "sheet_idx1": "EI0", - "props": [ - "font_name", - "font_color", - "font_bold", - "font_italic", - "font_underline", - "bgcolor" - ] - }, { "type": "sheet_data", "sheet_idx0": 0, diff --git a/evaluation_examples/examples/sheetcopilot/9b534cd8-d497-4ca8-8444-82105b87d6f4.json.old b/evaluation_examples/examples/sheetcopilot/9b534cd8-d497-4ca8-8444-82105b87d6f4.json.old new file mode 100644 index 0000000..64368f5 --- /dev/null +++ b/evaluation_examples/examples/sheetcopilot/9b534cd8-d497-4ca8-8444-82105b87d6f4.json.old @@ -0,0 +1,95 @@ +{ + "id": "9b534cd8-d497-4ca8-8444-82105b87d6f4", + "snapshot": "libreoffice_calc", + "instruction": "Calculate profit for each week in a new column with header \"Profit\". Then format all numbers with Accounting Number Format.", + "source": "SheetCopilot@198", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?id=1gRFurvUFCy-MpU7PQS_rMo9HUk5ZiDRj&export=download", + "path": "/home/user/WeeklySales.xlsx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/WeeklySales.xlsx" + } + } + ], + "trajectory": "trajectories/9b534cd8-d497-4ca8-8444-82105b87d6f4", + "related_apps": [ + "libreoffice_calc" + ], + "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "WeeklySales.xlsx - LibreOffice Calc", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + } + ], + "func": "compare_table", + "expected": { + "type": "cloud_file", + "path": "https://drive.google.com/uc?id=1w96DtYUzjnsBsBuFUYVK8PabzLUAzoJB&export=download", + "dest": "1_WeeklySales_gt1.xlsx" + }, + "result": { + "type": "vm_file", + "path": "/home/user/WeeklySales.xlsx", + "dest": "WeeklySales.xlsx" + }, + "options": { + "rules": [ + { + "type": "style", + "sheet_idx0": 0, + "sheet_idx1": "EI0", + "props": [ + "font_name", + "font_color", + "font_bold", + "font_italic", + "font_underline", + "bgcolor" + ] + }, + { + "type": "sheet_data", + "sheet_idx0": 0, + "sheet_idx1": "EI0" + } + ] + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/sheetcopilot/cd3c4994-b9e2-426b-8157-f7978ff55501.json b/evaluation_examples/examples/sheetcopilot/cd3c4994-b9e2-426b-8157-f7978ff55501.json index d3fe91c..4ec66fd 100644 --- a/evaluation_examples/examples/sheetcopilot/cd3c4994-b9e2-426b-8157-f7978ff55501.json +++ b/evaluation_examples/examples/sheetcopilot/cd3c4994-b9e2-426b-8157-f7978ff55501.json @@ -71,19 +71,6 @@ }, "options": { "rules": [ - { - "type": "style", - "sheet_idx0": 0, - "sheet_idx1": "EI0", - "props": [ - "font_name", - "font_color", - "font_bold", - "font_italic", - "font_underline", - "bgcolor" - ] - }, { "type": "sheet_data", "sheet_idx0": 0, diff --git a/evaluation_examples/examples/sheetcopilot/cd3c4994-b9e2-426b-8157-f7978ff55501.json.old b/evaluation_examples/examples/sheetcopilot/cd3c4994-b9e2-426b-8157-f7978ff55501.json.old new file mode 100644 index 0000000..d3fe91c --- /dev/null +++ b/evaluation_examples/examples/sheetcopilot/cd3c4994-b9e2-426b-8157-f7978ff55501.json.old @@ -0,0 +1,116 @@ +{ + "id": "cd3c4994-b9e2-426b-8157-f7978ff55501", + "snapshot": "libreoffice_calc", + "instruction": "In a new sheet, merge cells A1:B1 and write bold text \"Summer sales ($)\" with blue fill color and white text color. Then create a pivot table in this new sheet for my data in Sheet 1 and summarize the monthly revenue for each sales channel. Plot a stacked column chart illustrating the results. Set the chart titles as \"Revenue of Each Channel ($)\" and move the legends to the bottom.", + "source": "SheetCopilot@52", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?id=1I29YyV7zFdNqbsQw0SWD9pDuLz3oHlge&export=download", + "path": "/home/user/EntireSummerSales.xlsx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/EntireSummerSales.xlsx" + } + } + ], + "trajectory": "trajectories/cd3c4994-b9e2-426b-8157-f7978ff55501", + "related_apps": [ + "libreoffice_calc" + ], + "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "EntireSummerSales.xlsx - LibreOffice Calc", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + } + ], + "func": "compare_table", + "expected": { + "type": "cloud_file", + "path": "https://drive.google.com/uc?id=1MDqRVCrhUq_FKI0ZspIxcEyww_BZ-4Lb&export=download", + "dest": "3_EntireSummerSales_gt1.xlsx" + }, + "result": { + "type": "vm_file", + "path": "/home/user/EntireSummerSales.xlsx", + "dest": "EntireSummerSales.xlsx" + }, + "options": { + "rules": [ + { + "type": "style", + "sheet_idx0": 0, + "sheet_idx1": "EI0", + "props": [ + "font_name", + "font_color", + "font_bold", + "font_italic", + "font_underline", + "bgcolor" + ] + }, + { + "type": "sheet_data", + "sheet_idx0": 0, + "sheet_idx1": "EI0" + }, + { + "type": "chart", + "sheet_idx0": 0, + "sheet_idx1": "EI0", + "chart_props": [ + "type", + "legend", + "title" + ] + }, + { + "type": "pivot_table", + "sheet_idx0": 0, + "sheet_idx1": "EI0", + "pivot_props": [ + "col_fields", + "filter", + "row_fields", + "data_fields" + ] + } + ] + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/sheetcopilot/de7a24c3-7f47-45c7-bba9-ba1aaaf015f8.json b/evaluation_examples/examples/sheetcopilot/de7a24c3-7f47-45c7-bba9-ba1aaaf015f8.json index 8bb46b5..3a49e31 100644 --- a/evaluation_examples/examples/sheetcopilot/de7a24c3-7f47-45c7-bba9-ba1aaaf015f8.json +++ b/evaluation_examples/examples/sheetcopilot/de7a24c3-7f47-45c7-bba9-ba1aaaf015f8.json @@ -71,19 +71,6 @@ }, "options": { "rules": [ - { - "type": "style", - "sheet_idx0": 0, - "sheet_idx1": "EI0", - "props": [ - "font_name", - "font_color", - "font_bold", - "font_italic", - "font_underline", - "bgcolor" - ] - }, { "type": "sheet_data", "sheet_idx0": 0, diff --git a/evaluation_examples/examples/sheetcopilot/de7a24c3-7f47-45c7-bba9-ba1aaaf015f8.json.old b/evaluation_examples/examples/sheetcopilot/de7a24c3-7f47-45c7-bba9-ba1aaaf015f8.json.old new file mode 100644 index 0000000..8bb46b5 --- /dev/null +++ b/evaluation_examples/examples/sheetcopilot/de7a24c3-7f47-45c7-bba9-ba1aaaf015f8.json.old @@ -0,0 +1,118 @@ +{ + "id": "de7a24c3-7f47-45c7-bba9-ba1aaaf015f8", + "snapshot": "libreoffice_calc", + "instruction": "In a new sheet, merge cells A1:B1 and write bold text \"Summer sales ($)\" with blue fill color and white text color. Then create a pivot table in this new sheet for the Sheet 1 data to calculate the revenue sum for each product. Plot a horizontal bar chart illustrating the results. Set the chart title as \"Revenue of Each Product\" and remove the axis titles and legend.", + "source": "SheetCopilot@50", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?id=1I29YyV7zFdNqbsQw0SWD9pDuLz3oHlge&export=download", + "path": "/home/user/EntireSummerSales.xlsx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/EntireSummerSales.xlsx" + } + } + ], + "trajectory": "trajectories/de7a24c3-7f47-45c7-bba9-ba1aaaf015f8", + "related_apps": [ + "libreoffice_calc" + ], + "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "EntireSummerSales.xlsx - LibreOffice Calc", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + } + ], + "func": "compare_table", + "expected": { + "type": "cloud_file", + "path": "https://drive.google.com/uc?id=12QQfwtxURMSMJlP9IRvKJzSCAWu0TqsB&export=download", + "dest": "1_EntireSummerSales_gt1.xlsx" + }, + "result": { + "type": "vm_file", + "path": "/home/user/EntireSummerSales.xlsx", + "dest": "EntireSummerSales.xlsx" + }, + "options": { + "rules": [ + { + "type": "style", + "sheet_idx0": 0, + "sheet_idx1": "EI0", + "props": [ + "font_name", + "font_color", + "font_bold", + "font_italic", + "font_underline", + "bgcolor" + ] + }, + { + "type": "sheet_data", + "sheet_idx0": 0, + "sheet_idx1": "EI0" + }, + { + "type": "chart", + "sheet_idx0": 0, + "sheet_idx1": "EI0", + "chart_props": [ + "xtitle", + "ytitle", + "type", + "legend", + "title" + ] + }, + { + "type": "pivot_table", + "sheet_idx0": 0, + "sheet_idx1": "EI0", + "pivot_props": [ + "col_fields", + "filter", + "row_fields", + "data_fields" + ] + } + ] + } + } +} \ No newline at end of file diff --git a/quick_compare_table.py b/quick_compare_table.py new file mode 100644 index 0000000..6767f9a --- /dev/null +++ b/quick_compare_table.py @@ -0,0 +1,51 @@ +#!/usr/bin/python3 + +from desktop_env.evaluators.metrics import compare_table +import json +import sys +import os.path +from typing import Dict +from typing import Any + +import logging +import datetime + +logger = logging.getLogger() +logger.setLevel(logging.DEBUG) + +datetime_str: str = datetime.datetime.now().strftime("%Y%m%d@%H%M%S") + +file_handler = logging.FileHandler(os.path.join("logs", "normal-{:}.log".format(datetime_str))) +debug_handler = logging.FileHandler(os.path.join("logs", "debug-{:}.log".format(datetime_str))) +stdout_handler = logging.StreamHandler(sys.stdout) +sdebug_handler = logging.FileHandler(os.path.join("logs", "sdebug-{:}.log".format(datetime_str))) + +file_handler.setLevel(logging.INFO) +debug_handler.setLevel(logging.DEBUG) +stdout_handler.setLevel(logging.INFO) +sdebug_handler.setLevel(logging.DEBUG) + +formatter = logging.Formatter(fmt="\x1b[1;33m[%(asctime)s \x1b[31m%(levelname)s \x1b[32m%(module)s/%(lineno)d-%(processName)s\x1b[1;33m] \x1b[0m%(message)s") +file_handler.setFormatter(formatter) +debug_handler.setFormatter(formatter) +stdout_handler.setFormatter(formatter) +sdebug_handler.setFormatter(formatter) + +stdout_handler.addFilter(logging.Filter("desktopenv")) +sdebug_handler.addFilter(logging.Filter("desktopenv")) + +logger.addHandler(file_handler) +logger.addHandler(debug_handler) +logger.addHandler(stdout_handler) +logger.addHandler(sdebug_handler) + +config_file: str = sys.argv[1] + +with open(config_file) as f: + config: Dict[str, Any] = json.load(f) + +print( compare_table( os.path.join("cache/", config["id"], config["evaluator"]["result"]["dest"]) + , os.path.join("cache/", config["id"], config["evaluator"]["expected"]["dest"]) + , **config["evaluator"]["options"] + ) + )