From 506c375554a96b1885a0ff00c69983f367c85231 Mon Sep 17 00:00:00 2001 From: Timothyxxx <384084775@qq.com> Date: Sun, 25 Feb 2024 03:49:48 +0800 Subject: [PATCH 1/3] Fix some json typos from Chrome --- desktop_env/evaluators/getters/__init__.py | 1 - .../9f935cce-0a9f-435f-8007-817732bfc0a5.json | 115 ++++++++--------- .../b070486d-e161-459b-aa2b-ef442d973b92.json | 119 +++++++++--------- .../cabb3bae-cccb-41bd-9f5d-0f3a9fecd825.json | 119 +++++++++--------- 4 files changed, 178 insertions(+), 176 deletions(-) diff --git a/desktop_env/evaluators/getters/__init__.py b/desktop_env/evaluators/getters/__init__.py index d3a349d..43f4c1b 100644 --- a/desktop_env/evaluators/getters/__init__.py +++ b/desktop_env/evaluators/getters/__init__.py @@ -20,7 +20,6 @@ from .chrome import ( get_find_unpacked_extension_path, get_data_delete_automacally, get_active_tab_html_parse, - get_active_tab_html_parse_accTree, get_active_tab_url_parse, get_gotoRecreationPage_and_get_html_content, get_url_dashPart, diff --git a/evaluation_examples/examples/chrome/9f935cce-0a9f-435f-8007-817732bfc0a5.json b/evaluation_examples/examples/chrome/9f935cce-0a9f-435f-8007-817732bfc0a5.json index c465279..a868b34 100644 --- a/evaluation_examples/examples/chrome/9f935cce-0a9f-435f-8007-817732bfc0a5.json +++ b/evaluation_examples/examples/chrome/9f935cce-0a9f-435f-8007-817732bfc0a5.json @@ -1,62 +1,63 @@ { - "id": "9f935cce-0a9f-435f-8007-817732bfc0a5", - "snapshot": "chrome", - "instruction": "Browse list of Civil Division forms.", - "source": "online_tasks", - "config": [ - { - "type": "launch", - "parameters": { - "command": [ - "google-chrome", - "--remote-debugging-port=1337" - ] - } - }, - { - "type": "launch", - "parameters": { - "command": [ - "socat", - "tcp-listen:9222,fork", - "tcp:localhost:1337" - ] - } - }, - { - "type": "chrome_open_tabs", - "parameters": { - "urls_to_open": [ - "https://www.justice.gov/" - ] - } - }, - { - "type": "activate_window", - "parameters": { - "window_name": "Google Chrome" - } + "id": "9f935cce-0a9f-435f-8007-817732bfc0a5", + "snapshot": "chrome", + "instruction": "Browse list of Civil Division forms.", + "source": "online_tasks", + "config": [ + { + "type": "launch", + "parameters": { + "command": [ + "google-chrome", + "--remote-debugging-port=1337" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "socat", + "tcp-listen:9222,fork", + "tcp:localhost:1337" + ] + } + }, + { + "type": "chrome_open_tabs", + "parameters": { + "urls_to_open": [ + "https://www.justice.gov/" + ] + } + }, + { + "type": "activate_window", + "parameters": { + "window_name": "Google Chrome" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "chrome" + ], + "evaluator": { + "func": "is_expected_url_pattern_match", + "result": { + "type": "active_tab_info", + "selectors": [ + "application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]" + ], + "goto_prefix": "https://www." + }, + "expected": { + "type": "rule", + "rules": { + "expected": [ + "forms\\?title=&field_component_target_id=431" + ] } - ], - "trajectory": "trajectories/", - "related_apps": [ - "chrome" - ], - "evaluator": { - "func":"is_expected_url_pattern_match", - "result": { - "type": "active_tab_info", - "selectors":[ - "application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]" - ], - "goto_prefix": "https://www." - }, - "expected":{ - "type": "rule", - "rules":{ - "expected": ["forms\\?title=&field_component_target_id=431"] - } - } } } } diff --git a/evaluation_examples/examples/chrome/b070486d-e161-459b-aa2b-ef442d973b92.json b/evaluation_examples/examples/chrome/b070486d-e161-459b-aa2b-ef442d973b92.json index 40cc8cf..eee2c9b 100644 --- a/evaluation_examples/examples/chrome/b070486d-e161-459b-aa2b-ef442d973b92.json +++ b/evaluation_examples/examples/chrome/b070486d-e161-459b-aa2b-ef442d973b92.json @@ -1,65 +1,64 @@ { - "id": " b070486d-e161-459b-aa2b-ef442d973b92", - "snapshot": "chrome", - "instruction": "Show side effects of Tamiflu.", - "source": "online_tasks", - "config": [ - { - "type": "launch", - "parameters": { - "command": [ - "google-chrome", - "--remote-debugging-port=1337" - ] - } - }, - { - "type": "launch", - "parameters": { - "command": [ - "socat", - "tcp-listen:9222,fork", - "tcp:localhost:1337" - ] - } - }, - { - "type": "chrome_open_tabs", - "parameters": { - "urls_to_open": [ - "https://www.drugs.com/" - ] - } - }, - { - "type": "activate_window", - "parameters": { - "window_name": "Google Chrome" - } + "id": " b070486d-e161-459b-aa2b-ef442d973b92", + "snapshot": "chrome", + "instruction": "Show side effects of Tamiflu.", + "source": "online_tasks", + "config": [ + { + "type": "launch", + "parameters": { + "command": [ + "google-chrome", + "--remote-debugging-port=1337" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "socat", + "tcp-listen:9222,fork", + "tcp:localhost:1337" + ] + } + }, + { + "type": "chrome_open_tabs", + "parameters": { + "urls_to_open": [ + "https://www.drugs.com/" + ] + } + }, + { + "type": "activate_window", + "parameters": { + "window_name": "Google Chrome" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "chrome" + ], + "evaluator": { + "func": "exact_match", + "result": { + "type": "url_dashPart", + "selectors": [ + "application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]" + ], + "goto_prefix": "https://www.", + "partIndex": -1, + "needDeleteId": false, + "returnType": "string" + }, + "expected": { + "type": "rule", + "rules": { + "expected": "tamiflu.html#side-effects" } - ], - "trajectory": "trajectories/", - "related_apps": [ - "chrome" - ], - "evaluator": { - "func":"exact_match", - "result": { - "type": "url_dashPart", - "selectors":[ - "application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]" - ], - "goto_prefix": "https://www.", - "partIndex": -1, - "needDeleteId": false, - "returnType": "string" - }, - "expected":{ - "type": "rule", - "rules":{ - "expected": "tamiflu.html#side-effects" - } - } } } } \ No newline at end of file diff --git a/evaluation_examples/examples/chrome/cabb3bae-cccb-41bd-9f5d-0f3a9fecd825.json b/evaluation_examples/examples/chrome/cabb3bae-cccb-41bd-9f5d-0f3a9fecd825.json index 70586e3..a2d9b1b 100644 --- a/evaluation_examples/examples/chrome/cabb3bae-cccb-41bd-9f5d-0f3a9fecd825.json +++ b/evaluation_examples/examples/chrome/cabb3bae-cccb-41bd-9f5d-0f3a9fecd825.json @@ -1,62 +1,65 @@ { - "id": "cabb3bae-cccb-41bd-9f5d-0f3a9fecd825", - "snapshot": "chrome", - "instruction": "Browse spider-man toys for kids and sort by lowest price.", - "source": "online_tasks", - "config": [ - { - "type": "launch", - "parameters": { - "command": [ - "google-chrome", - "--remote-debugging-port=1337" - ] - } - }, - { - "type": "launch", - "parameters": { - "command": [ - "socat", - "tcp-listen:9222,fork", - "tcp:localhost:1337" - ] - } - }, - { - "type": "chrome_open_tabs", - "parameters": { - "urls_to_open": [ - "https://www.kohls.com/" - ] - } - }, - { - "type": "activate_window", - "parameters": { - "window_name": "Google Chrome" - } + "id": "cabb3bae-cccb-41bd-9f5d-0f3a9fecd825", + "snapshot": "chrome", + "instruction": "Browse spider-man toys for kids and sort by lowest price.", + "source": "online_tasks", + "config": [ + { + "type": "launch", + "parameters": { + "command": [ + "google-chrome", + "--remote-debugging-port=1337" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "socat", + "tcp-listen:9222,fork", + "tcp:localhost:1337" + ] + } + }, + { + "type": "chrome_open_tabs", + "parameters": { + "urls_to_open": [ + "https://www.kohls.com/" + ] + } + }, + { + "type": "activate_window", + "parameters": { + "window_name": "Google Chrome" } - ], - "trajectory": "trajectories/", - "related_apps": [ - "chrome" - ], - "evaluator": { - "func":"is_expected_url_pattern_match", - "result": { - "type": "active_url_from_accessTree", - "selectors":[ - "application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]" - ], - "goto_prefix": "https://www." - }, - "expected":{ - "type": "rule", - "rules":{ - "expected": ["AgeAppropriate:Kids", "search=spider-man%20toys", "S=4"] - } - } } - } + ], + "trajectory": "trajectories/", + "related_apps": [ + "chrome" + ], + "evaluator": { + "func": "is_expected_url_pattern_match", + "result": { + "type": "active_url_from_accessTree", + "selectors": [ + "application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]" + ], + "goto_prefix": "https://www." + }, + "expected": { + "type": "rule", + "rules": { + "expected": [ + "AgeAppropriate:Kids", + "search=spider-man%20toys", + "S=4" + ] + } + } + } } From f7e87f1ec34f31ba12011033d931feffebab1173 Mon Sep 17 00:00:00 2001 From: David Chang Date: Sun, 25 Feb 2024 18:49:51 +0800 Subject: [PATCH 2/3] ver Feb25th fixed bug caused by non-existing sheet names --- desktop_env/evaluators/metrics/table.py | 55 ++++++++++++++++--------- 1 file changed, 36 insertions(+), 19 deletions(-) diff --git a/desktop_env/evaluators/metrics/table.py b/desktop_env/evaluators/metrics/table.py index 82ad0ac..b52a533 100644 --- a/desktop_env/evaluators/metrics/table.py +++ b/desktop_env/evaluators/metrics/table.py @@ -59,24 +59,29 @@ SHEET = Union[pd.DataFrame, Worksheet, List[str]] def _load_sheet(book: BOOK, index: str) -> SHEET: # function _load_sheet {{{ # - if isinstance(book, str): - book: str = cast(str, book) - csv_name: str = "{:}-{:}.csv".format(os.path.splitext(book)[0], index) + try: + if isinstance(book, str): + book: str = cast(str, book) + csv_name: str = "{:}-{:}.csv".format(os.path.splitext(book)[0], index) - with open(csv_name) as f: - csv_lines: List[str] = list(itertools.dropwhile(lambda l: len(l) == 0 - , map(lambda l: l.strip() - , reversed(f.read().splitlines()) - ) - ) - ) - return csv_lines - if isinstance(book, pd.ExcelFile): - return pd.read_excel(book, index) - if isinstance(book, Workbook): - return book[index] - logger.error("Not supported workbook format") - raise NotImplementedError("Not supported workbook format") + with open(csv_name) as f: + csv_lines: List[str] = list(itertools.dropwhile(lambda l: len(l) == 0 + , map(lambda l: l.strip() + , reversed(f.read().splitlines()) + ) + ) + ) + return csv_lines + if isinstance(book, pd.ExcelFile): + return pd.read_excel(book, index) + if isinstance(book, Workbook): + return book[index] + logger.error("Not supported workbook format") + raise NotImplementedError("Not supported workbook format") + except NotImplementedError as e: + raise e + except: + return None # }}} function _load_sheet # @@ -140,6 +145,8 @@ def compare_table(result: str, expected: str = None, **options) -> float: error_limit: int = r.get("precision", 4) sheet1: pd.DataFrame = _load_sheet(*parse_idx(r["sheet_idx0"], pdworkbookr, pdworkbooke)).round(error_limit) + if sheet1 is None: + return 0. sheet2: pd.DataFrame = _load_sheet(*parse_idx(r["sheet_idx1"], pdworkbookr, pdworkbooke)).round(error_limit) metric: bool = sheet1.equals(sheet2) logger.debug("Sheet1: \n%s", str(sheet1)) @@ -158,6 +165,8 @@ def compare_table(result: str, expected: str = None, **options) -> float: # ignore_case: optional, defaults to False sheet1: List[str] = _load_sheet(*parse_idx(r["sheet_idx0"], result, expected)) + if sheet1 is None: + return 0. sheet2: List[str] = _load_sheet(*parse_idx(r["sheet_idx1"], result, expected)) if r.get("ignore_case", False): sheet1 = [l.lower() for l in sheet1] @@ -195,11 +204,11 @@ def compare_table(result: str, expected: str = None, **options) -> float: # sheet_idx1: as sheet_idx0 # props: list of str indicating concerned styles, see utils._read_cell_style - sheet_idx1: Tuple[Book, str] = parse_idx(r["sheet_idx0"], xlworkbookr, xlworkbooke) + sheet_idx1: Tuple[BOOK, str] = parse_idx(r["sheet_idx0"], xlworkbookr, xlworkbooke) book_name1: str = parse_idx(r["sheet_idx0"], result, expected)[0] styles1: Dict[str, List[Any]] = load_xlsx_styles(*sheet_idx1, book_name1, **r) - sheet_idx2: Tuple[Book, str] = parse_idx(r["sheet_idx1"], xlworkbookr, xlworkbooke) + sheet_idx2: Tuple[BOOK, str] = parse_idx(r["sheet_idx1"], xlworkbookr, xlworkbooke) book_name2: str = parse_idx(r["sheet_idx1"], result, expected)[0] styles2: Dict[str, List[Any]] = load_xlsx_styles(*sheet_idx2, book_name2, **r) # number_formats1: List[str] = [c.number_format.lower() for col in sheet1.iter_cols() for c in col if c.value is not None and c.data_type=="n"] @@ -214,6 +223,8 @@ def compare_table(result: str, expected: str = None, **options) -> float: # sheet_idx1: as sheet_idx0 sheet1: Worksheet = _load_sheet(*parse_idx(r["sheet_idx0"], xlworkbookr, xlworkbooke)) + if sheet1 is None: + return 0. sheet2: Worksheet = _load_sheet(*parse_idx(r["sheet_idx1"], xlworkbookr, xlworkbooke)) metric: bool = sheet1.freeze_panes == sheet2.freeze_panes logger.debug("Assertion: %s.freeze(%s) == %s.freeze(%s) - %s" @@ -230,6 +241,8 @@ def compare_table(result: str, expected: str = None, **options) -> float: # ref: value sheet: Worksheet = _load_sheet(*parse_idx(r["sheet_idx"], xlworkbookr, xlworkbooke)) + if sheet is None: + return 0. zoom_scale: Number = sheet.sheet_view.zoomScale or 100. metric: bool = _match_value_to_rule(zoom_scale, r) logger.debug("Assertion: %s.zoom(%.1f) %s %.1f - %s", r["sheet_idx"], zoom_scale, r["method"], r["ref"], @@ -258,6 +271,8 @@ def compare_table(result: str, expected: str = None, **options) -> float: # * imeMode sheet: Worksheet = _load_sheet(*parse_idx(r["sheet_idx"], xlworkbookr, xlworkbooke)) + if sheet is None: + return 0. data_validators: List[DataValidation] = sheet.data_validations.dataValidation total_metric = len(data_validators) >= len(r["dv_props"]) @@ -348,6 +363,8 @@ def compare_table(result: str, expected: str = None, **options) -> float: # supported attributes: value & those supported by utils._read_cell_style sheet: Worksheet = _load_sheet(*parse_idx(r["sheet_idx"], xlworkbookr, xlworkbooke)) + if sheet is None: + return 0. # data_frame: pd.DataFrame = _load_sheet(*parse_idx(r["sheet_idx"], pdworkbookr, pdworkbooke)) cell: Cell = sheet[r["coordinate"]] metric: bool = True From aa0098d142859d7921652bf1549c02289d6ad666 Mon Sep 17 00:00:00 2001 From: David Chang Date: Sun, 25 Feb 2024 18:55:24 +0800 Subject: [PATCH 3/3] ver Feb25thv2 fixed a latent bug w.r.t. converting int sheet index to str sheet name --- desktop_env/evaluators/metrics/table.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/desktop_env/evaluators/metrics/table.py b/desktop_env/evaluators/metrics/table.py index b52a533..e3dc881 100644 --- a/desktop_env/evaluators/metrics/table.py +++ b/desktop_env/evaluators/metrics/table.py @@ -33,16 +33,25 @@ def _parse_sheet_idx(sheet_idx: Union[int, str] ) -> Tuple[BOOK, str]: # function _parse_sheet_idx {{{ # if isinstance(sheet_idx, int): - index: str = result_sheet_names[sheet_idx] + try: + index: str = result_sheet_names[sheet_idx] + except: + index = "" book: BOOK = result elif sheet_idx.startswith("RI"): - index: str = result_sheet_names[int(sheet_idx[2:])] + try: + index: str = result_sheet_names[int(sheet_idx[2:])] + except: + index = "" book: BOOK = result elif sheet_idx.startswith("RN"): index: str = sheet_idx[2:] book: BOOK = result elif sheet_idx.startswith("EI"): - index: str = expected_sheet_names[int(sheet_idx[2:])] + try: + index: str = expected_sheet_names[int(sheet_idx[2:])] + except: + index = "" book: BOOK = expected elif sheet_idx.startswith("EN"): index: str = sheet_idx[2:]