import csv import functools import json import operator import re import pdfplumber import sqlite3 from numbers import Number from typing import Callable, Any, Union from typing import Dict, List, Pattern import datetime import pandas as pd import lxml.etree from lxml.cssselect import CSSSelector from lxml.etree import _Element from rapidfuzz import fuzz from docx import Document from .utils import _match_record, _match_value_to_rule def check_include_exclude(result: str, rules: Dict[str, List[str]]) -> float: if result is None: return 0. print(result, rules) include = rules.get("include", []) exclude = rules.get("exclude", []) if all(r in result for r in include) and all(r not in result for r in exclude): return 1. else: return 0. def exact_match(result, rules) -> float: expect = rules["expected"] print(result, expect) if result == expect: return 1. else: return 0. def is_in_list(result, rules) -> float: expect = rules["expected"] if expect in result: return 1. else: return 0. def fuzzy_match(result, rules) -> float: expect = rules["expected"] return fuzz.ratio(result, expect) / 100. def fuzzy_place_math(result_file_path, rules) -> float: expect = rules["expected"] # a list of possible answers # read list.docx, and get all texts out, overlook blank lines, remove blanks before and after each line doc = Document(result_file_path) words_list = [] for para in doc.paragraphs: words_list.extend(para.text.split()) # 打印出提取的单词列表 print(words_list) for word in words_list: if not any(ans in word for ans in expect): print("Wrong place:", word) return 0 return 1 def check_csv(result: str, rules: Dict[str, List[Dict[str, str]]]) -> float: """ Args: result (str): path to csv file rules (Dict[str, List[Dict[str, str]]]): dict like { "expect": [{key: value}] "unexpect": [{key: value}] } Returns: float """ if result is None: return 0. expect_metrics = [False] * len(rules.get("expect", [])) unexpect_metric = True with open(result) as f: reader = csv.DictReader(f) for rcd in reader: for i, r in enumerate(rules.get("expect", [])): expect_metrics[i] = expect_metrics[i] or _match_record(r, rcd) unexpect_metric = unexpect_metric and not any(_match_record(r, rcd) for r in rules.get("unexpect", [])) return float(all(expect_metrics) and unexpect_metric) def check_list(result: str, rules: Dict[str, List[str]]) -> float: """ Args: result (str): path to list file rules (Dict[str, List[str]]): dict like { "expect": list of str as regexes "unexpect": list of str as regexes } Returns: float """ if result is None: return 0. expect_patterns: List[Pattern[str]] = [re.compile(ptt) for ptt in rules.get("expect", [])] unexpect_patterns: List[Pattern[str]] = [re.compile(ptt) for ptt in rules.get("unexpect", [])] expect_metrics = [False] * len(expect_patterns) unexpect_metric = True with open(result) as f: for l in f: for i, r in enumerate(expect_patterns): expect_metrics[i] = expect_metrics[i] or (r.search(l) is not None) unexpect_metric = unexpect_metric and all(r.search(l) is None for r in unexpect_patterns) return float(all(expect_metrics) and unexpect_metric) _accessibility_ns_map = {"st": "uri:deskat:state.at-spi.gnome.org" , "attr": "uri:deskat:attributes.at-spi.gnome.org" , "cp": "uri:deskat:component.at-spi.gnome.org" , "doc": "uri:deskat:document.at-spi.gnome.org" , "docattr": "uri:deskat:attributes.document.at-spi.gnome.org" , "txt": "uri:deskat:text.at-spi.gnome.org" , "val": "uri:deskat:value.at-spi.gnome.org" , "act": "uri:deskat:action.at-spi.gnome.org" } def check_accessibility_tree(result: str, rules: Dict[str, Any]) -> float: """ Args: result (str): XML of GNOME Accessibility Tree rules (Dict[str, Any]): dict like { "selectors": list of str as CSS selectors, will be connected by ", " to form a composite selector. Only one from `selectors` and `xpath` is needed. If both are present, `xpath` takes the priority. "xpath": str as xpath. Only one from `selectors` and `xpath` is needed. If both are present, `xpath` takes the priority. "text": str as the expected text content of the selected element. "exact": bool specifying whether exact match or fuzzy match should be performed. defaults to True. } Returns: float """ at: _Element = lxml.etree.fromstring(result) if "xpath" in rules: elements: List[_Element] = at.xpath(rules["xpath"], namespaces=_accessibility_ns_map) elif "selectors" in rules: selector = CSSSelector(", ".join(rules["selectors"]), namespaces=_accessibility_ns_map) elements: List[_Element] = selector(at) else: raise ValueError("At least one of xpath and selectors is required") if len(elements) == 0: print("no elements") return 0. if "text" in rules: match_func: Callable[[str], Number] = functools.partial(operator.eq if rules["exact"] \ else (lambda a, b: fuzz.ratio(a, b) / 100.) , rules["text"] ) match_score: Number = 0 for elm in elements: match_score = max(match_score, match_func(elm.text or None)) else: match_score = 1. return float(match_score) # def check_existence(result: str, *args) -> float: # return 1. - (result is None) def run_sqlite3(result: str, rules: Dict[str, Any]) -> float: connection: sqlite3.Connection = sqlite3.connect(result) cursor: sqlite3.Cursor = connection.execute(rules["sql"]) return float(cursor.fetchone()[0] or 0) def check_json(result: str, rules: Dict[str, List[Dict[str, Union[List[str], str]]]]) -> float: """ Args: result (str): path to json file rules (Dict[str, List[Dict[str, Union[List[str], str]]]]): dict like { "expect": [ { "key": list of str "method": str "ref": something } ], "unexpect": float: """ One of the most commonly used function to evalute. Compare two json objects directly. """ if isinstance(result, str): # remove blanks before and after result result = result.strip() # replace all ' with " result = result.replace("'", '"') # load json object result = json.loads(result) print("result: ") print(result) print("expected: ") print(rules["expected"]) if result is None: return 0. expect_in_result = rules.get("expect_in_result", False) if not expect_in_result: expected_json = rules["expected"] for key in expected_json.keys(): expected_value = expected_json.get(key) if expected_value != result.get(key): return 0. return 1.0 else: expected_json = rules["expected"] for key in expected_json.keys(): expected_value = expected_json.get(key) if expected_value not in result.get(key): return 0. return 1.0 def compare_time_in_speedtest_results(speedtest_result_path, time_diff): # open the speedtest results file(csv) date_col = None with open(speedtest_result_path, 'r') as f: reader = pd.read_csv(f) for column in reader.columns: if column.startswith('TEST_DATE'): date_col = column break now_date_time = datetime.datetime.now().strftime('%H:%M') for date in reader[date_col]: date_time = date[-5:] # compare the date time with the current date time, if time diff less than time_diff para, then return true if not abs((datetime.datetime.strptime(date_time, '%H:%M') - datetime.datetime.strptime(now_date_time, '%H:%M')).total_seconds()) / 60 < int(time_diff): return False return True def is_included_all_json_objects(gold_file_path, result_file_path): print("gold_file_path: ") print(gold_file_path) print("result_file_path: ") print(result_file_path) # two json file, check if all the key-value pair in gold_file_path is included in result_file_path with open(gold_file_path, 'r') as f: gold_json = json.load(f) with open(result_file_path, 'r') as fr: result_json = json.load(fr) for key in gold_json.keys(): if key not in result_json.keys() or gold_json[key] != result_json[key]: return False return True def is_gold_text_included_in_pdf(pdf_file_path, gold_text_path): print("gold_text_path: ") print(gold_text_path) print("pdf_file_path: ") print(pdf_file_path) # gold file is a json file, we need to check all the value in json are included in pdf file. with open(gold_text_path, 'r') as f: gold_json = json.load(f) with pdfplumber.open(pdf_file_path) as pdf: text = '' for page in pdf.pages: text += page.extract_text() false_list = [] for key in gold_json.keys(): if gold_json[key] not in text: false_list.append(key) if len(false_list) > 0: print("false_list: ") print(false_list) return False else: return True def file_contains(file_path, config): # file_path ends with .txt if not file_path : return False with open(file_path, 'r') as f: file_text = f.read() for text in config["expected"]: if text not in file_text: return False return True def check_csv_line_number(file_path, line_number): # check file_path suffix if not file_path.endswith('.csv'): return False # check line number with open(file_path, 'r') as f: reader = csv.reader(f) line_count = sum(1 for row in reader) return True if line_count == int(line_number["expected"]) else False def compare_terminal_and_txt(txt_file_path, terminal_output): # read txt file content with open(txt_file_path, 'r') as f: txt_file_content = f.read() # compare terminal output with txt file content return True if terminal_output == txt_file_content else False def compare_python_pure_text(py_file_path, gold_file_path): # first, change the suffix of gold_file from .txt to .py print("py_file_path: ") print(py_file_path) print("gold_file_path: ") print(gold_file_path) # gold_file_path = gold_file_path.replace('.txt', '.py') def remove_whitespace(text): return ''.join(text.split()) with open(py_file_path, 'r') as file1: content1 = file1.read() with open(gold_file_path, 'r') as file2: content2 = file2.read() # 移除文件内容中的所有空白字符 content1_no_whitespace = remove_whitespace(content1) content2_no_whitespace = remove_whitespace(content2) # 比较处理后的文件内容 return content1_no_whitespace == content2_no_whitespace