xiaochuan's multiapp examples

desktop_env/evaluators/metrics/general.py

@@ -3,15 +3,18 @@ import functools
 import json
 import operator
 import re
+import pdfplumber
 import sqlite3
 from numbers import Number
 from typing import Callable, Any, Union
 from typing import Dict, List, Pattern
-
+import datetime
+import pandas as pd
 import lxml.etree
 from lxml.cssselect import CSSSelector
 from lxml.etree import _Element
 from rapidfuzz import fuzz
+from docx import Document
 
 from .utils import _match_record, _match_value_to_rule
 
@@ -46,13 +49,27 @@ def is_in_list(result, rules) -> float:
         return 0.
 
 
-
 def fuzzy_match(result, rules) -> float:
     expect = rules["expected"]
 
     return fuzz.ratio(result, expect) / 100.
 
 
+def fuzzy_place_math(result_file_path, rules) -> float:
+    expect = rules["expected"] # a list of possible answers
+    # read list.docx, and get all texts out, overlook blank lines, remove blanks before and after each line
+    doc = Document(result_file_path)
+    words_list = []
+    for para in doc.paragraphs:
+        words_list.extend(para.text.split())
+    # 打印出提取的单词列表
+    print(words_list)
+    for word in words_list:
+        if not any(ans in word for ans in expect):
+            print("Wrong place:", word)
+            return 0
+    return 1
+
 def check_csv(result: str, rules: Dict[str, List[Dict[str, str]]]) -> float:
     """
     Args:
@@ -227,15 +244,138 @@ def check_direct_json_object(result, rules)->float:
     One of the most commonly used function to evalute.
     Compare two json objects directly.
     """
+    if isinstance(result, str):
+        # remove blanks before and after result
+        result = result.strip()
+        # replace all ' with "
+        result = result.replace("'", '"')
+        # load json object
+        result = json.loads(result)
     print("result: ")
     print(result)
     print("expected: ")
     print(rules["expected"])
     if result is None:
         return 0.
-    expected_json = rules["expected"]
-    for key in expected_json.keys():
-        expected_value = expected_json.get(key)
-        if expected_value != result.get(key):
-            return 0.
-    return 1.0
+    expect_in_result = rules.get("expect_in_result", False)
+    if not expect_in_result:
+        expected_json = rules["expected"]
+        for key in expected_json.keys():
+            expected_value = expected_json.get(key)
+            if expected_value != result.get(key):
+                return 0.
+        return 1.0
+    else:
+        expected_json = rules["expected"]
+        for key in expected_json.keys():
+            expected_value = expected_json.get(key)
+            if expected_value not in result.get(key):
+                return 0.
+        return 1.0
+
+def compare_time_in_speedtest_results(speedtest_result_path, time_diff):
+    # open the speedtest results file(csv)
+    date_col = None
+    with open(speedtest_result_path, 'r') as f:
+        reader = pd.read_csv(f)
+        for column in reader.columns:
+            if column.startswith('TEST_DATE'):
+                date_col = column
+                break
+        now_date_time = datetime.datetime.now().strftime('%H:%M')
+        for date in reader[date_col]:
+            date_time = date[-5:]
+            # compare the date time with the current date time, if time diff less than time_diff para, then return true
+            if not abs((datetime.datetime.strptime(date_time, '%H:%M') - datetime.datetime.strptime(now_date_time, '%H:%M')).total_seconds()) / 60 < int(time_diff):
+                return False
+        return True
+
+
+def is_included_all_json_objects(gold_file_path, result_file_path):
+    print("gold_file_path: ")
+    print(gold_file_path)
+    print("result_file_path: ")
+    print(result_file_path)
+    # two json file, check if all the key-value pair in gold_file_path is included in result_file_path
+    with open(gold_file_path, 'r') as f:
+        gold_json = json.load(f)
+    with open(result_file_path, 'r') as fr:
+        result_json = json.load(fr)
+    for key in gold_json.keys():
+        if key not in result_json.keys() or gold_json[key] != result_json[key]:
+            return False
+    return True
+
+
+def is_gold_text_included_in_pdf(pdf_file_path, gold_text_path):
+    print("gold_text_path: ")
+    print(gold_text_path)
+    print("pdf_file_path: ")
+    print(pdf_file_path)
+    # gold file is a json file, we need to check all the value in json are included in pdf file.
+    with open(gold_text_path, 'r') as f:
+        gold_json = json.load(f)
+    with pdfplumber.open(pdf_file_path) as pdf:
+        text = ''
+        for page in pdf.pages:
+            text += page.extract_text()
+    false_list = []
+    for key in gold_json.keys():
+        if gold_json[key] not in text:
+            false_list.append(key)
+    if len(false_list) > 0:
+        print("false_list: ")
+        print(false_list)
+        return False
+    else:
+        return True
+
+
+def file_contains(file_path, config):
+    # file_path ends with .txt
+    if not file_path :
+        return False
+    with open(file_path, 'r') as f:
+        file_text = f.read()
+    for text in config["expected"]:
+        if text not in file_text:
+            return False
+    return True
+
+def check_csv_line_number(file_path, line_number):
+    # check file_path suffix
+    if not file_path.endswith('.csv'):
+        return False
+    # check line number
+    with open(file_path, 'r') as f:
+        reader = csv.reader(f)
+        line_count = sum(1 for row in reader)
+    return True if line_count == int(line_number["expected"]) else False
+
+
+def compare_terminal_and_txt(txt_file_path, terminal_output):
+    # read txt file content
+    with open(txt_file_path, 'r') as f:
+        txt_file_content = f.read()
+    # compare terminal output with txt file content
+    return True if terminal_output == txt_file_content else False
+
+
+def compare_python_pure_text(py_file_path, gold_file_path):
+    # first, change the suffix of gold_file from .txt to .py
+    print("py_file_path: ")
+    print(py_file_path)
+    print("gold_file_path: ")
+    print(gold_file_path)
+    # gold_file_path = gold_file_path.replace('.txt', '.py')
+    def remove_whitespace(text):
+        return ''.join(text.split())
+    with open(py_file_path, 'r') as file1:
+        content1 = file1.read()
+    with open(gold_file_path, 'r') as file2:
+        content2 = file2.read()
+    # 移除文件内容中的所有空白字符
+    content1_no_whitespace = remove_whitespace(content1)
+    content2_no_whitespace = remove_whitespace(content2)
+    # 比较处理后的文件内容
+    return content1_no_whitespace == content2_no_whitespace