Merge branch 'main' of https://github.com/xlang-ai/DesktopEnv

desktop_env/evaluators/metrics/__init__.py

@@ -48,7 +48,8 @@ from .docs import (
     check_tabstops,
     compare_contains_image,
     compare_docx_images,
-    compare_image_text
+    compare_image_text,
+    compare_references
 )
 from .general import (
     check_csv,

desktop_env/evaluators/metrics/chrome.py

@@ -143,6 +143,11 @@ def compare_pdfs(pdf1_path: Union[str, List[str]], pdf2_path: Union[str, List[st
 import fitz
 from PIL import Image
 from io import BytesIO
+from borb.pdf import Document
+from borb.pdf import PDF
+
+from pathlib import Path
+import typing
 
 def compare_pdf_images(pdf1_path: str, pdf2_path: str, **kwargs) -> float:
     def extract_images_from_pdf(pdf_path):
@@ -151,24 +156,32 @@ def compare_pdf_images(pdf1_path: str, pdf2_path: str, **kwargs) -> float:
 
         for page_number in range(pdf_document.page_count):
             page = pdf_document[page_number]
-            image_list = page.get_images(full=True)
+            pixmap = page.get_pixmap()
 
-            for img_index, img_info in enumerate(image_list):
-                base_image = pdf_document.extract_image(img_index)
-                image_bytes = base_image["image"]
+            img = Image.frombytes("RGB", [pixmap.width, pixmap.height], pixmap.samples)
 
-                images.append(BytesIO(image_bytes))
+            images.append(img)
 
         return images
     
+    def fix_pdf(in_path: Path, out_path: Path) -> None:
+        doc: typing.Optional[Document] = None
+        with open(in_path, "rb") as fh:
+            doc = PDF.loads(fh)
+        with open(out_path, "wb") as fh:
+            PDF.dumps(fh, doc)
+    
+    fix_pdf(Path(pdf1_path), Path(pdf1_path))
+    fix_pdf(Path(pdf2_path), Path(pdf2_path))
+
     images1 = extract_images_from_pdf(pdf1_path)
     images2 = extract_images_from_pdf(pdf2_path)
 
     if len(images1) != len(images2):
         return 0.
 
-    for i, (img1, img2) in enumerate(zip(images1, images2), 1):
-        if Image.open(img1).tobytes() != Image.open(img2).tobytes():
+    for img1, img2 in zip(images1, images2):
+        if img1.tobytes() != img2.tobytes():
             return 0.
     
     return 1.

desktop_env/evaluators/metrics/docs.py

@@ -11,9 +11,9 @@ from docx.shared import RGBColor
 from odf.opendocument import load
 from odf.text import P
 from odf.text import Span
+from rapidfuzz import fuzz
 from skimage.color import deltaE_ciede2000
 from skimage.color import rgb2lab
-from rapidfuzz import fuzz
 
 logger = logging.getLogger("desktopenv.metric.docs")
 
@@ -173,9 +173,11 @@ def compare_docx_tables(docx_file1, docx_file2):
 
     return 1
 
+
 from io import BytesIO
 from PIL import Image
 
+
 def compare_docx_images(docx_file1, docx_file2):
     doc1 = Document(docx_file1)
     doc2 = Document(docx_file2)
@@ -187,7 +189,7 @@ def compare_docx_images(docx_file1, docx_file2):
                 img_data = rel.target_part.blob
                 images.append(BytesIO(img_data))
         return images
-    
+
     images1 = extract_images(doc1)
     images2 = extract_images(doc2)
     if len(images1) != len(images2):
@@ -197,8 +199,10 @@ def compare_docx_images(docx_file1, docx_file2):
             return 0
     return 1
 
+
 import pytesseract
 
+
 def compare_image_text(image_path, rule):
     img = Image.open(image_path)
     img_text = pytesseract.image_to_string(img)
@@ -207,6 +211,7 @@ def compare_image_text(image_path, rule):
     else:
         raise ValueError("Unsupported rule type")
 
+
 def compare_line_spacing(docx_file1, docx_file2):
     if not compare_docx_files(docx_file1, docx_file2):
         return 0
@@ -313,7 +318,7 @@ def check_tabstops(docx_file1, docx_file2, **kwargs) -> float:
     section = doc2.sections[0]
     paragraph_width = section.page_width - section.left_margin - section.right_margin
     ignore_tabs = lambda x: x.alignment == WD_TAB_ALIGNMENT.CLEAR or (
-                x.alignment == WD_TAB_ALIGNMENT.LEFT and x.position == 0)
+            x.alignment == WD_TAB_ALIGNMENT.LEFT and x.position == 0)
     minus = .0
     for p1, p2 in zip(para1, para2):
         # filter CLEAR tabstop and default left-0 tabstop
@@ -583,3 +588,95 @@ def compare_highlighted_text(file1, file2):
         return 1
     else:
         return 0
+
+
+def compare_references(file1, file2, **options):
+    reference_indicator = options.get('reference_indicator', 'References')
+    reference_base_result = options.get('reference_base_result', 0.5)
+
+    # Determine file types and load documents
+    if file1.endswith('.docx') and file2.endswith('.docx'):
+        doc1 = Document(file1)
+        doc2 = Document(file2)
+        doc1_paragraphs = [p.text for p in doc1.paragraphs]
+        doc2_paragraphs = [p.text for p in doc2.paragraphs]
+    else:
+        # Unsupported file types or mismatch
+        print("Unsupported file types or mismatch between file types.")
+        return 0
+
+    # Find the references section in the paragraphs, find the idx of the last reference_indicator in the paragraph list
+    ref1_idx = doc1_paragraphs.index(reference_indicator) if reference_indicator in doc1_paragraphs else -1
+    ref2_idx = doc2_paragraphs.index(reference_indicator) if reference_indicator in doc2_paragraphs else -1
+
+    if ref1_idx == -1 and ref2_idx == -1:
+        return 1
+
+    if ref1_idx == -1 or ref2_idx == -1:
+        return 0
+
+    # split the reference section into reference items, and remove the empty string items
+    ref1 = [p for p in doc1_paragraphs[ref1_idx + 1:] if p.strip()]
+    ref2 = [p for p in doc2_paragraphs[ref2_idx + 1:] if p.strip()]
+
+    # Compare the references
+
+    if len(ref1) != len(ref2):
+        return 0
+
+    total_similarity = 0
+    for r1, r2 in zip(ref1, ref2):
+        # fuzzy match the references
+        similarity = fuzz.ratio(r1, r2) / 100.0
+        total_similarity += similarity
+
+    result = total_similarity / len(ref1)
+    if result >= reference_base_result:
+        return (result - reference_base_result) / (1 - reference_base_result)
+    else:
+        return 0
+
+
+def compare_answer(file1, file2, **options):
+    """This is a specific function to compare the """
+    # Determine file types and load documents
+    if file1.endswith('.docx') and file2.endswith('.docx'):
+        doc1 = Document(file1)
+        doc2 = Document(file2)
+        doc1_paragraphs = [p.text for p in doc1.paragraphs]
+        doc2_paragraphs = [p.text for p in doc2.paragraphs]
+    else:
+        # Unsupported file types or mismatch
+        print("Unsupported file types or mismatch between file types.")
+        return 0
+
+    # Find the references section in the paragraphs, find the idx of the last reference_indicator in the paragraph list
+    ref1_idx = doc1_paragraphs.index(reference_indicator) if reference_indicator in doc1_paragraphs else -1
+    ref2_idx = doc2_paragraphs.index(reference_indicator) if reference_indicator in doc2_paragraphs else -1
+
+    if ref1_idx == -1 and ref2_idx == -1:
+        return 1
+
+    if ref1_idx == -1 or ref2_idx == -1:
+        return 0
+
+    # split the reference section into reference items, and remove the empty string items
+    ref1 = [p for p in doc1_paragraphs[ref1_idx + 1:] if p.strip()]
+    ref2 = [p for p in doc2_paragraphs[ref2_idx + 1:] if p.strip()]
+
+    # Compare the references
+
+    if len(ref1) != len(ref2):
+        return 0
+
+    total_similarity = 0
+    for r1, r2 in zip(ref1, ref2):
+        # fuzzy match the references
+        similarity = fuzz.ratio(r1, r2) / 100.0
+        total_similarity += similarity
+
+    result = total_similarity / len(ref1)
+    if result >= reference_base_result:
+        return (result - reference_base_result) / (1 - reference_base_result)
+    else:
+        return 0

desktop_env/evaluators/metrics/general.py

@@ -17,6 +17,9 @@ import difflib
 
 from .utils import _match_record, _match_value_to_rule
 
+import logging
+
+logger = logging.getLogger("desktopenv.metric.general")
 
 def check_include_exclude(result: str, rules: Dict[str, List[str]]) -> float:
     if result is None:
@@ -184,7 +187,7 @@ def check_accessibility_tree(result: str, rules: List[Dict[str, Any]]) -> float:
             raise ValueError("At least one of xpath and selectors is required")
 
         if len(elements) == 0:
-            print("no elements")
+            logger.info("No elements: %s", r["xpath"] if "xpath" in r else r["selectors"])
             return 0.
 
         if "text" in r:

desktop_env/evaluators/metrics/others.py

@@ -81,7 +81,7 @@ def check_mp3_meta(result: str, meta: Dict[str, Dict[str, Union[str, V]]]) -> bo
     for k, r in meta.items():
         value = id3_dict.get(k, "")
         if isinstance(value, list):
-            value: str = value[0]
+            value: str = ",".join(value)
         logger.debug("%s.%s: %s", result, k, value)
         metric = metric and _match_value_to_rule(value, r)
     return float(metric)

desktop_env/evaluators/metrics/table.py

@@ -16,9 +16,9 @@ from openpyxl.utils import get_column_letter
 from openpyxl.worksheet.datavalidation import DataValidation
 from openpyxl.worksheet.worksheet import Worksheet
 
-from .utils import _match_value_to_rule, _read_cell_style, read_cell_value
-from .utils import load_charts, load_sparklines, load_rows_or_cols, load_xlsx_styles\
-                 , load_filters, load_pivot_tables
+from desktop_env.evaluators.metrics.utils import _match_value_to_rule, _read_cell_style, read_cell_value
+from desktop_env.evaluators.metrics.utils import load_charts, load_sparklines, load_rows_or_cols, load_xlsx_styles \
+    , load_filters, load_pivot_tables
 from rapidfuzz import fuzz
 
 # from openpyxl.utils import coordinate_to_tuple