finish multi-app examples

desktop_env/evaluators/metrics/chrome.py

@@ -1,6 +1,6 @@
-import logging, re
-from typing import Any, Dict, List
-
+import logging, re, os, shutil
+from typing import Any, Dict, List, Union
+from bs4 import BeautifulSoup, Tag
 import fitz  # PyMuPDF
 import rapidfuzz.fuzz as fuzz
 
@@ -14,7 +14,6 @@ def is_expected_tabs(open_tabs: List[Dict[str, str]], rule: Dict[str, Any]) -> f
     Checks if the expected tabs are open in Chrome.
     """
 
-    print(open_tabs, rule)
     match_type = rule['type']
 
     if match_type == "url":
@@ -53,10 +52,12 @@ def is_expected_search_query(active_tab_info: Dict[str, str], rules: Dict[str, A
     return 0.
 
 
-def compare_pdfs(pdf1_path, pdf2_path):
+def compare_pdfs(pdf1_path: Union[str, List[str]], pdf2_path: Union[str, List[str]]):
     """
     Compare two PDF files.
     """
+    if type(pdf2_path) != list:
+        pdf1_path, pdf2_path = [pdf1_path], [pdf2_path]
 
     def extract_text_from_pdf(pdf_path):
         """Extract text from each page of the PDF."""
@@ -65,14 +66,100 @@ def compare_pdfs(pdf1_path, pdf2_path):
             for page in pdf:
                 text += page.get_text()
         return text.strip()
-    try:
-        text1 = extract_text_from_pdf(pdf1_path)
-        text2 = extract_text_from_pdf(pdf2_path)
 
-        return fuzz.ratio(text1, text2) / 100
-    except Exception as e:
-        logger.info(f"[ERROR]: unexpected error occurred when comparing PDF files: {e}")
-        return 0.0
+    score = 0.
+    for path1, path2 in zip(pdf1_path, pdf2_path):
+        try:
+            text1 = extract_text_from_pdf(path1)
+            text2 = extract_text_from_pdf(path2)
+            score += fuzz.ratio(text1, text2) / 100
+        except Exception as e:
+            logger.info(f"[ERROR]: unexpected error occurred when comparing PDF files: {e}")
+    return score / len(pdf2_path)
+
+
+def compare_archive(pred_path: str, gold_path: str, **kwargs) -> float:
+    """
+    Compare two archives. Note that the files in the archives should be of the same type.
+    """
+    if not pred_path: return 0.
+    pred_folder = os.path.splitext(pred_path)[0] + '_pred'
+    gold_folder = os.path.splitext(gold_path)[0] + '_gold'
+
+    if os.path.exists(pred_folder): # remove existing folder for new predictions
+        shutil.rmtree(pred_folder, ignore_errors=True)
+    os.makedirs(pred_folder)
+    shutil.unpack_archive(pred_path, pred_folder)
+    if not os.path.exists(gold_folder): # use cache if exists
+        os.makedirs(gold_folder)
+        shutil.unpack_archive(gold_path, gold_folder)
+    
+    pred_files = sorted(os.listdir(pred_folder))
+    gold_files = sorted(os.listdir(gold_folder))
+    if pred_files != gold_files: return 0.
+
+    def get_compare_function():
+        file_type = kwargs.pop('file_type', 'text')
+        if file_type == 'text':
+            from .vscode import compare_text_file
+            return compare_text_file
+        elif file_type == 'pdf': return compare_pdfs
+        elif file_type == 'docx':
+            from .docs import compare_docx_files
+            return compare_docx_files
+        elif file_type == 'ppt':
+            from .slides import compare_pptx_files
+            return compare_pptx_files
+        elif file_type == 'image':
+            from .vlc import compare_images
+            return compare_images
+        elif file_type == 'csv':
+            from .table import compare_csv
+            return compare_csv
+        elif file_type == 'table':
+            from .table import compare_table
+            return compare_table
+        elif file_type == 'audio':
+            from .vlc import compare_audios
+            return compare_audios
+        elif file_type == 'video':
+            from .vlc import compare_videos
+            return compare_videos
+        else: raise ValueError('[ERROR]: not support file type: %s' % file_type)
+
+    score = 0
+    compare_function = get_compare_function()
+    for f1, f2 in zip(pred_files, gold_files):
+        fp1 = os.path.join(pred_folder, f1)
+        fp2 = os.path.join(gold_folder, f2)
+        score += compare_function(fp1, fp2, **kwargs)
+    return score / len(pred_files)
+
+
+def compare_htmls(html_path1: str, html_path2: str) -> float:
+    """
+    Compare two HTML files.
+    """
+    with open(html_path1, 'r', encoding='utf-8') as inf:
+        soup1 = BeautifulSoup(inf, 'lxml')
+    with open(html_path2, 'r', encoding='utf-8') as inf:
+        soup2 = BeautifulSoup(inf, 'lxml')
+    
+    def compare_elements(elem1, elem2):
+        if not (isinstance(elem1, Tag) and isinstance(elem2, Tag)):
+            return elem1 == elem2
+        if elem1.name != elem2.name:
+            return False
+        if elem1.text.strip() != elem2.text.strip():
+            return False
+        if elem1.attrs != elem2.attrs:
+            return False
+        return True
+
+    for elem1, elem2 in zip(soup1.recursiveChildGenerator(), soup2.recursiveChildGenerator()):
+        if not compare_elements(elem1, elem2):
+            return .0
+    return 1.
 
 
 def is_cookie_deleted(cookie_data, rule):