Fix a bug in multiple apps example

desktop_env/evaluators/metrics/chrome.py

@@ -2,9 +2,9 @@ import logging
 import os
 import re
 import shutil
+from itertools import product
 from typing import Any, Dict, List, Union
 
-import fitz  # PyMuPDF
 import rapidfuzz.fuzz as fuzz
 from bs4 import BeautifulSoup, Tag
 
@@ -99,7 +99,19 @@ def is_expected_bookmarks(bookmarks: List[str], rule: Dict[str, Any]) -> float:
             # Check if it contains the specified URLs
             liked_authors_urls = [bookmark['url'] for bookmark in liked_authors_folder['children'] if
                                   bookmark['type'] == 'url']
-            return 1. if set(liked_authors_urls) == set(rule['urls']) else 0.
+
+            urls = rule['urls']
+
+            for idx, url in enumerate(urls):
+                if isinstance(url, str):
+                    urls[idx] = [url]
+
+            combinations = product(*urls)
+
+            for combination in combinations:
+                if set(combination) == set(liked_authors_urls):
+                    return 1.
+            return 0.
         else:
             return 0.
     else:
@@ -140,15 +152,16 @@ def compare_pdfs(pdf1_path: Union[str, List[str]], pdf2_path: Union[str, List[st
             logger.info(f"[ERROR]: unexpected error occurred when comparing PDF files: {e}")
     return score / len(pdf2_path)
 
+
 import fitz
 from PIL import Image
-from io import BytesIO
 from borb.pdf import Document
 from borb.pdf import PDF
 
 from pathlib import Path
 import typing
 
+
 def compare_pdf_images(pdf1_path: str, pdf2_path: str, **kwargs) -> float:
     def extract_images_from_pdf(pdf_path):
         pdf_document = fitz.open(pdf_path)

evaluation_examples/examples/multi_apps/a82b78bb-7fde-4cb3-94a4-035baf10bcf0.json

@@ -23,6 +23,12 @@
         ]
       }
     },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 2
+      }
+    },
     {
       "type": "download",
       "parameters": {
@@ -75,10 +81,10 @@
           "Liked Authors"
         ],
         "urls": [
-          "https://jimfan.me/",
-          "https://ai.stanford.edu/~dahuang/",
-          "https://yukezhu.me/",
-          "https://www.eas.caltech.edu/people/anima"
+          ["https://jimfan.me/", "https://research.nvidia.com/person/linxi-jim-fan"],
+          ["https://research.nvidia.com/person/de-an-huang", "https://ai.stanford.edu/~dahuang/"],
+          ["https://yukezhu.me/", "https://www.cs.utexas.edu/people/faculty-researchers/yuke-zhu", "https://experts.utexas.edu/yuke_zhu", "https://research.nvidia.com/person/yuke-zhu"],
+          ["http://tensorlab.cms.caltech.edu/users/anima/", "https://www.eas.caltech.edu/people/anima"]
         ]
       }
     }