Fix multiple examples

desktop_env/evaluators/metrics/calc.py

@@ -1,8 +1,13 @@
+import logging
+from typing import List
+
 import openpyxl
 
-def compare_conference_city_in_order( actual_city_list_path, expected_city):
+logger = logging.getLogger("desktopenv.metrics.calc")
+
+
+def compare_conference_city_in_order(actual_city_list_path, expected_city):
     expected_city_list = expected_city["expected"]
-    print(f"Reading csv file from {actual_city_list_path}")
     wb = openpyxl.load_workbook(actual_city_list_path)
     sheet = wb.active
     actual_city_list = []
@@ -12,16 +17,25 @@ def compare_conference_city_in_order( actual_city_list_path, expected_city):
     # expected_city is the city that we want to compare with the actual city list
     # must in order index
     # debug
-    print("expected_city_list:")
-    print(expected_city_list)
-    print("actual_city_list_path:")
-    print(actual_city_list)
-    wrong_list = []
     try:
         for i in range(len(actual_city_list)):
-            if expected_city_list[i] not in actual_city_list[i]:
-                wrong_list.append(i)
-                print(f"Expected city {expected_city_list[i]}; Actual city {actual_city_list[i]}")
+            if isinstance(expected_city_list[i], str):
+                if expected_city_list[i] not in actual_city_list[i]:
+                    logger.debug(f"Expected city {expected_city_list[i]}; Actual city {actual_city_list[i]}")
+                    print(f"Expected city {expected_city_list[i]}; Actual city {actual_city_list[i]}")
+                    return 0.
+
+
+            elif isinstance(expected_city_list[i], List):
+                if not any(possible_str in actual_city_list[i] for possible_str in expected_city_list[i]):
+                    logger.debug(f"Expected city {expected_city_list[i]}; Actual city {actual_city_list[i]}")
+                    print(f"Expected city {expected_city_list[i]}; Actual city {actual_city_list[i]}")
+                    return 0.
+
+            else:
+                raise TypeError("Expected city should be a string or a list of strings")
+
     except:
-        return False
-    return True if len(wrong_list) == 0 else False
+        return 0.
+
+    return 1.

desktop_env/evaluators/metrics/chrome.py

@@ -65,7 +65,16 @@ def is_expected_installed_extensions(installed_extensions, expected) -> float:
     print("installed_extensions: ")
     print(installed_extensions)
     expected_extensions = expected["expected"]
-    return 1 if expected_extensions == installed_extensions else 0. # must equal, no additional extensions allowed  
+
+    # whether the expected extensions are installed
+    set_expected_extensions = set(expected_extensions)
+    set_installed_extensions = set(installed_extensions)
+
+    if set_expected_extensions.issubset(set_installed_extensions):
+        return 1.
+    else:
+        return 0.
+
 
 def is_expected_tabs(open_tabs: List[Dict[str, str]], rule: Dict[str, Any]) -> float:
     """