Fix multiple examples

desktop_env/evaluators/metrics/calc.py

@@ -1,8 +1,13 @@
+import logging
+from typing import List
+
 import openpyxl
 
-def compare_conference_city_in_order( actual_city_list_path, expected_city):
+logger = logging.getLogger("desktopenv.metrics.calc")
+
+
+def compare_conference_city_in_order(actual_city_list_path, expected_city):
     expected_city_list = expected_city["expected"]
-    print(f"Reading csv file from {actual_city_list_path}")
     wb = openpyxl.load_workbook(actual_city_list_path)
     sheet = wb.active
     actual_city_list = []
@@ -12,16 +17,25 @@ def compare_conference_city_in_order( actual_city_list_path, expected_city):
     # expected_city is the city that we want to compare with the actual city list
     # must in order index
     # debug
-    print("expected_city_list:")
-    print(expected_city_list)
-    print("actual_city_list_path:")
-    print(actual_city_list)
-    wrong_list = []
     try:
         for i in range(len(actual_city_list)):
-            if expected_city_list[i] not in actual_city_list[i]:
-                wrong_list.append(i)
-                print(f"Expected city {expected_city_list[i]}; Actual city {actual_city_list[i]}")
+            if isinstance(expected_city_list[i], str):
+                if expected_city_list[i] not in actual_city_list[i]:
+                    logger.debug(f"Expected city {expected_city_list[i]}; Actual city {actual_city_list[i]}")
+                    print(f"Expected city {expected_city_list[i]}; Actual city {actual_city_list[i]}")
+                    return 0.
+
+
+            elif isinstance(expected_city_list[i], List):
+                if not any(possible_str in actual_city_list[i] for possible_str in expected_city_list[i]):
+                    logger.debug(f"Expected city {expected_city_list[i]}; Actual city {actual_city_list[i]}")
+                    print(f"Expected city {expected_city_list[i]}; Actual city {actual_city_list[i]}")
+                    return 0.
+
+            else:
+                raise TypeError("Expected city should be a string or a list of strings")
+
     except:
-        return False
-    return True if len(wrong_list) == 0 else False
+        return 0.
+
+    return 1.

desktop_env/evaluators/metrics/chrome.py

@@ -65,7 +65,16 @@ def is_expected_installed_extensions(installed_extensions, expected) -> float:
     print("installed_extensions: ")
     print(installed_extensions)
     expected_extensions = expected["expected"]
-    return 1 if expected_extensions == installed_extensions else 0. # must equal, no additional extensions allowed  
+
+    # whether the expected extensions are installed
+    set_expected_extensions = set(expected_extensions)
+    set_installed_extensions = set(installed_extensions)
+
+    if set_expected_extensions.issubset(set_installed_extensions):
+        return 1.
+    else:
+        return 0.
+
 
 def is_expected_tabs(open_tabs: List[Dict[str, str]], rule: Dict[str, Any]) -> float:
     """

evaluation_examples/examples/multi_apps/6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a.json

@@ -51,7 +51,7 @@
 					"command": [
 						"python",
 						"-c",
-						"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5); pyautogui.press(\"enter\");"
+						"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5);"
 					]
 				}
 			}
@@ -60,7 +60,7 @@
       "expected": {
         "type": "rule",
         "rules":{
-          "expected": ["Scottsdale","Atlanta","Lake Tahoe","Banff","Beijing","Montreal","San Diego","Lille","Montreal","San Juan","New York","Barcelona","Toulon","Sydney","Long Beach","Vancouver","Stockholm","Montréal","New Orleans","Long Beach","Vancouver"]
+          "expected": ["Scottsdale","Atlanta","Lake Tahoe","Banff","Beijing",["Montreal", "Montréal"],"San Diego","Lille",["Montreal", "Montréal"],"San Juan",["New York", "New York City", "NYC"],"Barcelona","Toulon","Sydney","Long Beach","Vancouver","Stockholm",["Montreal", "Montréal"],"New Orleans","Long Beach","Vancouver"]
         }
       },
       "result": {

evaluation_examples/examples/multi_apps/873cafdd-a581-47f6-8b33-b9696ddb7b05.json

@@ -56,7 +56,7 @@
       "expected": {
         "type": "rule",
         "rules":{
-          "expected": ["Zoom Chrome Extension", "Speechify Text to Speech Voice Reader", "React Developer Tools", "Momentum", "Google Translate", "Web Store", "Chromium PDF Viewer", "Google Hangouts"]
+          "expected": ["Zoom Chrome Extension", "Speechify Text to Speech Voice Reader", "React Developer Tools", "Momentum", "Google Translate"]
         }
       },
       "result": {