ver Jan10thv2

a new example config for Thunderbird
fixed several bugs

desktop_env/evaluators/metrics/general.py

@@ -4,13 +4,14 @@ import lxml.etree
 from lxml.etree import _Element
 from lxml.cssselect import CSSSelector
 
-from typing import Dict, List
+from typing import Dict, List, Pattern
 from typing import Callable, Any
 from numbers import Number
 
 import operator
 from rapidfuzz import fuzz
 import functools
+import re
 
 def _match_record(pattern: Dict[str, str], item: Dict[str, str]) -> float:
     return all(k in item and item[k]==val for k, val in pattern.items())
@@ -37,7 +38,33 @@ def check_csv(result: str, rules: Dict[str, List[Dict[str, str]]]) -> float:
         for rcd in reader:
             for i, r in enumerate(rules.get("expect", [])):
                 expect_metrics[i] = expect_metrics[i] or _match_record(r, rcd)
-            unexpect_metric = unexpect_metric and all(_match_record(r, rcd) for r in rules.get("unexpect", []))
+            unexpect_metric = unexpect_metric and not any(_match_record(r, rcd) for r in rules.get("unexpect", []))
+    return float(all(expect_metrics) and unexpect_metric)
+
+def check_list(result: str, rules: Dict[str, List[str]]) -> float:
+    """
+    Args:
+        result (str): path to list file
+        rules (Dict[str, List[str]]): dict like
+          {
+            "expect": list of str as regexes
+            "unexpect": list of str as regexes
+          }
+
+    Returns:
+        float
+    """
+
+    expect_patterns: List[Pattern[str]] = [re.compile(ptt) for ptt in rules.get("expect", [])]
+    unexpect_patterns: List[Pattern[str]] = [re.compile(ptt) for ptt in rules.get("unexpect", [])]
+
+    expect_metrics = [False] * len(expect_patterns)
+    unexpect_metric = True
+    with open(result) as f:
+        for l in f:
+            for i, r in enumerate(expect_patterns):
+                expect_metrics[i] = expect_metrics[i] or (r.search(l) is not None)
+            unexpect_metric = unexpect_metric and all(r.search(l) is None for r in unexpect_patterns)
     return float(all(expect_metrics) and unexpect_metric)
 
 _accessibility_ns_map = { "st": "uri:deskat:state.at-spi.gnome.org"
@@ -93,3 +120,6 @@ def check_accessibility_tree(result: str, rules: Dict[str, Any]) -> float:
         match_score = 1.
 
     return float(match_score)
+
+#def check_existence(result: str, *args) -> float:
+    #return 1. - (result is None)