ver Feb28th

a new multi app task --- init a web extension project with web tool

desktop_env/evaluators/metrics/__init__.py

@@ -57,7 +57,8 @@ from .general import (
     is_in_list,
     fuzzy_match,
     check_include_exclude,
-    check_direct_json_object
+    check_direct_json_object,
+    diff_text_file
 )
 from .gimp import (
     check_brightness_decrease_and_structure_sim,

desktop_env/evaluators/metrics/general.py

@@ -12,6 +12,7 @@ import lxml.etree
 from lxml.cssselect import CSSSelector
 from lxml.etree import _Element
 from rapidfuzz import fuzz
+import difflib
 
 from .utils import _match_record, _match_value_to_rule
 
@@ -45,7 +46,15 @@ def is_in_list(result, rules) -> float:
     else:
         return 0.
 
+def diff_text_file(result: str, expect: str) -> float:
+    if result is None:
+        return 0.
 
+    with open(result) as f:
+        result_lines: List[str] = f.read().splitlines()
+    with open(expect) as f:
+        expected_lines: List[str] = f.read().splitlines()
+    return difflib.SequenceMatcher(a=result_lines, b=expected_lines).ratio()
 
 def fuzzy_match(result, rules) -> float:
     expect = rules["expected"]
@@ -212,14 +221,21 @@ def check_json(result: str, rules: Dict[str, List[Dict[str, Union[List[str], str
     for r in expect_rules:
         value = result
         for k in r["key"]:
-            value = value[k]
+            try:
+                value = value[k]
+            except KeyError:
+                return 0.
         metric = metric and _match_value_to_rule(value, r)
     for r in unexpect_rules:
         value = result
         for k in r["key"]:
-            value = value[k]
+            try:
+                value = value[k]
+            except KeyError:
+                value = None
+                break
         metric = metric and not _match_value_to_rule(value, r)
-    return metric
+    return float(metric)
 
 
 def check_direct_json_object(result, rules)->float:
@@ -238,4 +254,4 @@ def check_direct_json_object(result, rules)->float:
         expected_value = expected_json.get(key)
         if expected_value != result.get(key):
             return 0.
-    return 1.0
+    return 1.0

desktop_env/evaluators/metrics/utils.py

@@ -587,7 +587,7 @@ def _match_value_to_rule(value: V, rule: Dict[str, Union[str, V]]) -> bool:
         bool
     """
 
-    if rule["method"].startswith("re"):
+    if rule["method"].startswith("re"): # re.FLAGs
         flags: List[str] = rule["method"].split(".")[1:]
         flags: Iterable[re.RegexFlag] = (getattr(re, fl) for fl in flags)
         flag: re.RegexFlag = functools.reduce(operator.or_, flags, re.RegexFlag(0))
@@ -600,7 +600,7 @@ def _match_value_to_rule(value: V, rule: Dict[str, Union[str, V]]) -> bool:
         , "ge", "gt"
                           }:
         return getattr(operator, rule["method"])(value, rule["ref"])
-    if rule["method"].startswith("approx"):
+    if rule["method"].startswith("approx"): # approx:THRESHOLD
         threshold: float = float(rule["method"].split(":")[1])
         logger.debug("Approx: TH%f, REF%f, VAL%s", threshold, rule["ref"], repr(value))
         try: