ver Jan10thv2

a new example config for Thunderbird
fixed several bugs

desktop_env/controllers/setup.py

@@ -242,6 +242,9 @@ class SetupController:
             logger.error("An error occurred while trying to send the request: %s", e)
             traceback.print_exc()
 
+    def _command_setup(self, command: List[str], stdout: str = "", stderr: str = ""):
+        self._execute_setup(command, stdout, stderr)
+
     def _act_setup(self, action_seq: List[Union[Dict[str, Any], str]]):
         # TODO
         raise NotImplementedError()

desktop_env/evaluators/getters/file.py

@@ -1,4 +1,5 @@
 from typing import Dict
+from typing import Optional
 
 import os
 import requests
@@ -27,7 +28,7 @@ def get_cloud_file(env, config: Dict[str, str]) -> str:
     return _path
 
 
-def get_vm_file(env, config: Dict[str, str]) -> str:
+def get_vm_file(env, config: Dict[str, str]) -> Optional[str]:
     """
     Config:
         path (str): absolute path on the VM to fetch
@@ -37,6 +38,8 @@ def get_vm_file(env, config: Dict[str, str]) -> str:
     _path = os.path.join(env.cache_dir, config["dest"])
 
     file = env.controller.get_file(config["path"])
+    if file is None:
+        return None
     with open(_path, "wb") as f:
         f.write(file)
 

desktop_env/evaluators/metrics/__init__.py

@@ -6,4 +6,4 @@ from .docs import is_first_line_centered, check_file_exists, compare_contains_im
 from .pdf import check_pdf_pages
 from .libreoffice import check_libre_locale
 #from .vlc import is_vlc_playing
-from .general import check_csv, check_accessibility_tree
+from .general import check_csv, check_accessibility_tree, check_list

desktop_env/evaluators/metrics/general.py

@@ -4,13 +4,14 @@ import lxml.etree
 from lxml.etree import _Element
 from lxml.cssselect import CSSSelector
 
-from typing import Dict, List
+from typing import Dict, List, Pattern
 from typing import Callable, Any
 from numbers import Number
 
 import operator
 from rapidfuzz import fuzz
 import functools
+import re
 
 def _match_record(pattern: Dict[str, str], item: Dict[str, str]) -> float:
     return all(k in item and item[k]==val for k, val in pattern.items())
@@ -37,7 +38,33 @@ def check_csv(result: str, rules: Dict[str, List[Dict[str, str]]]) -> float:
         for rcd in reader:
             for i, r in enumerate(rules.get("expect", [])):
                 expect_metrics[i] = expect_metrics[i] or _match_record(r, rcd)
-            unexpect_metric = unexpect_metric and all(_match_record(r, rcd) for r in rules.get("unexpect", []))
+            unexpect_metric = unexpect_metric and not any(_match_record(r, rcd) for r in rules.get("unexpect", []))
+    return float(all(expect_metrics) and unexpect_metric)
+
+def check_list(result: str, rules: Dict[str, List[str]]) -> float:
+    """
+    Args:
+        result (str): path to list file
+        rules (Dict[str, List[str]]): dict like
+          {
+            "expect": list of str as regexes
+            "unexpect": list of str as regexes
+          }
+
+    Returns:
+        float
+    """
+
+    expect_patterns: List[Pattern[str]] = [re.compile(ptt) for ptt in rules.get("expect", [])]
+    unexpect_patterns: List[Pattern[str]] = [re.compile(ptt) for ptt in rules.get("unexpect", [])]
+
+    expect_metrics = [False] * len(expect_patterns)
+    unexpect_metric = True
+    with open(result) as f:
+        for l in f:
+            for i, r in enumerate(expect_patterns):
+                expect_metrics[i] = expect_metrics[i] or (r.search(l) is not None)
+            unexpect_metric = unexpect_metric and all(r.search(l) is None for r in unexpect_patterns)
     return float(all(expect_metrics) and unexpect_metric)
 
 _accessibility_ns_map = { "st": "uri:deskat:state.at-spi.gnome.org"
@@ -93,3 +120,6 @@ def check_accessibility_tree(result: str, rules: Dict[str, Any]) -> float:
         match_score = 1.
 
     return float(match_score)
+
+#def check_existence(result: str, *args) -> float:
+    #return 1. - (result is None)

desktop_env/evaluators/metrics/table.py

@@ -31,6 +31,9 @@ def compare_table(actual: str, expected: str, **options) -> float:
         float: the score
     """
 
+    if actual is None:
+        return 0.
+
     df1 = pd.read_excel(expected)
     df2 = pd.read_excel(actual)
     metric: bool = df1.equals(df2)
@@ -71,6 +74,9 @@ def compare_table(actual: str, expected: str, **options) -> float:
     return float(metric)
 
 def check_sheet_list(result: str, rules: List[Dict[str, Any]]) -> float:
+    if result is None:
+        return 0.
+
     # workbook: Workbook = openpyxl.load_workbook(filename=result)
     workbook = pd.ExcelFile(result)
     worksheet_names: List[str] = workbook.sheet_names
@@ -109,10 +115,16 @@ def check_sheet_list(result: str, rules: List[Dict[str, Any]]) -> float:
     return float(passes)
 
 def check_xlsx_freeze(result: str, rules: Dict[str, str]) -> float:
+    if result is None:
+        return 0.
+
     worksheet: Worksheet = openpyxl.load_workbook(filename=result).active
     return float(worksheet.freeze_panes == rules["position"])
 
 def check_xlsx_zoom(result: str, rules: Dict[str, Union[str, Number]]) -> float:
+    if result is None:
+        return 0.
+
     worksheet = openpyxl.load_workbook(filename=result).active
     zoom_scale: Number = worksheet.sheet_view.zoomScale or 100.
     return float( getattr(operator, rules["relation"])( zoom_scale