Update OS evals

This commit is contained in:
Liu Yitao
2024-01-25 10:45:51 +08:00
parent 09f3e776ae
commit 93b4ff7d95
10 changed files with 311 additions and 0 deletions

View File

@@ -16,6 +16,11 @@ from .utils import _match_record, _match_value_to_rule
import sqlite3
def check_include_exclude(result: str, rules: Dict[str, List[str]]) -> float:
include = rules.get("include", [])
exclude = rules.get("exclude", [])
return all(r in result for r in include) and all(r not in result for r in exclude)
def exact_match(result, rules) -> float:
expect = rules["expected"]
print(result, expect)
@@ -195,3 +200,4 @@ def check_json(result: str, rules: Dict[str, List[Dict[str, Union[List[str], str
value = value[k]
metric = metric and not _match_value_to_rule(value, r)
return metric