lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Code clean

Browse Source
This commit is contained in:
Timothyxxx
2024-03-14 12:54:10 +08:00
parent 26d52a7231
commit 0aae756538
12 changed files with 120 additions and 384 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
desktop_env/evaluators/metrics/__init__.py
View File

@@ -114,7 +114,8 @@ from .slides import (
) )
from .table import ( from .table import (
compare_table, compare_table,
compare_csv compare_csv,
compare_conference_city_in_order
) )
from .thunderbird import ( from .thunderbird import (
check_thunderbird_prefs, check_thunderbird_prefs,
@@ -148,7 +149,6 @@ from .vscode import (
check_html_background_image, check_html_background_image,
compare_zip_files compare_zip_files
) )
from .calc import compare_conference_city_in_order
from .others import compare_epub, check_mp3_meta from .others import compare_epub, check_mp3_meta
def infeasible(): def infeasible():

4
desktop_env/evaluators/metrics/basic_os.py
View File

@@ -1,6 +1,3 @@
import subprocess
def check_gnome_favorite_apps(apps_str: str, rule): def check_gnome_favorite_apps(apps_str: str, rule):
# parse the string like "['thunderbird.desktop', 'vim.desktop', 'google-chrome.desktop']" # parse the string like "['thunderbird.desktop', 'vim.desktop', 'google-chrome.desktop']"
# to a list of strings # to a list of strings
@@ -57,6 +54,7 @@ def check_moved_jpgs(directory_list, rule):
else: else:
return 0 return 0
def is_in_vm_clickboard(config, terminal_output): def is_in_vm_clickboard(config, terminal_output):
print("terminal_output: ") print("terminal_output: ")
print(terminal_output) print(terminal_output)

41
desktop_env/evaluators/metrics/calc.py
View File

@@ -1,41 +0,0 @@
import logging
from typing import List
import openpyxl
logger = logging.getLogger("desktopenv.metrics.calc")
def compare_conference_city_in_order(actual_city_list_path, expected_city):
expected_city_list = expected_city["expected"]
wb = openpyxl.load_workbook(actual_city_list_path)
sheet = wb.active
actual_city_list = []
for row in sheet["C2:C22"]:
for cell in row:
actual_city_list.append(cell.value)
# expected_city is the city that we want to compare with the actual city list
# must in order index
# debug
try:
for i in range(len(actual_city_list)):
if isinstance(expected_city_list[i], str):
if expected_city_list[i] not in actual_city_list[i]:
logger.debug(f"Expected city {expected_city_list[i]}; Actual city {actual_city_list[i]}")
print(f"Expected city {expected_city_list[i]}; Actual city {actual_city_list[i]}")
return 0.
elif isinstance(expected_city_list[i], List):
if not any(possible_str in actual_city_list[i] for possible_str in expected_city_list[i]):
logger.debug(f"Expected city {expected_city_list[i]}; Actual city {actual_city_list[i]}")
print(f"Expected city {expected_city_list[i]}; Actual city {actual_city_list[i]}")
return 0.
else:
raise TypeError("Expected city should be a string or a list of strings")
except:
return 0.
return 1.

28
desktop_env/evaluators/metrics/demo.py
View File

@@ -1,28 +0,0 @@
import fitz # PyMuPDF
def extract_answers_from_pdf(pdf_file):
# 打开PDF文件
doc = fitz.open(pdf_file)
answers = []
# 遍历每一页
for page in doc:
# 提取当前页的文本
text = page.get_text()
# 分割文本为行
lines = text.split('\n')
for line in lines:
if line.strip(): # 排除空白行
# 分割等号，提取答案
parts = line.split('=')
if len(parts) > 1:
answer = parts[-1].strip() # 取等号后的部分为答案
answers.append(answer)
return answers
# 假设你的文件名是'math_problems.pdf'
pdf_file = '/Users/lxc/Desktop/calculus.pdf'
answers = extract_answers_from_pdf(pdf_file)
for i, answer in enumerate(answers, 1):
print(f"题目{i}的答案是: {answer}")

44
desktop_env/evaluators/metrics/gimp.py
View File

@@ -568,47 +568,3 @@ def check_image_file_size(src_path, rule):
return 1.0 return 1.0
else: else:
return 0.0 return 0.0
if __name__ == "__main__":
# actual_config_path = "../../../cache/sessionrc_test"
# rule = {
# "key": "hide-docks",
# "value": "no"
# }
# print(check_config_status(actual_config_path, rule))
#
# actual_config_path = "../../../cache/action-history_test"
# rule = {
# "key": ["history-item", "\"filters-vignette\""],
# "value": "1"
# }
# print(check_config_status(actual_config_path, rule))
#
# actual_config_path = "../../../cache/gimprc_test"
# rule = {
# "key": "undo-levels",
# "value": "100"
# }
# print(check_config_status(actual_config_path, rule))
#
# src_path = "../../../cache/734d6579-c07d-47a8-9ae2-13339795476b/green_background_with_object.png"
# tgt_path = "../../../cache/734d6579-c07d-47a8-9ae2-13339795476b/white_background_with_object.png"
# print(check_green_background(src_path, tgt_path))
#
# tgt_path = "../../../cache/f4aec372-4fb0-4df5-a52b-79e0e2a5d6ce/Triangle_In_The_Middle.png"
# print(check_triangle_position(tgt_path))
#
# src_path = "../../../cache/bb7db4c2-30b5-4be7-8dd7-b8c4ec7d3108/anmi_sharper.png"
# tgt_path = "../../../cache/bb7db4c2-30b5-4be7-8dd7-b8c4ec7d3108/anmi.png"
# print(check_sharper(src_path, tgt_path))
#
# src_path = "../../../cache/3c8f201a-009d-4bbe-8b65-a6f8b35bb57f/compressed.jpeg"
# rule = {
# "max_size": 500000
# }
# print(check_image_file_size(src_path, rule))
src_path = "../../../cache/d68204bf-11c1-4b13-b48b-d303c73d4bf6/12ad623d-7f35-550e-9c44-6830386b20a0_rearranged_gold.png"
tgt_path = "../../../cache/d68204bf-11c1-4b13-b48b-d303c73d4bf6/d1c7c561-6e76-5d7b-9c10-4af0332dfa50_tilearray.png"
print(check_structure_sim(src_path, tgt_path))

10
desktop_env/evaluators/metrics/libreoffice.py
View File

@@ -26,13 +26,3 @@ def check_libre_locale(config_file: str, rules: Dict[str, List[str]]) -> float:
for ptn in rules["locale_set"] for ptn in rules["locale_set"]
) )
) )
if __name__ == "__main__":
path1 = "../../任务数据/LibreOffice Calc/registrymodifications.ru.xcu"
print(check_libre_locale(path1, {"locale_set": ["ru-*", "de-*", "fr-*"
, "pt-*", "es-*", "it-*"
]
}
)
)

72
desktop_env/evaluators/metrics/others.py
View File

@@ -1,20 +1,20 @@
import zipfile import logging
import os.path
import os import os
import os.path
import zipfile
from typing import List, Dict
from typing import Union, TypeVar
import lxml.html import lxml.html
from lxml.html import HtmlElement from lxml.html import HtmlElement
from typing import List, Dict
from typing import Union, TypeVar
from mutagen.easyid3 import EasyID3 from mutagen.easyid3 import EasyID3
from .general import diff_text_file from .general import diff_text_file
from .utils import _match_value_to_rule from .utils import _match_value_to_rule
import logging
logger = logging.getLogger("desktopenv.metric.others") logger = logging.getLogger("desktopenv.metric.others")
def process_epub(filename: str) -> List[str]: def process_epub(filename: str) -> List[str]:
file_list: List[str] = [] file_list: List[str] = []
@@ -23,7 +23,7 @@ def process_epub(filename: str) -> List[str]:
try: try:
with zipfile.ZipFile(filename, "r") as z_f: with zipfile.ZipFile(filename, "r") as z_f:
with z_f.open("toc.ncx") as in_f\ with z_f.open("toc.ncx") as in_f \
, open(os.path.join(base_dir, "toc.ncx"), "w") as out_f: , open(os.path.join(base_dir, "toc.ncx"), "w") as out_f:
contents: str = in_f.read().decode() contents: str = in_f.read().decode()
contents = contents.splitlines() contents = contents.splitlines()
@@ -31,7 +31,7 @@ def process_epub(filename: str) -> List[str]:
if "navPoint" not in l: if "navPoint" not in l:
out_f.write(l + "\n") out_f.write(l + "\n")
file_list.append(os.path.join(base_dir, "toc.ncx")) file_list.append(os.path.join(base_dir, "toc.ncx"))
with z_f.open("content.opf") as in_f\ with z_f.open("content.opf") as in_f \
, open(os.path.join(base_dir, "content.opf"), "w") as out_f: , open(os.path.join(base_dir, "content.opf"), "w") as out_f:
contents: str = in_f.read().decode() contents: str = in_f.read().decode()
contents = contents.splitlines() contents = contents.splitlines()
@@ -41,14 +41,14 @@ def process_epub(filename: str) -> List[str]:
file_list.append(os.path.join(base_dir, "content.opf")) file_list.append(os.path.join(base_dir, "content.opf"))
for f_n in z_f.namelist(): for f_n in z_f.namelist():
if f_n.endswith(".html"): if f_n.endswith(".html"):
with z_f.open(f_n) as in_f\ with z_f.open(f_n) as in_f \
, open(os.path.join(base_dir, f_n), "w") as out_f: , open(os.path.join(base_dir, f_n), "w") as out_f:
html: HtmlElement = lxml.html.fromstring( html: HtmlElement = lxml.html.fromstring(
''.join( filter( lambda ch: ch!="\n" and ch!="\r" ''.join(filter(lambda ch: ch != "\n" and ch != "\r"
, in_f.read().decode() , in_f.read().decode()
) )
).encode() ).encode()
) )
out_f.write(lxml.html.tostring(html, pretty_print=True, encoding="unicode")) out_f.write(lxml.html.tostring(html, pretty_print=True, encoding="unicode"))
file_list.append(os.path.join(base_dir, f_n)) file_list.append(os.path.join(base_dir, f_n))
logger.debug("%s: %s", filename, file_list) logger.debug("%s: %s", filename, file_list)
@@ -56,6 +56,7 @@ def process_epub(filename: str) -> List[str]:
except zipfile.BadZipFile: except zipfile.BadZipFile:
return [] return []
def compare_epub(result: str, expected: str) -> float: def compare_epub(result: str, expected: str) -> float:
if result is None: if result is None:
return 0. return 0.
@@ -69,8 +70,10 @@ def compare_epub(result: str, expected: str) -> float:
metric *= current_metric metric *= current_metric
return metric return metric
V = TypeVar("Value") V = TypeVar("Value")
def check_mp3_meta(result: str, meta: Dict[str, Dict[str, Union[str, V]]]) -> bool: def check_mp3_meta(result: str, meta: Dict[str, Dict[str, Union[str, V]]]) -> bool:
# checks using _match_value_to_rule # checks using _match_value_to_rule
if result is None: if result is None:
@@ -85,44 +88,3 @@ def check_mp3_meta(result: str, meta: Dict[str, Dict[str, Union[str, V]]]) -> bo
logger.debug("%s.%s: %s", result, k, value) logger.debug("%s.%s: %s", result, k, value)
metric = metric and _match_value_to_rule(value, r) metric = metric and _match_value_to_rule(value, r)
return float(metric) return float(metric)
if __name__ == "__main__":
import datetime
import sys
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
datetime_str: str = datetime.datetime.now().strftime("%Y%m%d@%H%M%S")
file_handler = logging.FileHandler(os.path.join("logs", "normal-{:}.log".format(datetime_str)))
debug_handler = logging.FileHandler(os.path.join("logs", "debug-{:}.log".format(datetime_str)))
stdout_handler = logging.StreamHandler(sys.stdout)
sdebug_handler = logging.FileHandler(os.path.join("logs", "sdebug-{:}.log".format(datetime_str)))
file_handler.setLevel(logging.INFO)
debug_handler.setLevel(logging.DEBUG)
stdout_handler.setLevel(logging.INFO)
sdebug_handler.setLevel(logging.DEBUG)
formatter = logging.Formatter(fmt="\x1b[1;33m[%(asctime)s \x1b[31m%(levelname)s \x1b[32m%(module)s/%(lineno)d-%(processName)s\x1b[1;33m] \x1b[0m%(message)s")
file_handler.setFormatter(formatter)
debug_handler.setFormatter(formatter)
stdout_handler.setFormatter(formatter)
sdebug_handler.setFormatter(formatter)
logger.addHandler(file_handler)
logger.addHandler(debug_handler)
logger.addHandler(stdout_handler)
logger.addHandler(sdebug_handler)
metric = check_mp3_meta( "snapshots/test/cache/3f05f3b9-29ba-4b6b-95aa-2204697ffc06/Cheng Xiang - Missing You - gt.mp3"
, { "title": { "method": "eq"
, "ref": "Missing You"
}
, "artist": { "method": "eq"
, "ref": "Cheng Xiang"
}
}
)
print(metric)

18
desktop_env/evaluators/metrics/pdf.py
View File

@@ -2,6 +2,7 @@ import operator
from typing import Any from typing import Any
from typing import Dict from typing import Dict
import fitz # PyMuPDF
from pypdf import PdfReader from pypdf import PdfReader
@@ -11,3 +12,20 @@ def check_pdf_pages(pdf_file: str, rules: Dict[str, Any]) -> float:
reader = PdfReader(pdf_file) reader = PdfReader(pdf_file)
nb_pages: int = len(reader.pages) nb_pages: int = len(reader.pages)
return float(getattr(operator, rules["relation"])(nb_pages, rules["ref_value"])) return float(getattr(operator, rules["relation"])(nb_pages, rules["ref_value"]))
def extract_answers_from_pdf(pdf_file):
doc = fitz.open(pdf_file)
answers = []
for page in doc:
text = page.get_text()
lines = text.split('\n')
for line in lines:
if line.strip():
parts = line.split('=')
if len(parts) > 1:
answer = parts[-1].strip()
answers.append(answer)
return answers

33
desktop_env/evaluators/metrics/slides.py
View File

@@ -165,6 +165,7 @@ def compare_pptx_files(file1_path, file2_path, **options):
# compare the content of each slide # compare the content of each slide
for slide1, slide2 in zip(prs1.slides, prs2.slides): for slide1, slide2 in zip(prs1.slides, prs2.slides):
slide_idx += 1 slide_idx += 1
def get_slide_background_color(slide): def get_slide_background_color(slide):
background = slide.background background = slide.background
if background.fill.background(): if background.fill.background():
@@ -207,17 +208,19 @@ def compare_pptx_files(file1_path, file2_path, **options):
if examine_top_position: if examine_top_position:
if slide_idx == 2 and shape1.shape_type == 13 and shape2.shape_type == 13: if slide_idx == 2 and shape1.shape_type == 13 and shape2.shape_type == 13:
if shape1.top >= shape2.top or shape1.top > 1980000: if shape1.top >= shape2.top or shape1.top > 1980000:
return 0 return 0
elif shape1.left != shape2.left or shape1.top != shape2.top or shape1.width != shape2.width or shape1.height != shape2.height: elif shape1.left != shape2.left or shape1.top != shape2.top or shape1.width != shape2.width or shape1.height != shape2.height:
return 0 return 0
if examine_shape_for_shift_size: if examine_shape_for_shift_size:
if shape1.left != shape2.left or shape1.top != shape2.top or shape1.width != shape2.width or shape1.height != shape2.height: if shape1.left != shape2.left or shape1.top != shape2.top or shape1.width != shape2.width or shape1.height != shape2.height:
if not (hasattr(shape1, "text") and hasattr(shape2, "text") and shape1.text == shape2.text and shape1.text == "Elaborate on what you want to discuss."): if not (hasattr(shape1, "text") and hasattr(shape2,
"text") and shape1.text == shape2.text and shape1.text == "Elaborate on what you want to discuss."):
return 0 return 0
if (shape1.left != shape2.left or shape1.top != shape2.top or shape1.width != shape2.width or shape1.height != shape2.height) and examine_shape: if (
shape1.left != shape2.left or shape1.top != shape2.top or shape1.width != shape2.width or shape1.height != shape2.height) and examine_shape:
return 0 return 0
if examine_image_size: if examine_image_size:
@@ -228,7 +231,8 @@ def compare_pptx_files(file1_path, file2_path, **options):
return 0 return 0
if examine_modify_height: if examine_modify_height:
if not hasattr(shape1, "text") and not hasattr(shape2, "text") or shape1.shape_type == 5 and shape2.shape_type == 5: if not hasattr(shape1, "text") and not hasattr(shape2,
"text") or shape1.shape_type == 5 and shape2.shape_type == 5:
if shape1.height != shape2.height: if shape1.height != shape2.height:
return 0 return 0
elif shape1.left != shape2.left or shape1.top != shape2.top or shape1.width != shape2.width or shape1.height != shape2.height: elif shape1.left != shape2.left or shape1.top != shape2.top or shape1.width != shape2.width or shape1.height != shape2.height:
@@ -238,7 +242,7 @@ def compare_pptx_files(file1_path, file2_path, **options):
if shape1.text.strip() != shape2.text.strip() and examine_text: if shape1.text.strip() != shape2.text.strip() and examine_text:
return 0 return 0
# check if the paragraphs are the same # check if the paragraphs are the same
for para1, para2 in zip(shape1.text_frame.paragraphs, shape2.text_frame.paragraphs): for para1, para2 in zip(shape1.text_frame.paragraphs, shape2.text_frame.paragraphs):
if para1.alignment != para2.alignment and examine_alignment: if para1.alignment != para2.alignment and examine_alignment:
return 0 return 0
@@ -305,11 +309,10 @@ def compare_pptx_files(file1_path, file2_path, **options):
return bullets return bullets
if examine_bullets and _extract_bullets(run1.part.blob.decode('utf-8')) != _extract_bullets(run2.part.blob.decode('utf-8')): if examine_bullets and _extract_bullets(run1.part.blob.decode('utf-8')) != _extract_bullets(
run2.part.blob.decode('utf-8')):
return 0 return 0
# fixme: Actually there are more properties to be compared, we can add them later via parsing the xml data # fixme: Actually there are more properties to be compared, we can add them later via parsing the xml data
return 1 return 1
@@ -524,15 +527,3 @@ def check_auto_saving_time(pptx_file, rules):
logger.error(f"Error parsing XML: {e}") logger.error(f"Error parsing XML: {e}")
except FileNotFoundError: except FileNotFoundError:
logger.error(f"File not found: {pptx_file}") logger.error(f"File not found: {pptx_file}")
if __name__ == '__main__':
# print(compare_pptx_files(
# r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\550ce7e7-747b-495f-b122-acdc4d0b8e54\New_Club_Spring_2018_Training_Gold.pptx",
# r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\550ce7e7-747b-495f-b122-acdc4d0b8e54\New_Club_Spring_2018_Training_Gold.pptx"))
# print(evaluate_presentation_fill_to_rgb_distance(r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\3b27600c-3668-4abd-8f84-7bcdebbccbdb\lec17-gui-events.pptx", {"rgb": (0, 0, 255)}))
# print(check_auto_saving_time(r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\2cd43775-7085-45d8-89fa-9e35c0a915cf\registrymodifications.xcu", {"minutes": 3}))
print(compare_pptx_files(
r"D:\NJU\HKUNLP\Desktop-Env\DesktopEnv\cache\08aced46-45a2-48d7-993b-ed3fb5b32302\22_6_Gold.pptx",
r"D:\NJU\HKUNLP\Desktop-Env\DesktopEnv\cache\08aced46-45a2-48d7-993b-ed3fb5b32302\22_6.pptx",
examine_shape=False))

121
desktop_env/evaluators/metrics/table.py
View File

@@ -11,15 +11,15 @@ import openpyxl
import pandas as pd import pandas as pd
from openpyxl import Workbook from openpyxl import Workbook
from openpyxl.cell.cell import Cell from openpyxl.cell.cell import Cell
from openpyxl.worksheet.cell_range import MultiCellRange
from openpyxl.utils import get_column_letter from openpyxl.utils import get_column_letter
from openpyxl.worksheet.cell_range import MultiCellRange
from openpyxl.worksheet.datavalidation import DataValidation from openpyxl.worksheet.datavalidation import DataValidation
from openpyxl.worksheet.worksheet import Worksheet from openpyxl.worksheet.worksheet import Worksheet
from rapidfuzz import fuzz
from desktop_env.evaluators.metrics.utils import _match_value_to_rule, _read_cell_style, read_cell_value from desktop_env.evaluators.metrics.utils import _match_value_to_rule, _read_cell_style, read_cell_value
from desktop_env.evaluators.metrics.utils import load_charts, load_sparklines, load_rows_or_cols, load_xlsx_styles \ from desktop_env.evaluators.metrics.utils import load_charts, load_sparklines, load_rows_or_cols, load_xlsx_styles \
, load_filters, load_pivot_tables , load_filters, load_pivot_tables
from rapidfuzz import fuzz
# from openpyxl.utils import coordinate_to_tuple # from openpyxl.utils import coordinate_to_tuple
@@ -165,7 +165,7 @@ def compare_table(result: str, expected: str = None, **options) -> float:
logger.debug("Sheet1: \n%s", str(sheet1)) logger.debug("Sheet1: \n%s", str(sheet1))
logger.debug("Sheet2: \n%s", str(sheet2)) logger.debug("Sheet2: \n%s", str(sheet2))
try: try:
logger.debug("Sheet1 =v= Sheet2: \n%s", str(sheet1==sheet2)) logger.debug("Sheet1 =v= Sheet2: \n%s", str(sheet1 == sheet2))
except: except:
logger.debug("Sheet1 =/v= Sheet2") logger.debug("Sheet1 =/v= Sheet2")
logger.debug("Assertion: %s =v= %s - %s", r["sheet_idx0"], r["sheet_idx1"], metric) logger.debug("Assertion: %s =v= %s - %s", r["sheet_idx0"], r["sheet_idx1"], metric)
@@ -231,14 +231,14 @@ def compare_table(result: str, expected: str = None, **options) -> float:
value1 = value1.lower() value1 = value1.lower()
value2 = value2.lower() value2 = value2.lower()
if rl["type"]=="includes": if rl["type"] == "includes":
metric: bool = value2 in value1 metric: bool = value2 in value1
elif rl["type"]=="included_by": elif rl["type"] == "included_by":
metric: bool = value1 in value2 metric: bool = value1 in value2
elif rl["type"]=="fuzzy_match": elif rl["type"] == "fuzzy_match":
metric: bool = fuzz.ratio(value1, value2) >= rl.get("threshold", 85.) metric: bool = fuzz.ratio(value1, value2) >= rl.get("threshold", 85.)
elif rl["type"]=="exact_match": elif rl["type"] == "exact_match":
metric: bool = value1==value2 metric: bool = value1 == value2
total_metric = total_metric and metric total_metric = total_metric and metric
metric: bool = total_metric metric: bool = total_metric
@@ -409,7 +409,7 @@ def compare_table(result: str, expected: str = None, **options) -> float:
filters1: Dict[str, Any] = load_filters(*parse_idx(r["sheet_idx0"], xlworkbookr, xlworkbooke), **r) filters1: Dict[str, Any] = load_filters(*parse_idx(r["sheet_idx0"], xlworkbookr, xlworkbooke), **r)
filters2: Dict[str, Any] = load_filters(*parse_idx(r["sheet_idx1"], xlworkbookr, xlworkbooke), **r) filters2: Dict[str, Any] = load_filters(*parse_idx(r["sheet_idx1"], xlworkbookr, xlworkbooke), **r)
metric: bool = filters1==filters2 metric: bool = filters1 == filters2
logger.debug("Assertion: %s[filter] == %s[filter] - %s", r["sheet_idx0"], r["sheet_idx1"], metric) logger.debug("Assertion: %s[filter] == %s[filter] - %s", r["sheet_idx0"], r["sheet_idx1"], metric)
# }}} Compare Filters # # }}} Compare Filters #
@@ -421,7 +421,7 @@ def compare_table(result: str, expected: str = None, **options) -> float:
pivots1: Dict[str, Any] = load_pivot_tables(*parse_idx(r["sheet_idx0"], xlworkbookr, xlworkbooke), **r) pivots1: Dict[str, Any] = load_pivot_tables(*parse_idx(r["sheet_idx0"], xlworkbookr, xlworkbooke), **r)
pivots2: Dict[str, Any] = load_pivot_tables(*parse_idx(r["sheet_idx1"], xlworkbookr, xlworkbooke), **r) pivots2: Dict[str, Any] = load_pivot_tables(*parse_idx(r["sheet_idx1"], xlworkbookr, xlworkbooke), **r)
metric: bool = pivots1==pivots2 metric: bool = pivots1 == pivots2
logger.debug("Assertion: %s[pivot]==%s[pivot] - %s", r["sheet_idx0"], r["sheet_idx1"], metric) logger.debug("Assertion: %s[pivot]==%s[pivot] - %s", r["sheet_idx0"], r["sheet_idx1"], metric)
# }}} Compare Pivot Tables # # }}} Compare Pivot Tables #
@@ -482,81 +482,36 @@ def compare_csv(result: str, expected: str, **options) -> float:
return float(metric) return float(metric)
if __name__ == '__main__': def compare_conference_city_in_order(actual_city_list_path, expected_city):
import datetime expected_city_list = expected_city["expected"]
import sys wb = openpyxl.load_workbook(actual_city_list_path)
sheet = wb.active
actual_city_list = []
for row in sheet["C2:C22"]:
for cell in row:
actual_city_list.append(cell.value)
# expected_city is the city that we want to compare with the actual city list
# must in order index
# debug
try:
for i in range(len(actual_city_list)):
if isinstance(expected_city_list[i], str):
if expected_city_list[i] not in actual_city_list[i]:
logger.debug(f"Expected city {expected_city_list[i]}; Actual city {actual_city_list[i]}")
print(f"Expected city {expected_city_list[i]}; Actual city {actual_city_list[i]}")
return 0.
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
datetime_str: str = datetime.datetime.now().strftime("%Y%m%d@%H%M%S") elif isinstance(expected_city_list[i], List):
if not any(possible_str in actual_city_list[i] for possible_str in expected_city_list[i]):
logger.debug(f"Expected city {expected_city_list[i]}; Actual city {actual_city_list[i]}")
print(f"Expected city {expected_city_list[i]}; Actual city {actual_city_list[i]}")
return 0.
file_handler = logging.FileHandler(os.path.join("logs", "normal-{:}.log".format(datetime_str))) else:
debug_handler = logging.FileHandler(os.path.join("logs", "debug-{:}.log".format(datetime_str))) raise TypeError("Expected city should be a string or a list of strings")
stdout_handler = logging.StreamHandler(sys.stdout)
sdebug_handler = logging.FileHandler(os.path.join("logs", "sdebug-{:}.log".format(datetime_str)))
file_handler.setLevel(logging.INFO) except:
debug_handler.setLevel(logging.DEBUG) return 0.
stdout_handler.setLevel(logging.INFO)
sdebug_handler.setLevel(logging.DEBUG)
formatter = logging.Formatter( return 1.
fmt="\x1b[1;33m[%(asctime)s \x1b[31m%(levelname)s \x1b[32m%(module)s/%(lineno)d-%(processName)s\x1b[1;33m] \x1b[0m%(message)s")
file_handler.setFormatter(formatter)
debug_handler.setFormatter(formatter)
stdout_handler.setFormatter(formatter)
sdebug_handler.setFormatter(formatter)
stdout_handler.addFilter(logging.Filter("desktopenv"))
sdebug_handler.addFilter(logging.Filter("desktopenv"))
logger.addHandler(file_handler)
logger.addHandler(debug_handler)
logger.addHandler(stdout_handler)
logger.addHandler(sdebug_handler)
path1 = "snapshots/test/cache/4e6fcf72-daf3-439f-a232-c434ce416af6/Employee_Age_By_Birthday.xlsx"
path2 = "snapshots/test/cache/4e6fcf72-daf3-439f-a232-c434ce416af6/Employee_Age_By_Birthday_gold.xlsx"
rules = [ { "type": "sheet_data"
, "sheet_idx0": 0
, "sheet_idx1": "EI0"
}
]
print(compare_table(path1, path2
, rules=rules
)
)
print(compare_table(path2, path2
, rules=rules
)
)
# Row Properties
# path1 = "../../任务数据/LibreOffice Calc/Date_Budget_Variance_HideNA.xlsx"
# path2 = "../../任务数据/LibreOffice Calc/Date_Budget_Variance_HideNA_gold.xlsx"
# workbook: Workbook = openpyxl.load_workbook(filename=path1)
# worksheet: Worksheet = workbook.active
# for r_no, dms in worksheet.column_dimensions.items():
# print(r_no, type(r_no), type(dms), dms.hidden)
# Conditional Formats
# import formulas
# path1 = "../../任务数据/LibreOffice Calc/Calendar_Highlight_Weekend_Days.xlsx"
# path2 = "../../任务数据/LibreOffice Calc/Calendar_Highlight_Weekend_Days_gold.xlsx"
# path3 = "../../任务数据/LibreOffice Calc/Calendar_Highlight_Weekend_Days_gold_test.xlsx"
# workbook: Workbook = openpyxl.load_workbook(filename=path2)
# worksheet: Worksheet = workbook.active
# print(worksheet.conditional_formatting)
# for itm in worksheet.conditional_formatting:
# print(itm.cells)
# for r in itm.rules:
# print( r.type, r.formula, r.dxf.font.color.rgb
# , r.dxf.fill.fgColor.rgb, r.dxf.fill.bgColor.rgb
# )
# condition = formulas.Parser().ast("=" + r.formula[0])[1].compile()
##print(r.type, r.operator, r.dxfId, r.dxf)
# for r in itm.cells:
# for c in r.cells:
# value = worksheet.cell(row=c[0], column=c[1]).value
# print(value, condition(str(value)))

92
desktop_env/evaluators/metrics/thunderbird.py
View File

@@ -1,17 +1,19 @@
import json
import logging
import re
from typing import List, Pattern, Dict, Match from typing import List, Pattern, Dict, Match
from typing import Union, Any, TypeVar, Callable from typing import Union, Any, TypeVar, Callable
import re
import json
from .utils import _match_record from .utils import _match_record
from .utils import _match_value_to_rule as _match_pref from .utils import _match_value_to_rule as _match_pref
import logging
logger = logging.getLogger("desktopenv.metric.thunderbird") logger = logging.getLogger("desktopenv.metric.thunderbird")
V = TypeVar("Value") V = TypeVar("Value")
_pref_pattern: Pattern[str] = re.compile(r'^user_pref\("(?P<key>(?:[^"]|\\")+)\", (?P<val>.+)\);$'); _pref_pattern: Pattern[str] = re.compile(r'^user_pref\("(?P<key>(?:[^"]|\\")+)\", (?P<val>.+)\);$');
def check_thunderbird_prefs(result: str, rule: Dict[str, Dict[str, Dict[str, Any]]]): def check_thunderbird_prefs(result: str, rule: Dict[str, Dict[str, Dict[str, Any]]]):
""" """
Args: Args:
@@ -51,10 +53,10 @@ def check_thunderbird_prefs(result: str, rule: Dict[str, Dict[str, Dict[str, Any
continue continue
key: str = match_.group("key") key: str = match_.group("key")
#value: str = match_.group("val") # value: str = match_.group("val")
#if value in {"true", "false"}: # if value in {"true", "false"}:
#value = value.title() # value = value.title()
#value: V = eval(value) # value: V = eval(value)
value = json.loads(match_.group("val")) value = json.loads(match_.group("val"))
if key in expect_rules: if key in expect_rules:
logger.debug("K: %s, V: %s", key, repr(value)) logger.debug("K: %s, V: %s", key, repr(value))
@@ -64,9 +66,13 @@ def check_thunderbird_prefs(result: str, rule: Dict[str, Dict[str, Dict[str, Any
return float(all(expect_metrics.values()) and unexpect_metric) return float(all(expect_metrics.values()) and unexpect_metric)
_value_processor: Callable[[str], str] = lambda val: val.replace("\\\"", "\"").replace("\\\\", "\\") _value_processor: Callable[[str], str] = lambda val: val.replace("\\\"", "\"").replace("\\\\", "\\")
#_condition_pattern: Pattern[str] = re.compile(r'(?P<type>AND|OR) \((?P<key>[\w ]+),(?P<rel>[\w ' + '\'' + r']+),(?:"(?P<val2>(?:[^"]|\")+)"|(?P<val1>[^)]+))\)') # _condition_pattern: Pattern[str] = re.compile(r'(?P<type>AND|OR) \((?P<key>[\w ]+),(?P<rel>[\w ' + '\'' + r']+),(?:"(?P<val2>(?:[^"]|\")+)"|(?P<val1>[^)]+))\)')
_condition_pattern: Pattern[str] = re.compile(r'\b(?:AND|OR) \((?:[\w ]+),(?:[\w ' + '\'' + r']+),(?:"(?:(?:[^"]|\")+)"|(?:[^)]+))\)|\bALL\b') _condition_pattern: Pattern[str] = re.compile(
r'\b(?:AND|OR) \((?:[\w ]+),(?:[\w ' + '\'' + r']+),(?:"(?:(?:[^"]|\")+)"|(?:[^)]+))\)|\bALL\b')
def check_thunderbird_filter(result: str, rules: Dict[str, List[Dict[str, str]]]) -> float: def check_thunderbird_filter(result: str, rules: Dict[str, List[Dict[str, str]]]) -> float:
""" """
Args: Args:
@@ -112,8 +118,8 @@ def check_thunderbird_filter(result: str, rules: Dict[str, List[Dict[str, str]]]
condition_str: str = _value_processor(l[11:-2]) condition_str: str = _value_processor(l[11:-2])
logger.debug("FILTER CONDITION: %s", condition_str) logger.debug("FILTER CONDITION: %s", condition_str)
conditions: List[str] =\ conditions: List[str] = \
_condition_pattern.findall(condition_str) _condition_pattern.findall(condition_str)
logger.debug("FILTER CONDITIONS: %s", repr(conditions)) logger.debug("FILTER CONDITIONS: %s", repr(conditions))
filter_["condition"] = conditions filter_["condition"] = conditions
@@ -138,6 +144,7 @@ def check_thunderbird_folder(result: Union[str, List[str]], reference: Union[str
remove_deleted (bool): ignore deleted messages which has status code 0008 or 0009. default: True remove_deleted (bool): ignore deleted messages which has status code 0008 or 0009. default: True
remove_duplicate (bool): remove duplicate messages. default: True remove_duplicate (bool): remove duplicate messages. default: True
""" """
def normalize_msg(msg, options): def normalize_msg(msg, options):
ignore_status = options.get('ignore_status', False) ignore_status = options.get('ignore_status', False)
ignore_keys = options.get('ignore_keys', False) ignore_keys = options.get('ignore_keys', False)
@@ -167,66 +174,3 @@ def check_thunderbird_folder(result: Union[str, List[str]], reference: Union[str
mail2 = read_thunderbird_folder_file(gold) mail2 = read_thunderbird_folder_file(gold)
if mail1 != mail2: return .0 if mail1 != mail2: return .0
return 1.0 return 1.0
if __name__ == "__main__":
#import lxml.etree
#from lxml.cssselect import CSSSelector
#from lxml.etree import _Element
#xml = "../../任务数据/Thunderbird/vertical-card-view.xml"
#xml = "../../任务数据/Thunderbird/vertical-table-view.xml"
#at: _Element = lxml.etree.parse(xml)
#elements: List[_Element] = CSSSelector('application[name=Thunderbird] page-tab-list')(at) # page tab tags
#elements: List[_Element] = CSSSelector('application[name=Thunderbird] panel>scroll-pane>internal-frame>panel[name$="anonym-x2024@outlook.com"]')(at) # email tag page
#elements: List[_Element] = CSSSelector('application[name=Thunderbird] panel>scroll-pane>internal-frame>panel[name$="anonym-x2024@outlook.com"]>section:nth-child(3)')(at) # email tag page
#elements: List[_Element] = CSSSelector('application[name=Thunderbird] panel>scroll-pane>internal-frame>panel[name$="anonym-x2024@outlook.com"]>section[attr|id=threadPane]>section[attr|id="threadTree"]>table[attr|class="tree-table"]>section[attr|class~="tree-table-header"]>table-row>column-header[name=Subject]>push-button', namespaces={"attr": "uri:deskat:attributes.at-spi.gnome.org"})(at) # table view, column header
#elements: List[_Element] = CSSSelector('application[name=Thunderbird] panel>scroll-pane>internal-frame>panel[name$="anonym-x2024@outlook.com"]>section[attr|id=threadPane]>section[attr|id="threadTree"]>table[attr|class="tree-table"]>tree>tree-item>section[name="Subject"]>section>section', namespaces={"attr": "uri:deskat:attributes.at-spi.gnome.org"})(at) # table view, column header
#print(len(elements))
#for elm in elements:
#print(lxml.etree.tostring(elm, encoding="unicode", pretty_print=True))
import datetime
import os
import sys
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
datetime_str: str = datetime.datetime.now().strftime("%Y%m%d@%H%M%S")
file_handler = logging.FileHandler(os.path.join("logs", "normal-{:}.log".format(datetime_str)))
debug_handler = logging.FileHandler(os.path.join("logs", "debug-{:}.log".format(datetime_str)))
stdout_handler = logging.StreamHandler(sys.stdout)
sdebug_handler = logging.FileHandler(os.path.join("logs", "sdebug-{:}.log".format(datetime_str)))
file_handler.setLevel(logging.INFO)
debug_handler.setLevel(logging.DEBUG)
stdout_handler.setLevel(logging.INFO)
sdebug_handler.setLevel(logging.DEBUG)
formatter = logging.Formatter(fmt="\x1b[1;33m[%(asctime)s \x1b[31m%(levelname)s \x1b[32m%(module)s/%(lineno)d-%(processName)s\x1b[1;33m] \x1b[0m%(message)s")
file_handler.setFormatter(formatter)
debug_handler.setFormatter(formatter)
stdout_handler.setFormatter(formatter)
sdebug_handler.setFormatter(formatter)
stdout_handler.addFilter(logging.Filter("desktopenv"))
sdebug_handler.addFilter(logging.Filter("desktopenv"))
logger.addHandler(file_handler)
logger.addHandler(debug_handler)
logger.addHandler(stdout_handler)
logger.addHandler(sdebug_handler)
print( check_thunderbird_filter( "../../任务数据/Thunderbird/msgFilterRules.dat"
, { "expect": [ { "enabled": "yes"
, "action": "Move to folder"
, "actionValue": "mailbox://nobody@Local%20Folders/Promotions"
, "condition": ["AND (subject,contains,discount)"]
}
]
}
)
)

9
desktop_env/evaluators/metrics/vscode.py
View File

@@ -271,12 +271,3 @@ def compare_result_files(src_path, tgt_path):
if src_content == tgt_content: if src_content == tgt_content:
return 1.0 return 1.0
return 0.0 return 0.0
if __name__ == "__main__":
src_path = "../../../cache/bb7db4c2-30b5-4be7-8dd7-b8c4ec7d3108/index.html"
rule = {
"type:": "value",
"value": "anmi_sharper.png"
}
print(check_html_background_image(src_path, rule))