lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'zdy'

Browse Source
This commit is contained in:
David Chang
2024-01-18 18:06:00 +08:00
parent 91824f754c 119a79e4fa
commit 4af19fb777
9 changed files with 458 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
desktop_env/evaluators/metrics/__init__.py
View File

@@ -6,7 +6,7 @@ from .docs import is_first_line_centered, check_file_exists, compare_contains_im
from .general import exact_match, fuzzy_match, check_csv, check_accessibility_tree, check_list
from .libreoffice import check_libre_locale
from .pdf import check_pdf_pages
from .table import check_sheet_list, check_xlsx_freeze, check_xlsx_zoom
from .table import check_sheet_list, check_xlsx_freeze, check_xlsx_zoom, check_data_validations
from .table import compare_table
from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, compare_images, compare_audios, \
compare_videos

86
desktop_env/evaluators/metrics/table.py
View File

@@ -10,8 +10,10 @@ import openpyxl
import pandas as pd
from openpyxl import Workbook
from openpyxl.worksheet.worksheet import Worksheet
#from openpyxl.worksheet.cell_range import MultiCellRange
from openpyxl.worksheet.datavalidation import DataValidation
from .utils import load_charts, load_sparklines
from .utils import load_charts, load_sparklines, _match_value_to_rule
logger = logging.getLogger("desktopenv.metric.table")
@@ -27,7 +29,7 @@ def compare_table(actual: str, expected: str, **options) -> float:
* sparkline
* chart
* number_format
"chart_props": list of str, giving the converned chart properties
"chart_props": list of str, giving the concerned chart properties
"as_shown": bool, TODO
}
@@ -164,6 +166,57 @@ def check_xlsx_zoom(result: str, rules: Dict[str, Union[str, Number]]) -> float:
)
)
def check_data_validations(result: str, rules: List[Dict[str, Dict[str, Any]]]) -> float:
"""
Args:
result (str): path to the concerned xlsx file
rules (List[Dict[str, Dict[str, Any]]]): list of dict like
{
<str as attribute>: {
"method": str
"ref": something
}
}
Available attributes:
* ranges
* type
* formula1
* formula2
* operator
* allowBlank
* showDropDown
* showInputMessage
* showErrorMessage
* error
* errorTitle
* errorStyle
* prompt
* promptTitle
* imeMode
Returns:
float
"""
workbook: Workbook = openpyxl.load_workbook(result)
worksheet: Worksheet = workbook.active
data_validators: List[DataValidation] = worksheet.data_validations.dataValidation
total_metric = True
for dat_vldt in data_validators:
metric = False
for r in rules:
metric = metric or all( _match_value_to_rule( getattr(dat_vldt, attrbt)
, mr
)\
for attrbt, mr in r.items()
)
if metric:
break
total_metric = total_metric and metric
if not total_metric:
break
return float(total_metric)
if __name__ == '__main__':
# path1 = ""
@@ -247,8 +300,31 @@ if __name__ == '__main__':
# print(check_zoom(path1, {"relation": "lt", "ref_value": 100}))
# print(check_zoom(path2, {"relation": "lt", "ref_value": 100}))
path1 = "../../任务数据/LibreOffice Calc/Customers_New_7digit_Id.xlsx"
path2 = "../../任务数据/LibreOffice Calc/Customers_New_7digit_Id_gold.xlsx"
#path1 = "../../任务数据/LibreOffice Calc/Customers_New_7digit_Id.xlsx"
#path2 = "../../任务数据/LibreOffice Calc/Customers_New_7digit_Id_gold.xlsx"
#data_frame: pd.DataFrame = pd.read_excel(path1)
#print(data_frame)
print(compare_table(path1, path2, as_shown=True))
#print(compare_table(path1, path2, as_shown=True))
#from openpyxl.worksheet.cell_range import MultiCellRange
path = "../../任务数据/LibreOffice Calc/Order_Id_Mark_Pass_Fail_gold.xlsx"
#worksheet: Worksheet = openpyxl.load_workbook(filename=path).active
##print(worksheet.data_validations)
#print(type(worksheet.data_validations.dataValidation))
#for dat_vldt in worksheet.data_validations.dataValidation:
#print(dat_vldt.sqref)
#print(all(r in MultiCellRange("D2:D30 B1:B60") for r in dat_vldt.sqref))
print( check_data_validations( path, [ { "ranges": { "method": "spreadsheet_range"
, "ref": ["D2:D29", "D2:D1048576"]
}
, "type": { "method": "eq"
, "ref": "list"
}
, "formula1": { "method": "str_set_eq"
, "ref": ["Pass", "Fail", "Held"]
}
}
]
)
)

21
desktop_env/evaluators/metrics/utils.py
View File

@@ -6,6 +6,7 @@ from urllib.parse import urlparse, urlunparse
import re
import functools
import operator
import builtins
import lxml.cssselect
import lxml.etree
@@ -15,6 +16,7 @@ from lxml.etree import _Element
from openpyxl import Workbook
from openpyxl.chart._chart import ChartBase
from openpyxl.worksheet.worksheet import Worksheet
from openpyxl.worksheet.cell_range import MultiCellRange
V = TypeVar("Value")
@@ -138,6 +140,8 @@ def load_charts(xlsx_file: Workbook, **options) -> Dict[str, Any]:
def _match_record(pattern: Dict[str, Any], item: Dict[str, Any]) -> bool:
return all(k in item and item[k] == val for k, val in pattern.items())
def _multicellrange_containsby(subset_candidate: MultiCellRange, superset_candidate: MultiCellRange) -> bool:
return all(r in superset_candidate for r in subset_candidate)
def _match_value_to_rule(value: V, rule: Dict[str, Union[str, V]]) -> bool:
"""
Args:
@@ -165,6 +169,23 @@ def _match_value_to_rule(value: V, rule: Dict[str, Union[str, V]]) -> bool:
, "ge", "gt"
}:
return getattr(operator, rule["method"])(value, rule["ref"])
if rule["method"] == "spreadsheet_range":
subset_limit = MultiCellRange(rule["ref"][0])
superset_limit = MultiCellRange(rule["ref"][1])
return _multicellrange_containsby(subset_limit, value)\
and _multicellrange_containsby(value, superset_limit)
if rule["method"].startswith("range."): # e.g., range.te [0, 2] -> 0 < x <= 2
left_et = rule["method"][6]
right_et = rule["method"][7]
return getattr(operator, "l" + left_et)(rule["ref"][0], value)\
and getattr(operator, "l" + right_et)(value, rule["ref"][1])
if rule["method"] in {"str_list_eq", "str_set_eq"}:
container_type_str: str = rule["method"][4:-3]
container_type = getattr(builtins, container_type_str)
value: container_type = container_type(value.strip("\"'").split(","))
ref: container_type = container_type(rule["ref"])
return value==ref
raise NotImplementedError()
def are_lists_equal(list1, list2, comparison_func):

69
evaluation_examples/examples/libreoffice_calc/7a4e4bc8-922c-4c84-865c-25ba34136be1.json
View File

@@ -3,20 +3,65 @@
"snapshot": "libreoffice_calc",
"instruction": "Reorder the columns to be \"Data\", \"First Name\", \"Last Name\", \"Order ID\", \"Sales\"",
"source": "https://www.youtube.com/shorts/bvUhr1AHs44",
"config": {
"download": [
[
"",
"C:\\Users\\tianbaox\\Desktop\\Name_Order_Id_move_column.xlsx"
]
],
"open": [
"C:\\Users\\tianbaox\\Desktop\\Name_Order_Id_move_column.xlsx"
]
},
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1jS159dHRdeZh54A8WgqJn_somQpKP1K_&export=download&authuser=0&confirm=t&uuid=06aa5391-5f3e-499b-9073-9ce24109f91f&at=APZUnTWFCuiN8b5ilNmxx3aPYGUF:1705551767314",
"path": "/home/user/Name_Order_Id_move_column.xlsx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/Name_Order_Id_move_column.xlsx"
}
}
],
"trajectory": "trajectories/7a4e4bc8-922c-4c84-865c-25ba34136be1",
"related_apps": [
"libreoffice calc"
],
"evaluator": "evaluation_dir"
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "Name_Order_Id_move_column.xlsx - LibreOffice Calc",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
]
}
}
],
"func": "compare_table",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1jAK4--qLRdUOA57MBsmERpR205GGIMij&export=download&authuser=0&confirm=t&uuid=4ca454c9-d9d5-4e5f-95e1-cb0e849932e5&at=APZUnTVgkUbmLJLC4O7ACKmzfFbq:1705551965635",
"dest": "Name_Order_Id_move_column_gold.xlsx"
},
"result": {
"type": "vm_file",
"path": "/home/user/Name_Order_Id_move_column.xlsx",
"dest": "Name_Order_Id_move_column.xlsx"
}
}
}

71
evaluation_examples/examples/libreoffice_calc/7efeb4b1-3d19-4762-b163-63328d66303b.json
View File

@@ -1,22 +1,67 @@
{
"id": "7efeb4b1-3d19-4762-b163-63328d66303b",
"snapshot": "libreoffice_calc",
"instruction": "Fill in the Serieal Numbers in \"Serial #\" column",
"instruction": "Fill the Sequence Numbers as \"No. #\" in the \"Seq No.\" column",
"source": "https://www.youtube.com/shorts/4jzXfZNhfmk",
"config": {
"download": [
[
"",
"C:\\Users\\tianbaox\\Desktop\\Order_Sales_Serial#.xlsx"
]
],
"open": [
"C:\\Users\\tianbaox\\Desktop\\Order_Sales_Serial#.xlsx"
]
},
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1muvWVg44C6EtBpBQkZ6_ylg3M8r3jjOC&export=download&authuser=0&confirm=t&uuid=41354a7c-d199-4044-960e-0d146fe6f12a&at=APZUnTW3nj5RV3SBJt5tdeVxM4mM:1705553238027",
"path": "/home/user/Order_Sales_Serial#.xlsx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/Order_Sales_Serial#.xlsx"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"libreoffice calc"
],
"evaluator": "evaluation_dir"
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "Order_Sales_Serial#.xlsx - LibreOffice Calc",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
]
}
}
],
"func": "compare_table",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1K37qrxab2cib1SkKPthYkb83fzaj9ETa&export=download&authuser=0&confirm=t&uuid=66b763e6-2fb1-46fc-9753-7f8533366b82&at=APZUnTU1nCKKXBwCbyI8aKpdhE5W:1705560437314",
"dest": "Order_Sales_Serial#_gold.xlsx"
},
"result": {
"type": "vm_file",
"path": "/home/user/Order_Sales_Serial#.xlsx",
"dest": "Order_Sales_Serial#.xlsx"
}
}
}

71
evaluation_examples/examples/libreoffice_calc/a9f325aa-8c05-4e4f-8341-9e4358565f4f.json
View File

@@ -1,22 +1,67 @@
{
"id": "a9f325aa-8c05-4e4f-8341-9e4358565f4f",
"snapshot": "libreoffice_calc",
"instruction": "Clean the messy movie titles and put them in the cleaned column",
"instruction": "Remove the adundant whitespaces and canonicalize the letter cases by capitalizing the first letter of each words and leave other letters as lower case.",
"source": "https://www.youtube.com/shorts/A0gmEBRKXWs",
"config": {
"download": [
[
"",
"C:\\Users\\tianbaox\\Desktop\\"
]
],
"open": [
"C:\\Users\\tianbaox\\Desktop\\"
]
},
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1PGJxDM5QglZFdLd7yLQnA2MASf2so14S&export=download&authuser=0&confirm=t&uuid=725f7d90-1e30-4579-b946-7c8932aac440&at=APZUnTUpha_2nl3sAOmM1AeCCir_:1705561280883",
"path": "/home/user/Movie_title_garbage_clean.xlsx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/Movie_title_garbage_clean.xlsx"
}
}
],
"trajectory": "trajectories/a9f325aa-8c05-4e4f-8341-9e4358565f4f",
"related_apps": [
"libreoffice calc"
],
"evaluator": "evaluation_dir"
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "Movie_title_garbage_clean.xlsx - LibreOffice Calc",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
]
}
}
],
"func": "compare_table",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1hrmRQig53UW069MEN_V23xJXFrIooylm&export=download&authuser=0&confirm=t&uuid=82a01881-c72c-4463-b258-c8c66f8d72af&at=APZUnTVZVuQj91twmMdH1plMcABA:1705561377705",
"dest": "Movie_title_garbage_clean_gold.xlsx"
},
"result": {
"type": "vm_file",
"path": "/home/user/Movie_title_garbage_clean.xlsx",
"dest": "Movie_title_garbage_clean.xlsx"
}
}
}

73
evaluation_examples/examples/libreoffice_calc/d681960f-7bc3-4286-9913-a8812ba3261a.json
View File

@@ -1,34 +1,67 @@
{
"id": "d681960f-7bc3-4286-9913-a8812ba3261a",
"snapshot": "libreoffice_calc",
"instruction": "According to the green table shown above, calculate and give each student a grade",
"instruction": "According to the scale table shown above, calculate and give each student a grade in the table below",
"source": "https://www.youtube.com/shorts/d7U1S_IsTVM",
"config": {
"download": [
[
"https://drive.usercontent.google.com/download?id=1wodZjx1KjThUsrtF6ZJaCTy1fQX4E9vA&export=download&authuser=0&confirm=t&uuid=d07ca312-1abc-40f2-81cd-d06e27119854&at=APZUnTWwjnxsHQYapSvpLR8NmlfV:1701785087048",
"C:\\Users\\tianbaox\\Desktop\\Student_Grades_and_Remarks.xlsx"
]
],
"open": [
"C:\\Users\\tianbaox\\Desktop\\Student_Grades_and_Remarks.xlsx"
]
},
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1wodZjx1KjThUsrtF6ZJaCTy1fQX4E9vA&export=download&authuser=0&confirm=t&uuid=d07ca312-1abc-40f2-81cd-d06e27119854&at=APZUnTWwjnxsHQYapSvpLR8NmlfV:1701785087048",
"path": "/home/user/Student_Grades_and_Remarks.xlsx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/Student_Grades_and_Remarks.xlsx"
}
}
],
"trajectory": "trajectories/d681960f-7bc3-4286-9913-a8812ba3261a",
"related_apps": [
"libreoffice calc"
],
"evaluator": {
"func": "compare_table(expected, actual)",
"paths": {
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1kfEHJH1n0yCsQp443IIFvdD9uWv0DWMr&export=download&authuser=0&confirm=t&uuid=d9907f65-8d39-4ecc-8747-b4ed7e6011f5&at=APZUnTXpPAnlh5sD6q-R8oQtqL6g:1702362952170"
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "Student_Grades_and_Remarks.xlsx - LibreOffice Calc",
"strict": true
}
},
"actual": {
"type": "vm_file",
"path": "C:\\Users\\tianbaox\\Desktop\\Student_Grades_and_Remarks.xlsx"
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
]
}
}
],
"func": "compare_table",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1kfEHJH1n0yCsQp443IIFvdD9uWv0DWMr&export=download&authuser=0&confirm=t&uuid=d9907f65-8d39-4ecc-8747-b4ed7e6011f5&at=APZUnTXpPAnlh5sD6q-R8oQtqL6g:1702362952170",
"dest": "Student_Grades_and_Remarks_gold.xlsx"
},
"result": {
"type": "vm_file",
"path": "/home/user/Student_Grades_and_Remarks.xlsx",
"dest": "Student_Grades_and_Remarks.xlsx"
}
}
}

71
evaluation_examples/examples/libreoffice_calc/eb03d19a-b88d-4de4-8a64-ca0ac66f426b.json
View File

@@ -1,22 +1,67 @@
{
"id": "eb03d19a-b88d-4de4-8a64-ca0ac66f426b",
"snapshot": "libreoffice_calc",
"instruction": "Traverse the table and paste it below",
"instruction": "Transpose the table and paste it starting from B8",
"source": "https://www.youtube.com/shorts/t9JLUaT55UQ",
"config": {
"download": [
[
"",
"C:\\Users\\tianbaox\\Desktop\\"
]
],
"open": [
"C:\\Users\\tianbaox\\Desktop\\"
]
},
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1Zl6ZUkbiV9sISjJRfracd8-xKrW2G3yv&export=download&authuser=0&confirm=t&uuid=6799cca6-62d2-4cfa-b28a-b8da486b5d01&at=APZUnTVy7-CQMjvuFyu7ZMxztfT6:1705563660974",
"path": "/home/user/Students_Class_Subject_Marks.xlsx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/Students_Class_Subject_Marks.xlsx"
}
}
],
"trajectory": "trajectories/eb03d19a-b88d-4de4-8a64-ca0ac66f426b",
"related_apps": [
"libreoffice calc"
],
"evaluator": "evaluation_dir"
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "Students_Class_Subject_Marks.xlsx - LibreOffice Calc",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
]
}
}
],
"func": "compare_table",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1GwqF52n6cvmQ-ivhmfNCep9mZboQ706-&export=download&authuser=0&confirm=t&uuid=4c750a20-73b8-4072-a4bf-3a4aa77b314a&at=APZUnTXYm-3NVdfneACqfDLYCWko:1705563733494",
"dest": "Students_Class_Subject_Marks_gold.xlsx"
},
"result": {
"type": "vm_file",
"path": "/home/user/Students_Class_Subject_Marks.xlsx",
"dest": "Students_Class_Subject_Marks.xlsx"
}
}
}

83
evaluation_examples/examples/libreoffice_calc/ecb0df7a-4e8d-4a03-b162-053391d3afaf.json
View File

@@ -3,20 +3,79 @@
"snapshot": "libreoffice_calc",
"instruction": "Enable each cell in the column\"Pass/Fail/Held\" is a drop down list",
"source": "https://www.youtube.com/shorts/tXOovKn0H68",
"config": {
"download": [
[
"",
"C:\\Users\\tianbaox\\Desktop\\"
]
],
"open": [
"C:\\Users\\tianbaox\\Desktop\\"
]
},
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1vKtnFG-sL7Ba0UCcUh4dWJDfZeAHAP6l&export=download&authuser=0&confirm=t&uuid=70a267fa-dc71-4893-8fe3-a7254e50c567&at=APZUnTVfnhipGIdrD39159Eqv9lf:1705567650653",
"path": "/home/user/Order_Id_Mark_Pass_Fail.xlsx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/Order_Id_Mark_Pass_Fail.xlsx"
}
}
],
"trajectory": "trajectories/ecb0df7a-4e8d-4a03-b162-053391d3afaf",
"related_apps": [
"libreoffice calc"
],
"evaluator": "evaluation_dir"
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "Order_Id_Mark_Pass_Fail.xlsx - LibreOffice Calc",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
]
}
}
],
"func": "check_data_validations",
"expected": {
"type": "rule",
"rules": [
{
"ranges": {
"method": "spreadsheet_range",
"ref": ["D2:D29", "D2:D1048576"]
},
"type": {
"method": "eq",
"ref": "list"
},
"formula1": {
"method": "str_set_eq",
"ref": ["Pass", "Fail", "Held"]
}
}
]
},
"result": {
"type": "vm_file",
"path": "/home/user/Order_Id_Mark_Pass_Fail.xlsx",
"dest": "Order_Id_Mark_Pass_Fail.xlsx"
}
}
}