lzy/sci-gui-agent-benchmark
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ver Jan21stv2

Browse Source 
fixed bugs
updated parts of configs
This commit is contained in:
David Chang
2024-01-21 23:55:04 +08:00
parent 4514c32269
commit 552491f765
7 changed files with 104 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
desktop_env/evaluators/README.md
View File

@@ -7,6 +7,25 @@
pip install python-docx pip install python-docx
``` ```
## LibreOffice Calc
### Required Libraries
```
openpyxl
pandas
lxml
xmltodict
```
### How to Generate CSV from XLSX
```sh
libreoffice --convert-to "csv:Text - txt - csv (StarCalc):44,34,UTF8,,,,false,true,true,false,false,1" --out-dir /home/user /home/user/abc.xlsx
```
This command will generate `abc-Sheet1.csv` under `/home/user`. The last `1` in the conversion options indicates the sheet number (starting from 1) to export. Detailed usage should be referred to at [CSV Filter Options](https://help.libreoffice.org/latest/ro/text/shared/guide/csv_params.html).
## Chrome ## Chrome
### Starting Chrome with Remote Debugging for Python ### Starting Chrome with Remote Debugging for Python

2
desktop_env/evaluators/metrics/__init__.py
View File

@@ -6,7 +6,7 @@ from .docs import is_first_line_centered, check_file_exists, compare_contains_im
from .general import exact_match, fuzzy_match, check_csv, check_accessibility_tree, check_list from .general import exact_match, fuzzy_match, check_csv, check_accessibility_tree, check_list
from .libreoffice import check_libre_locale from .libreoffice import check_libre_locale
from .pdf import check_pdf_pages from .pdf import check_pdf_pages
from .table import check_sheet_list, check_xlsx_freeze, check_xlsx_zoom, check_data_validations #from .table import check_sheet_list, check_xlsx_freeze, check_xlsx_zoom, check_data_validations
from .table import compare_table from .table import compare_table
from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, compare_images, compare_audios, \ from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, compare_images, compare_audios, \
compare_videos compare_videos

143
desktop_env/evaluators/metrics/table.py
View File

@@ -107,11 +107,8 @@ def compare_table(result: str, expected: str, **options) -> float:
for r in options["rules"]: for r in options["rules"]:
if r["type"] == "sheet_name": if r["type"] == "sheet_name":
# Compare Sheet Names {{{ # # Compare Sheet Names {{{ #
# sheet_idx0: 0 == "RI0" == "RNSheet1" | "EI0" == "ENSheet1"
# sheet_idx1: as sheet_idx0
metric: bool = worksheetr_names==worksheete_names metric: bool = worksheetr_names==worksheete_names
logger.debug("Assertion: %s.sheet_names == %s.sheet_names - %s", r["sheet_idx0"], r["sheet_idx1"], metric) logger.debug("Assertion: %s.sheet_names == %s.sheet_names - %s", result, expected, metric)
# }}} Compare Sheet Names # # }}} Compare Sheet Names #
elif r["type"] == "sheet_data": elif r["type"] == "sheet_data":
@@ -255,112 +252,34 @@ def compare_table(result: str, expected: str, **options) -> float:
# }}} function compare_table # # }}} function compare_table #
if __name__ == '__main__': if __name__ == '__main__':
# path1 = "" path1 = "../../任务数据/LibreOffice Calc/Represent_in_millions_billions.xlsx"
# path2 = "" path2 = "../../任务数据/LibreOffice Calc/Represent_in_millions_billions_gold.xlsx"
# print(compare_table(path1, path2)) rules = [ { "type": "sheet_print"
, "sheet_idx0": 0
# path1 = "../../../../../任务数据/LibreOffice Calc/OrderId_Month_Chart_gold.xlsx" , "sheet_idx1": "EI0"
# path2 = "../../../../../任务数据/LibreOffice Calc/OrderId_Month_Chart.xlsx" , "ignore_case": "true"
# print(compare_with_sparklines(path1, path2)) }
]
# path1 = "../../../../../任务数据/LibreOffice Calc/Freeze_row_column_gold.xlsx" print( compare_table( path1, path2
# path2 = "../../../../../任务数据/LibreOffice Calc/Freeze_row_column.xlsx" , rules=rules
# workbook1: Workbook = openpyxl.load_workbook(filename=path1) )
# worksheet1: Worksheet = workbook1.active
# print(worksheet1.freeze_panes)
# workbook2: Workbook = openpyxl.load_workbook(filename=path2)
# worksheet2: Worksheet = workbook2.active
# print(worksheet2.freeze_panes)
# rule = {"position": "C6"}
# print(check_xlsx_freeze(path1, rule))
# path1 = "../../../../../任务数据/LibreOffice Calc/copy_sheet_insert_gold.xlsx"
# rule = [ { "type": "sheet_name"
# , "sheet_idx": 0
# , "sheet_name": "Sheet1"
# }
# , { "type": "sheet_data"
# , "sheet_idx0": "../../../../../任务数据/LibreOffice Calc/copy_sheet_insert.xlsx@0"
# , "sheet_idx1": 1
# }
# , { "type": "sheet_name"
# , "sheet_idx": 2
# , "sheet_name": "Sheet2"
# }
# ]
# print(check_sheet_list(path1, rule))
# path1 = "../../任务数据/LibreOffice Calc/Create_column_charts_using_statistics_gold.xlsx"
# path2 = "../../任务数据/LibreOffice Calc/Create_column_charts_using_statistics_gold2.xlsx"
# print(compare_table(path1, path2, features=["chart"], chart_props=["type", "direction"]))
# path1 = "../../任务数据/LibreOffice Calc/Represent_in_millions_billions_gold.xlsx"
# path2 = "../../任务数据/LibreOffice Calc/Represent_in_millions_billions_gold3.xlsx"
# path1 = "../../任务数据/LibreOffice Calc/Set_Decimal_Separator_Dot.xlsx"
# path2 = "../../任务数据/LibreOffice Calc/Set_Decimal_Separator_Dot_gold.xlsx"
# workbook1: Workbook = openpyxl.load_workbook(filename=path1)
# worksheet1: Worksheet = workbook1.active
# import itertools
# for col, r in itertools.product( ['A', 'B']
# , range(1, 20)
# ):
# position: str = "{:}{:d}".format(col, r)
# print(worksheet1[position])
# print(worksheet1[position].value)
# print(worksheet1[position].number_format)
# workbook2: Workbook = openpyxl.load_workbook(filename=path2)
# worksheet2: Worksheet = workbook2.active
# for col, r in itertools.product( ['A', 'B']
# , range(1, 20)
# ):
# position: str = "{:}{:d}".format(col, r)
# print(worksheet2[position])
# print(worksheet2[position].value)
# print(worksheet2[position].number_format)
# print(compare_table(path1, path2, features=["number_format"]))
# path1 = "../../任务数据/LibreOffice Calc/Zoom_Out_Oversized_Cells_gold.xlsx"
# path2 = "../../任务数据/LibreOffice Calc/Zoom_Out_Oversized_Cells.xlsx"
# workbook1: Workbook = openpyxl.load_workbook(filename=path1)
# worksheet1: Worksheet = workbook1.active
# print(worksheet1.sheet_view.zoomScale)
# print(type(worksheet1.sheet_view.zoomScale))
#
# import os
# import os.path
# for wb in filter( lambda f: f.endswith(".xlsx")
# , os.listdir("../../任务数据/LibreOffice Calc/")
# ):
# path = os.path.join("../../任务数据/LibreOffice Calc/", wb)
# print(wb, openpyxl.load_workbook(filename=path).active.sheet_view.zoomScale)
# print(check_zoom(path1, {"relation": "lt", "ref_value": 100}))
# print(check_zoom(path2, {"relation": "lt", "ref_value": 100}))
#path1 = "../../任务数据/LibreOffice Calc/Customers_New_7digit_Id.xlsx"
#path2 = "../../任务数据/LibreOffice Calc/Customers_New_7digit_Id_gold.xlsx"
#data_frame: pd.DataFrame = pd.read_excel(path1)
#print(data_frame)
#print(compare_table(path1, path2, as_shown=True))
#from openpyxl.worksheet.cell_range import MultiCellRange
path = "../../任务数据/LibreOffice Calc/Order_Id_Mark_Pass_Fail_gold.xlsx"
#worksheet: Worksheet = openpyxl.load_workbook(filename=path).active
##print(worksheet.data_validations)
#print(type(worksheet.data_validations.dataValidation))
#for dat_vldt in worksheet.data_validations.dataValidation:
#print(dat_vldt.sqref)
#print(all(r in MultiCellRange("D2:D30 B1:B60") for r in dat_vldt.sqref))
print( check_data_validations( path, [ { "ranges": { "method": "spreadsheet_range"
, "ref": ["D2:D29", "D2:D1048576"]
}
, "type": { "method": "eq"
, "ref": "list"
}
, "formula1": { "method": "str_set_eq"
, "ref": ["Pass", "Fail", "Held"]
}
}
]
)
) )
print( compare_table( path2, path2
, rules=rules
)
)
#path = "../../任务数据/LibreOffice Calc/Order_Id_Mark_Pass_Fail_gold.xlsx"
#print( check_data_validations( path, [ { "ranges": { "method": "spreadsheet_range"
#, "ref": ["D2:D29", "D2:D1048576"]
#}
#, "type": { "method": "eq"
#, "ref": "list"
#}
#, "formula1": { "method": "str_set_eq"
#, "ref": ["Pass", "Fail", "Held"]
#}
#}
#]
#)
#)

16
evaluation_examples/examples/libreoffice_calc/0bf05a7d-b28b-44d2-955a-50b41e24012a.json
View File

@@ -63,7 +63,7 @@
"command": [ "command": [
"libreoffice", "libreoffice",
"--convert-to", "--convert-to",
"csv:Text - txt - csv (StarCalc):44,34,UTF-8,,,,false,true,true", "csv:Text - txt - csv (StarCalc):44,34,UTF-8,,,,false,true,true,false,false,1",
"--outdir", "--outdir",
"/home/user", "/home/user",
"/home/user/Customers_New_7digit_Id.xlsx" "/home/user/Customers_New_7digit_Id.xlsx"
@@ -76,11 +76,11 @@
"type": "vm_file", "type": "vm_file",
"path": [ "path": [
"/home/user/Customers_New_7digit_Id.xlsx", "/home/user/Customers_New_7digit_Id.xlsx",
"/home/user/Customers_New_7digit_Id.csv" "/home/user/Customers_New_7digit_Id-Sheet1.csv"
], ],
"dest": [ "dest": [
"Customers_New_7digit_Id.xlsx", "Customers_New_7digit_Id.xlsx",
"Customers_New_7digit_Id.csv" "Customers_New_7digit_Id-Sheet1.csv"
], ],
"multi": true "multi": true
}, },
@@ -92,12 +92,18 @@
], ],
"dest": [ "dest": [
"Customers_New_7digit_Id_gold.xlsx", "Customers_New_7digit_Id_gold.xlsx",
"Customers_New_7digit_Id_gold.csv" "Customers_New_7digit_Id_gold-Sheet1.csv"
], ],
"multi": true "multi": true
}, },
"options": { "options": {
"as_shown": true "rules": [
{
"type": "sheet_print",
"sheet_idx0": 0,
"sheet_idx1": "EI0"
}
]
} }
} }
} }

20
evaluation_examples/examples/libreoffice_calc/0cecd4f3-74de-457b-ba94-29ad6b5dafb6.json
View File

@@ -1,7 +1,7 @@
{ {
"id": "0cecd4f3-74de-457b-ba94-29ad6b5dafb6", "id": "0cecd4f3-74de-457b-ba94-29ad6b5dafb6",
"snapshot": "libreoffice_calc", "snapshot": "libreoffice_calc",
"instruction": "Copy sheet1 and insert it before sheet2", "instruction": "Copy Sheet1 and insert it before Sheet2 and rename it as Sheet1.bak",
"source": "https://www.libreofficehelp.com/add-insert-delete-copy-move-rename-a-worksheet-in-libreoffice-calc/", "source": "https://www.libreofficehelp.com/add-insert-delete-copy-move-rename-a-worksheet-in-libreoffice-calc/",
"config": [ "config": [
{ {
@@ -52,19 +52,21 @@
} }
} }
], ],
"func": "check_sheet_list", "func": "compare_table",
"result": { "result": {
"type": "vm_file", "type": "vm_file",
"path": "/home/user/copy_sheet_insert.xlsx", "path": "/home/user/copy_sheet_insert.xlsx",
"dest": "copy_sheet_insert.xlsx" "dest": "copy_sheet_insert.xlsx"
}, },
"expected": { "expected": {
"type": "rule", "type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=15SjZNQtdt55SW6FIpi2JmWmVOpoZGjEB&export=download&authuser=0&confirm=t&uuid=182ac3dc-e0e9-4d30-b800-b842d5fcd665&at=APZUnTXVlvE4vv1-QO7HKqQOrikJ:1705850636082",
"dest": "copy_sheet_insert_gold.xlsx"
},
"options": {
"rules": [ "rules": [
{ {
"type": "sheet_name", "type": "sheet_name"
"sheet_idx": 0,
"sheet_name": "Sheet1"
}, },
{ {
"type": "sheet_data", "type": "sheet_data",
@@ -72,9 +74,9 @@
"sheet_idx1": 1 "sheet_idx1": 1
}, },
{ {
"type": "sheet_name", "type": "sheet_data",
"sheet_idx": 2, "sheet_idx0": 1,
"sheet_name": "Sheet2" "sheet_idx1": "EI1"
} }
] ]
} }

25
evaluation_examples/examples/libreoffice_calc/1334ca3e-f9e3-4db8-9ca7-b4c653be7d17.json
View File

@@ -52,18 +52,31 @@
} }
} }
], ],
"func": "check_xlsx_zoom", "func": "compare_table",
"result": { "result": {
"type": "vm_file", "type": "vm_file",
"path": "/home/user/Zoom_Out_Oversized_Cells.xlsx", "path": "/home/user/Zoom_Out_Oversized_Cells.xlsx",
"dest": "Zoom_Out_Oversized_Cells.xlsx" "dest": "Zoom_Out_Oversized_Cells.xlsx"
}, },
"expected": { "expected": {
"type": "rule", "type": "cloud_file",
"rules": { "path": "https://drive.usercontent.google.com/download?id=1sX_roln3dNbXat0umlAIwZrW2eBmLkLE&export=download&authuser=0&confirm=t&uuid=a8f92992-86bd-43f4-92ce-9e4d99dca980&at=APZUnTUDPJVW_y6QeI9vlxlHAS0Q:1705851308643",
"relation": "lt", "dest": "Zoom_Out_Oversized_Cells_gold.xlsx"
"ref_value": 260 },
} "options": {
"rules": [
{
"type": "zoom",
"sheet_idx": 0,
"method": "lt",
"ref": 260
},
{
"type": "table_data",
"sheet_idx0": 0,
"sheet_idx1": "EI0"
}
]
} }
} }
} }

18
evaluation_examples/examples/libreoffice_calc/21df9241-f8d7-4509-b7f1-37e501a823f7.json
View File

@@ -63,7 +63,7 @@
"command": [ "command": [
"libreoffice", "libreoffice",
"--convert-to", "--convert-to",
"csv:Text - txt - csv (StarCalc):44,34,UTF-8,,,,false,true,true", "csv:Text - txt - csv (StarCalc):44,34,UTF-8,,,,false,true,true,false,false,1",
"--outdir", "--outdir",
"/home/user", "/home/user",
"/home/user/Represent_in_millions_billions.xlsx" "/home/user/Represent_in_millions_billions.xlsx"
@@ -76,11 +76,11 @@
"type": "vm_file", "type": "vm_file",
"path": [ "path": [
"/home/user/Represent_in_millions_billions.xlsx", "/home/user/Represent_in_millions_billions.xlsx",
"/home/user/Represent_in_millions_billions.csv" "/home/user/Represent_in_millions_billions-Sheet1.csv"
], ],
"dest": [ "dest": [
"Represent_in_millions_billions.xlsx", "Represent_in_millions_billions.xlsx",
"Represent_in_millions_billions.csv" "Represent_in_millions_billions-Sheet1.csv"
], ],
"multi": true "multi": true
}, },
@@ -92,13 +92,19 @@
], ],
"dest": [ "dest": [
"Represent_in_millions_billions_gold.xlsx", "Represent_in_millions_billions_gold.xlsx",
"Represent_in_millions_billions_gold.csv" "Represent_in_millions_billions_gold-Sheet1.csv"
], ],
"multi": true "multi": true
}, },
"options": { "options": {
"as_shown": true, "rules": [
"ignore_case": true {
"type": "sheet_print",
"sheet_idx0": 0,
"sheet_idx1": "EI0",
"ignore_case": true
}
]
} }
} }
} }