lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix some errors found in calc examples

Browse Source
This commit is contained in:
Timothyxxx
2024-01-28 21:19:18 +08:00
parent 353ab6607d
commit cc21c3a6b1
9 changed files with 186 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
desktop_env/evaluators/metrics/__init__.py
View File

@@ -19,7 +19,7 @@ from .slides import check_presenter_console_disable, check_image_stretch_and_cen
compare_pptx_files, check_strikethrough, \ compare_pptx_files, check_strikethrough, \
check_slide_orientation_Portrait, evaluate_presentation_fill_to_rgb_distance, check_left_panel check_slide_orientation_Portrait, evaluate_presentation_fill_to_rgb_distance, check_left_panel
# from .table import check_sheet_list, check_xlsx_freeze, check_xlsx_zoom, check_data_validations # from .table import check_sheet_list, check_xlsx_freeze, check_xlsx_zoom, check_data_validations
from .table import compare_table from .table import compare_table, compare_csv
from .thunderbird import check_thunderbird_prefs, check_thunderbird_filter from .thunderbird import check_thunderbird_prefs, check_thunderbird_filter
from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, compare_images, compare_audios, \ from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, compare_images, compare_audios, \

2
desktop_env/evaluators/metrics/pdf.py
View File

@@ -6,6 +6,8 @@ from pypdf import PdfReader
def check_pdf_pages(pdf_file: str, rules: Dict[str, Any]) -> float: def check_pdf_pages(pdf_file: str, rules: Dict[str, Any]) -> float:
if pdf_file is None:
return 0.0
reader = PdfReader(pdf_file) reader = PdfReader(pdf_file)
nb_pages: int = len(reader.pages) nb_pages: int = len(reader.pages)
return float(getattr(operator, rules["relation"])(nb_pages, rules["ref_value"])) return float(getattr(operator, rules["relation"])(nb_pages, rules["ref_value"]))

44
desktop_env/evaluators/metrics/table.py
View File

@@ -1,27 +1,30 @@
import functools
import itertools
import logging import logging
import os.path
# import operator # import operator
from numbers import Number from numbers import Number
from typing import Any, Union, cast, Callable, Iterable from typing import Any, Union, cast, Callable, Iterable
from typing import Dict, List, Tuple from typing import Dict, List, Tuple
import os.path
import itertools
import functools
import openpyxl import openpyxl
import pandas as pd import pandas as pd
from openpyxl import Workbook from openpyxl import Workbook
from openpyxl.worksheet.worksheet import Worksheet from openpyxl.cell.cell import Cell
# from openpyxl.worksheet.cell_range import MultiCellRange # from openpyxl.worksheet.cell_range import MultiCellRange
from openpyxl.worksheet.datavalidation import DataValidation from openpyxl.worksheet.datavalidation import DataValidation
from openpyxl.cell.cell import Cell from openpyxl.worksheet.worksheet import Worksheet
#from openpyxl.utils import coordinate_to_tuple
from .utils import load_charts, load_sparklines, load_rows_or_cols, load_xlsx_styles
from .utils import _match_value_to_rule, _read_cell_style, read_cell_value from .utils import _match_value_to_rule, _read_cell_style, read_cell_value
from .utils import load_charts, load_sparklines, load_rows_or_cols, load_xlsx_styles
# from openpyxl.utils import coordinate_to_tuple
logger = logging.getLogger("desktopenv.metric.table") logger = logging.getLogger("desktopenv.metric.table")
BOOK = Union[pd.ExcelFile, Workbook, str] BOOK = Union[pd.ExcelFile, Workbook, str]
def _parse_sheet_idx(sheet_idx: Union[int, str] def _parse_sheet_idx(sheet_idx: Union[int, str]
, result: BOOK, expected: BOOK , result: BOOK, expected: BOOK
, result_sheet_names: List[str] , result_sheet_names: List[str]
@@ -49,7 +52,10 @@ def _parse_sheet_idx( sheet_idx: Union[int, str]
return book, index return book, index
# }}} function _parse_sheet_idx # # }}} function _parse_sheet_idx #
SHEET = Union[pd.DataFrame, Worksheet, List[str]] SHEET = Union[pd.DataFrame, Worksheet, List[str]]
def _load_sheet(book: BOOK, index: str) -> SHEET: def _load_sheet(book: BOOK, index: str) -> SHEET:
# function _load_sheet {{{ # # function _load_sheet {{{ #
if isinstance(book, str): if isinstance(book, str):
@@ -72,7 +78,8 @@ def _load_sheet(book: BOOK, index: str) -> SHEET:
raise NotImplementedError("Not supported workbook format") raise NotImplementedError("Not supported workbook format")
# }}} function _load_sheet # # }}} function _load_sheet #
def compare_table(result: str, expected: str, **options) -> float:
def compare_table(result: str, expected: str = None, **options) -> float:
# function compare_table {{{ # # function compare_table {{{ #
""" """
Args: Args:
@@ -99,14 +106,21 @@ def compare_table(result: str, expected: str, **options) -> float:
return 0. return 0.
worksheetr_names: List[str] = pdworkbookr.sheet_names worksheetr_names: List[str] = pdworkbookr.sheet_names
if expected is not None:
xlworkbooke: Workbook = openpyxl.load_workbook(filename=expected) xlworkbooke: Workbook = openpyxl.load_workbook(filename=expected)
pdworkbooke = pd.ExcelFile(expected) pdworkbooke = pd.ExcelFile(expected)
worksheete_names: List[str] = pdworkbooke.sheet_names worksheete_names: List[str] = pdworkbooke.sheet_names
else:
xlworkbooke: Workbook = None
pdworkbooke = None
worksheete_names: List[str] = None
parse_idx: Callable[[Union[str, int], BOOK, BOOK], BOOK] = \ parse_idx: Callable[[Union[str, int], BOOK, BOOK], BOOK] = \
functools.partial( _parse_sheet_idx functools.partial(
, result_sheet_names=worksheetr_names _parse_sheet_idx,
, expected_sheet_names=worksheete_names result_sheet_names=worksheetr_names,
expected_sheet_names=worksheete_names
) )
passes = True passes = True
@@ -204,7 +218,8 @@ def compare_table(result: str, expected: str, **options) -> float:
sheet: Worksheet = _load_sheet(*parse_idx(r["sheet_idx"], xlworkbookr, xlworkbooke)) sheet: Worksheet = _load_sheet(*parse_idx(r["sheet_idx"], xlworkbookr, xlworkbooke))
zoom_scale: Number = sheet.sheet_view.zoomScale or 100. zoom_scale: Number = sheet.sheet_view.zoomScale or 100.
metric: bool = _match_value_to_rule(zoom_scale, r) metric: bool = _match_value_to_rule(zoom_scale, r)
logger.debug("Assertion: %s.zoom(%.1f) %s %.1f - %s", r["sheet_idx"], zoom_scale, r["method"], r["ref"], metric) logger.debug("Assertion: %s.zoom(%.1f) %s %.1f - %s", r["sheet_idx"], zoom_scale, r["method"], r["ref"],
metric)
# }}} Check Zooming # # }}} Check Zooming #
elif r["type"] == "data_validation": elif r["type"] == "data_validation":
@@ -323,6 +338,7 @@ def compare_table(result: str, expected: str, **options) -> float:
return float(passes) return float(passes)
# }}} function compare_table # # }}} function compare_table #
def compare_csv(result: str, expected: str, **options) -> float: def compare_csv(result: str, expected: str, **options) -> float:
if result is None: if result is None:
return 0. return 0.
@@ -341,6 +357,7 @@ def compare_csv(result: str, expected: str, **options) -> float:
metric: bool = list(result_lines) == list(expected_lines) metric: bool = list(result_lines) == list(expected_lines)
return float(metric) return float(metric)
if __name__ == '__main__': if __name__ == '__main__':
import datetime import datetime
import sys import sys
@@ -360,7 +377,8 @@ if __name__ == '__main__':
stdout_handler.setLevel(logging.INFO) stdout_handler.setLevel(logging.INFO)
sdebug_handler.setLevel(logging.DEBUG) sdebug_handler.setLevel(logging.DEBUG)
formatter = logging.Formatter(fmt="\x1b[1;33m[%(asctime)s \x1b[31m%(levelname)s \x1b[32m%(module)s/%(lineno)d-%(processName)s\x1b[1;33m] \x1b[0m%(message)s") formatter = logging.Formatter(
fmt="\x1b[1;33m[%(asctime)s \x1b[31m%(levelname)s \x1b[32m%(module)s/%(lineno)d-%(processName)s\x1b[1;33m] \x1b[0m%(message)s")
file_handler.setFormatter(formatter) file_handler.setFormatter(formatter)
debug_handler.setFormatter(formatter) debug_handler.setFormatter(formatter)
stdout_handler.setFormatter(formatter) stdout_handler.setFormatter(formatter)

5
evaluation_examples/examples/libreoffice_calc/4f07fbe9-70de-4927-a4d5-bb28bc12c52c.json
View File

@@ -92,8 +92,9 @@
], ],
"dest": [ "dest": [
"Padding_Decimals_In_Formular_gold.xlsx", "Padding_Decimals_In_Formular_gold.xlsx",
"Padding_Decimals_In_Formular_gold-Sheet1.xlsx" "Padding_Decimals_In_Formular_gold-Sheet1.csv"
] ],
"multi": true
}, },
"options": { "options": {
"rules": [ "rules": [

2
evaluation_examples/examples/libreoffice_calc/6e99a1ad-07d2-4b66-a1ce-ece6d99c20a5.json
View File

@@ -88,7 +88,7 @@
"type": "vm_file", "type": "vm_file",
"path": [ "path": [
"/home/user/Keep_Two_decimal_points.xlsx", "/home/user/Keep_Two_decimal_points.xlsx",
"/home/user/Keep_Two_decimal_points-Sheet1.xlsx" "/home/user/Keep_Two_decimal_points-Sheet1.csv"
], ],
"dest": [ "dest": [
"Keep_Two_decimal_points.xlsx", "Keep_Two_decimal_points.xlsx",

2
evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json
View File

@@ -7,7 +7,7 @@
{ {
"type": "download", "type": "download",
"parameters": { "parameters": {
"file": [ "files": [
{ {
"url": "https://drive.usercontent.google.com/download?id=1uT0axjo9lwkKu6hYVnsAL2FCrdH0DLUv&export=download&authuser=0&confirm=t&uuid=e7da6304-9c7a-4862-8a30-9f2284b843da&at=APZUnTVNHThpAZJmF6IuPckFvslw:1704187618838", "url": "https://drive.usercontent.google.com/download?id=1uT0axjo9lwkKu6hYVnsAL2FCrdH0DLUv&export=download&authuser=0&confirm=t&uuid=e7da6304-9c7a-4862-8a30-9f2284b843da&at=APZUnTVNHThpAZJmF6IuPckFvslw:1704187618838",
"path": "/home/user/Set_Decimal_Separator_Dot.xlsx" "path": "/home/user/Set_Decimal_Separator_Dot.xlsx"

27
experiment_a11y_tree.py
View File

@@ -186,24 +186,21 @@ if __name__ == '__main__':
] ]
calc_list = [ calc_list = [
# "eb03d19a-b88d-4de4-8a64-ca0ac66f426b", "eb03d19a-b88d-4de4-8a64-ca0ac66f426b",
# "0bf05a7d-b28b-44d2-955a-50b41e24012a", "0bf05a7d-b28b-44d2-955a-50b41e24012a",
# "7a4e4bc8-922c-4c84-865c-25ba34136be1", "7a4e4bc8-922c-4c84-865c-25ba34136be1",
# "2bd59342-0664-4ccb-ba87-79379096cc08", "2bd59342-0664-4ccb-ba87-79379096cc08",
# "ecb0df7a-4e8d-4a03-b162-053391d3afaf", "ecb0df7a-4e8d-4a03-b162-053391d3afaf",
# "7efeb4b1-3d19-4762-b163-63328d66303b", "7efeb4b1-3d19-4762-b163-63328d66303b",
# "4e6fcf72-daf3-439f-a232-c434ce416af6", "4e6fcf72-daf3-439f-a232-c434ce416af6",
# "6054afcb-5bab-4702-90a0-b259b5d3217c", "6054afcb-5bab-4702-90a0-b259b5d3217c",
# "abed40dc-063f-4598-8ba5-9fe749c0615d", "abed40dc-063f-4598-8ba5-9fe749c0615d",
# "01b269ae-2111-4a07-81fd-3fcd711993b0", "01b269ae-2111-4a07-81fd-3fcd711993b0",
# "8b1ce5f2-59d2-4dcc-b0b0-666a714b9a14", "8b1ce5f2-59d2-4dcc-b0b0-666a714b9a14",
# "0cecd4f3-74de-457b-ba94-29ad6b5dafb6", "0cecd4f3-74de-457b-ba94-29ad6b5dafb6",
"4188d3a4-077d-46b7-9c86-23e1a036f6c1", "4188d3a4-077d-46b7-9c86-23e1a036f6c1",
"51b11269-2ca8-4b2a-9163-f21758420e78", "51b11269-2ca8-4b2a-9163-f21758420e78",
"7e429b8d-a3f0-4ed0-9b58-08957d00b127", "7e429b8d-a3f0-4ed0-9b58-08957d00b127",
"f5a90742-3fa2-40fc-a564-f29b054e0337",
"22df9241-f8d7-4509-b7f1-37e501a823f7",
"1434ca3e-f9e3-4db8-9ca7-b4c653be7d17",
"347ef137-7eeb-4c80-a3bb-0951f26a8aff", "347ef137-7eeb-4c80-a3bb-0951f26a8aff",
"6e99a1ad-07d2-4b66-a1ce-ece6d99c20a5", "6e99a1ad-07d2-4b66-a1ce-ece6d99c20a5",
"3aaa4e37-dc91-482e-99af-132a612d40f3", "3aaa4e37-dc91-482e-99af-132a612d40f3",

34
experiment_screenshot.py
View File

@@ -167,5 +167,35 @@ if __name__ == '__main__':
"480bcfea-d68f-4aaa-a0a9-2589ef319381", "480bcfea-d68f-4aaa-a0a9-2589ef319381",
"af630914-714e-4a24-a7bb-f9af687d3b91" "af630914-714e-4a24-a7bb-f9af687d3b91"
] ]
for example_id in chrome_list: calc_list = [
main("chrome", example_id) "eb03d19a-b88d-4de4-8a64-ca0ac66f426b",
"0bf05a7d-b28b-44d2-955a-50b41e24012a",
"7a4e4bc8-922c-4c84-865c-25ba34136be1",
"2bd59342-0664-4ccb-ba87-79379096cc08",
"ecb0df7a-4e8d-4a03-b162-053391d3afaf",
"7efeb4b1-3d19-4762-b163-63328d66303b",
"4e6fcf72-daf3-439f-a232-c434ce416af6",
"6054afcb-5bab-4702-90a0-b259b5d3217c",
"abed40dc-063f-4598-8ba5-9fe749c0615d",
"01b269ae-2111-4a07-81fd-3fcd711993b0",
"8b1ce5f2-59d2-4dcc-b0b0-666a714b9a14",
"0cecd4f3-74de-457b-ba94-29ad6b5dafb6",
"4188d3a4-077d-46b7-9c86-23e1a036f6c1",
"51b11269-2ca8-4b2a-9163-f21758420e78",
"7e429b8d-a3f0-4ed0-9b58-08957d00b127",
"347ef137-7eeb-4c80-a3bb-0951f26a8aff",
"6e99a1ad-07d2-4b66-a1ce-ece6d99c20a5",
"3aaa4e37-dc91-482e-99af-132a612d40f3",
"37608790-6147-45d0-9f20-1137bb35703d",
"f9584479-3d0d-4c79-affa-9ad7afdd8850",
"d681960f-7bc3-4286-9913-a8812ba3261a",
"21df9241-f8d7-4509-b7f1-37e501a823f7",
"1334ca3e-f9e3-4db8-9ca7-b4c653be7d17",
"357ef137-7eeb-4c80-a3bb-0951f26a8aff",
"aa3a8974-2e85-438b-b29e-a64df44deb4b",
"a01fbce3-2793-461f-ab86-43680ccbae25",
"4f07fbe9-70de-4927-a4d5-bb28bc12c52c",
]
for example_id in calc_list:
main("libreoffice_calc", example_id)

13
mm_agents/gpt_4v_agent.py
View File

@@ -505,6 +505,7 @@ class GPT4v_Agent:
@backoff.on_exception( @backoff.on_exception(
backoff.expo, backoff.expo,
(APIError, RateLimitError, APIConnectionError, ServiceUnavailableError, InvalidRequestError), (APIError, RateLimitError, APIConnectionError, ServiceUnavailableError, InvalidRequestError),
max_tries=3
) )
def call_llm(self, payload): def call_llm(self, payload):
response = requests.post( response = requests.post(
@@ -514,6 +515,18 @@ class GPT4v_Agent:
) )
if response.status_code != 200: if response.status_code != 200:
if response.json()['error']['code'] == "context_length_exceeded":
print("Context length exceeded. Retrying with a smaller context.")
payload["messages"] = payload["messages"][-1:]
retry_response = requests.post(
"https://api.openai.com/v1/chat/completions",
headers=self.headers,
json=payload
)
if retry_response.status_code != 200:
print("Failed to call LLM: " + retry_response.text)
return ""
print("Failed to call LLM: " + response.text) print("Failed to call LLM: " + response.text)
return "" return ""
else: else: