lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into fix_chrome

Browse Source
This commit is contained in:
yuanmengqi
2025-07-13 10:16:08 +00:00
parent a16b54c175
commit 572a94b6df
15 changed files with 228 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
desktop_env/evaluators/metrics/chrome.py
View File

@@ -29,8 +29,8 @@ def is_expected_active_tab(active_tab_info: Dict[str, str], rule: Dict[str, Any]
actual_url = active_tab_info.get('url', None)
else:
actual_url = active_tab_info
print("expected_url: {}".format(expected_url))
print("actual_url: {}".format(actual_url))
logger.info("expected_url: {}".format(expected_url))
logger.info("actual_url: {}".format(actual_url))
return 1 if compare_urls(expected_url, actual_url) else 0
else:
logger.error(f"Unknown type: {match_type}")
@@ -76,23 +76,26 @@ def is_expected_url_pattern_match(result, rules) -> float:
if type(result) == dict:
result_url = result["url"]
print("result url: {}".format(result_url))
logger.info("result url: {}".format(result_url))
else:
result_url = result
# expect_regex = re.compile(rules["expected"])
patterns = rules["expected"]
print("expected_regex: {}".format(patterns))
logger.info("expected_regex: {}".format(patterns))
for pattern in patterns:
match = re.search(pattern, result_url)
print(match)
logger.info("match: {}".format(match))
if not match:
return 0.
return 1.
def is_expected_installed_extensions(installed_extensions, expected) -> float:
print("installed_extensions: ")
print(installed_extensions)
if not installed_extensions:
return 0.
logger.info("installed_extensions: ")
logger.info(installed_extensions)
expected_extensions = expected["expected"]
# whether the expected extensions are installed
@@ -109,6 +112,8 @@ def is_expected_tabs(open_tabs: List[Dict[str, str]], rule: Dict[str, Any]) -> f
"""
Checks if the expected tabs are open in Chrome.
"""
if not open_tabs:
return 0.
match_type = rule['type']
@@ -146,8 +151,10 @@ def is_expected_bookmarks(bookmarks: List[str], rule: Dict[str, Any]) -> float:
bookmark['type'] == 'folder' and bookmark['name'] == 'Liked Authors'), None)
if liked_authors_folder:
# Check if it contains the specified URLs
logger.info("'Liked Authors' folder exists")
liked_authors_urls = [bookmark['url'] for bookmark in liked_authors_folder['children'] if
bookmark['type'] == 'url']
logger.info("Here is the 'Liked Authors' folder's urls: {}".format(liked_authors_urls))
urls = rule['urls']
@@ -168,6 +175,9 @@ def is_expected_bookmarks(bookmarks: List[str], rule: Dict[str, Any]) -> float:
def is_expected_search_query(active_tab_info: Dict[str, str], rules: Dict[str, Any]) -> float:
if not active_tab_info:
return 0.
expected = rules['expect']
pattern = expected['pattern']
matched = re.search(pattern, active_tab_info['url'])

5
desktop_env/evaluators/metrics/gimp.py
View File

@@ -396,7 +396,10 @@ def check_structure_sim_resized(src_path, tgt_path):
# Check if the structure is similar
structure_same = structure_check_by_ssim(img_src_resized, img_tgt)
return structure_same
if structure_same:
return 1.
else:
return 0.
def check_contrast_increase_and_structure_sim(src_path, tgt_path):

63
desktop_env/evaluators/metrics/table.py
View File

@@ -463,23 +463,60 @@ def compare_table(result: str, expected: str = None, **options) -> float:
# }}} function compare_table #
def compare_csv(result: str, expected: str, **options) -> float:
def compare_csv(result: str, expected: Union[str, List[str]], **options) -> float:
"""
Compare CSV files. If expected is a list, returns 1.0 if result matches any of the expected files.
Args:
result: Path to result CSV file
expected: Path to expected CSV file or list of paths to expected CSV files
options: Additional options (strict, ignore_case)
Returns:
1.0 if result matches expected (or any file in expected list), 0.0 otherwise
"""
if result is None:
return 0.
with open(result) as f:
result_lines: Iterable[str] = f.read().splitlines()
with open(expected) as f:
expected_lines: Iterable[str] = f.read().splitlines()
if not options.get("strict", True):
result_lines = map(str.strip, result_lines)
expected_lines = map(str.strip, expected_lines)
if options.get("ignore_case", False):
result_lines = map(str.lower, result_lines)
expected_lines = map(str.lower, expected_lines)
try:
with open(result) as f:
result_lines: Iterable[str] = f.read().splitlines()
except (FileNotFoundError, IOError):
return 0.
metric: bool = list(result_lines) == list(expected_lines)
return float(metric)
# Convert expected to list if it's a single string (for backward compatibility)
if isinstance(expected, str):
expected_files = [expected]
else:
expected_files = expected
# Try to match against each expected file
for expected_file in expected_files:
try:
with open(expected_file) as f:
expected_lines: Iterable[str] = f.read().splitlines()
# Process lines based on options
current_result_lines = result_lines
current_expected_lines = expected_lines
if not options.get("strict", True):
current_result_lines = map(str.strip, current_result_lines)
current_expected_lines = map(str.strip, current_expected_lines)
if options.get("ignore_case", False):
current_result_lines = map(str.lower, current_result_lines)
current_expected_lines = map(str.lower, current_expected_lines)
# Check if this expected file matches
if list(current_result_lines) == list(current_expected_lines):
return 1.0
except (FileNotFoundError, IOError):
# If this expected file doesn't exist, continue to next one
continue
# No match found
return 0.0
def compare_conference_city_in_order(actual_city_list_path, expected_city):