lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Check and fix on Chrome tasks

Browse Source 
- Added `pytz` dependency to `requirements.txt` for timezone handling.
- Introduced `get_macys_product_url_parse` function to replace the old `get_url_path_parse` for better clarity and maintain backward compatibility.
- Enhanced logging throughout the `get_active_tab_html_parse` and `get_rule_relativeTime` functions for improved debugging and traceability.
- Updated JSON examples to reflect changes in expected keys and added new fields for better evaluation context.
- Removed deprecated execution commands from JSON examples to streamline the evaluation process.
This commit is contained in:
yuanmengqi
2025-07-06 07:52:37 +00:00
parent 1b40a458de
commit 9be6fcd688
20 changed files with 521 additions and 171 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
desktop_env/evaluators/metrics/general.py
View File

@@ -307,6 +307,9 @@ def check_direct_json_object(result, rules) -> float:
One of the most commonly used function to evalute.
Compare two json objects directly.
"""
logger.info(f"[DEBUG] check_direct_json_object called with result: {result}")
logger.info(f"[DEBUG] check_direct_json_object called with rules: {rules}")
if isinstance(result, str):
# remove blanks before and after result
result = result.strip()
@@ -314,45 +317,84 @@ def check_direct_json_object(result, rules) -> float:
result = result.replace("'", '"')
# load json object
result = json.loads(result)
logger.info(f"[DEBUG] Processed result: {result}")
if result is None:
logger.info("[DEBUG] Result is None, returning 0.0")
return 0.
# Check if expected value contains evaluation failure indicator
try:
expected_json = rules.get("expected", {})
if expected_json:
for key, value in expected_json.items():
if value == "__EVALUATION_FAILED__":
logger.error(f"[DEBUG] Expected value for key '{key}' indicates evaluation failure, returning 0.0")
return 0.
except Exception as e:
logger.error(f"[DEBUG] Error checking for evaluation failure indicator: {e}")
return 0.
try:
expect_in_result = rules.get("expect_in_result", False)
logger.info(f"[DEBUG] expect_in_result: {expect_in_result}")
if not expect_in_result:
expected_json = rules["expected"]
logger.info(f"[DEBUG] Expected JSON: {expected_json}")
for key in expected_json.keys():
expected_value = expected_json.get(key)
actual_value = result.get(key)
logger.info(f"[DEBUG] Checking key '{key}': expected='{expected_value}', actual='{actual_value}'")
if expected_json.get("ignore_list_order", False):
expected_value = sorted(expected_value)
result_value = sorted(result.get(key))
logger.info(f"[DEBUG] Comparing lists (sorted): expected={expected_value}, actual={result_value}")
if expected_value != result_value:
logger.info(f"[DEBUG] List comparison failed for key '{key}', returning 0.0")
return 0.
else:
if expected_value != result.get(key):
if expected_value != actual_value:
logger.info(f"[DEBUG] Value comparison failed for key '{key}': expected='{expected_value}', actual='{actual_value}', returning 0.0")
return 0.
else:
logger.info(f"[DEBUG] Value comparison passed for key '{key}'")
logger.info("[DEBUG] All comparisons passed, returning 1.0")
return 1.0
else:
expected_json = rules["expected"]
logger.info(f"[DEBUG] Expected JSON (expect_in_result mode): {expected_json}")
for key in expected_json.keys():
if isinstance(expected_json.get(key), list):
flag = 0
expected_value_list = expected_json.get(key)
logger.info(f"[DEBUG] Checking list key '{key}': expected_list={expected_value_list}, actual='{result.get(key)}'")
for each_expected_value in expected_value_list:
if isinstance(result.get(key), list) and each_expected_value in result.get(key):
flag = 1
logger.info(f"[DEBUG] Found expected value '{each_expected_value}' in result list for key '{key}'")
break
if flag == 0:
logger.info(f"[DEBUG] No expected values found in result list for key '{key}', returning 0.0")
return 0.
elif isinstance(expected_json.get(key), str):
if expected_json.get(key) not in result.get(key):
expected_str = expected_json.get(key)
actual_str = result.get(key)
logger.info(f"[DEBUG] Checking string key '{key}': expected='{expected_str}', actual='{actual_str}'")
if expected_str not in actual_str:
logger.info(f"[DEBUG] Expected string '{expected_str}' not found in actual string '{actual_str}' for key '{key}', returning 0.0")
return 0.
else:
logger.debug("check_direct_json_object: expected value type not supported")
return 0.
logger.info("[DEBUG] All expect_in_result comparisons passed, returning 1.0")
return 1.0
except:
logger.debug("check_direct_json_object: result is not a valid json object")
except Exception as e:
logger.debug(f"check_direct_json_object: result is not a valid json object, error: {e}")
return 0.