lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into fix_chrome

Browse Source
This commit is contained in:
yuanmengqi
2025-07-13 10:16:08 +00:00
parent a16b54c175
commit 572a94b6df
15 changed files with 228 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files

90
desktop_env/evaluators/getters/chrome.py
View File

@@ -52,6 +52,11 @@ def get_info_from_website(env, config: Dict[Any, Any]) -> Any:
- attribute (str): optional for 'attribute' and 'click_and_attribute', the attribute to be extracted.
- backups (Any): The backup information to be returned if the extraction fails.
"""
# 添加函数开始日志
logger.info(f"[INFO_FROM_WEBSITE] Starting to get information from website: {config.get('url', 'N/A')}")
logger.info(f"[INFO_FROM_WEBSITE] Total info operations to perform: {len(config.get('infos', []))}")
logger.debug(f"[INFO_FROM_WEBSITE] Full config: {config}")
try:
host = env.vm_ip
port = env.chromium_port # fixme: this port is hard-coded, need to be changed from config file
@@ -59,11 +64,18 @@ def get_info_from_website(env, config: Dict[Any, Any]) -> Any:
remote_debugging_url = f"http://{host}:{port}"
backend_url = f"http://{host}:{server_port}"
use_proxy = env.current_use_proxy
logger.info(f"[INFO_FROM_WEBSITE] Connecting to Chrome at {remote_debugging_url}")
with sync_playwright() as p:
# connect to remote Chrome instance
try:
browser = p.chromium.connect_over_cdp(remote_debugging_url)
logger.info(f"[INFO_FROM_WEBSITE] Successfully connected to existing Chrome instance")
except Exception as e:
logger.warning(f"[INFO_FROM_WEBSITE] Failed to connect to existing Chrome instance: {e}")
logger.info(f"[INFO_FROM_WEBSITE] Starting new Chrome instance...")
# If the connection fails (e.g., the agent close the browser instance), start a new browser instance
app = 'chromium' if 'arm' in platform.machine() else 'google-chrome'
command = [
@@ -72,52 +84,116 @@ def get_info_from_website(env, config: Dict[Any, Any]) -> Any:
]
if use_proxy:
command.append(f"--proxy-server=127.0.0.1:18888")
logger.info(f"[INFO_FROM_WEBSITE] Using proxy server: 127.0.0.1:18888")
logger.info(f"[INFO_FROM_WEBSITE] Starting browser with command: {' '.join(command)}")
payload = json.dumps({"command": command, "shell": False})
headers = {"Content-Type": "application/json"}
#requests.post("http://" + host + ":" + server_port + "/setup" + "/launch", headers=headers, data=payload)
requests.post(backend_url + "/setup" + "/launch", headers=headers, data=payload)
time.sleep(5)
browser = p.chromium.connect_over_cdp(remote_debugging_url)
logger.info(f"[INFO_FROM_WEBSITE] Successfully connected to new Chrome instance")
page = browser.contexts[0].new_page()
logger.info(f"[INFO_FROM_WEBSITE] Created new page, navigating to: {config['url']}")
page.goto(config["url"])
page.wait_for_load_state('load')
# 记录页面加载完成后的信息
logger.info(f"[INFO_FROM_WEBSITE] Page loaded successfully")
logger.info(f"[INFO_FROM_WEBSITE] Page title: '{page.title()}'")
logger.info(f"[INFO_FROM_WEBSITE] Current URL: '{page.url}'")
infos = []
for info_dict in config.get('infos', []):
for idx, info_dict in enumerate(config.get('infos', [])):
logger.info(f"[INFO_FROM_WEBSITE] Processing info operation {idx + 1}/{len(config.get('infos', []))}")
logger.debug(f"[INFO_FROM_WEBSITE] Info config: {info_dict}")
if page.url != config["url"]:
logger.info(f"[INFO_FROM_WEBSITE] Page URL changed, navigating back to: {config['url']}")
page.goto(config["url"])
page.wait_for_load_state('load')
logger.info(f"[INFO_FROM_WEBSITE] Back to original page")
action = info_dict.get('action', 'inner_text')
selector = info_dict.get('selector')
logger.info(f"[INFO_FROM_WEBSITE] Action: {action}, Selector: {selector}")
if action == "inner_text":
logger.debug(f"[INFO_FROM_WEBSITE] Waiting for element with selector: {selector}")
ele = page.wait_for_selector(info_dict['selector'], state='attached', timeout=10000)
infos.append(ele.inner_text())
extracted_text = ele.inner_text()
logger.info(f"[INFO_FROM_WEBSITE] Successfully extracted inner_text: '{extracted_text}'")
infos.append(extracted_text)
elif action == "attribute":
attribute = info_dict.get('attribute')
logger.debug(f"[INFO_FROM_WEBSITE] Waiting for element with selector: {selector}")
logger.debug(f"[INFO_FROM_WEBSITE] Extracting attribute: {attribute}")
ele = page.wait_for_selector(info_dict['selector'], state='attached', timeout=10000)
infos.append(ele.get_attribute(info_dict['attribute']))
extracted_attr = ele.get_attribute(info_dict['attribute'])
logger.info(f"[INFO_FROM_WEBSITE] Successfully extracted attribute '{attribute}': '{extracted_attr}'")
infos.append(extracted_attr)
elif action == 'click_and_inner_text':
logger.debug(f"[INFO_FROM_WEBSITE] Performing click_and_inner_text with {len(info_dict['selector'])} selectors")
for idx, sel in enumerate(info_dict['selector']):
logger.debug(f"[INFO_FROM_WEBSITE] Processing selector {idx + 1}/{len(info_dict['selector'])}: {sel}")
if idx != len(info_dict['selector']) - 1:
logger.debug(f"[INFO_FROM_WEBSITE] Clicking element with selector: {sel}")
link = page.wait_for_selector(sel, state='attached', timeout=10000)
link.click()
page.wait_for_load_state('load')
logger.info(f"[INFO_FROM_WEBSITE] Successfully clicked element, page loaded")
logger.debug(f"[INFO_FROM_WEBSITE] New page URL: {page.url}")
else:
logger.debug(f"[INFO_FROM_WEBSITE] Extracting inner_text from final element: {sel}")
ele = page.wait_for_selector(sel, state='attached', timeout=10000)
infos.append(ele.inner_text())
extracted_text = ele.inner_text()
logger.info(f"[INFO_FROM_WEBSITE] Successfully extracted inner_text after clicks: '{extracted_text}'")
infos.append(extracted_text)
elif action == 'click_and_attribute':
attribute = info_dict.get('attribute')
logger.debug(f"[INFO_FROM_WEBSITE] Performing click_and_attribute with {len(info_dict['selector'])} selectors")
logger.debug(f"[INFO_FROM_WEBSITE] Target attribute: {attribute}")
for idx, sel in enumerate(info_dict['selector']):
logger.debug(f"[INFO_FROM_WEBSITE] Processing selector {idx + 1}/{len(info_dict['selector'])}: {sel}")
if idx != len(info_dict['selector']) - 1:
logger.debug(f"[INFO_FROM_WEBSITE] Clicking element with selector: {sel}")
link = page.wait_for_selector(sel, state='attached', timeout=10000)
link.click()
page.wait_for_load_state('load')
logger.info(f"[INFO_FROM_WEBSITE] Successfully clicked element, page loaded")
logger.debug(f"[INFO_FROM_WEBSITE] New page URL: {page.url}")
else:
logger.debug(f"[INFO_FROM_WEBSITE] Extracting attribute from final element: {sel}")
ele = page.wait_for_selector(sel, state='attached')
infos.append(ele.get_attribute(info_dict['attribute']))
extracted_attr = ele.get_attribute(info_dict['attribute'])
logger.info(f"[INFO_FROM_WEBSITE] Successfully extracted attribute '{attribute}' after clicks: '{extracted_attr}'")
infos.append(extracted_attr)
else:
logger.error(f"[INFO_FROM_WEBSITE] Unsupported action: {action}")
raise NotImplementedError(f'The action {action} is not supported yet.')
logger.info(f"[INFO_FROM_WEBSITE] Completed info operation {idx + 1}")
# 记录最终提取的所有信息
logger.info(f"[INFO_FROM_WEBSITE] All operations completed successfully")
logger.info(f"[INFO_FROM_WEBSITE] Total extracted information count: {len(infos)}")
logger.info(f"[INFO_FROM_WEBSITE] Final extracted information: {infos}")
return infos
except Exception as e:
logger.error(f'[ERROR]: failed to obtain information from the website: {config["url"]}. Use backup results instead.')
return config.get('backups', None)
logger.error(f'[INFO_FROM_WEBSITE] ERROR: Failed to obtain information from website: {config.get("url", "N/A")}')
logger.error(f'[INFO_FROM_WEBSITE] Exception details: {str(e)}')
logger.error(f'[INFO_FROM_WEBSITE] Exception type: {type(e).__name__}')
logger.info(f'[INFO_FROM_WEBSITE] Using backup results instead')
backup_data = config.get('backups', None)
logger.info(f'[INFO_FROM_WEBSITE] Backup data: {backup_data}')
return backup_data
# The following ones just need to load info from the files of software, no need to connect to the software