lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix one bug in Chrome getter; fix one erro for corner case in doc

Browse Source
This commit is contained in:
Timothyxxx
2024-04-02 14:50:29 +08:00
parent af04a305a9
commit 2d8eeaad58
2 changed files with 49 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
desktop_env/evaluators/getters/chrome.py
View File

@@ -558,39 +558,54 @@ def get_active_url_from_accessTree(env, config):
Return
url: str
"""
accessibility_tree: str = env.controller.get_accessibility_tree()
# download accessibility tree to "/home/user/Desktop"
logger.debug("AT@eval: %s", accessibility_tree)
# first, use accessibility API to get the active tab URL
at: _Element = lxml.etree.fromstring(accessibility_tree)
arch = platform.machine()
print("Your architecture is: {}".format(arch))
try:
if "arm" in arch:
selector = CSSSelector("application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]",
namespaces=_accessibility_ns_map)
else:
selector = CSSSelector("application[name=Google\\ Chrome] entry[name=Address\\ and\\ search\\ bar]",
namespaces=_accessibility_ns_map)
except:
logger.error("Failed to parse the selector for active tab URL")
return None
elements: List[_Element] = selector(at)
# if "xpath" in config:
# elements: List[_Element] = at.xpath(config["xpath"], namespaces=_accessibility_ns_map)
# elif "selectors" in config:
# selector = CSSSelector(", ".join(config["selectors"]), namespaces=_accessibility_ns_map)
# elements: List[_Element] = selector(at)
if len(elements) == 0:
print("no elements found")
return None
elif elements[-1].text is None:
print("no text found")
# Ensure the controller and its method are accessible and return a valid result
if hasattr(env, 'controller') and callable(getattr(env.controller, 'get_accessibility_tree', None)):
accessibility_tree = env.controller.get_accessibility_tree()
if accessibility_tree is None:
print("Failed to get the accessibility tree.")
return None
else:
print("Controller or method 'get_accessibility_tree' not found.")
return None
active_tab_url = config["goto_prefix"] + elements[0].text if "goto_prefix" in config.keys() else "https://" + \
elements[0].text
print("active tab url now: {}".format(active_tab_url))
logger.debug("AT@eval: %s", accessibility_tree)
at = None
try:
at = lxml.etree.fromstring(accessibility_tree)
except ValueError as e:
logger.error(f"Error parsing accessibility tree: {e}")
return None
# Determine the correct selector based on system architecture
selector = None
arch = platform.machine()
print(f"Your architecture is: {arch}")
if "arm" in arch:
selector_string = "application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
else:
selector_string = "application[name=Google\\ Chrome] entry[name=Address\\ and\\ search\\ bar]"
try:
selector = CSSSelector(selector_string, namespaces=_accessibility_ns_map)
except Exception as e:
logger.error(f"Failed to parse the selector for active tab URL: {e}")
return None
elements = selector(at) if selector else []
if not elements:
print("No elements found.")
return None
elif not elements[-1].text:
print("No text found in the latest element.")
return None
# Use a default prefix if 'goto_prefix' is not specified in the config
goto_prefix = config.get("goto_prefix", "https://")
active_tab_url = f"{goto_prefix}{elements[0].text}"
print(f"Active tab url now: {active_tab_url}")
return active_tab_url

4
desktop_env/evaluators/metrics/docs.py
View File

@@ -6,6 +6,7 @@ import zipfile
from io import BytesIO
from typing import List, Dict, Any
import easyocr
from PIL import Image
from docx import Document
from docx.enum.text import WD_PARAGRAPH_ALIGNMENT, WD_TAB_ALIGNMENT
@@ -247,8 +248,9 @@ def compare_docx_images(docx_file1, docx_file2):
return 1
import easyocr
def compare_image_text(image_path, rule):
if not image_path:
return 0
reader = easyocr.Reader(['en'])
result = reader.readtext(image_path)
extracted_text = ' '.join([entry[1] for entry in result])