diff --git a/desktop_env/evaluators/getters/chrome.py b/desktop_env/evaluators/getters/chrome.py index 619219b..f856704 100644 --- a/desktop_env/evaluators/getters/chrome.py +++ b/desktop_env/evaluators/getters/chrome.py @@ -2,27 +2,29 @@ import json import logging import os import platform -import time import sqlite3 +import time from typing import Dict, Any, List -from pydrive.auth import GoogleAuth -from pydrive.drive import GoogleDrive, GoogleDriveFileList, GoogleDriveFile -from playwright.sync_api import sync_playwright from urllib.parse import urlparse, parse_qs import lxml.etree +import requests from lxml.cssselect import CSSSelector from lxml.etree import _Element +from playwright.sync_api import sync_playwright +from pydrive.auth import GoogleAuth +from pydrive.drive import GoogleDrive, GoogleDriveFileList, GoogleDriveFile -_accessibility_ns_map = {"st": "uri:deskat:state.at-spi.gnome.org" - , "attr": "uri:deskat:attributes.at-spi.gnome.org" - , "cp": "uri:deskat:component.at-spi.gnome.org" - , "doc": "uri:deskat:document.at-spi.gnome.org" - , "docattr": "uri:deskat:attributes.document.at-spi.gnome.org" - , "txt": "uri:deskat:text.at-spi.gnome.org" - , "val": "uri:deskat:value.at-spi.gnome.org" - , "act": "uri:deskat:action.at-spi.gnome.org" - } +_accessibility_ns_map = { + "st": "uri:deskat:state.at-spi.gnome.org", + "attr": "uri:deskat:attributes.at-spi.gnome.org", + "cp": "uri:deskat:component.at-spi.gnome.org", + "doc": "uri:deskat:document.at-spi.gnome.org", + "docattr": "uri:deskat:attributes.document.at-spi.gnome.org", + "txt": "uri:deskat:text.at-spi.gnome.org", + "val": "uri:deskat:value.at-spi.gnome.org", + "act": "uri:deskat:action.at-spi.gnome.org" +} logger = logging.getLogger("desktopenv.getters.chrome") @@ -44,9 +46,12 @@ def get_default_search_engine(env, config: Dict[str, str]): "import os; print(os.path.join(os.getenv('HOME'), 'Library/Application Support/Google/Chrome/Default/Preferences'))")[ 'output'].strip() elif os_type == 'Linux': - preference_file_path = env.controller.execute_python_command( - "import os; print(os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Preferences'))")[ - 'output'].strip() + if "arm" in platform.machine(): + raise NotImplementedError + else: + preference_file_path = env.controller.execute_python_command( + "import os; print(os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Preferences'))")[ + 'output'].strip() else: raise Exception('Unsupported operating system') @@ -77,9 +82,12 @@ def get_cookie_data(env, config: Dict[str, str]): "import os; print(os.path.join(os.getenv('HOME'), 'Library/Application Support/Google/Chrome/Default/Cookies'))")[ 'output'].strip() elif os_type == 'Linux': - chrome_cookie_file_path = env.controller.execute_python_command( - "import os; print(os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Cookies'))")[ - 'output'].strip() + if "arm" in platform.machine(): + raise NotImplementedError + else: + chrome_cookie_file_path = env.controller.execute_python_command( + "import os; print(os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Cookies'))")[ + 'output'].strip() else: raise Exception('Unsupported operating system') @@ -113,9 +121,12 @@ def get_history(env, config: Dict[str, str]): """import os; print(os.path.join(os.getenv('HOME'), "Library", "Application Support", "Google", "Chrome", "Default", "History"))""")[ 'output'].strip() elif os_type == 'Linux': - chrome_history_path = env.controller.execute_python_command( - "import os; print(os.path.join(os.getenv('HOME'), '.config', 'google-chrome', 'Default', 'History'))")[ - 'output'].strip() + if "arm" in platform.machine(): + raise NotImplementedError + else: + chrome_history_path = env.controller.execute_python_command( + "import os; print(os.path.join(os.getenv('HOME'), '.config', 'google-chrome', 'Default', 'History'))")[ + 'output'].strip() else: raise Exception('Unsupported operating system') @@ -149,9 +160,12 @@ def get_enabled_experiments(env, config: Dict[str, str]): "import os; print(os.path.join(os.getenv('HOME'), 'Library/Application Support/Google/Chrome/Local State'))")[ 'output'].strip() elif os_type == 'Linux': - preference_file_path = env.controller.execute_python_command( - "import os; print(os.path.join(os.getenv('HOME'), '.config/google-chrome/Local State'))")[ - 'output'].strip() + if "arm" in platform.machine(): + raise NotImplementedError + else: + preference_file_path = env.controller.execute_python_command( + "import os; print(os.path.join(os.getenv('HOME'), '.config/google-chrome/Local State'))")[ + 'output'].strip() else: raise Exception('Unsupported operating system') @@ -181,9 +195,12 @@ def get_profile_name(env, config: Dict[str, str]): "import os; print(os.path.join(os.getenv('HOME'), 'Library/Application Support/Google/Chrome/Default/Preferences'))")[ 'output'].strip() elif os_type == 'Linux': - preference_file_path = env.controller.execute_python_command( - "import os; print(os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Preferences'))")[ - 'output'].strip() + if "arm" in platform.machine(): + raise NotImplementedError + else: + preference_file_path = env.controller.execute_python_command( + "import os; print(os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Preferences'))")[ + 'output'].strip() else: raise Exception('Unsupported operating system') @@ -210,9 +227,12 @@ def get_chrome_language(env, config: Dict[str, str]): "import os; print(os.path.join(os.getenv('HOME'), 'Library/Application Support/Google/Chrome/Local State'))")[ 'output'].strip() elif os_type == 'Linux': - preference_file_path = env.controller.execute_python_command( - "import os; print(os.path.join(os.getenv('HOME'), '.config/google-chrome/Local State'))")[ - 'output'].strip() + if "arm" in platform.machine(): + raise NotImplementedError + else: + preference_file_path = env.controller.execute_python_command( + "import os; print(os.path.join(os.getenv('HOME'), '.config/google-chrome/Local State'))")[ + 'output'].strip() else: raise Exception('Unsupported operating system') @@ -239,9 +259,12 @@ def get_chrome_font_size(env, config: Dict[str, str]): "import os; print(os.path.join(os.getenv('HOME'), 'Library/Application Support/Google/Chrome/Default/Preferences'))")[ 'output'].strip() elif os_type == 'Linux': - preference_file_path = env.controller.execute_python_command( - "import os; print(os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Preferences'))")[ - 'output'].strip() + if "arm" in platform.machine(): + raise NotImplementedError + else: + preference_file_path = env.controller.execute_python_command( + "import os; print(os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Preferences'))")[ + 'output'].strip() else: raise Exception('Unsupported operating system') @@ -274,9 +297,12 @@ def get_bookmarks(env, config: Dict[str, str]): "import os; print(os.path.join(os.getenv('HOME'), 'Library/Application Support/Google/Chrome/Default/Bookmarks'))")[ 'output'].strip() elif os_type == 'Linux': - preference_file_path = env.controller.execute_python_command( - "import os; print(os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Bookmarks'))")[ - 'output'].strip() + if "arm" in platform.machine(): + raise NotImplementedError + else: + preference_file_path = env.controller.execute_python_command( + "import os; print(os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Bookmarks'))")[ + 'output'].strip() else: raise Exception('Unsupported operating system') @@ -301,8 +327,11 @@ def get_extensions_installed_from_shop(env, config: Dict[str, str]): """os.path.expanduser('~') + '/Library/Application Support/Google/Chrome/Default/Extensions/'""")[ 'output'].strip() elif os_type == 'Linux': - chrome_extension_dir = env.controller.execute_python_command( - """os.path.expanduser('~') + '/.config/google-chrome/Default/Extensions/'""")['output'].strip() + if "arm" in platform.machine(): + raise NotImplementedError + else: + chrome_extension_dir = env.controller.execute_python_command( + """os.path.expanduser('~') + '/.config/google-chrome/Default/Extensions/'""")['output'].strip() else: raise Exception('Unsupported operating system') @@ -335,7 +364,28 @@ def get_page_info(env, config: Dict[str, str]): remote_debugging_url = f"http://{host}:{port}" with sync_playwright() as p: # connect to remote Chrome instance - browser = p.chromium.connect_over_cdp(remote_debugging_url) + try: + browser = p.chromium.connect_over_cdp(remote_debugging_url) + except Exception as e: + # If the connection fails, start a new browser instance + platform.machine() + if "arm" in platform.machine(): + # start a new browser instance if the connection fails + payload = json.dumps({"command": [ + "chromium", + "--remote-debugging-port=1337" + ], "shell": False}) + else: + payload = json.dumps({"command": [ + "google-chrome", + "--remote-debugging-port=1337" + ], "shell": False}) + + headers = {"Content-Type": "application/json"} + requests.post("http://" + host + ":5000/setup" + "/launch", headers=headers, data=payload) + time.sleep(5) + browser = p.chromium.connect_over_cdp(remote_debugging_url) + page = browser.contexts[0].new_page() page.goto(url) @@ -364,7 +414,30 @@ def get_open_tabs_info(env, config: Dict[str, str]): remote_debugging_url = f"http://{host}:{port}" with sync_playwright() as p: # connect to remote Chrome instance - browser = p.chromium.connect_over_cdp(remote_debugging_url) + try: + browser = p.chromium.connect_over_cdp(remote_debugging_url) + except Exception as e: + # If the connection fails, start a new browser instance + platform.machine() + if "arm" in platform.machine(): + # start a new browser instance if the connection fails + payload = json.dumps({"command": [ + "chromium", + "--remote-debugging-port=1337" + ], "shell": False}) + else: + payload = json.dumps({"command": [ + "google-chrome", + "--remote-debugging-port=1337" + ], "shell": False}) + + headers = {"Content-Type": "application/json"} + requests.post("http://" + host + ":5000/setup" + "/launch", headers=headers, data=payload) + time.sleep(5) + try: + browser = p.chromium.connect_over_cdp(remote_debugging_url) + except Exception as e: + return [] tabs_info = [] for context in browser.contexts: @@ -406,7 +479,7 @@ def get_active_url_from_accessTree(env, config): } Return url: str - """ + """ accessibility_tree: str = env.controller.get_accessibility_tree() # download accessibility tree to "/home/user/Desktop" logger.debug("AT@eval: %s", accessibility_tree) @@ -414,9 +487,11 @@ def get_active_url_from_accessTree(env, config): at: _Element = lxml.etree.fromstring(accessibility_tree) arch = platform.machine() if "arm" in arch: - selector = CSSSelector(", application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]", namespaces=_accessibility_ns_map) + selector = CSSSelector(", application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]", + namespaces=_accessibility_ns_map) else: - selector = CSSSelector(", application[name=Google\\ Chrome] entry[name=Address\\ and\\ search\\ bar]", namespaces=_accessibility_ns_map) + selector = CSSSelector("application[name=Google\\ Chrome] entry[name=Address\\ and\\ search\\ bar]", + namespaces=_accessibility_ns_map) elements: List[_Element] = selector(at) # if "xpath" in config: # elements: List[_Element] = at.xpath(config["xpath"], namespaces=_accessibility_ns_map) @@ -425,8 +500,9 @@ def get_active_url_from_accessTree(env, config): # elements: List[_Element] = selector(at) if len(elements) == 0: print("no elements found") - return 0. - active_tab_url = config["goto_prefix"]+elements[0].text if "goto_prefix" in config.keys() else "https://" + elements[0].text + return None + active_tab_url = config["goto_prefix"] + elements[0].text if "goto_prefix" in config.keys() else "https://" + \ + elements[0].text print("active tab url now: {}".format(active_tab_url)) return active_tab_url @@ -448,8 +524,12 @@ def get_active_tab_info(env, config: Dict[str, str]): remote_debugging_url = f"http://{host}:{port}" with sync_playwright() as p: - # connect to remote Chrome instance - browser = p.chromium.connect_over_cdp(remote_debugging_url) + # connect to remote Chrome instance, since it is supposed to be the active one, we won't start a new one if failed + try: + browser = p.chromium.connect_over_cdp(remote_debugging_url) + except Exception as e: + return None + active_tab_info = {} # go to the target URL page page = browser.new_page() @@ -481,7 +561,28 @@ def get_pdf_from_url(env, config: Dict[str, str]) -> str: remote_debugging_url = f"http://{host}:{port}" with sync_playwright() as p: - browser = p.chromium.connect_over_cdp(remote_debugging_url) + try: + browser = p.chromium.connect_over_cdp(remote_debugging_url) + except Exception as e: + # If the connection fails, start a new browser instance + platform.machine() + if "arm" in platform.machine(): + # start a new browser instance if the connection fails + payload = json.dumps({"command": [ + "chromium", + "--remote-debugging-port=1337" + ], "shell": False}) + else: + payload = json.dumps({"command": [ + "google-chrome", + "--remote-debugging-port=1337" + ], "shell": False}) + + headers = {"Content-Type": "application/json"} + requests.post("http://" + host + ":5000/setup" + "/launch", headers=headers, data=payload) + time.sleep(5) + browser = p.chromium.connect_over_cdp(remote_debugging_url) + page = browser.new_page() page.goto(_url) page.pdf(path=_path) @@ -498,7 +599,27 @@ def get_chrome_saved_address(env, config: Dict[str, str]): remote_debugging_url = f"http://{host}:{port}" with sync_playwright() as p: # connect to remote Chrome instance - browser = p.chromium.connect_over_cdp(remote_debugging_url) + try: + browser = p.chromium.connect_over_cdp(remote_debugging_url) + except Exception as e: + # If the connection fails, start a new browser instance + platform.machine() + if "arm" in platform.machine(): + # start a new browser instance if the connection fails + payload = json.dumps({"command": [ + "chromium", + "--remote-debugging-port=1337" + ], "shell": False}) + else: + payload = json.dumps({"command": [ + "google-chrome", + "--remote-debugging-port=1337" + ], "shell": False}) + + headers = {"Content-Type": "application/json"} + requests.post("http://" + host + ":5000/setup" + "/launch", headers=headers, data=payload) + time.sleep(5) + browser = p.chromium.connect_over_cdp(remote_debugging_url) page = browser.new_page() @@ -556,7 +677,27 @@ def get_number_of_search_results(env, config: Dict[str, str]): remote_debugging_url = f"http://{host}:{port}" with sync_playwright() as p: - browser = p.chromium.connect_over_cdp(remote_debugging_url) + try: + browser = p.chromium.connect_over_cdp(remote_debugging_url) + except Exception as e: + # If the connection fails, start a new browser instance + platform.machine() + if "arm" in platform.machine(): + # start a new browser instance if the connection fails + payload = json.dumps({"command": [ + "chromium", + "--remote-debugging-port=1337" + ], "shell": False}) + else: + payload = json.dumps({"command": [ + "google-chrome", + "--remote-debugging-port=1337" + ], "shell": False}) + + headers = {"Content-Type": "application/json"} + requests.post("http://" + host + ":5000/setup" + "/launch", headers=headers, data=payload) + time.sleep(5) + browser = p.chromium.connect_over_cdp(remote_debugging_url) page = browser.new_page() page.goto(url) search_results = page.query_selector_all(result_selector) @@ -634,12 +775,15 @@ def get_enable_do_not_track(env, config: Dict[str, str]): "import os; print(os.path.join(os.getenv('HOME'), 'Library/Application Support/Google/Chrome/Default/Preferences'))")[ 'output'].strip() elif os_type == 'Linux': - preference_file_path = env.controller.execute_python_command( - "import os; print(os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Preferences'))")[ - 'output'].strip() - # preference_file_path = env.controller.execute_python_command( - # "import os; print(os.path.join(os.getenv('HOME'), 'snap/chromium/common/chromium/Default/Preferences'))")[ - # 'output'].strip() + if "arm" in platform.machine(): + preference_file_path = env.controller.execute_python_command( + "import os; print(os.path.join(os.getenv('HOME'), 'snap/chromium/common/chromium/Default/Preferences'))")[ + 'output'].strip() + else: + preference_file_path = env.controller.execute_python_command( + "import os; print(os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Preferences'))")[ + 'output'].strip() + else: raise Exception('Unsupported operating system') @@ -664,12 +808,15 @@ def get_enable_enhanced_safety_browsing(env, config: Dict[str, str]): "import os; print(os.path.join(os.getenv('HOME'), 'Library/Application Support/Google/Chrome/Default/Preferences'))")[ 'output'].strip() elif os_type == 'Linux': - preference_file_path = env.controller.execute_python_command( - "import os; print(os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Preferences'))")[ - 'output'].strip() - # preference_file_path = env.controller.execute_python_command( - # "import os; print(os.path.join(os.getenv('HOME'), 'snap/chromium/common/chromium/Default/Preferences'))")[ - # 'output'].strip() + if "arm" in platform.machine(): + preference_file_path = env.controller.execute_python_command( + "import os; print(os.path.join(os.getenv('HOME'), 'snap/chromium/common/chromium/Default/Preferences'))")[ + 'output'].strip() + else: + preference_file_path = env.controller.execute_python_command( + "import os; print(os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Preferences'))")[ + 'output'].strip() + else: raise Exception('Unsupported operating system') @@ -694,12 +841,15 @@ def get_new_startup_page(env, config: Dict[str, str]): "import os; print(os.path.join(os.getenv('HOME'), 'Library/Application Support/Google/Chrome/Default/Preferences'))")[ 'output'].strip() elif os_type == 'Linux': - preference_file_path = env.controller.execute_python_command( - "import os; print(os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Preferences'))")[ - 'output'].strip() - # preference_file_path = env.controller.execute_python_command( - # "import os; print(os.path.join(os.getenv('HOME'), 'snap/chromium/common/chromium/Default/Preferences'))")[ - # 'output'].strip() + if "arm" in platform.machine(): + preference_file_path = env.controller.execute_python_command( + "import os; print(os.path.join(os.getenv('HOME'), 'snap/chromium/common/chromium/Default/Preferences'))")[ + 'output'].strip() + else: + preference_file_path = env.controller.execute_python_command( + "import os; print(os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Preferences'))")[ + 'output'].strip() + else: raise Exception('Unsupported operating system') @@ -729,12 +879,15 @@ def get_find_unpacked_extension_path(env, config: Dict[str, str]): "import os; print(os.path.join(os.getenv('HOME'), 'Library/Application Support/Google/Chrome/Default/Preferences'))")[ 'output'].strip() elif os_type == 'Linux': - preference_file_path = env.controller.execute_python_command( - "import os; print(os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Preferences'))")[ - 'output'].strip() - # preference_file_path = env.controller.execute_python_command( - # "import os; print(os.path.join(os.getenv('HOME'), 'snap/chromium/common/chromium/Default/Preferences'))")[ - # 'output'].strip() + if "arm" in platform.machine(): + preference_file_path = env.controller.execute_python_command( + "import os; print(os.path.join(os.getenv('HOME'), 'snap/chromium/common/chromium/Default/Preferences'))")[ + 'output'].strip() + else: + preference_file_path = env.controller.execute_python_command( + "import os; print(os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Preferences'))")[ + 'output'].strip() + else: raise Exception('Unsupported operating system') @@ -766,12 +919,14 @@ def get_data_delete_automacally(env, config: Dict[str, str]): "import os; print(os.path.join(os.getenv('HOME'), 'Library/Application Support/Google/Chrome/Default/Preferences'))")[ 'output'].strip() elif os_type == 'Linux': - # preference_file_path = env.controller.execute_python_command( - # "import os; print(os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Preferences'))")[ - # 'output'].strip() - preference_file_path = env.controller.execute_python_command( - "import os; print(os.path.join(os.getenv('HOME'), 'snap/chromium/common/chromium/Default/Preferences'))")[ - 'output'].strip() + if "arm" in platform.machine(): + preference_file_path = env.controller.execute_python_command( + "import os; print(os.path.join(os.getenv('HOME'), 'snap/chromium/common/chromium/Default/Preferences'))")[ + 'output'].strip() + else: + preference_file_path = env.controller.execute_python_command( + "import os; print(os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Preferences'))")[ + 'output'].strip() else: raise Exception('Unsupported operating system') @@ -779,7 +934,7 @@ def get_data_delete_automacally(env, config: Dict[str, str]): content = env.controller.get_file(preference_file_path) data = json.loads(content) data_delete_state = data["profile"]["exit_type"] - return data_delete_state + return data_delete_state except Exception as e: logger.error(f"Error: {e}") return "Google" @@ -823,7 +978,27 @@ def get_active_tab_html_parse(env, config: Dict[str, Any]): remote_debugging_url = f"http://{host}:{port}" with sync_playwright() as p: # connect to remote Chrome instance - browser = p.chromium.connect_over_cdp(remote_debugging_url) + try: + browser = p.chromium.connect_over_cdp(remote_debugging_url) + except Exception as e: + # If the connection fails, start a new browser instance + platform.machine() + if "arm" in platform.machine(): + # start a new browser instance if the connection fails + payload = json.dumps({"command": [ + "chromium", + "--remote-debugging-port=1337" + ], "shell": False}) + else: + payload = json.dumps({"command": [ + "google-chrome", + "--remote-debugging-port=1337" + ], "shell": False}) + + headers = {"Content-Type": "application/json"} + requests.post("http://" + host + ":5000/setup" + "/launch", headers=headers, data=payload) + time.sleep(5) + browser = p.chromium.connect_over_cdp(remote_debugging_url) target_page = None for context in browser.contexts: for page in context.pages: @@ -831,31 +1006,34 @@ def get_active_tab_html_parse(env, config: Dict[str, Any]): if page.url == active_tab_url: target_page = page break + if target_page is None: + return {} return_json = {} if config["category"] == "class": - #find the text of elements in html with specific class name + # find the text of elements in html with specific class name class_multiObject = config["class_multiObject"] for key in class_multiObject.keys(): object_dict = class_multiObject[key] for order_key in object_dict.keys(): - return_json[object_dict[order_key]] = target_page.query_selector_all("."+key)[int(order_key)].text_content().strip() + return_json[object_dict[order_key]] = target_page.query_selector_all("." + key)[ + int(order_key)].text_content().strip() class_singleObject = config["class_singleObject"] for key in class_singleObject.keys(): - return_json[class_singleObject[key]] = target_page.query_selector("."+key).text_content().strip() + return_json[class_singleObject[key]] = target_page.query_selector("." + key).text_content().strip() elif config['category'] == "label": - #find the text of elements in html with specific label name + # find the text of elements in html with specific label name labelObject = config["labelObject"] for key in labelObject.keys(): return_json[labelObject[key]] = target_page.get_by_label(key).text_content().strip() elif config["category"] == "xpath": - #find the text of elements in html with specific xpath + # find the text of elements in html with specific xpath xpathObject = config["xpathObject"] for key in xpathObject.keys(): - return_json[xpathObject[key]] = target_page.locator("xpath="+key).text_content().strip() + return_json[xpathObject[key]] = target_page.locator("xpath=" + key).text_content().strip() elif config["category"] == "input": inputObject = config["inputObject"] for key in inputObject.keys(): - return_json[inputObject[key]] = target_page.locator("xpath="+key).input_value().strip() + return_json[inputObject[key]] = target_page.locator("xpath=" + key).input_value().strip() browser.close() return return_json @@ -869,7 +1047,27 @@ def get_gotoRecreationPage_and_get_html_content(env, config: Dict[str, Any]): remote_debugging_url = f"http://{host}:{port}" with sync_playwright() as p: - browser = p.chromium.connect_over_cdp(remote_debugging_url) + try: + browser = p.chromium.connect_over_cdp(remote_debugging_url) + except Exception as e: + # If the connection fails, start a new browser instance + platform.machine() + if "arm" in platform.machine(): + # start a new browser instance if the connection fails + payload = json.dumps({"command": [ + "chromium", + "--remote-debugging-port=1337" + ], "shell": False}) + else: + payload = json.dumps({"command": [ + "google-chrome", + "--remote-debugging-port=1337" + ], "shell": False}) + + headers = {"Content-Type": "application/json"} + requests.post("http://" + host + ":5000/setup" + "/launch", headers=headers, data=payload) + time.sleep(5) + browser = p.chromium.connect_over_cdp(remote_debugging_url) page = browser.new_page() page.goto("https://www.recreation.gov/") page.fill("input#hero-search-input", "Albion Basin") @@ -888,22 +1086,22 @@ def get_gotoRecreationPage_and_get_html_content(env, config: Dict[str, Any]): newpage.click("button.next-available") print("after third click") - return_json = {} - return_json["expected"]={} - #find the text of elements in html with specific class name + return_json["expected"] = {} + # find the text of elements in html with specific class name if config["selector"] == "class": if "order" in config.keys(): className = config["class"] - return_json["expected"][className]=newpage.query_selector_all("."+className)[int(config["order"])].text_content().strip() + return_json["expected"][className] = newpage.query_selector_all("." + className)[ + int(config["order"])].text_content().strip() else: className = config["class"] - return_json["expected"][className] = newpage.query_selector("."+className).text_content().strip() + return_json["expected"][className] = newpage.query_selector("." + className).text_content().strip() browser.close() - return return_json + return return_json -def get_active_tab_url_parse(env, config:Dict[str, Any]): +def get_active_tab_url_parse(env, config: Dict[str, Any]): """ This function is used to parse the url according to config["parse_keys"]. config: @@ -914,6 +1112,8 @@ def get_active_tab_url_parse(env, config:Dict[str, Any]): ( { "original key": "new key" } ) """ active_tab_url = get_active_url_from_accessTree(env, config) + if active_tab_url is None: + return None # connect to remote Chrome instance # parse in a hard-coded way to find the specific info about task @@ -944,6 +1144,8 @@ def get_url_dashPart(env, config: Dict[str, str]): a string, used to indicate the return type, "string" or "json". """ active_tab_url = get_active_url_from_accessTree(env, config) + if active_tab_url is None: + return None # extract the last dash-separated part of the URL, and delete all the characters after "id" dash_part = active_tab_url.split("/")[config["partIndex"]] diff --git a/desktop_env/evaluators/metrics/chrome.py b/desktop_env/evaluators/metrics/chrome.py index 820327b..0a76bf0 100644 --- a/desktop_env/evaluators/metrics/chrome.py +++ b/desktop_env/evaluators/metrics/chrome.py @@ -17,11 +17,17 @@ def is_expected_active_tab(active_tab_info: Dict[str, str], rule: Dict[str, Any] """ Checks if the expected active tab is open in Chrome. """ + if not active_tab_info: + return 0. + match_type = rule['type'] if match_type == "url": expected_url = rule['url'] - actual_url = active_tab_info.get('url', None) + if isinstance(active_tab_info, Dict): + actual_url = active_tab_info.get('url', None) + else: + actual_url = active_tab_info print("expected_url: {}".format(expected_url)) print("actual_url: {}".format(actual_url)) return 1 if compare_urls(expected_url, actual_url) else 0 @@ -29,13 +35,17 @@ def is_expected_active_tab(active_tab_info: Dict[str, str], rule: Dict[str, Any] logger.error(f"Unknown type: {match_type}") return 0 + # rules[expected] is a string-formatted regex def is_expected_url_pattern_match(result, rules) -> float: """ This function is used to search the expected pattern in the url using regex. result is the return value of function "activte_tab_info" or return value of function "get_active_url_from_accessTree" """ - if type(result)== dict: + if not result: + return 0. + + if type(result) == dict: result_url = result["url"] print("result url: {}".format(result_url)) else: diff --git a/desktop_env/evaluators/metrics/general.py b/desktop_env/evaluators/metrics/general.py index 6c3e7b8..550c54b 100644 --- a/desktop_env/evaluators/metrics/general.py +++ b/desktop_env/evaluators/metrics/general.py @@ -231,8 +231,11 @@ def check_direct_json_object(result, rules)->float: print(result) print("expected: ") print(rules["expected"]) + if result is None: + return 0. expected_json = rules["expected"] for key in expected_json.keys(): - if expected_json[key] != result[key]: + expected_value = expected_json.get(key) + if expected_value != result.get(key): return 0. return 1.0 \ No newline at end of file diff --git a/evaluation_examples/examples/chrome/0d8b7de3-e8de-4d86-b9fd-dd2dce58a217.json b/evaluation_examples/examples/chrome/0d8b7de3-e8de-4d86-b9fd-dd2dce58a217.json index 4555083..a3630a8 100644 --- a/evaluation_examples/examples/chrome/0d8b7de3-e8de-4d86-b9fd-dd2dce58a217.json +++ b/evaluation_examples/examples/chrome/0d8b7de3-e8de-4d86-b9fd-dd2dce58a217.json @@ -55,7 +55,8 @@ "evaluator": { "func": "is_expected_active_tab", "result": { - "type": "active_tab_info" + "type": "active_url_from_accessTree", + "goto_prefix": "https://www." }, "expected": { "type": "rule", diff --git a/evaluation_examples/examples/chrome/12086550-11c0-466b-b367-1d9e75b3910e.json b/evaluation_examples/examples/chrome/12086550-11c0-466b-b367-1d9e75b3910e.json index ec22f9a..478f50c 100644 --- a/evaluation_examples/examples/chrome/12086550-11c0-466b-b367-1d9e75b3910e.json +++ b/evaluation_examples/examples/chrome/12086550-11c0-466b-b367-1d9e75b3910e.json @@ -30,7 +30,8 @@ "evaluator": { "func": "is_expected_active_tab", "result": { - "type": "active_tab_info" + "type": "active_url_from_accessTree", + "goto_prefix": "" }, "expected": { "type": "rule", diff --git a/evaluation_examples/examples/chrome/368d9ba4-203c-40c1-9fa3-da2f1430ce63.json b/evaluation_examples/examples/chrome/368d9ba4-203c-40c1-9fa3-da2f1430ce63.json index ed93803..a73acb2 100644 --- a/evaluation_examples/examples/chrome/368d9ba4-203c-40c1-9fa3-da2f1430ce63.json +++ b/evaluation_examples/examples/chrome/368d9ba4-203c-40c1-9fa3-da2f1430ce63.json @@ -53,7 +53,7 @@ "key":"time" }, { - "type": "active_tab_info", + "type": "active_url_from_accessTree", "goto_prefix": "https://www." }], "expected":[ diff --git a/evaluation_examples/examples/chrome/59155008-fe71-45ec-8a8f-dc35497b6aa8.json b/evaluation_examples/examples/chrome/59155008-fe71-45ec-8a8f-dc35497b6aa8.json index 8c97446..a7461b4 100644 --- a/evaluation_examples/examples/chrome/59155008-fe71-45ec-8a8f-dc35497b6aa8.json +++ b/evaluation_examples/examples/chrome/59155008-fe71-45ec-8a8f-dc35497b6aa8.json @@ -55,7 +55,8 @@ "evaluator": { "func": "is_expected_active_tab", "result": { - "type": "active_tab_info" + "type": "active_url_from_accessTree", + "goto_prefix": "https://www." }, "expected": { "type": "rule", diff --git a/evaluation_examples/examples/chrome/9f935cce-0a9f-435f-8007-817732bfc0a5.json b/evaluation_examples/examples/chrome/9f935cce-0a9f-435f-8007-817732bfc0a5.json index ea1543d..257b11b 100644 --- a/evaluation_examples/examples/chrome/9f935cce-0a9f-435f-8007-817732bfc0a5.json +++ b/evaluation_examples/examples/chrome/9f935cce-0a9f-435f-8007-817732bfc0a5.json @@ -45,10 +45,7 @@ "evaluator": { "func": "is_expected_url_pattern_match", "result": { - "type": "active_tab_info", - "selectors": [ - "application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]" - ], + "type": "active_url_from_accessTree", "goto_prefix": "https://www." }, "expected": { @@ -58,22 +55,6 @@ "forms\\?title=&field_component_target_id=431" ] } - ], - "trajectory": "trajectories/", - "related_apps": [ - "chrome" - ], - "evaluator": { - "func":"is_expected_url_pattern_match", - "result": { - "type": "active_tab_info", - "goto_prefix": "https://www." - }, - "expected":{ - "type": "rule", - "rules":{ - "expected": ["forms\\?title=&field_component_target_id=431"] - } - } } + } } \ No newline at end of file diff --git a/evaluation_examples/examples/chrome/a728a36e-8bf1-4bb6-9a03-ef039a5233f0.json b/evaluation_examples/examples/chrome/a728a36e-8bf1-4bb6-9a03-ef039a5233f0.json index 76ad5f8..1cdf900 100644 --- a/evaluation_examples/examples/chrome/a728a36e-8bf1-4bb6-9a03-ef039a5233f0.json +++ b/evaluation_examples/examples/chrome/a728a36e-8bf1-4bb6-9a03-ef039a5233f0.json @@ -55,7 +55,8 @@ "evaluator": { "func": "is_expected_active_tab", "result": { - "type": "active_tab_info" + "type": "active_url_from_accessTree", + "goto_prefix": "https://www." }, "expected": { "type": "rule", diff --git a/evaluation_examples/examples/chrome/b070486d-e161-459b-aa2b-ef442d973b92.json b/evaluation_examples/examples/chrome/b070486d-e161-459b-aa2b-ef442d973b92.json index 56a66e4..0b12421 100644 --- a/evaluation_examples/examples/chrome/b070486d-e161-459b-aa2b-ef442d973b92.json +++ b/evaluation_examples/examples/chrome/b070486d-e161-459b-aa2b-ef442d973b92.json @@ -1,5 +1,5 @@ { - "id": " b070486d-e161-459b-aa2b-ef442d973b92", + "id": "b070486d-e161-459b-aa2b-ef442d973b92", "snapshot": "chrome", "instruction": "Show side effects of Tamiflu.", "source": "online_tasks", @@ -43,41 +43,19 @@ "chrome" ], "evaluator": { - "func": "exact_match", - "result": { - "type": "url_dashPart", - "selectors": [ - "application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]" - ], - "goto_prefix": "https://www.", - "partIndex": -1, - "needDeleteId": false, - "returnType": "string" - }, - "expected": { - "type": "rule", - "rules": { - "expected": "tamiflu.html#side-effects" - } - ], - "trajectory": "trajectories/", - "related_apps": [ - "chrome" - ], - "evaluator": { - "func":"exact_match", - "result": { - "type": "url_dashPart", - "goto_prefix": "https://www.", - "partIndex": -1, - "needDeleteId": false, - "returnType": "string" - }, - "expected":{ - "type": "rule", - "rules":{ - "expected": "tamiflu.html#side-effects" - } + "func": "exact_match", + "result": { + "type": "url_dashPart", + "goto_prefix": "https://www.", + "partIndex": -1, + "needDeleteId": false, + "returnType": "string" + }, + "expected": { + "type": "rule", + "rules": { + "expected": "tamiflu.html#side-effects" } + } } - } \ No newline at end of file +} \ No newline at end of file diff --git a/evaluation_examples/examples/chrome/cabb3bae-cccb-41bd-9f5d-0f3a9fecd825.json b/evaluation_examples/examples/chrome/cabb3bae-cccb-41bd-9f5d-0f3a9fecd825.json index 6b1809e..460c60d 100644 --- a/evaluation_examples/examples/chrome/cabb3bae-cccb-41bd-9f5d-0f3a9fecd825.json +++ b/evaluation_examples/examples/chrome/cabb3bae-cccb-41bd-9f5d-0f3a9fecd825.json @@ -43,39 +43,20 @@ "chrome" ], "evaluator": { - "func": "is_expected_url_pattern_match", - "result": { - "type": "active_url_from_accessTree", - "selectors": [ - "application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]" - ], - "goto_prefix": "https://www." - }, - "expected": { - "type": "rule", - "rules": { - "expected": [ - "AgeAppropriate:Kids", - "search=spider-man%20toys", - "S=4" - ] - } - ], - "trajectory": "trajectories/", - "related_apps": [ - "chrome" - ], - "evaluator": { - "func":"is_expected_url_pattern_match", - "result": { - "type": "active_url_from_accessTree", - "goto_prefix": "https://www." - }, - "expected":{ - "type": "rule", - "rules":{ - "expected": ["AgeAppropriate:Kids", "search=spider-man%20toys", "S=4"] - } + "func": "is_expected_url_pattern_match", + "result": { + "type": "active_url_from_accessTree", + "goto_prefix": "https://www." + }, + "expected": { + "type": "rule", + "rules": { + "expected": [ + "AgeAppropriate:Kids", + "search=spider-man%20toys", + "S=4" + ] } + } } - } +} diff --git a/evaluation_examples/examples/chrome/f0b971a1-6831-4b9b-a50e-22a6e47f45ba.json b/evaluation_examples/examples/chrome/f0b971a1-6831-4b9b-a50e-22a6e47f45ba.json index b2da339..05d4f8e 100644 --- a/evaluation_examples/examples/chrome/f0b971a1-6831-4b9b-a50e-22a6e47f45ba.json +++ b/evaluation_examples/examples/chrome/f0b971a1-6831-4b9b-a50e-22a6e47f45ba.json @@ -55,7 +55,8 @@ "evaluator": { "func": "is_expected_active_tab", "result": { - "type": "active_tab_info" + "type": "active_url_from_accessTree", + "goto_prefix": "https://www." }, "expected": { "type": "rule", diff --git a/evaluation_examples/examples/chrome/f5d96daf-83a8-4c86-9686-bada31fc66ab.json b/evaluation_examples/examples/chrome/f5d96daf-83a8-4c86-9686-bada31fc66ab.json index f91d346..22e4a64 100644 --- a/evaluation_examples/examples/chrome/f5d96daf-83a8-4c86-9686-bada31fc66ab.json +++ b/evaluation_examples/examples/chrome/f5d96daf-83a8-4c86-9686-bada31fc66ab.json @@ -55,7 +55,8 @@ "evaluator": { "func": "is_expected_active_tab", "result": { - "type": "active_tab_info" + "type": "active_url_from_accessTree", + "goto_prefix": "https://www." }, "expected": { "type": "rule",