diff --git a/desktop_env/evaluators/getters/__init__.py b/desktop_env/evaluators/getters/__init__.py index 3535ddc..7348fec 100644 --- a/desktop_env/evaluators/getters/__init__.py +++ b/desktop_env/evaluators/getters/__init__.py @@ -12,7 +12,11 @@ from .chrome import ( get_profile_name, get_number_of_search_results, get_googledrive_file, - get_active_tab_info + get_active_tab_info, + get_enable_do_not_track, + get_enable_enhanced_safety_browsing, + get_new_startup_page, + get_find_unpacked_extension_path ) from .file import get_cloud_file, get_vm_file, get_cache_file from .general import get_vm_command_line, get_vm_terminal_output diff --git a/desktop_env/evaluators/getters/chrome.py b/desktop_env/evaluators/getters/chrome.py index 6322238..323fb46 100644 --- a/desktop_env/evaluators/getters/chrome.py +++ b/desktop_env/evaluators/getters/chrome.py @@ -363,6 +363,9 @@ def get_active_tab_info(env, config: Dict[str, str]): break browser.close() + print("active_tab_title: {}".format(active_tab_info.get('title', 'None'))) + print("active_tab_url: {}".format(active_tab_info.get('url', 'None'))) + print("active_tab_content: {}".format(active_tab_info.get('content', 'None'))) return active_tab_info @@ -516,4 +519,133 @@ def get_googledrive_file(env, config: Dict[str, Any]) -> str: else f"title = '{fp}' and trashed = false" for jdx, fp in enumerate(path)] dest = config['dest'][idx] _path_list.append(get_single_file(query, os.path.join(env.cache_dir, dest))) - return _path_list \ No newline at end of file + return _path_list + + +def get_enable_do_not_track(env, config: Dict[str, str]): + os_type = env.vm_platform + if os_type == 'Windows': + preference_file_path = env.controller.execute_python_command("""import os; print(os.path.join(os.getenv('LOCALAPPDATA'), + 'Google\\Chrome\\User Data\\Default\\Preferences'))""")['output'].strip() + elif os_type == 'Darwin': + preference_file_path = env.controller.execute_python_command( + "import os; print(os.path.join(os.getenv('HOME'), 'Library/Application Support/Google/Chrome/Default/Preferences'))")[ + 'output'].strip() + elif os_type == 'Linux': + # preference_file_path = env.controller.execute_python_command( + # "import os; print(os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Preferences'))")[ + # 'output'].strip() + preference_file_path = env.controller.execute_python_command( + "import os; print(os.path.join(os.getenv('HOME'), 'snap/chromium/common/chromium/Default/Preferences'))")[ + 'output'].strip() + else: + raise Exception('Unsupported operating system') + + try: + content = env.controller.get_file(preference_file_path) + data = json.loads(content) + + if_enable_do_not_track = data.get('enable_do_not_track', {}) # bool + return "true" if if_enable_do_not_track else "false" + except Exception as e: + logger.error(f"Error: {e}") + return "Google" + + +def get_enable_enhanced_safety_browsing(env, config: Dict[str, str]): + os_type = env.vm_platform + if os_type == 'Windows': + preference_file_path = env.controller.execute_python_command("""import os; print(os.path.join(os.getenv('LOCALAPPDATA'), + 'Google\\Chrome\\User Data\\Default\\Preferences'))""")['output'].strip() + elif os_type == 'Darwin': + preference_file_path = env.controller.execute_python_command( + "import os; print(os.path.join(os.getenv('HOME'), 'Library/Application Support/Google/Chrome/Default/Preferences'))")[ + 'output'].strip() + elif os_type == 'Linux': + # preference_file_path = env.controller.execute_python_command( + # "import os; print(os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Preferences'))")[ + # 'output'].strip() + preference_file_path = env.controller.execute_python_command( + "import os; print(os.path.join(os.getenv('HOME'), 'snap/chromium/common/chromium/Default/Preferences'))")[ + 'output'].strip() + else: + raise Exception('Unsupported operating system') + + try: + content = env.controller.get_file(preference_file_path) + data = json.loads(content) + + if_enable_do_not_track = data.get('safebrowsing', {}).get('enhanced', {}) # bool + return "true" if if_enable_do_not_track else "false" + except Exception as e: + logger.error(f"Error: {e}") + return "Google" + + +def get_new_startup_page(env, config: Dict[str, str]): + os_type = env.vm_platform + if os_type == 'Windows': + preference_file_path = env.controller.execute_python_command("""import os; print(os.path.join(os.getenv('LOCALAPPDATA'), + 'Google\\Chrome\\User Data\\Default\\Preferences'))""")['output'].strip() + elif os_type == 'Darwin': + preference_file_path = env.controller.execute_python_command( + "import os; print(os.path.join(os.getenv('HOME'), 'Library/Application Support/Google/Chrome/Default/Preferences'))")[ + 'output'].strip() + elif os_type == 'Linux': + # preference_file_path = env.controller.execute_python_command( + # "import os; print(os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Preferences'))")[ + # 'output'].strip() + preference_file_path = env.controller.execute_python_command( + "import os; print(os.path.join(os.getenv('HOME'), 'snap/chromium/common/chromium/Default/Preferences'))")[ + 'output'].strip() + else: + raise Exception('Unsupported operating system') + + try: + content = env.controller.get_file(preference_file_path) + data = json.loads(content) + + # if data has no key called 'session', it means the chrome is on a fresh-start mode, which is a true state; + # otherwise, try to find the code number in 'restored_on_startup' in 'session' + if "session" not in data.keys(): + return "true" + else: + if_enable_do_not_track = data.get('session', {}).get('restore_on_startup', {}) # int, need to be 5 + return "true" if if_enable_do_not_track == 5 else "false" + except Exception as e: + logger.error(f"Error: {e}") + return "Google" + + +def get_find_unpacked_extension_path(env, config: Dict[str, str]): + os_type = env.vm_platform + if os_type == 'Windows': + preference_file_path = env.controller.execute_python_command("""import os; print(os.path.join(os.getenv('LOCALAPPDATA'), + 'Google\\Chrome\\User Data\\Default\\Preferences'))""")['output'].strip() + elif os_type == 'Darwin': + preference_file_path = env.controller.execute_python_command( + "import os; print(os.path.join(os.getenv('HOME'), 'Library/Application Support/Google/Chrome/Default/Preferences'))")[ + 'output'].strip() + elif os_type == 'Linux': + # preference_file_path = env.controller.execute_python_command( + # "import os; print(os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Preferences'))")[ + # 'output'].strip() + preference_file_path = env.controller.execute_python_command( + "import os; print(os.path.join(os.getenv('HOME'), 'snap/chromium/common/chromium/Default/Preferences'))")[ + 'output'].strip() + else: + raise Exception('Unsupported operating system') + + try: + content = env.controller.get_file(preference_file_path) + data = json.loads(content) + # Preferences store all the path of installed extensions, return them all and let metrics try to find one matches the targeted extension path + all_extensions_path = [] + all_extensions = data.get('extensions', {}).get('settings', {}) + for id in all_extensions.keys(): + path = all_extensions[id]["path"] + all_extensions_path.append(path) + return all_extensions_path + except Exception as e: + logger.error(f"Error: {e}") + return "Google" \ No newline at end of file diff --git a/desktop_env/evaluators/metrics/__init__.py b/desktop_env/evaluators/metrics/__init__.py index ec562a0..bfb5c54 100644 --- a/desktop_env/evaluators/metrics/__init__.py +++ b/desktop_env/evaluators/metrics/__init__.py @@ -15,7 +15,8 @@ from .chrome import ( check_font_size, check_enabled_experiments, check_history_deleted, - is_expected_search_query + is_expected_search_query, + is_expected_active_tab ) from .docs import ( compare_font_names, @@ -53,6 +54,7 @@ from .general import ( check_json, check_list, exact_match, + is_in_list, fuzzy_match, check_include_exclude ) diff --git a/desktop_env/evaluators/metrics/chrome.py b/desktop_env/evaluators/metrics/chrome.py index b48519d..69d3dcd 100644 --- a/desktop_env/evaluators/metrics/chrome.py +++ b/desktop_env/evaluators/metrics/chrome.py @@ -9,6 +9,22 @@ from desktop_env.evaluators.metrics.utils import are_lists_equal, compare_urls logger = logging.getLogger("desktopenv.metrics.chrome") +def is_expected_active_tab(active_tab_info: Dict[str, str], rule: Dict[str, Any]) -> float: + """ + Checks if the expected active tab is open in Chrome. + """ + match_type = rule['type'] + + if match_type == "url": + expected_url = rule['url'] + actual_url = active_tab_info['url'] + print("expected_url: {}".format(expected_url)) + print("actual_url: {}".format(actual_url)) + return 1 if compare_urls(expected_url, actual_url) else 0 + else: + logger.error(f"Unknown type: {match_type}") + return 0 + def is_expected_tabs(open_tabs: List[Dict[str, str]], rule: Dict[str, Any]) -> float: """ Checks if the expected tabs are open in Chrome. diff --git a/desktop_env/evaluators/metrics/general.py b/desktop_env/evaluators/metrics/general.py index 4e1f50a..9681c07 100644 --- a/desktop_env/evaluators/metrics/general.py +++ b/desktop_env/evaluators/metrics/general.py @@ -38,7 +38,13 @@ def exact_match(result, rules) -> float: else: return 0. - +def is_in_list(result, rules) -> float: + expect = rules["expected"] + if expect in result: + return 1. + else: + return 0. + def fuzzy_match(result, rules) -> float: expect = rules["expected"] diff --git a/evaluation_examples/examples/chrome/030eeff7-b492-4218-b312-701ec99ee0cc.json b/evaluation_examples/examples/chrome/030eeff7-b492-4218-b312-701ec99ee0cc.json index 8d3c6b4..b03b588 100644 --- a/evaluation_examples/examples/chrome/030eeff7-b492-4218-b312-701ec99ee0cc.json +++ b/evaluation_examples/examples/chrome/030eeff7-b492-4218-b312-701ec99ee0cc.json @@ -3,16 +3,41 @@ "snapshot": "chrome", "instruction": "Can you enable the 'Do Not Track' feature in Chrome to enhance my online privacy?", "source": "https://www.surreycc.gov.uk/website/cookies/do-not-track", - "config": [], + "config": [ + { + "type": "launch", + "parameters": { + "command": [ + "google-chrome", + "--remote-debugging-port=9222" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "socat", + "tcp-listen:9222,fork", + "tcp:localhost:1337" + ] + } + } + ], "trajectory": "trajectories/", "related_apps": [ "chrome" ], "evaluator": { - "func": "", + "func": "exact_match", "result": { + "type": "enable_do_not_track" }, "expected": { + "type": "rule", + "rules": { + "expected": "true" + } } } } diff --git a/evaluation_examples/examples/chrome/12086550-11c0-466b-b367-1d9e75b3910e.json b/evaluation_examples/examples/chrome/12086550-11c0-466b-b367-1d9e75b3910e.json index 6e2660d..ec22f9a 100644 --- a/evaluation_examples/examples/chrome/12086550-11c0-466b-b367-1d9e75b3910e.json +++ b/evaluation_examples/examples/chrome/12086550-11c0-466b-b367-1d9e75b3910e.json @@ -3,16 +3,41 @@ "snapshot": "chrome", "instruction": "Computer, please navigate to the area in my browser settings where my passwords are stored. I want to check my login information for Etsy without revealing it just yet.", "source": "https://www.quora.com/What-are-the-cool-tricks-to-use-Google-Chrome", - "config": [], + "config": [ + { + "type": "launch", + "parameters": { + "command": [ + "google-chrome", + "--remote-debugging-port=1337" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "socat", + "tcp-listen:9222,fork", + "tcp:localhost:1337" + ] + } + }], "trajectory": "trajectories/", "related_apps": [ "chrome" ], "evaluator": { - "func": "", + "func": "is_expected_active_tab", "result": { + "type": "active_tab_info" }, "expected": { + "type": "rule", + "rules": { + "type": "url", + "url":"chrome://password-manager/passwords" + } } } } diff --git a/evaluation_examples/examples/chrome/3299584d-8f11-4457-bf4c-ce98f7600250.json b/evaluation_examples/examples/chrome/3299584d-8f11-4457-bf4c-ce98f7600250.json index 297b062..06a6cd9 100644 --- a/evaluation_examples/examples/chrome/3299584d-8f11-4457-bf4c-ce98f7600250.json +++ b/evaluation_examples/examples/chrome/3299584d-8f11-4457-bf4c-ce98f7600250.json @@ -3,16 +3,55 @@ "snapshot": "chrome", "instruction": "On my surface pro whenever I launch Chrome it always opens \"funbrain.com.\" I don't want this. I cleared my cache but it still happens. What should I do?", "source": "https://www.reddit.com/r/techsupport/comments/12zwymy/comment/jhtri65/?utm_source=share&utm_medium=web2x&context=3", - "config": [], + "config": [ + { + "type": "execute", + "parameters": { + "command": "echo lixiaochuan20 | sudo -S apt update -y && echo lixiaochuan20 | sudo -S apt install jq -y", + "shell": true + } + }, + { + "type": "execute", + "parameters": { + "command": "cd /home/user/snap/chromium/common/chromium/Default && jq '. + {\"session\":{\"restore_on_startup\":4, \"startup_urls\":[\"http://funbrain.com/\"]}}' Preferences > temp && mv temp Preferences", + "shell": true + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "google-chrome", + "--remote-debugging-port=9222" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "socat", + "tcp-listen:9222,fork", + "tcp:localhost:1337" + ] + } + } + ], "trajectory": "trajectories/", "related_apps": [ "chrome" ], "evaluator": { - "func": "", + "func": "exact_match", "result": { + "type": "new_startup_page" }, "expected": { + "type": "rule", + "rules": { + "expected": "true" + } } } } diff --git a/evaluation_examples/examples/chrome/6766f2b8-8a72-417f-a9e5-56fcaa735837.json b/evaluation_examples/examples/chrome/6766f2b8-8a72-417f-a9e5-56fcaa735837.json index 277ba84..ddf8177 100644 --- a/evaluation_examples/examples/chrome/6766f2b8-8a72-417f-a9e5-56fcaa735837.json +++ b/evaluation_examples/examples/chrome/6766f2b8-8a72-417f-a9e5-56fcaa735837.json @@ -3,16 +3,59 @@ "snapshot": "chrome", "instruction": "Could you help me install the unpacked extension at /to/path in Chrome?", "source": "https://support.google.com/chrome/thread/205881926/it-s-possible-to-load-unpacked-extension-automatically-in-chrome?hl=en", - "config": [], + "config": [ + { + "type": "download", + "parameters": { + "files":[ + { + "url":"https://drive.google.com/uc?id=1t-6Qrpgox0UvqQ0EAhkCnSV8Rqsp4cMF&export=download", + "path":"/home/user/Desktop/helloExtension.zip" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": "echo lixiaochuan20 | sudo -S apt-get update -y && echo lixiaochuan20 | sudo -S apt-get install unzip -y && unzip /home/user/Desktop/helloExtension.zip -d /home/user/Desktop/ && rm /home/user/Desktop/helloExtension.zip", + "shell": true + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "google-chrome", + "--remote-debugging-port=9222" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "socat", + "tcp-listen:9222,fork", + "tcp:localhost:1337" + ] + } + } + ], "trajectory": "trajectories/", "related_apps": [ "chrome" ], "evaluator": { - "func": "", + "func": "is_in_list", "result": { + "type": "find_unpacked_extension_path" }, "expected": { + "type": "rule", + "rules": { + "expected": "/home/user/Desktop/helloExtension" + } } } } diff --git a/evaluation_examples/examples/chrome/9656a811-9b5b-4ddf-99c7-5117bcef0626.json b/evaluation_examples/examples/chrome/9656a811-9b5b-4ddf-99c7-5117bcef0626.json index 075b60b..da8a2a0 100644 --- a/evaluation_examples/examples/chrome/9656a811-9b5b-4ddf-99c7-5117bcef0626.json +++ b/evaluation_examples/examples/chrome/9656a811-9b5b-4ddf-99c7-5117bcef0626.json @@ -3,16 +3,41 @@ "snapshot": "chrome", "instruction": "I want Chrome to warn me whenever I visit a potentially harmful or unsafe website. Can you enable this safety feature?", "source": "https://www.quora.com/How-do-I-set-the-security-settings-for-the-Google-Chrome-browser-for-the-best-security#:~:text=Enable%20Safe%20Browsing:%20Chrome%20has%20a%20built%2Din,Security%20%3E%20Security%20%3E%20Enable%20Safe%20Browsing.", - "config": [], + "config": [ + { + "type": "launch", + "parameters": { + "command": [ + "google-chrome", + "--remote-debugging-port=9222" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "socat", + "tcp-listen:9222,fork", + "tcp:localhost:1337" + ] + } + } + ], "trajectory": "trajectories/", "related_apps": [ "chrome" ], "evaluator": { - "func": "", + "func": "exact_match", "result": { + "type": "enable_enhanced_safety_browsing" }, "expected": { + "type": "rule", + "rules": { + "expected": "true" + } } } } diff --git a/evaluation_examples/examples/os/c56de254-a3ec-414e-81a6-83d2ce8c41fa.json b/evaluation_examples/examples/os/c56de254-a3ec-414e-81a6-83d2ce8c41fa.json index bf4f500..59b3d0e 100644 --- a/evaluation_examples/examples/os/c56de254-a3ec-414e-81a6-83d2ce8c41fa.json +++ b/evaluation_examples/examples/os/c56de254-a3ec-414e-81a6-83d2ce8c41fa.json @@ -7,7 +7,7 @@ { "type": "execute", "parameters": { - "command": "sudo apt-get update && sudo apt-get install -y python3", + "command": "sudo aptget update && sudo apt-get install -y python3", "shell": true } } diff --git a/requirements.txt b/requirements.txt index bce1ae0..a6082f9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ fabric gymnasium~=0.28.1 requests~=2.31.0 transformers~=4.35.2 -torch~=2.1.1+cu118 +torch~=2.1.1 accelerate opencv-python~=4.8.1.78 matplotlib~=3.7.4