lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #6 from xlang-ai/xiaochuanli/addChromeExtensions

Browse Source 
Xiaochuanli/add chrome extensions
This commit is contained in:
Tianbao Xie
2024-02-19 17:24:16 +08:00
committed by GitHub
parent ad9d8f51c6 e31e1dacde
commit c35bcc476b
12 changed files with 333 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
desktop_env/evaluators/getters/__init__.py
View File

@@ -12,7 +12,11 @@ from .chrome import (
get_profile_name,
get_number_of_search_results,
get_googledrive_file,
get_active_tab_info
get_active_tab_info,
get_enable_do_not_track,
get_enable_enhanced_safety_browsing,
get_new_startup_page,
get_find_unpacked_extension_path
)
from .file import get_cloud_file, get_vm_file, get_cache_file
from .general import get_vm_command_line, get_vm_terminal_output

134
desktop_env/evaluators/getters/chrome.py
View File

@@ -363,6 +363,9 @@ def get_active_tab_info(env, config: Dict[str, str]):
break
browser.close()
print("active_tab_title: {}".format(active_tab_info.get('title', 'None')))
print("active_tab_url: {}".format(active_tab_info.get('url', 'None')))
print("active_tab_content: {}".format(active_tab_info.get('content', 'None')))
return active_tab_info
@@ -516,4 +519,133 @@ def get_googledrive_file(env, config: Dict[str, Any]) -> str:
else f"title = '{fp}' and trashed = false" for jdx, fp in enumerate(path)]
dest = config['dest'][idx]
_path_list.append(get_single_file(query, os.path.join(env.cache_dir, dest)))
return _path_list
return _path_list
def get_enable_do_not_track(env, config: Dict[str, str]):
os_type = env.vm_platform
if os_type == 'Windows':
preference_file_path = env.controller.execute_python_command("""import os; print(os.path.join(os.getenv('LOCALAPPDATA'),
'Google\\Chrome\\User Data\\Default\\Preferences'))""")['output'].strip()
elif os_type == 'Darwin':
preference_file_path = env.controller.execute_python_command(
"import os; print(os.path.join(os.getenv('HOME'), 'Library/Application Support/Google/Chrome/Default/Preferences'))")[
'output'].strip()
elif os_type == 'Linux':
# preference_file_path = env.controller.execute_python_command(
# "import os; print(os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Preferences'))")[
# 'output'].strip()
preference_file_path = env.controller.execute_python_command(
"import os; print(os.path.join(os.getenv('HOME'), 'snap/chromium/common/chromium/Default/Preferences'))")[
'output'].strip()
else:
raise Exception('Unsupported operating system')
try:
content = env.controller.get_file(preference_file_path)
data = json.loads(content)
if_enable_do_not_track = data.get('enable_do_not_track', {}) # bool
return "true" if if_enable_do_not_track else "false"
except Exception as e:
logger.error(f"Error: {e}")
return "Google"
def get_enable_enhanced_safety_browsing(env, config: Dict[str, str]):
os_type = env.vm_platform
if os_type == 'Windows':
preference_file_path = env.controller.execute_python_command("""import os; print(os.path.join(os.getenv('LOCALAPPDATA'),
'Google\\Chrome\\User Data\\Default\\Preferences'))""")['output'].strip()
elif os_type == 'Darwin':
preference_file_path = env.controller.execute_python_command(
"import os; print(os.path.join(os.getenv('HOME'), 'Library/Application Support/Google/Chrome/Default/Preferences'))")[
'output'].strip()
elif os_type == 'Linux':
# preference_file_path = env.controller.execute_python_command(
# "import os; print(os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Preferences'))")[
# 'output'].strip()
preference_file_path = env.controller.execute_python_command(
"import os; print(os.path.join(os.getenv('HOME'), 'snap/chromium/common/chromium/Default/Preferences'))")[
'output'].strip()
else:
raise Exception('Unsupported operating system')
try:
content = env.controller.get_file(preference_file_path)
data = json.loads(content)
if_enable_do_not_track = data.get('safebrowsing', {}).get('enhanced', {}) # bool
return "true" if if_enable_do_not_track else "false"
except Exception as e:
logger.error(f"Error: {e}")
return "Google"
def get_new_startup_page(env, config: Dict[str, str]):
os_type = env.vm_platform
if os_type == 'Windows':
preference_file_path = env.controller.execute_python_command("""import os; print(os.path.join(os.getenv('LOCALAPPDATA'),
'Google\\Chrome\\User Data\\Default\\Preferences'))""")['output'].strip()
elif os_type == 'Darwin':
preference_file_path = env.controller.execute_python_command(
"import os; print(os.path.join(os.getenv('HOME'), 'Library/Application Support/Google/Chrome/Default/Preferences'))")[
'output'].strip()
elif os_type == 'Linux':
# preference_file_path = env.controller.execute_python_command(
# "import os; print(os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Preferences'))")[
# 'output'].strip()
preference_file_path = env.controller.execute_python_command(
"import os; print(os.path.join(os.getenv('HOME'), 'snap/chromium/common/chromium/Default/Preferences'))")[
'output'].strip()
else:
raise Exception('Unsupported operating system')
try:
content = env.controller.get_file(preference_file_path)
data = json.loads(content)
# if data has no key called 'session', it means the chrome is on a fresh-start mode, which is a true state;
# otherwise, try to find the code number in 'restored_on_startup' in 'session'
if "session" not in data.keys():
return "true"
else:
if_enable_do_not_track = data.get('session', {}).get('restore_on_startup', {}) # int, need to be 5
return "true" if if_enable_do_not_track == 5 else "false"
except Exception as e:
logger.error(f"Error: {e}")
return "Google"
def get_find_unpacked_extension_path(env, config: Dict[str, str]):
os_type = env.vm_platform
if os_type == 'Windows':
preference_file_path = env.controller.execute_python_command("""import os; print(os.path.join(os.getenv('LOCALAPPDATA'),
'Google\\Chrome\\User Data\\Default\\Preferences'))""")['output'].strip()
elif os_type == 'Darwin':
preference_file_path = env.controller.execute_python_command(
"import os; print(os.path.join(os.getenv('HOME'), 'Library/Application Support/Google/Chrome/Default/Preferences'))")[
'output'].strip()
elif os_type == 'Linux':
# preference_file_path = env.controller.execute_python_command(
# "import os; print(os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Preferences'))")[
# 'output'].strip()
preference_file_path = env.controller.execute_python_command(
"import os; print(os.path.join(os.getenv('HOME'), 'snap/chromium/common/chromium/Default/Preferences'))")[
'output'].strip()
else:
raise Exception('Unsupported operating system')
try:
content = env.controller.get_file(preference_file_path)
data = json.loads(content)
# Preferences store all the path of installed extensions, return them all and let metrics try to find one matches the targeted extension path
all_extensions_path = []
all_extensions = data.get('extensions', {}).get('settings', {})
for id in all_extensions.keys():
path = all_extensions[id]["path"]
all_extensions_path.append(path)
return all_extensions_path
except Exception as e:
logger.error(f"Error: {e}")
return "Google"

4
desktop_env/evaluators/metrics/__init__.py
View File

@@ -15,7 +15,8 @@ from .chrome import (
check_font_size,
check_enabled_experiments,
check_history_deleted,
is_expected_search_query
is_expected_search_query,
is_expected_active_tab
)
from .docs import (
compare_font_names,
@@ -51,6 +52,7 @@ from .general import (
check_json,
check_list,
exact_match,
is_in_list,
fuzzy_match,
check_include_exclude
)

16
desktop_env/evaluators/metrics/chrome.py
View File

@@ -9,6 +9,22 @@ from desktop_env.evaluators.metrics.utils import are_lists_equal, compare_urls
logger = logging.getLogger("desktopenv.metrics.chrome")
def is_expected_active_tab(active_tab_info: Dict[str, str], rule: Dict[str, Any]) -> float:
"""
Checks if the expected active tab is open in Chrome.
"""
match_type = rule['type']
if match_type == "url":
expected_url = rule['url']
actual_url = active_tab_info['url']
print("expected_url: {}".format(expected_url))
print("actual_url: {}".format(actual_url))
return 1 if compare_urls(expected_url, actual_url) else 0
else:
logger.error(f"Unknown type: {match_type}")
return 0
def is_expected_tabs(open_tabs: List[Dict[str, str]], rule: Dict[str, Any]) -> float:
"""
Checks if the expected tabs are open in Chrome.

8
desktop_env/evaluators/metrics/general.py
View File

@@ -38,7 +38,13 @@ def exact_match(result, rules) -> float:
else:
return 0.
def is_in_list(result, rules) -> float:
expect = rules["expected"]
if expect in result:
return 1.
else:
return 0.
def fuzzy_match(result, rules) -> float:
expect = rules["expected"]

29
evaluation_examples/examples/chrome/030eeff7-b492-4218-b312-701ec99ee0cc.json
View File

@@ -3,16 +3,41 @@
"snapshot": "chrome",
"instruction": "Can you enable the 'Do Not Track' feature in Chrome to enhance my online privacy?",
"source": "https://www.surreycc.gov.uk/website/cookies/do-not-track",
"config": [],
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=9222"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "",
"func": "exact_match",
"result": {
"type": "enable_do_not_track"
},
"expected": {
"type": "rule",
"rules": {
"expected": "true"
}
}
}
}

29
evaluation_examples/examples/chrome/12086550-11c0-466b-b367-1d9e75b3910e.json
View File

@@ -3,16 +3,41 @@
"snapshot": "chrome",
"instruction": "Computer, please navigate to the area in my browser settings where my passwords are stored. I want to check my login information for Etsy without revealing it just yet.",
"source": "https://www.quora.com/What-are-the-cool-tricks-to-use-Google-Chrome",
"config": [],
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
}],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "",
"func": "is_expected_active_tab",
"result": {
"type": "active_tab_info"
},
"expected": {
"type": "rule",
"rules": {
"type": "url",
"url":"chrome://password-manager/passwords"
}
}
}
}

43
evaluation_examples/examples/chrome/3299584d-8f11-4457-bf4c-ce98f7600250.json
View File

@@ -3,16 +3,55 @@
"snapshot": "chrome",
"instruction": "On my surface pro whenever I launch Chrome it always opens \"funbrain.com.\" I don't want this. I cleared my cache but it still happens. What should I do?",
"source": "https://www.reddit.com/r/techsupport/comments/12zwymy/comment/jhtri65/?utm_source=share&utm_medium=web2x&context=3",
"config": [],
"config": [
{
"type": "execute",
"parameters": {
"command": "echo lixiaochuan20 | sudo -S apt update -y && echo lixiaochuan20 | sudo -S apt install jq -y",
"shell": true
}
},
{
"type": "execute",
"parameters": {
"command": "cd /home/user/snap/chromium/common/chromium/Default && jq '. + {\"session\":{\"restore_on_startup\":4, \"startup_urls\":[\"http://funbrain.com/\"]}}' Preferences > temp && mv temp Preferences",
"shell": true
}
},
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=9222"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "",
"func": "exact_match",
"result": {
"type": "new_startup_page"
},
"expected": {
"type": "rule",
"rules": {
"expected": "true"
}
}
}
}

47
evaluation_examples/examples/chrome/6766f2b8-8a72-417f-a9e5-56fcaa735837.json
View File

@@ -3,16 +3,59 @@
"snapshot": "chrome",
"instruction": "Could you help me install the unpacked extension at /to/path in Chrome?",
"source": "https://support.google.com/chrome/thread/205881926/it-s-possible-to-load-unpacked-extension-automatically-in-chrome?hl=en",
"config": [],
"config": [
{
"type": "download",
"parameters": {
"files":[
{
"url":"https://drive.google.com/uc?id=1t-6Qrpgox0UvqQ0EAhkCnSV8Rqsp4cMF&export=download",
"path":"/home/user/Desktop/helloExtension.zip"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": "echo lixiaochuan20 | sudo -S apt-get update -y && echo lixiaochuan20 | sudo -S apt-get install unzip -y && unzip /home/user/Desktop/helloExtension.zip -d /home/user/Desktop/ && rm /home/user/Desktop/helloExtension.zip",
"shell": true
}
},
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=9222"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "",
"func": "is_in_list",
"result": {
"type": "find_unpacked_extension_path"
},
"expected": {
"type": "rule",
"rules": {
"expected": "/home/user/Desktop/helloExtension"
}
}
}
}

29
evaluation_examples/examples/chrome/9656a811-9b5b-4ddf-99c7-5117bcef0626.json
View File

@@ -3,16 +3,41 @@
"snapshot": "chrome",
"instruction": "I want Chrome to warn me whenever I visit a potentially harmful or unsafe website. Can you enable this safety feature?",
"source": "https://www.quora.com/How-do-I-set-the-security-settings-for-the-Google-Chrome-browser-for-the-best-security#:~:text=Enable%20Safe%20Browsing:%20Chrome%20has%20a%20built%2Din,Security%20%3E%20Security%20%3E%20Enable%20Safe%20Browsing.",
"config": [],
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=9222"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "",
"func": "exact_match",
"result": {
"type": "enable_enhanced_safety_browsing"
},
"expected": {
"type": "rule",
"rules": {
"expected": "true"
}
}
}
}

2
evaluation_examples/examples/os/c56de254-a3ec-414e-81a6-83d2ce8c41fa.json
View File

@@ -7,7 +7,7 @@
{
"type": "execute",
"parameters": {
"command": "sudo apt-get update && sudo apt-get install -y python3",
"command": "sudo aptget update && sudo apt-get install -y python3",
"shell": true
}
}

2
requirements.txt
View File

@@ -4,7 +4,7 @@ fabric
gymnasium~=0.28.1
requests~=2.31.0
transformers~=4.35.2
torch~=2.1.1+cu118
torch~=2.1.1
accelerate
opencv-python~=4.8.1.78
matplotlib~=3.7.4