lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Finish Chrome v2 loading

Browse Source
This commit is contained in:
Timothyxxx
2024-01-24 23:05:28 +08:00
parent acfb232cba
commit 5dea912d01
8 changed files with 408 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
desktop_env/envs/desktop_env.py
View File

@@ -104,6 +104,9 @@ class DesktopEnv(gym.Env):
self.action_history: List[Dict[str, any]] = []
def _config_screen_size(self):
"""
fixme: Experimental features, will cause unexpected error when system corrupt
"""
def calculate_vram_size(width, height, bits_per_pixel=32):
"""
Calculate VRAM size for given width, height, and color depth.

6
desktop_env/evaluators/getters/__init__.py
View File

@@ -1,10 +1,12 @@
from .chrome import get_default_search_engine, get_cookie_data, get_bookmarks, get_open_tabs_info, get_pdf_from_url, \
get_shortcuts_on_desktop
get_shortcuts_on_desktop, get_history, get_enabled_experiments, get_chrome_language, get_chrome_font_size, \
get_profile_name, \
get_number_of_search_results
from .file import get_cloud_file, get_vm_file, get_cache_file
from .general import get_vm_command_line
from .impress import get_audio_in_slide
from .info import get_vm_screen_size, get_vm_window_size, get_vm_wallpaper
from .misc import get_rule, get_accessibility_tree
from .replay import get_replay
from .vlc import get_vlc_playing_info, get_vlc_config
from .vscode import get_vscode_config
from .impress import get_audio_in_slide

186
desktop_env/evaluators/getters/chrome.py
View File

@@ -84,6 +84,168 @@ def get_cookie_data(env, config: Dict[str, str]):
return None
def get_history(env, config: Dict[str, str]):
os_type = env.vm_platform
if os_type == 'Windows':
chrome_history_path = env.controller.execute_python_command(
"""import os; print(os.path.join(os.getenv('USERPROFILE'), "AppData", "Local", "Google", "Chrome", "User Data", "Default", "History"))""")[
'output'].strip()
elif os_type == 'Darwin':
chrome_history_path = env.controller.execute_python_command(
"""import os; print(os.path.join(os.getenv('HOME'), "Library", "Application Support", "Google", "Chrome", "Default", "History"))""")[
'output'].strip()
elif os_type == 'Linux':
chrome_history_path = env.controller.execute_python_command(
"import os; print(os.path.join(os.getenv('HOME'), '.config', 'google-chrome', 'Default', 'History'))")[
'output'].strip()
else:
raise Exception('Unsupported operating system')
try:
content = env.controller.get_file(chrome_history_path)
_path = os.path.join(env.cache_dir, config["dest"])
with open(_path, "wb") as f:
f.write(content)
conn = sqlite3.connect(_path)
cursor = conn.cursor()
# Query to check for OpenAI cookies
cursor.execute("SELECT url, title, last_visit_time FROM urls")
history_items = cursor.fetchall()
return history_items
except Exception as e:
logger.error(f"Error: {e}")
return None
def get_enabled_experiments(env, config: Dict[str, str]):
os_type = env.vm_platform
if os_type == 'Windows':
preference_file_path = env.controller.execute_python_command("""import os; print(os.path.join(os.getenv('LOCALAPPDATA'),
'Google\\Chrome\\User Data\\Local State'))""")[
'output'].strip()
elif os_type == 'Darwin':
preference_file_path = env.controller.execute_python_command(
"import os; print(os.path.join(os.getenv('HOME'), 'Library/Application Support/Google/Chrome/Local State'))")[
'output'].strip()
elif os_type == 'Linux':
preference_file_path = env.controller.execute_python_command(
"import os; print(os.path.join(os.getenv('HOME'), '.config/google-chrome/Local State'))")[
'output'].strip()
else:
raise Exception('Unsupported operating system')
try:
content = env.controller.get_file(preference_file_path)
data = json.loads(content)
# The path within the JSON data to the default search engine might vary
enabled_labs_experiments = data.get('browser', {}).get('enabled_labs_experiments', [])
return enabled_labs_experiments
except Exception as e:
logger.error(f"Error: {e}")
return []
def get_profile_name(env, config: Dict[str, str]):
"""
Get the username from the Chrome browser.
Assume the cookies are stored in the default location, not encrypted and not large in size.
"""
os_type = env.vm_platform
if os_type == 'Windows':
preference_file_path = env.controller.execute_python_command("""import os; print(os.path.join(os.getenv('LOCALAPPDATA'),
'Google\\Chrome\\User Data\\Default\\Preferences'))""")['output'].strip()
elif os_type == 'Darwin':
preference_file_path = env.controller.execute_python_command(
"import os; print(os.path.join(os.getenv('HOME'), 'Library/Application Support/Google/Chrome/Default/Preferences'))")[
'output'].strip()
elif os_type == 'Linux':
preference_file_path = env.controller.execute_python_command(
"import os; print(os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Preferences'))")[
'output'].strip()
else:
raise Exception('Unsupported operating system')
try:
content = env.controller.get_file(preference_file_path)
data = json.loads(content)
# The path within the JSON data to the default search engine might vary
profile_name = data.get('profile', {}).get('name', None)
return profile_name
except Exception as e:
logger.error(f"Error: {e}")
return None
def get_chrome_language(env, config: Dict[str, str]):
os_type = env.vm_platform
if os_type == 'Windows':
preference_file_path = env.controller.execute_python_command("""import os; print(os.path.join(os.getenv('LOCALAPPDATA'),
'Google\\Chrome\\User Data\\Local State'))""")[
'output'].strip()
elif os_type == 'Darwin':
preference_file_path = env.controller.execute_python_command(
"import os; print(os.path.join(os.getenv('HOME'), 'Library/Application Support/Google/Chrome/Local State'))")[
'output'].strip()
elif os_type == 'Linux':
preference_file_path = env.controller.execute_python_command(
"import os; print(os.path.join(os.getenv('HOME'), '.config/google-chrome/Local State'))")[
'output'].strip()
else:
raise Exception('Unsupported operating system')
try:
content = env.controller.get_file(preference_file_path)
data = json.loads(content)
# The path within the JSON data to the default search engine might vary
enabled_labs_experiments = data.get('intl', {}).get('app_locale', "en-US")
return enabled_labs_experiments
except Exception as e:
logger.error(f"Error: {e}")
return "en-US"
def get_chrome_font_size(env, config: Dict[str, str]):
os_type = env.vm_platform
if os_type == 'Windows':
preference_file_path = env.controller.execute_python_command("""import os; print(os.path.join(os.getenv('LOCALAPPDATA'),
'Google\\Chrome\\User Data\\Default\\Preferences'))""")[
'output'].strip()
elif os_type == 'Darwin':
preference_file_path = env.controller.execute_python_command(
"import os; print(os.path.join(os.getenv('HOME'), 'Library/Application Support/Google/Chrome/Default/Preferences'))")[
'output'].strip()
elif os_type == 'Linux':
preference_file_path = env.controller.execute_python_command(
"import os; print(os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Preferences'))")[
'output'].strip()
else:
raise Exception('Unsupported operating system')
try:
content = env.controller.get_file(preference_file_path)
data = json.loads(content)
# The path within the JSON data to the default search engine might vary
search_engine = data.get('webkit', {}).get('webprefs', {
"default_fixed_font_size": 13,
"default_font_size": 16,
"minimum_font_size": 13
})
return search_engine
except Exception as e:
logger.error(f"Error: {e}")
return {
"default_fixed_font_size": 13,
"default_font_size": 16
}
def get_bookmarks(env, config: Dict[str, str]):
os_type = env.vm_platform
if os_type == 'Windows':
@@ -227,8 +389,7 @@ def get_pdf_from_url(env, config: Dict[str, str]) -> str:
# fixme: needs to be changed (maybe through post-processing) since it's not working
def get_chrome_saved_address(env, config: Dict[str, str]):
# host = env.vm_ip
host = "192.168.13.130"
host = env.vm_ip
port = 9222 # fixme: this port is hard-coded, need to be changed from config file
remote_debugging_url = f"http://{host}:{port}"
@@ -278,6 +439,25 @@ def get_shortcuts_on_desktop(env, config: Dict[str, str]):
for shortcut_path in shortcuts_paths:
short_cuts[shortcut_path] = env.controller.get_file(env.controller.execute_python_command(
f"import os; print(os.path.join(os.path.expanduser('~'), 'Desktop', '{shortcut_path}'))")['output'].strip()).decode('utf-8')
f"import os; print(os.path.join(os.path.expanduser('~'), 'Desktop', '{shortcut_path}'))")[
'output'].strip()).decode('utf-8')
return short_cuts
def get_number_of_search_results(env, config: Dict[str, str]):
# todo: move into the config file
url, result_selector = "https://google.com/search?q=query", '.search-result'
host = env.vm_ip
port = 9222 # fixme: this port is hard-coded, need to be changed from config file
remote_debugging_url = f"http://{host}:{port}"
with sync_playwright() as p:
browser = p.chromium.connect_over_cdp(remote_debugging_url)
page = browser.new_page()
page.goto(url)
search_results = page.query_selector_all(result_selector)
actual_count = len(search_results)
browser.close()
return actual_count

42
desktop_env/evaluators/metrics/chrome.py
View File

@@ -95,3 +95,45 @@ def is_shortcut_on_desktop(shortcuts: Dict[str, str], rule):
raise TypeError(f"{rule['type']} not support yet!")
else:
raise TypeError(f"{rule['type']} not support yet!")
def check_history_deleted(history_data, rule):
"""
Check if the history is deleted.
"""
if rule['type'] == 'keywords':
history_domains = [history[0] for history in history_data]
for keyword in rule['keywords']:
for history_domain in history_domains:
if keyword in history_domain:
return 0.
return 1.
else:
raise TypeError(f"{rule['type']} not support yet!")
def check_enabled_experiments(enabled_experiments, rule):
"""
Check if the enabled experiments are as expected.
"""
enabled_experiments_names = [experiment.split("@")[0] for experiment in enabled_experiments]
if rule['type'] == 'names':
return 1. if enabled_experiments_names == rule['names'] else 0.
else:
raise TypeError(f"{rule['type']} not support yet!")
def check_font_size(font_size, rule):
"""
Check if the font size is as expected.
"""
default_font_size = font_size['default_font_size']
if rule['type'] == 'value':
return 1. if default_font_size == rule['value'] else 0.
elif rule['type'] == 'range':
return 1. if rule['min'] < default_font_size < rule['max'] else 0.
else:
raise TypeError(f"{rule['type']} not support yet!")

49
evaluation_examples/examples/chrome/2ae9ba84-3a0d-4d4c-8338-3a1478dc5fe3.json
View File

@@ -3,16 +3,61 @@
"snapshot": "chrome",
"instruction": "Lately I have changed my English name to Thomas. I want to update my username. Could you help me change the username in chrome profiles to Thomas?",
"source": "https://superuser.com/questions/1393683/how-to-change-the-username-in-google-chrome-profiles?rq=1",
"config": [],
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "",
"postconfig": [
{
"type": "launch",
"parameters": {
"command": [
"pkill",
"chrome"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
}
],
"func": "exact_match",
"result": {
"type": "profile_name"
},
"expected": {
"type": "rule",
"rules": {
"expected": "Thomas"
}
}
}
}

53
evaluation_examples/examples/chrome/44ee5668-ecd5-4366-a6ce-c1c9b8d4e938.json
View File

@@ -3,16 +3,65 @@
"snapshot": "chrome",
"instruction": "I am looking for an website address I accessed a month ago, but Youtube websites which take almost all of my browsing history are interrupting my search. This is too annoying. I want to remove all my Youtube browsing history first to facilitate my search. Could you help me clear browsing history from Youtube?",
"source": "https://superuser.com/questions/1787991/clear-browsing-history-from-specific-site-on-chrome",
"config": [],
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "",
"postconfig": [
{
"type": "launch",
"parameters": {
"command": [
"pkill",
"chrome"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
}
],
"func": "check_history_deleted",
"result": {
"type": "history",
"dest": "history.sqlite"
},
"expected": {
"type": "rule",
"rules": {
"type": "keywords",
"keywords": [
"youtube"
]
}
}
}
}

53
evaluation_examples/examples/chrome/480bcfea-d68f-4aaa-a0a9-2589ef319381.json
View File

@@ -3,16 +3,65 @@
"snapshot": "chrome",
"instruction": "I do not like the design of the new 2023 chrome UI. I want to keep using the original one. Can you disable the new 2023 version chrome UI for me? ",
"source": "https://bugartisan.medium.com/disable-the-new-chrome-ui-round-in-2023-db168271f71e",
"config": [],
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "",
"postconfig": [
{
"type": "launch",
"parameters": {
"command": [
"pkill",
"chrome"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
}
],
"func": "check_enabled_experiments",
"result": {
"type": "enabled_experiments"
},
"expected": {
"type": "rule",
"rules": {
"type": "names",
"names": [
"chrome-refresh-2023",
"chrome-webui-refresh-2023"
]
}
}
}
}

29
evaluation_examples/examples/chrome/ae78f875-5b98-4907-bbb5-9c737fc68c03.json
View File

@@ -3,16 +3,41 @@
"snapshot": "chrome",
"instruction": "Could you please change the number of search results displayed on one page to 50? I find that having more results visible at once significantly enhances my research efficiency, as it reduces the need to constantly click through multiple pages. ",
"source": "https://support.google.com/chrome/thread/219988391/increase-search-results-per-page?hl=en",
"config": [],
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "",
"func": "exact_match",
"result": {
"type": "number_of_search_results"
},
"expected": {
"type": "rule",
"rules": {
"expected": 50
}
}
}
}