diff --git a/desktop_env/assets/history_empty.sqlite b/desktop_env/assets/history_empty.sqlite new file mode 100644 index 0000000..c68c17c Binary files /dev/null and b/desktop_env/assets/history_empty.sqlite differ diff --git a/desktop_env/controllers/setup.py b/desktop_env/controllers/setup.py index a29021b..1f081a9 100644 --- a/desktop_env/controllers/setup.py +++ b/desktop_env/controllers/setup.py @@ -1,24 +1,29 @@ import json import logging +import os import os.path +import sqlite3 +import tempfile import time import traceback import uuid -import tempfile +from datetime import datetime, timedelta from typing import Any, Union, Optional from typing import Dict, List -import os +import shutil import requests +from playwright.sync_api import sync_playwright, TimeoutError from pydrive.auth import GoogleAuth from pydrive.drive import GoogleDrive, GoogleDriveFile, GoogleDriveFileList -from playwright.sync_api import sync_playwright, TimeoutError from requests_toolbelt.multipart.encoder import MultipartEncoder +from desktop_env.controllers.python import PythonController from desktop_env.evaluators.metrics.utils import compare_urls logger = logging.getLogger("desktopenv.setup") +FILE_PATH = os.path.dirname(os.path.abspath(__file__)) class SetupController: def __init__(self, vm_ip: str, cache_dir: str): @@ -130,7 +135,8 @@ class SetupController: break except requests.RequestException as e: - logger.error(f"Failed to download {url} caused by {e}. Retrying... ({max_retries - i - 1} attempts left)") + logger.error( + f"Failed to download {url} caused by {e}. Retrying... ({max_retries - i - 1} attempts left)") if not downloaded: raise requests.RequestException(f"Failed to download {url}. No retries left. Error: {e}") @@ -349,18 +355,18 @@ class SetupController: logger.info("Connect to Chrome @: %s", remote_debugging_url) logger.debug("PLAYWRIGHT ENV: %s", repr(os.environ)) for attempt in range(15): - if attempt>0: + if attempt > 0: time.sleep(5) browser = None with sync_playwright() as p: try: browser = p.chromium.connect_over_cdp(remote_debugging_url) - #break + # break except Exception as e: if attempt < 14: logger.error(f"Attempt {attempt + 1}: Failed to connect, retrying. Error: {e}") - #time.sleep(10) + # time.sleep(10) continue else: logger.error(f"Failed to connect after multiple attempts: {e}") @@ -379,7 +385,7 @@ class SetupController: try: page.goto(url, timeout=60000) except: - logger.warning("Opening %s exceeds time limit", url) # only for human test + logger.warning("Opening %s exceeds time limit", url) # only for human test logger.info(f"Opened tab {i + 1}: {url}") if i == 0: @@ -458,16 +464,17 @@ class SetupController: for p in paths: q = f'"{parent_id}" in parents and title = "{p}" and mimeType = "application/vnd.google-apps.folder" and trashed = false' folder = drive.ListFile({'q': q}).GetList() - if len(folder) == 0: # not exists, create it + if len(folder) == 0: # not exists, create it parents = {} if parent_id == 'root' else {'parents': [{'id': parent_id}]} - file = drive.CreateFile({'title': p, 'mimeType':'application/vnd.google-apps.folder', **parents}) + file = drive.CreateFile({'title': p, 'mimeType': 'application/vnd.google-apps.folder', **parents}) file.Upload() parent_id = file['id'] - else: parent_id = folder[0]['id'] + else: + parent_id = folder[0]['id'] return parent_id for oid, operation in enumerate(config['operation']): - if operation == 'delete': # delete a specific file + if operation == 'delete': # delete a specific file # query pattern string, by default, remove all files/folders not in the trash to the trash params = config['args'][oid] q = params.get('query', '') @@ -476,15 +483,19 @@ class SetupController: filelist: GoogleDriveFileList = drive.ListFile({'q': q_file}).GetList() q_folder = f"( {q} ) and mimeType = 'application/vnd.google-apps.folder'" if q.strip() else "mimeType = 'application/vnd.google-apps.folder'" folderlist: GoogleDriveFileList = drive.ListFile({'q': q_folder}).GetList() - for file in filelist: # first delete file, then folder + for file in filelist: # first delete file, then folder file: GoogleDriveFile - if trash: file.Trash() - else: file.Delete() + if trash: + file.Trash() + else: + file.Delete() for folder in folderlist: folder: GoogleDriveFile # note that, if a folder is trashed/deleted, all files and folders in it will be trashed/deleted - if trash: folder.Trash() - else: folder.Delete() + if trash: + folder.Trash() + else: + folder.Delete() elif operation == 'mkdirs': params = config['args'][oid] mkdir_in_googledrive(params['path']) @@ -508,7 +519,6 @@ class SetupController: else: raise ValueError('[ERROR]: not implemented clean type!') - def _login_setup(self, **config): """ Login to a website with account and password information. @args: @@ -537,7 +547,7 @@ class SetupController: raise e if not browser: return - + context = browser.contexts[0] platform = config['platform'] @@ -565,3 +575,82 @@ class SetupController: raise NotImplementedError return browser, context + + def _update_browse_history_setup(self, **config): + db_path = os.path.join("desktop_env", "assets", "history_empty.sqlite") + + # copy a new history file in the tmp folder + cache_path = os.path.join(self.cache_dir, "history_new.sqlite") + shutil.copyfile(db_path, cache_path) + db_path = cache_path + + history = config['history'] + + for history_item in history: + url = history_item['url'] + title = history_item['title'] + visit_time = datetime.now() - timedelta(seconds=history_item['visit_time_from_now_in_seconds']) + + # Chrome use ms from 1601-01-01 as timestamp + epoch_start = datetime(1601, 1, 1) + chrome_timestamp = int((visit_time - epoch_start).total_seconds() * 1000000) + + conn = sqlite3.connect(db_path) + cursor = conn.cursor() + + cursor.execute(''' + INSERT INTO urls (url, title, visit_count, typed_count, last_visit_time, hidden) + VALUES (?, ?, ?, ?, ?, ?) + ''', (url, title, 1, 0, chrome_timestamp, 0)) + + url_id = cursor.lastrowid + + cursor.execute(''' + INSERT INTO visits (url, visit_time, from_visit, transition, segment_id, visit_duration) + VALUES (?, ?, ?, ?, ?, ?) + ''', (url_id, chrome_timestamp, 0, 805306368, 0, 0)) + + conn.commit() + conn.close() + + logger.info('Fake browsing history added successfully.') + + controller = PythonController(self.vm_ip) + + # get the path of the history file according to the platform + os_type = controller.get_vm_platform() + + if os_type == 'Windows': + chrome_history_path = controller.execute_python_command( + """import os; print(os.path.join(os.getenv('USERPROFILE'), "AppData", "Local", "Google", "Chrome", "User Data", "Default", "History"))""")[ + 'output'].strip() + elif os_type == 'Darwin': + chrome_history_path = controller.execute_python_command( + """import os; print(os.path.join(os.getenv('HOME'), "Library", "Application Support", "Google", "Chrome", "Default", "History"))""")[ + 'output'].strip() + elif os_type == 'Linux': + chrome_history_path = controller.execute_python_command( + "import os; print(os.path.join(os.getenv('HOME'), '.config', 'google-chrome', 'Default', 'History'))")[ + 'output'].strip() + else: + raise Exception('Unsupported operating system') + + form = MultipartEncoder({ + "file_path": chrome_history_path, + "file_data": (os.path.basename(chrome_history_path), open(db_path, "rb")) + }) + headers = {"Content-Type": form.content_type} + logger.debug(form.content_type) + + # send request to server to upload file + try: + logger.debug("REQUEST ADDRESS: %s", self.http_server + "/setup" + "/upload") + response = requests.post(self.http_server + "/setup" + "/upload", headers=headers, data=form) + if response.status_code == 200: + logger.info("Command executed successfully: %s", response.text) + else: + logger.error("Failed to upload file. Status code: %s", response.text) + except requests.exceptions.RequestException as e: + logger.error("An error occurred while trying to send the request: %s", e) + + self._execute_setup(["sudo chown -R user:user /home/user/.config/google-chrome/Default/History"], shell=True) diff --git a/desktop_env/evaluators/metrics/slides.py b/desktop_env/evaluators/metrics/slides.py index d3b8542..63e1e39 100644 --- a/desktop_env/evaluators/metrics/slides.py +++ b/desktop_env/evaluators/metrics/slides.py @@ -139,6 +139,7 @@ def compare_pptx_files(file1_path, file2_path, **options): examine_number_of_slides = options.get("examine_number_of_slides", True) examine_shape = options.get("examine_shape", True) examine_text = options.get("examine_text", True) + examine_indent = options.get("examine_indent", True) examine_font_name = options.get("examine_font_name", True) examine_font_size = options.get("examine_font_size", True) examine_font_bold = options.get("examine_font_bold", True) @@ -146,6 +147,7 @@ def compare_pptx_files(file1_path, file2_path, **options): examine_color_rgb = options.get("examine_color_rgb", True) examine_font_underline = options.get("examine_font_underline", True) examine_strike_through = options.get("examine_strike_through", True) + examine_bullets = options.get("examine_bullets", True) # compare the number of slides if len(prs1.slides) != len(prs2.slides) and examine_number_of_slides: @@ -167,6 +169,12 @@ def compare_pptx_files(file1_path, file2_path, **options): # check if the paragraphs are the same for para1, para2 in zip(shape1.text_frame.paragraphs, shape2.text_frame.paragraphs): # check if the runs are the same + if para1.text != para2.text and examine_text: + return 0 + + if para1.level != para2.level and examine_indent: + return 0 + for run1, run2 in zip(para1.runs, para2.runs): # check if the font properties are the same @@ -192,7 +200,40 @@ def compare_pptx_files(file1_path, file2_path, **options): 'strike', 'noStrike') and examine_strike_through: return 0 - # fixme: Actually there are more properties to be compared, but we cannot get them through pptx + def _extract_bullets(xml_data): + root = ET.fromstring(xml_data) + + namespaces = { + 'a': 'http://schemas.openxmlformats.org/drawingml/2006/main', + 'p': 'http://schemas.openxmlformats.org/presentationml/2006/main', + } + + bullets = [] + + for paragraph in root.findall('.//a:p', namespaces): + pPr = paragraph.find('a:pPr', namespaces) + if pPr is not None: + lvl = pPr.get('lvl') + buChar = pPr.find('a:buChar', namespaces) + char = buChar.get('char') if buChar is not None else "No Bullet" + buClr = pPr.find('a:buClr/a:srgbClr', namespaces) + color = buClr.get('val') if buClr is not None else "No Color" + else: + lvl = "No Level" + char = "No Bullet" + color = "No Color" + + text = "".join(t.text for t in paragraph.findall('.//a:t', namespaces)) + + bullets.append((lvl, char, text, color)) + + return bullets + + if _extract_bullets(run1.part.blob.decode('utf-8')) != _extract_bullets( + run2.part.blob.decode('utf-8')) and examine_bullets: + return 0 + + # fixme: Actually there are more properties to be compared, we can add them later via parsing the xml data return 1 @@ -414,6 +455,7 @@ if __name__ == '__main__': # r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\550ce7e7-747b-495f-b122-acdc4d0b8e54\New_Club_Spring_2018_Training_Gold.pptx")) # print(evaluate_presentation_fill_to_rgb_distance(r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\3b27600c-3668-4abd-8f84-7bcdebbccbdb\lec17-gui-events.pptx", {"rgb": (0, 0, 255)})) # print(check_auto_saving_time(r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\2cd43775-7085-45d8-89fa-9e35c0a915cf\registrymodifications.xcu", {"minutes": 3})) - print(compare_pptx_files(r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\a669ef01-ded5-4099-9ea9-25e99b569840\Writing-Outlines.pptx", - r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\a669ef01-ded5-4099-9ea9-25e99b569840\Writing-Outlines_Gold.pptx", - examine_shape=False)) + print(compare_pptx_files( + r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\a669ef01-ded5-4099-9ea9-25e99b569840\Writing-Outlines.pptx", + r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\a669ef01-ded5-4099-9ea9-25e99b569840\Writing-Outlines_Gold.pptx", + examine_shape=False)) diff --git a/evaluation_examples/examples/chrome/44ee5668-ecd5-4366-a6ce-c1c9b8d4e938.json b/evaluation_examples/examples/chrome/44ee5668-ecd5-4366-a6ce-c1c9b8d4e938.json index 3a31855..def3f9e 100644 --- a/evaluation_examples/examples/chrome/44ee5668-ecd5-4366-a6ce-c1c9b8d4e938.json +++ b/evaluation_examples/examples/chrome/44ee5668-ecd5-4366-a6ce-c1c9b8d4e938.json @@ -4,6 +4,213 @@ "instruction": "I am looking for an website address I accessed a month ago, but Youtube websites which take almost all of my browsing history are interrupting my search. This is too annoying. I want to remove all my Youtube browsing history first to facilitate my search. Could you help me clear browsing history from Youtube?", "source": "https://superuser.com/questions/1787991/clear-browsing-history-from-specific-site-on-chrome", "config": [ + { + "type": "update_browse_history", + "parameters": { + "history": [ + { + "url": "https://www.youtube.com/watch?v=dQw4w9WgXcQ", + "title": "Rick Astley - Never Gonna Give You Up (Official Music Video)", + "visit_time_from_now_in_seconds": 3600 + }, + { + "url": "https://www.youtube.com/watch?v=9bZkp7q19f0", + "title": "PSY - GANGNAM STYLE(강남스타일) M/V", + "visit_time_from_now_in_seconds": 1631 + }, + { + "url": "https://www.youtube.com/watch?v=3tmd-ClpJxA", + "title": "Maroon 5 - Sugar (Official Music Video)", + "visit_time_from_now_in_seconds": 900 + }, + { + "url": "https://www.nytimes.com/", + "title": "The New York Times", + "visit_time_from_now_in_seconds": 300 + }, + { + "url": "https://www.youtube.com/watch?v=OPf0YbXqDm0", + "title": "Ed Sheeran - Shape of You [Official Music Video]", + "visit_time_from_now_in_seconds": 1200 + }, + { + "url": "https://www.youtube.com/watch?v=JGwWNGJdvx8", + "title": "Taylor Swift - Shake It Off", + "visit_time_from_now_in_seconds": 2400 + }, + { + "url": "https://www.bbc.co.uk/", + "title": "BBC", + "visit_time_from_now_in_seconds": 1500 + }, + { + "url": "https://www.youtube.com/watch?v=2Vv-BfVoq4g", + "title": "Adele - Hello", + "visit_time_from_now_in_seconds": 1800 + }, + { + "url": "https://www.youtube.com/watch?v=YQHsXMglC9A", + "title": "Katy Perry - Roar (Official Music Video)", + "visit_time_from_now_in_seconds": 2100 + }, + { + "url": "https://www.cnn.com/", + "title": "CNN", + "visit_time_from_now_in_seconds": 2700 + }, + { + "url": "https://www.youtube.com/watch?v=ru0K8uYEZWw", + "title": "Justin Bieber - Baby ft. Ludacris (Official Music Video)", + "visit_time_from_now_in_seconds": 3200 + }, + { + "url": "https://www.youtube.com/watch?v=9bZkp7q19f0", + "title": "PSY - GANGNAM STYLE(강남스타일) M/V", + "visit_time_from_now_in_seconds": 3700 + }, + { + "url": "https://www.nationalgeographic.com/", + "title": "National Geographic", + "visit_time_from_now_in_seconds": 4000 + }, + { + "url": "https://www.youtube.com/watch?v=OPf0YbXqDm0", + "title": "Ed Sheeran - Shape of You [Official Music Video]", + "visit_time_from_now_in_seconds": 4300 + }, + { + "url": "https://www.youtube.com/watch?v=JGwWNGJdvx8", + "title": "Taylor Swift - Shake It Off", + "visit_time_from_now_in_seconds": 4700 + }, + { + "url": "https://www.bbc.co.uk/", + "title": "BBC", + "visit_time_from_now_in_seconds": 5000 + }, + { + "url": "https://www.youtube.com/watch?v=2Vv-BfVoq4g", + "title": "Adele - Hello", + "visit_time_from_now_in_seconds": 5300 + }, + { + "url": "https://www.youtube.com/watch?v=YQHsXMglC9A", + "title": "Katy Perry - Roar (Official Music Video)", + "visit_time_from_now_in_seconds": 5600 + }, + { + "url": "https://www.cnn.com/", + "title": "CNN", + "visit_time_from_now_in_seconds": 5900 + }, + { + "url": "https://www.youtube.com/watch?v=ru0K8uYEZWw", + "title": "Justin Bieber - Baby ft. Ludacris (Official Music Video)", + "visit_time_from_now_in_seconds": 6300 + }, + { + "url": "https://www.youtube.com/watch?v=dQw4w9WgXcQ", + "title": "Rick Astley - Never Gonna Give You Up (Official Music Video)", + "visit_time_from_now_in_seconds": 6700 + }, + { + "url": "https://www.nationalgeographic.com/", + "title": "National Geographic", + "visit_time_from_now_in_seconds": 7000 + }, + { + "url": "https://www.youtube.com/watch?v=OPf0YbXqDm0", + "title": "Ed Sheeran - Shape of You [Official Music Video]", + "visit_time_from_now_in_seconds": 7300 + }, + { + "url": "https://www.youtube.com/watch?v=JGwWNGJdvx8", + "title": "Taylor Swift - Shake It Off", + "visit_time_from_now_in_seconds": 7600 + }, + { + "url": "https://www.bbc.co.uk/", + "title": "BBC", + "visit_time_from_now_in_seconds": 7900 + }, + { + "url": "https://www.youtube.com/watch?v=2Vv-BfVoq4g", + "title": "Adele - Hello", + "visit_time_from_now_in_seconds": 8200 + }, + { + "url": "https://www.youtube.com/watch?v=YQHsXMglC9A", + "title": "Katy Perry - Roar (Official Music Video)", + "visit_time_from_now_in_seconds": 8500 + }, + { + "url": "https://www.cnn.com/", + "title": "CNN", + "visit_time_from_now_in_seconds": 8800 + }, + { + "url": "https://www.youtube.com/watch?v=ru0K8uYEZWw", + "title": "Justin Bieber - Baby ft. Ludacris (Official Music Video)", + "visit_time_from_now_in_seconds": 9100 + }, + { + "url": "https://www.youtube.com/watch?v=dQw4w9WgXcQ", + "title": "Rick Astley - Never Gonna Give You Up (Official Music Video)", + "visit_time_from_now_in_seconds": 9400 + }, + { + "url": "https://www.nationalgeographic.com/", + "title": "National Geographic", + "visit_time_from_now_in_seconds": 9700 + }, + { + "url": "https://www.youtube.com/watch?v=OPf0YbXqDm0", + "title": "Ed Sheeran - Shape of You [Official Music Video]", + "visit_time_from_now_in_seconds": 10000 + }, + { + "url": "https://www.youtube.com/watch?v=JGwWNGJdvx8", + "title": "Taylor Swift - Shake It Off", + "visit_time_from_now_in_seconds": 10300 + }, + { + "url": "https://www.bbc.co.uk/", + "title": "BBC", + "visit_time_from_now_in_seconds": 10600 + }, + { + "url": "https://www.youtube.com/watch?v=2Vv-BfVoq4g", + "title": "Adele - Hello", + "visit_time_from_now_in_seconds": 10900 + }, + { + "url": "https://www.youtube.com/watch?v=YQHsXMglC9A", + "title": "Katy Perry - Roar (Official Music Video)", + "visit_time_from_now_in_seconds": 11200 + }, + { + "url": "https://www.cnn.com/", + "title": "CNN", + "visit_time_from_now_in_seconds": 11500 + }, + { + "url": "https://www.youtube.com/watch?v=ru0K8uYEZWw", + "title": "Justin Bieber - Baby ft. Ludacris (Official Music Video)", + "visit_time_from_now_in_seconds": 11800 + }, + { + "url": "https://www.youtube.com/watch?v=dQw4w9WgXcQ", + "title": "Rick Astley - Never Gonna Give You Up (Official Music Video)", + "visit_time_from_now_in_seconds": 12100 + }, + { + "url": "https://www.nationalgeographic.com/", + "title": "National Geographic", + "visit_time_from_now_in_seconds": 12400 + } + ] + } + }, { "type": "launch", "parameters": { diff --git a/evaluation_examples/examples/libreoffice_impress/358aa0a7-6677-453f-ae35-e440f004c31e.json b/evaluation_examples/examples/libreoffice_impress/358aa0a7-6677-453f-ae35-e440f004c31e.json index 227002d..b594213 100644 --- a/evaluation_examples/examples/libreoffice_impress/358aa0a7-6677-453f-ae35-e440f004c31e.json +++ b/evaluation_examples/examples/libreoffice_impress/358aa0a7-6677-453f-ae35-e440f004c31e.json @@ -54,7 +54,7 @@ { "type": "sleep", "parameters": { - "seconds": 0.5 + "seconds": 15 } } ], diff --git a/evaluation_examples/examples/libreoffice_impress/a669ef01-ded5-4099-9ea9-25e99b569840.json b/evaluation_examples/examples/libreoffice_impress/a669ef01-ded5-4099-9ea9-25e99b569840.json index 853f01a..25c8780 100644 --- a/evaluation_examples/examples/libreoffice_impress/a669ef01-ded5-4099-9ea9-25e99b569840.json +++ b/evaluation_examples/examples/libreoffice_impress/a669ef01-ded5-4099-9ea9-25e99b569840.json @@ -9,7 +9,7 @@ "parameters": { "files": [ { - "url": "https://drive.usercontent.google.com/download?id=1C0u-Qvvwa6UbJVTzzQHfdNIgp2i051xA&export=download&authuser=0&confirm=t&uuid=5551a43c-3ff7-424f-b82c-50a5c96b5809&at=APZUnTViShb8pJUviOkmVtn7Pums:1707299959829", + "url": "https://drive.usercontent.google.com/download?id=1hr2flq5iSyMYSps6Jd-3pDOEfZoHFCbb&export=download&authuser=0&confirm=t&uuid=02746987-6ea8-4fbb-8817-8051dab152e7&at=APZUnTWaH071WARB_12CQDvjfg6b:1707314868059", "path": "/home/user/Desktop/Writing-Outlines.pptx" } ] @@ -71,16 +71,13 @@ "func": "compare_pptx_files", "expected": { "type": "cloud_file", - "path": "https://drive.usercontent.google.com/download?id=1d4WJwm7KDhDIOJ9r9vKhRTrt_bF3PSln&export=download&authuser=0&confirm=t&uuid=a8ec87de-96cf-49f0-98e4-faa1218354fe&at=APZUnTWiOoiHFjyM4jTunLP4t5wE:1707299961717", + "path": "https://drive.usercontent.google.com/download?id=15mnwwGTDlelIf27C1HdJOiMgfEWNIoLl&export=download&authuser=0&confirm=t&uuid=b53e5fbb-565b-4498-9dc5-071eded307e0&at=APZUnTUdfXCDVFEQPCTYckB-H2Fn:1707314644205", "dest": "Writing-Outlines_Gold.pptx" }, "result": { "type": "vm_file", "path": "/home/user/Desktop/Writing-Outlines.pptx", "dest": "Writing-Outlines.pptx" - }, - "options": { - "examine_shape": false } } }