Update chrome examples

desktop_env/controllers/setup.py

@@ -1,24 +1,29 @@
 import json
 import logging
+import os
 import os.path
+import sqlite3
+import tempfile
 import time
 import traceback
 import uuid
-import tempfile
+from datetime import datetime, timedelta
 from typing import Any, Union, Optional
 from typing import Dict, List
-import os
 
+import shutil
 import requests
+from playwright.sync_api import sync_playwright, TimeoutError
 from pydrive.auth import GoogleAuth
 from pydrive.drive import GoogleDrive, GoogleDriveFile, GoogleDriveFileList
-from playwright.sync_api import sync_playwright, TimeoutError
 from requests_toolbelt.multipart.encoder import MultipartEncoder
 
+from desktop_env.controllers.python import PythonController
 from desktop_env.evaluators.metrics.utils import compare_urls
 
 logger = logging.getLogger("desktopenv.setup")
 
+FILE_PATH = os.path.dirname(os.path.abspath(__file__))
 
 class SetupController:
     def __init__(self, vm_ip: str, cache_dir: str):
@@ -130,7 +135,8 @@ class SetupController:
                         break
 
                     except requests.RequestException as e:
-                        logger.error(f"Failed to download {url} caused by {e}. Retrying... ({max_retries - i - 1} attempts left)")
+                        logger.error(
+                            f"Failed to download {url} caused by {e}. Retrying... ({max_retries - i - 1} attempts left)")
                 if not downloaded:
                     raise requests.RequestException(f"Failed to download {url}. No retries left. Error: {e}")
 
@@ -349,18 +355,18 @@ class SetupController:
         logger.info("Connect to Chrome @: %s", remote_debugging_url)
         logger.debug("PLAYWRIGHT ENV: %s", repr(os.environ))
         for attempt in range(15):
-            if attempt>0:
+            if attempt > 0:
                 time.sleep(5)
 
             browser = None
             with sync_playwright() as p:
                 try:
                     browser = p.chromium.connect_over_cdp(remote_debugging_url)
-                    #break
+                    # break
                 except Exception as e:
                     if attempt < 14:
                         logger.error(f"Attempt {attempt + 1}: Failed to connect, retrying. Error: {e}")
-                        #time.sleep(10)
+                        # time.sleep(10)
                         continue
                     else:
                         logger.error(f"Failed to connect after multiple attempts: {e}")
@@ -379,7 +385,7 @@ class SetupController:
                     try:
                         page.goto(url, timeout=60000)
                     except:
-                        logger.warning("Opening %s exceeds time limit", url) # only for human test
+                        logger.warning("Opening %s exceeds time limit", url)  # only for human test
                     logger.info(f"Opened tab {i + 1}: {url}")
 
                     if i == 0:
@@ -458,16 +464,17 @@ class SetupController:
             for p in paths:
                 q = f'"{parent_id}" in parents and title = "{p}" and mimeType = "application/vnd.google-apps.folder" and trashed = false'
                 folder = drive.ListFile({'q': q}).GetList()
-                if len(folder) == 0: # not exists, create it
+                if len(folder) == 0:  # not exists, create it
                     parents = {} if parent_id == 'root' else {'parents': [{'id': parent_id}]}
-                    file = drive.CreateFile({'title': p, 'mimeType':'application/vnd.google-apps.folder', **parents})
+                    file = drive.CreateFile({'title': p, 'mimeType': 'application/vnd.google-apps.folder', **parents})
                     file.Upload()
                     parent_id = file['id']
-                else: parent_id = folder[0]['id']
+                else:
+                    parent_id = folder[0]['id']
             return parent_id
 
         for oid, operation in enumerate(config['operation']):
-            if operation == 'delete': # delete a specific file
+            if operation == 'delete':  # delete a specific file
                 # query pattern string, by default, remove all files/folders not in the trash to the trash
                 params = config['args'][oid]
                 q = params.get('query', '')
@@ -476,15 +483,19 @@ class SetupController:
                 filelist: GoogleDriveFileList = drive.ListFile({'q': q_file}).GetList()
                 q_folder = f"( {q} ) and mimeType = 'application/vnd.google-apps.folder'" if q.strip() else "mimeType = 'application/vnd.google-apps.folder'"
                 folderlist: GoogleDriveFileList = drive.ListFile({'q': q_folder}).GetList()
-                for file in filelist: # first delete file, then folder
+                for file in filelist:  # first delete file, then folder
                     file: GoogleDriveFile
-                    if trash: file.Trash()
+                    if trash:
-                    else: file.Delete()
+                        file.Trash()
+                    else:
+                        file.Delete()
                 for folder in folderlist:
                     folder: GoogleDriveFile
                     # note that, if a folder is trashed/deleted, all files and folders in it will be trashed/deleted
-                    if trash: folder.Trash()
+                    if trash:
-                    else: folder.Delete()
+                        folder.Trash()
+                    else:
+                        folder.Delete()
             elif operation == 'mkdirs':
                 params = config['args'][oid]
                 mkdir_in_googledrive(params['path'])
@@ -508,7 +519,6 @@ class SetupController:
             else:
                 raise ValueError('[ERROR]: not implemented clean type!')
 
-
     def _login_setup(self, **config):
         """ Login to a website with account and password information.
         @args:
@@ -537,7 +547,7 @@ class SetupController:
                         raise e
             if not browser:
                 return
-            
+
             context = browser.contexts[0]
             platform = config['platform']
 
@@ -565,3 +575,82 @@ class SetupController:
                 raise NotImplementedError
 
             return browser, context
+
+    def _update_browse_history_setup(self, **config):
+        db_path = os.path.join("desktop_env", "assets", "history_empty.sqlite")
+
+        # copy a new history file in the tmp folder
+        cache_path = os.path.join(self.cache_dir, "history_new.sqlite")
+        shutil.copyfile(db_path, cache_path)
+        db_path = cache_path
+
+        history = config['history']
+
+        for history_item in history:
+            url = history_item['url']
+            title = history_item['title']
+            visit_time = datetime.now() - timedelta(seconds=history_item['visit_time_from_now_in_seconds'])
+
+            # Chrome use ms from 1601-01-01 as timestamp
+            epoch_start = datetime(1601, 1, 1)
+            chrome_timestamp = int((visit_time - epoch_start).total_seconds() * 1000000)
+
+            conn = sqlite3.connect(db_path)
+            cursor = conn.cursor()
+
+            cursor.execute('''
+                   INSERT INTO urls (url, title, visit_count, typed_count, last_visit_time, hidden)
+                   VALUES (?, ?, ?, ?, ?, ?)
+               ''', (url, title, 1, 0, chrome_timestamp, 0))
+
+            url_id = cursor.lastrowid
+
+            cursor.execute('''
+                   INSERT INTO visits (url, visit_time, from_visit, transition, segment_id, visit_duration)
+                   VALUES (?, ?, ?, ?, ?, ?)
+               ''', (url_id, chrome_timestamp, 0, 805306368, 0, 0))
+
+            conn.commit()
+            conn.close()
+
+        logger.info('Fake browsing history added successfully.')
+
+        controller = PythonController(self.vm_ip)
+
+        # get the path of the history file according to the platform
+        os_type = controller.get_vm_platform()
+
+        if os_type == 'Windows':
+            chrome_history_path = controller.execute_python_command(
+                """import os; print(os.path.join(os.getenv('USERPROFILE'), "AppData", "Local", "Google", "Chrome", "User Data", "Default", "History"))""")[
+                'output'].strip()
+        elif os_type == 'Darwin':
+            chrome_history_path = controller.execute_python_command(
+                """import os; print(os.path.join(os.getenv('HOME'), "Library", "Application Support", "Google", "Chrome", "Default", "History"))""")[
+                'output'].strip()
+        elif os_type == 'Linux':
+            chrome_history_path = controller.execute_python_command(
+                "import os; print(os.path.join(os.getenv('HOME'), '.config', 'google-chrome', 'Default', 'History'))")[
+                'output'].strip()
+        else:
+            raise Exception('Unsupported operating system')
+
+        form = MultipartEncoder({
+            "file_path": chrome_history_path,
+            "file_data": (os.path.basename(chrome_history_path), open(db_path, "rb"))
+        })
+        headers = {"Content-Type": form.content_type}
+        logger.debug(form.content_type)
+
+        # send request to server to upload file
+        try:
+            logger.debug("REQUEST ADDRESS: %s", self.http_server + "/setup" + "/upload")
+            response = requests.post(self.http_server + "/setup" + "/upload", headers=headers, data=form)
+            if response.status_code == 200:
+                logger.info("Command executed successfully: %s", response.text)
+            else:
+                logger.error("Failed to upload file. Status code: %s", response.text)
+        except requests.exceptions.RequestException as e:
+            logger.error("An error occurred while trying to send the request: %s", e)
+
+        self._execute_setup(["sudo chown -R user:user /home/user/.config/google-chrome/Default/History"], shell=True)

desktop_env/evaluators/metrics/slides.py

@@ -139,6 +139,7 @@ def compare_pptx_files(file1_path, file2_path, **options):
     examine_number_of_slides = options.get("examine_number_of_slides", True)
     examine_shape = options.get("examine_shape", True)
     examine_text = options.get("examine_text", True)
+    examine_indent = options.get("examine_indent", True)
     examine_font_name = options.get("examine_font_name", True)
     examine_font_size = options.get("examine_font_size", True)
     examine_font_bold = options.get("examine_font_bold", True)
@@ -146,6 +147,7 @@ def compare_pptx_files(file1_path, file2_path, **options):
     examine_color_rgb = options.get("examine_color_rgb", True)
     examine_font_underline = options.get("examine_font_underline", True)
     examine_strike_through = options.get("examine_strike_through", True)
+    examine_bullets = options.get("examine_bullets", True)
 
     # compare the number of slides
     if len(prs1.slides) != len(prs2.slides) and examine_number_of_slides:
@@ -167,6 +169,12 @@ def compare_pptx_files(file1_path, file2_path, **options):
                 # check if the paragraphs are the same
                 for para1, para2 in zip(shape1.text_frame.paragraphs, shape2.text_frame.paragraphs):
                     # check if the runs are the same
+                    if para1.text != para2.text and examine_text:
+                        return 0
+
+                    if para1.level != para2.level and examine_indent:
+                        return 0
+
                     for run1, run2 in zip(para1.runs, para2.runs):
 
                         # check if the font properties are the same
@@ -192,7 +200,40 @@ def compare_pptx_files(file1_path, file2_path, **options):
                                 'strike', 'noStrike') and examine_strike_through:
                             return 0
 
-                        # fixme: Actually there are more properties to be compared, but we cannot get them through pptx
+                        def _extract_bullets(xml_data):
+                            root = ET.fromstring(xml_data)
+
+                            namespaces = {
+                                'a': 'http://schemas.openxmlformats.org/drawingml/2006/main',
+                                'p': 'http://schemas.openxmlformats.org/presentationml/2006/main',
+                            }
+
+                            bullets = []
+
+                            for paragraph in root.findall('.//a:p', namespaces):
+                                pPr = paragraph.find('a:pPr', namespaces)
+                                if pPr is not None:
+                                    lvl = pPr.get('lvl')
+                                    buChar = pPr.find('a:buChar', namespaces)
+                                    char = buChar.get('char') if buChar is not None else "No Bullet"
+                                    buClr = pPr.find('a:buClr/a:srgbClr', namespaces)
+                                    color = buClr.get('val') if buClr is not None else "No Color"
+                                else:
+                                    lvl = "No Level"
+                                    char = "No Bullet"
+                                    color = "No Color"
+
+                                text = "".join(t.text for t in paragraph.findall('.//a:t', namespaces))
+
+                                bullets.append((lvl, char, text, color))
+
+                            return bullets
+
+                    if _extract_bullets(run1.part.blob.decode('utf-8')) != _extract_bullets(
+                            run2.part.blob.decode('utf-8')) and examine_bullets:
+                        return 0
+
+                    # fixme: Actually there are more properties to be compared, we can add them later via parsing the xml data
 
     return 1
 
@@ -414,6 +455,7 @@ if __name__ == '__main__':
     #     r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\550ce7e7-747b-495f-b122-acdc4d0b8e54\New_Club_Spring_2018_Training_Gold.pptx"))
     # print(evaluate_presentation_fill_to_rgb_distance(r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\3b27600c-3668-4abd-8f84-7bcdebbccbdb\lec17-gui-events.pptx", {"rgb": (0, 0, 255)}))
     # print(check_auto_saving_time(r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\2cd43775-7085-45d8-89fa-9e35c0a915cf\registrymodifications.xcu", {"minutes": 3}))
-    print(compare_pptx_files(r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\a669ef01-ded5-4099-9ea9-25e99b569840\Writing-Outlines.pptx",
+    print(compare_pptx_files(
-                             r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\a669ef01-ded5-4099-9ea9-25e99b569840\Writing-Outlines_Gold.pptx",
+        r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\a669ef01-ded5-4099-9ea9-25e99b569840\Writing-Outlines.pptx",
-                             examine_shape=False))
+        r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\a669ef01-ded5-4099-9ea9-25e99b569840\Writing-Outlines_Gold.pptx",
+        examine_shape=False))

evaluation_examples/examples/chrome/44ee5668-ecd5-4366-a6ce-c1c9b8d4e938.json

@@ -4,6 +4,213 @@
   "instruction": "I am looking for an website address I accessed a month ago, but Youtube websites which take almost all of my browsing history are interrupting my search. This is too annoying. I want to remove all my Youtube browsing history first to facilitate my search. Could you help me clear browsing history from Youtube?",
   "source": "https://superuser.com/questions/1787991/clear-browsing-history-from-specific-site-on-chrome",
   "config": [
+    {
+      "type": "update_browse_history",
+      "parameters": {
+        "history": [
+          {
+            "url": "https://www.youtube.com/watch?v=dQw4w9WgXcQ",
+            "title": "Rick Astley - Never Gonna Give You Up (Official Music Video)",
+            "visit_time_from_now_in_seconds": 3600
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=9bZkp7q19f0",
+            "title": "PSY - GANGNAM STYLE(강남스타일) M/V",
+            "visit_time_from_now_in_seconds": 1631
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=3tmd-ClpJxA",
+            "title": "Maroon 5 - Sugar (Official Music Video)",
+            "visit_time_from_now_in_seconds": 900
+          },
+          {
+            "url": "https://www.nytimes.com/",
+            "title": "The New York Times",
+            "visit_time_from_now_in_seconds": 300
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=OPf0YbXqDm0",
+            "title": "Ed Sheeran - Shape of You [Official Music Video]",
+            "visit_time_from_now_in_seconds": 1200
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=JGwWNGJdvx8",
+            "title": "Taylor Swift - Shake It Off",
+            "visit_time_from_now_in_seconds": 2400
+          },
+          {
+            "url": "https://www.bbc.co.uk/",
+            "title": "BBC",
+            "visit_time_from_now_in_seconds": 1500
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=2Vv-BfVoq4g",
+            "title": "Adele - Hello",
+            "visit_time_from_now_in_seconds": 1800
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=YQHsXMglC9A",
+            "title": "Katy Perry - Roar (Official Music Video)",
+            "visit_time_from_now_in_seconds": 2100
+          },
+          {
+            "url": "https://www.cnn.com/",
+            "title": "CNN",
+            "visit_time_from_now_in_seconds": 2700
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=ru0K8uYEZWw",
+            "title": "Justin Bieber - Baby ft. Ludacris (Official Music Video)",
+            "visit_time_from_now_in_seconds": 3200
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=9bZkp7q19f0",
+            "title": "PSY - GANGNAM STYLE(강남스타일) M/V",
+            "visit_time_from_now_in_seconds": 3700
+          },
+          {
+            "url": "https://www.nationalgeographic.com/",
+            "title": "National Geographic",
+            "visit_time_from_now_in_seconds": 4000
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=OPf0YbXqDm0",
+            "title": "Ed Sheeran - Shape of You [Official Music Video]",
+            "visit_time_from_now_in_seconds": 4300
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=JGwWNGJdvx8",
+            "title": "Taylor Swift - Shake It Off",
+            "visit_time_from_now_in_seconds": 4700
+          },
+          {
+            "url": "https://www.bbc.co.uk/",
+            "title": "BBC",
+            "visit_time_from_now_in_seconds": 5000
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=2Vv-BfVoq4g",
+            "title": "Adele - Hello",
+            "visit_time_from_now_in_seconds": 5300
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=YQHsXMglC9A",
+            "title": "Katy Perry - Roar (Official Music Video)",
+            "visit_time_from_now_in_seconds": 5600
+          },
+          {
+            "url": "https://www.cnn.com/",
+            "title": "CNN",
+            "visit_time_from_now_in_seconds": 5900
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=ru0K8uYEZWw",
+            "title": "Justin Bieber - Baby ft. Ludacris (Official Music Video)",
+            "visit_time_from_now_in_seconds": 6300
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=dQw4w9WgXcQ",
+            "title": "Rick Astley - Never Gonna Give You Up (Official Music Video)",
+            "visit_time_from_now_in_seconds": 6700
+          },
+          {
+            "url": "https://www.nationalgeographic.com/",
+            "title": "National Geographic",
+            "visit_time_from_now_in_seconds": 7000
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=OPf0YbXqDm0",
+            "title": "Ed Sheeran - Shape of You [Official Music Video]",
+            "visit_time_from_now_in_seconds": 7300
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=JGwWNGJdvx8",
+            "title": "Taylor Swift - Shake It Off",
+            "visit_time_from_now_in_seconds": 7600
+          },
+          {
+            "url": "https://www.bbc.co.uk/",
+            "title": "BBC",
+            "visit_time_from_now_in_seconds": 7900
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=2Vv-BfVoq4g",
+            "title": "Adele - Hello",
+            "visit_time_from_now_in_seconds": 8200
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=YQHsXMglC9A",
+            "title": "Katy Perry - Roar (Official Music Video)",
+            "visit_time_from_now_in_seconds": 8500
+          },
+          {
+            "url": "https://www.cnn.com/",
+            "title": "CNN",
+            "visit_time_from_now_in_seconds": 8800
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=ru0K8uYEZWw",
+            "title": "Justin Bieber - Baby ft. Ludacris (Official Music Video)",
+            "visit_time_from_now_in_seconds": 9100
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=dQw4w9WgXcQ",
+            "title": "Rick Astley - Never Gonna Give You Up (Official Music Video)",
+            "visit_time_from_now_in_seconds": 9400
+          },
+          {
+            "url": "https://www.nationalgeographic.com/",
+            "title": "National Geographic",
+            "visit_time_from_now_in_seconds": 9700
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=OPf0YbXqDm0",
+            "title": "Ed Sheeran - Shape of You [Official Music Video]",
+            "visit_time_from_now_in_seconds": 10000
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=JGwWNGJdvx8",
+            "title": "Taylor Swift - Shake It Off",
+            "visit_time_from_now_in_seconds": 10300
+          },
+          {
+            "url": "https://www.bbc.co.uk/",
+            "title": "BBC",
+            "visit_time_from_now_in_seconds": 10600
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=2Vv-BfVoq4g",
+            "title": "Adele - Hello",
+            "visit_time_from_now_in_seconds": 10900
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=YQHsXMglC9A",
+            "title": "Katy Perry - Roar (Official Music Video)",
+            "visit_time_from_now_in_seconds": 11200
+          },
+          {
+            "url": "https://www.cnn.com/",
+            "title": "CNN",
+            "visit_time_from_now_in_seconds": 11500
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=ru0K8uYEZWw",
+            "title": "Justin Bieber - Baby ft. Ludacris (Official Music Video)",
+            "visit_time_from_now_in_seconds": 11800
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=dQw4w9WgXcQ",
+            "title": "Rick Astley - Never Gonna Give You Up (Official Music Video)",
+            "visit_time_from_now_in_seconds": 12100
+          },
+          {
+            "url": "https://www.nationalgeographic.com/",
+            "title": "National Geographic",
+            "visit_time_from_now_in_seconds": 12400
+          }
+        ]
+      }
+    },
     {
       "type": "launch",
       "parameters": {

evaluation_examples/examples/libreoffice_impress/358aa0a7-6677-453f-ae35-e440f004c31e.json

@@ -54,7 +54,7 @@
       {
         "type": "sleep",
         "parameters": {
-          "seconds": 0.5
+          "seconds": 15
         }
       }
     ],

evaluation_examples/examples/libreoffice_impress/a669ef01-ded5-4099-9ea9-25e99b569840.json

@@ -9,7 +9,7 @@
       "parameters": {
         "files": [
           {
-            "url": "https://drive.usercontent.google.com/download?id=1C0u-Qvvwa6UbJVTzzQHfdNIgp2i051xA&export=download&authuser=0&confirm=t&uuid=5551a43c-3ff7-424f-b82c-50a5c96b5809&at=APZUnTViShb8pJUviOkmVtn7Pums:1707299959829",
+            "url": "https://drive.usercontent.google.com/download?id=1hr2flq5iSyMYSps6Jd-3pDOEfZoHFCbb&export=download&authuser=0&confirm=t&uuid=02746987-6ea8-4fbb-8817-8051dab152e7&at=APZUnTWaH071WARB_12CQDvjfg6b:1707314868059",
             "path": "/home/user/Desktop/Writing-Outlines.pptx"
           }
         ]
@@ -71,16 +71,13 @@
     "func": "compare_pptx_files",
     "expected": {
       "type": "cloud_file",
-      "path": "https://drive.usercontent.google.com/download?id=1d4WJwm7KDhDIOJ9r9vKhRTrt_bF3PSln&export=download&authuser=0&confirm=t&uuid=a8ec87de-96cf-49f0-98e4-faa1218354fe&at=APZUnTWiOoiHFjyM4jTunLP4t5wE:1707299961717",
+      "path": "https://drive.usercontent.google.com/download?id=15mnwwGTDlelIf27C1HdJOiMgfEWNIoLl&export=download&authuser=0&confirm=t&uuid=b53e5fbb-565b-4498-9dc5-071eded307e0&at=APZUnTUdfXCDVFEQPCTYckB-H2Fn:1707314644205",
       "dest": "Writing-Outlines_Gold.pptx"
     },
     "result": {
       "type": "vm_file",
       "path": "/home/user/Desktop/Writing-Outlines.pptx",
       "dest": "Writing-Outlines.pptx"
-    },
-    "options": {
-      "examine_shape": false
     }
   }
 }