Merge branch 'xiaochuanli/addChromeExtensions' of github.com:xlang-ai/DesktopEnv into xiaochuanli/addChromeExtensions

desktop_env/controllers/setup.py

@@ -1,24 +1,29 @@
 import json
 import logging
+import os
 import os.path
+import sqlite3
+import tempfile
 import time
 import traceback
 import uuid
-import tempfile
+from datetime import datetime, timedelta
 from typing import Any, Union, Optional
 from typing import Dict, List
-import os
 
+import shutil
 import requests
+from playwright.sync_api import sync_playwright, TimeoutError
 from pydrive.auth import GoogleAuth
 from pydrive.drive import GoogleDrive, GoogleDriveFile, GoogleDriveFileList
-from playwright.sync_api import sync_playwright, TimeoutError
 from requests_toolbelt.multipart.encoder import MultipartEncoder
 
+from desktop_env.controllers.python import PythonController
 from desktop_env.evaluators.metrics.utils import compare_urls
 
 logger = logging.getLogger("desktopenv.setup")
 
+FILE_PATH = os.path.dirname(os.path.abspath(__file__))
 
 class SetupController:
     def __init__(self, vm_ip: str, cache_dir: str):
@@ -130,7 +135,8 @@ class SetupController:
                         break
 
                     except requests.RequestException as e:
-                        logger.error(f"Failed to download {url} caused by {e}. Retrying... ({max_retries - i - 1} attempts left)")
+                        logger.error(
+                            f"Failed to download {url} caused by {e}. Retrying... ({max_retries - i - 1} attempts left)")
                 if not downloaded:
                     raise requests.RequestException(f"Failed to download {url}. No retries left. Error: {e}")
 
@@ -349,18 +355,18 @@ class SetupController:
         logger.info("Connect to Chrome @: %s", remote_debugging_url)
         logger.debug("PLAYWRIGHT ENV: %s", repr(os.environ))
         for attempt in range(15):
-            if attempt>0:
+            if attempt > 0:
                 time.sleep(5)
 
             browser = None
             with sync_playwright() as p:
                 try:
                     browser = p.chromium.connect_over_cdp(remote_debugging_url)
-                    #break
+                    # break
                 except Exception as e:
                     if attempt < 14:
                         logger.error(f"Attempt {attempt + 1}: Failed to connect, retrying. Error: {e}")
-                        #time.sleep(10)
+                        # time.sleep(10)
                         continue
                     else:
                         logger.error(f"Failed to connect after multiple attempts: {e}")
@@ -379,7 +385,7 @@ class SetupController:
                     try:
                         page.goto(url, timeout=60000)
                     except:
-                        logger.warning("Opening %s exceeds time limit", url) # only for human test
+                        logger.warning("Opening %s exceeds time limit", url)  # only for human test
                     logger.info(f"Opened tab {i + 1}: {url}")
 
                     if i == 0:
@@ -458,16 +464,17 @@ class SetupController:
             for p in paths:
                 q = f'"{parent_id}" in parents and title = "{p}" and mimeType = "application/vnd.google-apps.folder" and trashed = false'
                 folder = drive.ListFile({'q': q}).GetList()
-                if len(folder) == 0: # not exists, create it
+                if len(folder) == 0:  # not exists, create it
                     parents = {} if parent_id == 'root' else {'parents': [{'id': parent_id}]}
-                    file = drive.CreateFile({'title': p, 'mimeType':'application/vnd.google-apps.folder', **parents})
+                    file = drive.CreateFile({'title': p, 'mimeType': 'application/vnd.google-apps.folder', **parents})
                     file.Upload()
                     parent_id = file['id']
-                else: parent_id = folder[0]['id']
+                else:
+                    parent_id = folder[0]['id']
             return parent_id
 
         for oid, operation in enumerate(config['operation']):
-            if operation == 'delete': # delete a specific file
+            if operation == 'delete':  # delete a specific file
                 # query pattern string, by default, remove all files/folders not in the trash to the trash
                 params = config['args'][oid]
                 q = params.get('query', '')
@@ -476,15 +483,19 @@ class SetupController:
                 filelist: GoogleDriveFileList = drive.ListFile({'q': q_file}).GetList()
                 q_folder = f"( {q} ) and mimeType = 'application/vnd.google-apps.folder'" if q.strip() else "mimeType = 'application/vnd.google-apps.folder'"
                 folderlist: GoogleDriveFileList = drive.ListFile({'q': q_folder}).GetList()
-                for file in filelist: # first delete file, then folder
+                for file in filelist:  # first delete file, then folder
                     file: GoogleDriveFile
-                    if trash: file.Trash()
-                    else: file.Delete()
+                    if trash:
+                        file.Trash()
+                    else:
+                        file.Delete()
                 for folder in folderlist:
                     folder: GoogleDriveFile
                     # note that, if a folder is trashed/deleted, all files and folders in it will be trashed/deleted
-                    if trash: folder.Trash()
-                    else: folder.Delete()
+                    if trash:
+                        folder.Trash()
+                    else:
+                        folder.Delete()
             elif operation == 'mkdirs':
                 params = config['args'][oid]
                 mkdir_in_googledrive(params['path'])
@@ -508,7 +519,6 @@ class SetupController:
             else:
                 raise ValueError('[ERROR]: not implemented clean type!')
 
-
     def _login_setup(self, **config):
         """ Login to a website with account and password information.
         @args:
@@ -537,7 +547,7 @@ class SetupController:
                         raise e
             if not browser:
                 return
-            
+
             context = browser.contexts[0]
             platform = config['platform']
 
@@ -565,3 +575,82 @@ class SetupController:
                 raise NotImplementedError
 
             return browser, context
+
+    def _update_browse_history_setup(self, **config):
+        db_path = os.path.join("desktop_env", "assets", "history_empty.sqlite")
+
+        # copy a new history file in the tmp folder
+        cache_path = os.path.join(self.cache_dir, "history_new.sqlite")
+        shutil.copyfile(db_path, cache_path)
+        db_path = cache_path
+
+        history = config['history']
+
+        for history_item in history:
+            url = history_item['url']
+            title = history_item['title']
+            visit_time = datetime.now() - timedelta(seconds=history_item['visit_time_from_now_in_seconds'])
+
+            # Chrome use ms from 1601-01-01 as timestamp
+            epoch_start = datetime(1601, 1, 1)
+            chrome_timestamp = int((visit_time - epoch_start).total_seconds() * 1000000)
+
+            conn = sqlite3.connect(db_path)
+            cursor = conn.cursor()
+
+            cursor.execute('''
+                   INSERT INTO urls (url, title, visit_count, typed_count, last_visit_time, hidden)
+                   VALUES (?, ?, ?, ?, ?, ?)
+               ''', (url, title, 1, 0, chrome_timestamp, 0))
+
+            url_id = cursor.lastrowid
+
+            cursor.execute('''
+                   INSERT INTO visits (url, visit_time, from_visit, transition, segment_id, visit_duration)
+                   VALUES (?, ?, ?, ?, ?, ?)
+               ''', (url_id, chrome_timestamp, 0, 805306368, 0, 0))
+
+            conn.commit()
+            conn.close()
+
+        logger.info('Fake browsing history added successfully.')
+
+        controller = PythonController(self.vm_ip)
+
+        # get the path of the history file according to the platform
+        os_type = controller.get_vm_platform()
+
+        if os_type == 'Windows':
+            chrome_history_path = controller.execute_python_command(
+                """import os; print(os.path.join(os.getenv('USERPROFILE'), "AppData", "Local", "Google", "Chrome", "User Data", "Default", "History"))""")[
+                'output'].strip()
+        elif os_type == 'Darwin':
+            chrome_history_path = controller.execute_python_command(
+                """import os; print(os.path.join(os.getenv('HOME'), "Library", "Application Support", "Google", "Chrome", "Default", "History"))""")[
+                'output'].strip()
+        elif os_type == 'Linux':
+            chrome_history_path = controller.execute_python_command(
+                "import os; print(os.path.join(os.getenv('HOME'), '.config', 'google-chrome', 'Default', 'History'))")[
+                'output'].strip()
+        else:
+            raise Exception('Unsupported operating system')
+
+        form = MultipartEncoder({
+            "file_path": chrome_history_path,
+            "file_data": (os.path.basename(chrome_history_path), open(db_path, "rb"))
+        })
+        headers = {"Content-Type": form.content_type}
+        logger.debug(form.content_type)
+
+        # send request to server to upload file
+        try:
+            logger.debug("REQUEST ADDRESS: %s", self.http_server + "/setup" + "/upload")
+            response = requests.post(self.http_server + "/setup" + "/upload", headers=headers, data=form)
+            if response.status_code == 200:
+                logger.info("Command executed successfully: %s", response.text)
+            else:
+                logger.error("Failed to upload file. Status code: %s", response.text)
+        except requests.exceptions.RequestException as e:
+            logger.error("An error occurred while trying to send the request: %s", e)
+
+        self._execute_setup(["sudo chown -R user:user /home/user/.config/google-chrome/Default/History"], shell=True)

desktop_env/envs/desktop_env.py

@@ -175,10 +175,16 @@ class DesktopEnv(gym.Env):
         if isinstance(self.evaluator["func"], list) \
         else getattr(metrics, self.evaluator["func"])
         self.metric_conj: str = self.evaluator.get("conj", "and")  # take conjunction of multiple metrics
-        self.result_getter: Getter = [getattr(getters, "get_{:}".format(res["type"])) for res in
+        if "result" in self.evaluator:
+            self.result_getter: Getter = [getattr(getters, "get_{:}".format(res["type"])) for res in
                                       self.evaluator["result"]] \
             if isinstance(self.evaluator["result"], list) \
             else getattr(getters, "get_{:}".format(self.evaluator["result"]["type"]))
+        else:
+            self.result_getter = [None] * len(self.metric) \
+                if isinstance(self.metric, list) \
+                else None
+
         if "expected" in self.evaluator:
             self.expected_getter: Getter = [getattr(getters, "get_{:}".format(exp["type"])) if exp else None for exp in
                                             self.evaluator["expected"]] \
@@ -293,6 +299,12 @@ class DesktopEnv(gym.Env):
 
         self.setup_controller.setup(self.evaluator.get("postconfig", []))
 
+        if self.metric == "infeasible":
+            if self.action_history[-1] == "FAIL":
+                return 1
+            else:
+                return 0
+
         if type(self.metric) == list:
             results = []
             for idx, metric in enumerate(self.metric):
@@ -315,7 +327,8 @@ class DesktopEnv(gym.Env):
                     return 0
                 elif self.metric_conj == 'or' and float(metric) == 1.0:
                     return 1
-                else: results.append(metric)
+                else:
+                    results.append(metric)
             return sum(results) / len(results) if self.metric_conj == 'and' else max(results)
         else:
             try:

desktop_env/evaluators/getters/general.py

@@ -13,6 +13,8 @@ def get_vm_command_line(env, config: Dict[str, str]):
 
     response = requests.post(f"http://{vm_ip}:{port}/execute", json={"command": command, "shell": shell})
 
+    print(response.json())
+
     if response.status_code == 200:
         return response.json()["output"]
     else:

desktop_env/evaluators/metrics/__init__.py

@@ -32,9 +32,7 @@ from .docs import (
     evaluate_alignment,
     get_unique_train_ids,
     check_no_duplicates,
-    compare_init_lines
-)
-from .docs import (
+    compare_init_lines,
     find_default_font,
     contains_page_break,
     compare_docx_files,
@@ -89,7 +87,8 @@ from .slides import (
     evaluate_presentation_fill_to_rgb_distance,
     check_left_panel,
     check_transition,
-    check_page_number_colors
+    check_page_number_colors,
+    check_auto_saving_time
 )
 from .table import (
     compare_table,

desktop_env/evaluators/metrics/basic_os.py

@@ -46,7 +46,6 @@ def check_text_enlarged(scaling_factor_str):
 
 
 def check_moved_jpgs(directory_list, rule):
-
     expected_jpgs = rule["expected"]
     moved_jpgs = [node['name'] for node in directory_list['children']]
 

desktop_env/evaluators/metrics/slides.py

@@ -65,7 +65,6 @@ def check_image_stretch_and_center(modified_ppt, original_ppt):
 
 def is_red_color(color):
     # judge if the color is red
-    print(color.rgb)
     return color and color.rgb == (255, 0, 0)
 
 
@@ -94,7 +93,6 @@ def check_slide_numbers_color(pptx_file_path):
                     # "SlidePlaceholder" is the name of the placeholder in the master slide
                     page_number_text = shape.text
                     font_color = get_master_placeholder_color(presentation)
-                    print(font_color)
                     return 1 if font_color is not None and is_red_color(font_color) else 0
 
 
@@ -139,6 +137,7 @@ def compare_pptx_files(file1_path, file2_path, **options):
     examine_number_of_slides = options.get("examine_number_of_slides", True)
     examine_shape = options.get("examine_shape", True)
     examine_text = options.get("examine_text", True)
+    examine_indent = options.get("examine_indent", True)
     examine_font_name = options.get("examine_font_name", True)
     examine_font_size = options.get("examine_font_size", True)
     examine_font_bold = options.get("examine_font_bold", True)
@@ -146,6 +145,13 @@ def compare_pptx_files(file1_path, file2_path, **options):
     examine_color_rgb = options.get("examine_color_rgb", True)
     examine_font_underline = options.get("examine_font_underline", True)
     examine_strike_through = options.get("examine_strike_through", True)
+    examine_alignment = options.get("examine_alignment", True)
+    examine_bottom_position = options.get("examine_bottom_position", False)
+    examine_right_position = options.get("examine_right_position", False)
+    examine_image_size = options.get("examine_image_size", True)
+    examine_bullets = options.get("examine_bullets", True)
+    examine_background_color = options.get("examine_background_color", True)
+    examine_note = options.get("examine_note", True)
 
     # compare the number of slides
     if len(prs1.slides) != len(prs2.slides) and examine_number_of_slides:
@@ -153,26 +159,67 @@ def compare_pptx_files(file1_path, file2_path, **options):
 
     # compare the content of each slide
     for slide1, slide2 in zip(prs1.slides, prs2.slides):
-
+        def get_slide_background_color(slide):
+            background = slide.background
+            if background.fill.background():
+                return background.fill.fore_color.rgb
+            else:
+                return None
+            
+        if get_slide_background_color(slide1) != get_slide_background_color(slide2) and examine_background_color:
+            return 0
+        
+        def get_slide_notes(slide):
+            notes_slide = slide.notes_slide
+            if notes_slide:
+                return notes_slide.notes_text_frame.text
+            else:
+                return None
+        
+        if get_slide_notes(slide1) != get_slide_notes(slide2) and examine_note:
+            return 0
         # check if the shapes are the same
         for shape1, shape2 in zip(slide1.shapes, slide2.shapes):
-            if (
-                    shape1.left != shape2.left or shape1.top != shape2.top or shape1.width != shape2.width or shape1.height != shape2.height) and examine_shape:
+            if examine_bottom_position and shape1.top != shape2.top:
+                if hasattr(shape1, "text") and hasattr(shape2, "text") and shape1.text == shape2.text and shape1.text == "Product Comparison":
+                    if shape1.top >= shape2.top:
+                        return 0
+                    
+            if examine_right_position and shape1.left != shape2.left:
+                if not hasattr(shape1, "text") and not hasattr(shape2, "text"):
+                    if shape1.left >= shape2.left:
+                        return 0
+                
+            if (shape1.left != shape2.left or shape1.top != shape2.top or shape1.width != shape2.width or shape1.height != shape2.height) and examine_shape:
                 return 0
+            
+            if examine_image_size:
+                if shape1.shape_type == 13 and shape2.shape_type == 13:
+                    if shape1.width != shape2.width or shape1.height != shape2.height:
+                        return 0
+                elif shape1.left != shape2.left or shape1.top != shape2.top or shape1.width != shape2.width or shape1.height != shape2.height:
+                    return 0
 
             if hasattr(shape1, "text") and hasattr(shape2, "text"):
                 if shape1.text != shape2.text and examine_text:
-                    return 0
-
+                    return 0    
+                
                 # check if the paragraphs are the same
                 for para1, para2 in zip(shape1.text_frame.paragraphs, shape2.text_frame.paragraphs):
+                    if para1.alignment != para2.alignment and examine_alignment:
+                        return 0
+                        
                     # check if the runs are the same
-                    for run1, run2 in zip(para1.runs, para2.runs):
-                        if run1.text != run2.text and examine_text:
-                            return 0
+                    if para1.text != para2.text and examine_text:
+                        return 0
 
-                        # check if the font properties are the same
-                        if run1.font.name != run2.font.name and examine_font_name:
+                    if para1.level != para2.level and examine_indent:
+                        return 0
+
+                    for run1, run2 in zip(para1.runs, para2.runs):
+
+                        # check if the font properties are the same                        
+                        if run1.font.name != run2.font.name and examine_font_name:                            
                             return 0
 
                         if run1.font.size != run2.font.size and examine_font_size:
@@ -184,16 +231,52 @@ def compare_pptx_files(file1_path, file2_path, **options):
                         if run1.font.italic != run2.font.italic and examine_font_italic:
                             return 0
 
-                        if run1.font.color.rgb != run2.font.color.rgb and examine_color_rgb:
-                            return 0
+                        if hasattr(run1.font.color, "rgb") and hasattr(run2.font.color, "rgb"):
+                            if run1.font.color.rgb != run2.font.color.rgb and examine_color_rgb:
+                                return 0
 
                         if run1.font.underline != run2.font.underline and examine_font_underline:
                             return 0
 
-                        if run1.font._element.attrib.get('strike', 'noStrike') != run2.font._element.attrib.get('strike', 'noStrike') and examine_strike_through:
+                        if run1.font._element.attrib.get('strike', 'noStrike') != run2.font._element.attrib.get(
+                                'strike', 'noStrike') and examine_strike_through:
                             return 0
 
-                        # fixme: Actually there are more properties to be compared, but we cannot get them through pptx
+                        def _extract_bullets(xml_data):
+                            root = ET.fromstring(xml_data)
+
+                            namespaces = {
+                                'a': 'http://schemas.openxmlformats.org/drawingml/2006/main',
+                                'p': 'http://schemas.openxmlformats.org/presentationml/2006/main',
+                            }
+
+                            bullets = []
+
+                            for paragraph in root.findall('.//a:p', namespaces):
+                                pPr = paragraph.find('a:pPr', namespaces)
+                                if pPr is not None:
+                                    lvl = pPr.get('lvl')
+                                    buChar = pPr.find('a:buChar', namespaces)
+                                    char = buChar.get('char') if buChar is not None else "No Bullet"
+                                    buClr = pPr.find('a:buClr/a:srgbClr', namespaces)
+                                    color = buClr.get('val') if buClr is not None else "No Color"
+                                else:
+                                    lvl = "No Level"
+                                    char = "No Bullet"
+                                    color = "No Color"
+
+                                text = "".join(t.text for t in paragraph.findall('.//a:t', namespaces))
+
+                                bullets.append((lvl, char, text, color))
+
+                            return bullets
+
+                        if examine_bullets and _extract_bullets(run1.part.blob.decode('utf-8')) != _extract_bullets(run2.part.blob.decode('utf-8')):
+                            return 0
+                        
+
+
+                    # fixme: Actually there are more properties to be compared, we can add them later via parsing the xml data
 
     return 1
 
@@ -371,6 +454,51 @@ def check_page_number_colors(pptx_file, rules):
 
     return 1
 
+
+def check_auto_saving_time(pptx_file, rules):
+    minutes = rules["minutes"]
+
+    # open and parse xml file
+    try:
+        tree = ET.parse(pptx_file)
+        root = tree.getroot()
+
+        # Traverse the XML tree to find the autosave time setting
+        autosave_time = None
+        for item in root.findall(".//item"):
+            # Check the path attribute
+            path = item.get('{http://openoffice.org/2001/registry}path')
+            if path == "/org.openoffice.Office.Common/Save/Document":
+                # Once the correct item is found, look for the prop element with the name "AutoSaveTimeIntervall"
+                for prop in item.findall(".//prop"):
+                    name = prop.get('{http://openoffice.org/2001/registry}name')
+                    if name == "AutoSaveTimeIntervall":
+                        # Extract the value of the autosave time interval
+                        autosave_time = prop.find(".//value").text
+                        break
+
+        if autosave_time is None:
+            return 0
+        else:
+            autosave_time = int(autosave_time)
+            if autosave_time == minutes:
+                return 1
+            else:
+                return 0
+
+    except ET.ParseError as e:
+        logger.error(f"Error parsing XML: {e}")
+    except FileNotFoundError:
+        logger.error(f"File not found: {pptx_file}")
+
+
 if __name__ == '__main__':
-    print(compare_pptx_files(r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\550ce7e7-747b-495f-b122-acdc4d0b8e54\New_Club_Spring_2018_Training_Gold.pptx", r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\550ce7e7-747b-495f-b122-acdc4d0b8e54\New_Club_Spring_2018_Training_Gold.pptx"))
-    # print(evaluate_presentation_fill_to_rgb_distance(r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\3b27600c-3668-4abd-8f84-7bcdebbccbdb\lec17-gui-events.pptx", {"rgb": (0, 0, 255)}))
+    # print(compare_pptx_files(
+    #     r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\550ce7e7-747b-495f-b122-acdc4d0b8e54\New_Club_Spring_2018_Training_Gold.pptx",
+    #     r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\550ce7e7-747b-495f-b122-acdc4d0b8e54\New_Club_Spring_2018_Training_Gold.pptx"))
+    # print(evaluate_presentation_fill_to_rgb_distance(r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\3b27600c-3668-4abd-8f84-7bcdebbccbdb\lec17-gui-events.pptx", {"rgb": (0, 0, 255)}))
+    # print(check_auto_saving_time(r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\2cd43775-7085-45d8-89fa-9e35c0a915cf\registrymodifications.xcu", {"minutes": 3}))
+    print(compare_pptx_files(
+        r"D:\NJU\HKUNLP\Desktop-Env\DesktopEnv\cache\08aced46-45a2-48d7-993b-ed3fb5b32302\22_6_Gold.pptx",
+        r"D:\NJU\HKUNLP\Desktop-Env\DesktopEnv\cache\08aced46-45a2-48d7-993b-ed3fb5b32302\22_6.pptx",
+        examine_shape=False))