update pptc as libreoffice impress

desktop_env/controllers/setup.py

@@ -1,24 +1,29 @@
 import json
 import logging
+import os
 import os.path
+import sqlite3
+import tempfile
 import time
 import traceback
 import uuid
-import tempfile
+from datetime import datetime, timedelta
 from typing import Any, Union, Optional
 from typing import Dict, List
-import os
 
+import shutil
 import requests
+from playwright.sync_api import sync_playwright, TimeoutError
 from pydrive.auth import GoogleAuth
 from pydrive.drive import GoogleDrive, GoogleDriveFile, GoogleDriveFileList
-from playwright.sync_api import sync_playwright, TimeoutError
 from requests_toolbelt.multipart.encoder import MultipartEncoder
 
+from desktop_env.controllers.python import PythonController
 from desktop_env.evaluators.metrics.utils import compare_urls
 
 logger = logging.getLogger("desktopenv.setup")
 
+FILE_PATH = os.path.dirname(os.path.abspath(__file__))
 
 class SetupController:
     def __init__(self, vm_ip: str, cache_dir: str):
@@ -130,7 +135,8 @@ class SetupController:
                         break
 
                     except requests.RequestException as e:
-                        logger.error(f"Failed to download {url} caused by {e}. Retrying... ({max_retries - i - 1} attempts left)")
+                        logger.error(
+                            f"Failed to download {url} caused by {e}. Retrying... ({max_retries - i - 1} attempts left)")
                 if not downloaded:
                     raise requests.RequestException(f"Failed to download {url}. No retries left. Error: {e}")
 
@@ -349,18 +355,18 @@ class SetupController:
         logger.info("Connect to Chrome @: %s", remote_debugging_url)
         logger.debug("PLAYWRIGHT ENV: %s", repr(os.environ))
         for attempt in range(15):
-            if attempt>0:
+            if attempt > 0:
                 time.sleep(5)
 
             browser = None
             with sync_playwright() as p:
                 try:
                     browser = p.chromium.connect_over_cdp(remote_debugging_url)
-                    #break
+                    # break
                 except Exception as e:
                     if attempt < 14:
                         logger.error(f"Attempt {attempt + 1}: Failed to connect, retrying. Error: {e}")
-                        #time.sleep(10)
+                        # time.sleep(10)
                         continue
                     else:
                         logger.error(f"Failed to connect after multiple attempts: {e}")
@@ -379,7 +385,7 @@ class SetupController:
                     try:
                         page.goto(url, timeout=60000)
                     except:
-                        logger.warning("Opening %s exceeds time limit", url) # only for human test
+                        logger.warning("Opening %s exceeds time limit", url)  # only for human test
                     logger.info(f"Opened tab {i + 1}: {url}")
 
                     if i == 0:
@@ -458,16 +464,17 @@ class SetupController:
             for p in paths:
                 q = f'"{parent_id}" in parents and title = "{p}" and mimeType = "application/vnd.google-apps.folder" and trashed = false'
                 folder = drive.ListFile({'q': q}).GetList()
-                if len(folder) == 0: # not exists, create it
+                if len(folder) == 0:  # not exists, create it
                     parents = {} if parent_id == 'root' else {'parents': [{'id': parent_id}]}
-                    file = drive.CreateFile({'title': p, 'mimeType':'application/vnd.google-apps.folder', **parents})
+                    file = drive.CreateFile({'title': p, 'mimeType': 'application/vnd.google-apps.folder', **parents})
                     file.Upload()
                     parent_id = file['id']
-                else: parent_id = folder[0]['id']
+                else:
+                    parent_id = folder[0]['id']
             return parent_id
 
         for oid, operation in enumerate(config['operation']):
-            if operation == 'delete': # delete a specific file
+            if operation == 'delete':  # delete a specific file
                 # query pattern string, by default, remove all files/folders not in the trash to the trash
                 params = config['args'][oid]
                 q = params.get('query', '')
@@ -476,15 +483,19 @@ class SetupController:
                 filelist: GoogleDriveFileList = drive.ListFile({'q': q_file}).GetList()
                 q_folder = f"( {q} ) and mimeType = 'application/vnd.google-apps.folder'" if q.strip() else "mimeType = 'application/vnd.google-apps.folder'"
                 folderlist: GoogleDriveFileList = drive.ListFile({'q': q_folder}).GetList()
-                for file in filelist: # first delete file, then folder
+                for file in filelist:  # first delete file, then folder
                     file: GoogleDriveFile
-                    if trash: file.Trash()
+                    if trash:
-                    else: file.Delete()
+                        file.Trash()
+                    else:
+                        file.Delete()
                 for folder in folderlist:
                     folder: GoogleDriveFile
                     # note that, if a folder is trashed/deleted, all files and folders in it will be trashed/deleted
-                    if trash: folder.Trash()
+                    if trash:
-                    else: folder.Delete()
+                        folder.Trash()
+                    else:
+                        folder.Delete()
             elif operation == 'mkdirs':
                 params = config['args'][oid]
                 mkdir_in_googledrive(params['path'])
@@ -508,7 +519,6 @@ class SetupController:
             else:
                 raise ValueError('[ERROR]: not implemented clean type!')
 
-
     def _login_setup(self, **config):
         """ Login to a website with account and password information.
         @args:
@@ -537,7 +547,7 @@ class SetupController:
                         raise e
             if not browser:
                 return
-            
+
             context = browser.contexts[0]
             platform = config['platform']
 
@@ -565,3 +575,82 @@ class SetupController:
                 raise NotImplementedError
 
             return browser, context
+
+    def _update_browse_history_setup(self, **config):
+        db_path = os.path.join("desktop_env", "assets", "history_empty.sqlite")
+
+        # copy a new history file in the tmp folder
+        cache_path = os.path.join(self.cache_dir, "history_new.sqlite")
+        shutil.copyfile(db_path, cache_path)
+        db_path = cache_path
+
+        history = config['history']
+
+        for history_item in history:
+            url = history_item['url']
+            title = history_item['title']
+            visit_time = datetime.now() - timedelta(seconds=history_item['visit_time_from_now_in_seconds'])
+
+            # Chrome use ms from 1601-01-01 as timestamp
+            epoch_start = datetime(1601, 1, 1)
+            chrome_timestamp = int((visit_time - epoch_start).total_seconds() * 1000000)
+
+            conn = sqlite3.connect(db_path)
+            cursor = conn.cursor()
+
+            cursor.execute('''
+                   INSERT INTO urls (url, title, visit_count, typed_count, last_visit_time, hidden)
+                   VALUES (?, ?, ?, ?, ?, ?)
+               ''', (url, title, 1, 0, chrome_timestamp, 0))
+
+            url_id = cursor.lastrowid
+
+            cursor.execute('''
+                   INSERT INTO visits (url, visit_time, from_visit, transition, segment_id, visit_duration)
+                   VALUES (?, ?, ?, ?, ?, ?)
+               ''', (url_id, chrome_timestamp, 0, 805306368, 0, 0))
+
+            conn.commit()
+            conn.close()
+
+        logger.info('Fake browsing history added successfully.')
+
+        controller = PythonController(self.vm_ip)
+
+        # get the path of the history file according to the platform
+        os_type = controller.get_vm_platform()
+
+        if os_type == 'Windows':
+            chrome_history_path = controller.execute_python_command(
+                """import os; print(os.path.join(os.getenv('USERPROFILE'), "AppData", "Local", "Google", "Chrome", "User Data", "Default", "History"))""")[
+                'output'].strip()
+        elif os_type == 'Darwin':
+            chrome_history_path = controller.execute_python_command(
+                """import os; print(os.path.join(os.getenv('HOME'), "Library", "Application Support", "Google", "Chrome", "Default", "History"))""")[
+                'output'].strip()
+        elif os_type == 'Linux':
+            chrome_history_path = controller.execute_python_command(
+                "import os; print(os.path.join(os.getenv('HOME'), '.config', 'google-chrome', 'Default', 'History'))")[
+                'output'].strip()
+        else:
+            raise Exception('Unsupported operating system')
+
+        form = MultipartEncoder({
+            "file_path": chrome_history_path,
+            "file_data": (os.path.basename(chrome_history_path), open(db_path, "rb"))
+        })
+        headers = {"Content-Type": form.content_type}
+        logger.debug(form.content_type)
+
+        # send request to server to upload file
+        try:
+            logger.debug("REQUEST ADDRESS: %s", self.http_server + "/setup" + "/upload")
+            response = requests.post(self.http_server + "/setup" + "/upload", headers=headers, data=form)
+            if response.status_code == 200:
+                logger.info("Command executed successfully: %s", response.text)
+            else:
+                logger.error("Failed to upload file. Status code: %s", response.text)
+        except requests.exceptions.RequestException as e:
+            logger.error("An error occurred while trying to send the request: %s", e)
+
+        self._execute_setup(["sudo chown -R user:user /home/user/.config/google-chrome/Default/History"], shell=True)

desktop_env/envs/desktop_env.py

@@ -175,10 +175,16 @@ class DesktopEnv(gym.Env):
         if isinstance(self.evaluator["func"], list) \
         else getattr(metrics, self.evaluator["func"])
         self.metric_conj: str = self.evaluator.get("conj", "and")  # take conjunction of multiple metrics
-        self.result_getter: Getter = [getattr(getters, "get_{:}".format(res["type"])) for res in
+        if "result" in self.evaluator:
+            self.result_getter: Getter = [getattr(getters, "get_{:}".format(res["type"])) for res in
                                       self.evaluator["result"]] \
             if isinstance(self.evaluator["result"], list) \
             else getattr(getters, "get_{:}".format(self.evaluator["result"]["type"]))
+        else:
+            self.result_getter = [None] * len(self.metric) \
+                if isinstance(self.metric, list) \
+                else None
+
         if "expected" in self.evaluator:
             self.expected_getter: Getter = [getattr(getters, "get_{:}".format(exp["type"])) if exp else None for exp in
                                             self.evaluator["expected"]] \
@@ -293,6 +299,12 @@ class DesktopEnv(gym.Env):
 
         self.setup_controller.setup(self.evaluator.get("postconfig", []))
 
+        if self.metric == "infeasible":
+            if self.action_history[-1] == "FAIL":
+                return 1
+            else:
+                return 0
+
         if type(self.metric) == list:
             results = []
             for idx, metric in enumerate(self.metric):
@@ -315,7 +327,8 @@ class DesktopEnv(gym.Env):
                     return 0
                 elif self.metric_conj == 'or' and float(metric) == 1.0:
                     return 1
-                else: results.append(metric)
+                else:
+                    results.append(metric)
             return sum(results) / len(results) if self.metric_conj == 'and' else max(results)
         else:
             try:

desktop_env/evaluators/metrics/slides.py

@@ -139,6 +139,7 @@ def compare_pptx_files(file1_path, file2_path, **options):
     examine_number_of_slides = options.get("examine_number_of_slides", True)
     examine_shape = options.get("examine_shape", True)
     examine_text = options.get("examine_text", True)
+    examine_indent = options.get("examine_indent", True)
     examine_font_name = options.get("examine_font_name", True)
     examine_font_size = options.get("examine_font_size", True)
     examine_font_bold = options.get("examine_font_bold", True)
@@ -148,6 +149,7 @@ def compare_pptx_files(file1_path, file2_path, **options):
     examine_strike_through = options.get("examine_strike_through", True)
     examine_alignment = options.get("examine_alignment", True)
     examine_bottom_position = options.get("examine_bottom_position", False)
+    examine_bullets = options.get("examine_bullets", True)
 
     # compare the number of slides
     if len(prs1.slides) != len(prs2.slides) and examine_number_of_slides:
@@ -174,6 +176,12 @@ def compare_pptx_files(file1_path, file2_path, **options):
                     if para1.alignment != para2.alignment and examine_alignment:
                         return 0
                     # check if the runs are the same
+                    if para1.text != para2.text and examine_text:
+                        return 0
+
+                    if para1.level != para2.level and examine_indent:
+                        return 0
+
                     for run1, run2 in zip(para1.runs, para2.runs):
 
                         # check if the font properties are the same
@@ -200,7 +208,40 @@ def compare_pptx_files(file1_path, file2_path, **options):
                                 'strike', 'noStrike') and examine_strike_through:
                             return 0
 
-                        # fixme: Actually there are more properties to be compared, but we cannot get them through pptx
+                        def _extract_bullets(xml_data):
+                            root = ET.fromstring(xml_data)
+
+                            namespaces = {
+                                'a': 'http://schemas.openxmlformats.org/drawingml/2006/main',
+                                'p': 'http://schemas.openxmlformats.org/presentationml/2006/main',
+                            }
+
+                            bullets = []
+
+                            for paragraph in root.findall('.//a:p', namespaces):
+                                pPr = paragraph.find('a:pPr', namespaces)
+                                if pPr is not None:
+                                    lvl = pPr.get('lvl')
+                                    buChar = pPr.find('a:buChar', namespaces)
+                                    char = buChar.get('char') if buChar is not None else "No Bullet"
+                                    buClr = pPr.find('a:buClr/a:srgbClr', namespaces)
+                                    color = buClr.get('val') if buClr is not None else "No Color"
+                                else:
+                                    lvl = "No Level"
+                                    char = "No Bullet"
+                                    color = "No Color"
+
+                                text = "".join(t.text for t in paragraph.findall('.//a:t', namespaces))
+
+                                bullets.append((lvl, char, text, color))
+
+                            return bullets
+
+                    if _extract_bullets(run1.part.blob.decode('utf-8')) != _extract_bullets(
+                            run2.part.blob.decode('utf-8')) and examine_bullets:
+                        return 0
+
+                    # fixme: Actually there are more properties to be compared, we can add them later via parsing the xml data
 
     return 1
 
@@ -422,6 +463,7 @@ if __name__ == '__main__':
     #     r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\550ce7e7-747b-495f-b122-acdc4d0b8e54\New_Club_Spring_2018_Training_Gold.pptx"))
     # print(evaluate_presentation_fill_to_rgb_distance(r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\3b27600c-3668-4abd-8f84-7bcdebbccbdb\lec17-gui-events.pptx", {"rgb": (0, 0, 255)}))
     # print(check_auto_saving_time(r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\2cd43775-7085-45d8-89fa-9e35c0a915cf\registrymodifications.xcu", {"minutes": 3}))
-    print(compare_pptx_files(r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\a669ef01-ded5-4099-9ea9-25e99b569840\Writing-Outlines.pptx",
+    print(compare_pptx_files(
-                             r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\a669ef01-ded5-4099-9ea9-25e99b569840\Writing-Outlines_Gold.pptx",
+        r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\a669ef01-ded5-4099-9ea9-25e99b569840\Writing-Outlines.pptx",
-                             examine_shape=False))
+        r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\a669ef01-ded5-4099-9ea9-25e99b569840\Writing-Outlines_Gold.pptx",
+        examine_shape=False))

evaluation_examples/examples/chrome/3720f614-37fd-4d04-8a6b-76f54f8c222d.json

@@ -0,0 +1,19 @@
+{
+  "id": "3720f614-37fd-4d04-8a6b-76f54f8c222d",
+  "snapshot": "chrome",
+  "instruction": "I am more familiar with Korean as I am from Korea. I want to use chrome with my mother tongue. Could you help me change the Chrome interface language to Korean? ",
+  "source": "https://superuser.com/questions/984668/change-interface-language-of-chrome-to-english",
+  "config": [
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "chrome"
+  ],
+  "evaluator": {
+    "func": "infeasible",
+    "result": {
+    },
+    "expected": {
+    }
+  }
+}

evaluation_examples/examples/chrome/44ee5668-ecd5-4366-a6ce-c1c9b8d4e938.json

@@ -4,6 +4,213 @@
   "instruction": "I am looking for an website address I accessed a month ago, but Youtube websites which take almost all of my browsing history are interrupting my search. This is too annoying. I want to remove all my Youtube browsing history first to facilitate my search. Could you help me clear browsing history from Youtube?",
   "source": "https://superuser.com/questions/1787991/clear-browsing-history-from-specific-site-on-chrome",
   "config": [
+    {
+      "type": "update_browse_history",
+      "parameters": {
+        "history": [
+          {
+            "url": "https://www.youtube.com/watch?v=dQw4w9WgXcQ",
+            "title": "Rick Astley - Never Gonna Give You Up (Official Music Video)",
+            "visit_time_from_now_in_seconds": 3600
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=9bZkp7q19f0",
+            "title": "PSY - GANGNAM STYLE(강남스타일) M/V",
+            "visit_time_from_now_in_seconds": 1631
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=3tmd-ClpJxA",
+            "title": "Maroon 5 - Sugar (Official Music Video)",
+            "visit_time_from_now_in_seconds": 900
+          },
+          {
+            "url": "https://www.nytimes.com/",
+            "title": "The New York Times",
+            "visit_time_from_now_in_seconds": 300
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=OPf0YbXqDm0",
+            "title": "Ed Sheeran - Shape of You [Official Music Video]",
+            "visit_time_from_now_in_seconds": 1200
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=JGwWNGJdvx8",
+            "title": "Taylor Swift - Shake It Off",
+            "visit_time_from_now_in_seconds": 2400
+          },
+          {
+            "url": "https://www.bbc.co.uk/",
+            "title": "BBC",
+            "visit_time_from_now_in_seconds": 1500
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=2Vv-BfVoq4g",
+            "title": "Adele - Hello",
+            "visit_time_from_now_in_seconds": 1800
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=YQHsXMglC9A",
+            "title": "Katy Perry - Roar (Official Music Video)",
+            "visit_time_from_now_in_seconds": 2100
+          },
+          {
+            "url": "https://www.cnn.com/",
+            "title": "CNN",
+            "visit_time_from_now_in_seconds": 2700
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=ru0K8uYEZWw",
+            "title": "Justin Bieber - Baby ft. Ludacris (Official Music Video)",
+            "visit_time_from_now_in_seconds": 3200
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=9bZkp7q19f0",
+            "title": "PSY - GANGNAM STYLE(강남스타일) M/V",
+            "visit_time_from_now_in_seconds": 3700
+          },
+          {
+            "url": "https://www.nationalgeographic.com/",
+            "title": "National Geographic",
+            "visit_time_from_now_in_seconds": 4000
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=OPf0YbXqDm0",
+            "title": "Ed Sheeran - Shape of You [Official Music Video]",
+            "visit_time_from_now_in_seconds": 4300
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=JGwWNGJdvx8",
+            "title": "Taylor Swift - Shake It Off",
+            "visit_time_from_now_in_seconds": 4700
+          },
+          {
+            "url": "https://www.bbc.co.uk/",
+            "title": "BBC",
+            "visit_time_from_now_in_seconds": 5000
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=2Vv-BfVoq4g",
+            "title": "Adele - Hello",
+            "visit_time_from_now_in_seconds": 5300
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=YQHsXMglC9A",
+            "title": "Katy Perry - Roar (Official Music Video)",
+            "visit_time_from_now_in_seconds": 5600
+          },
+          {
+            "url": "https://www.cnn.com/",
+            "title": "CNN",
+            "visit_time_from_now_in_seconds": 5900
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=ru0K8uYEZWw",
+            "title": "Justin Bieber - Baby ft. Ludacris (Official Music Video)",
+            "visit_time_from_now_in_seconds": 6300
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=dQw4w9WgXcQ",
+            "title": "Rick Astley - Never Gonna Give You Up (Official Music Video)",
+            "visit_time_from_now_in_seconds": 6700
+          },
+          {
+            "url": "https://www.nationalgeographic.com/",
+            "title": "National Geographic",
+            "visit_time_from_now_in_seconds": 7000
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=OPf0YbXqDm0",
+            "title": "Ed Sheeran - Shape of You [Official Music Video]",
+            "visit_time_from_now_in_seconds": 7300
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=JGwWNGJdvx8",
+            "title": "Taylor Swift - Shake It Off",
+            "visit_time_from_now_in_seconds": 7600
+          },
+          {
+            "url": "https://www.bbc.co.uk/",
+            "title": "BBC",
+            "visit_time_from_now_in_seconds": 7900
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=2Vv-BfVoq4g",
+            "title": "Adele - Hello",
+            "visit_time_from_now_in_seconds": 8200
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=YQHsXMglC9A",
+            "title": "Katy Perry - Roar (Official Music Video)",
+            "visit_time_from_now_in_seconds": 8500
+          },
+          {
+            "url": "https://www.cnn.com/",
+            "title": "CNN",
+            "visit_time_from_now_in_seconds": 8800
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=ru0K8uYEZWw",
+            "title": "Justin Bieber - Baby ft. Ludacris (Official Music Video)",
+            "visit_time_from_now_in_seconds": 9100
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=dQw4w9WgXcQ",
+            "title": "Rick Astley - Never Gonna Give You Up (Official Music Video)",
+            "visit_time_from_now_in_seconds": 9400
+          },
+          {
+            "url": "https://www.nationalgeographic.com/",
+            "title": "National Geographic",
+            "visit_time_from_now_in_seconds": 9700
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=OPf0YbXqDm0",
+            "title": "Ed Sheeran - Shape of You [Official Music Video]",
+            "visit_time_from_now_in_seconds": 10000
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=JGwWNGJdvx8",
+            "title": "Taylor Swift - Shake It Off",
+            "visit_time_from_now_in_seconds": 10300
+          },
+          {
+            "url": "https://www.bbc.co.uk/",
+            "title": "BBC",
+            "visit_time_from_now_in_seconds": 10600
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=2Vv-BfVoq4g",
+            "title": "Adele - Hello",
+            "visit_time_from_now_in_seconds": 10900
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=YQHsXMglC9A",
+            "title": "Katy Perry - Roar (Official Music Video)",
+            "visit_time_from_now_in_seconds": 11200
+          },
+          {
+            "url": "https://www.cnn.com/",
+            "title": "CNN",
+            "visit_time_from_now_in_seconds": 11500
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=ru0K8uYEZWw",
+            "title": "Justin Bieber - Baby ft. Ludacris (Official Music Video)",
+            "visit_time_from_now_in_seconds": 11800
+          },
+          {
+            "url": "https://www.youtube.com/watch?v=dQw4w9WgXcQ",
+            "title": "Rick Astley - Never Gonna Give You Up (Official Music Video)",
+            "visit_time_from_now_in_seconds": 12100
+          },
+          {
+            "url": "https://www.nationalgeographic.com/",
+            "title": "National Geographic",
+            "visit_time_from_now_in_seconds": 12400
+          }
+        ]
+      }
+    },
     {
       "type": "launch",
       "parameters": {

evaluation_examples/examples/chrome/93eabf48-6a27-4cb6-b963-7d5fe1e0d3a9.json

@@ -0,0 +1,19 @@
+{
+  "id": "93eabf48-6a27-4cb6-b963-7d5fe1e0d3a9",
+  "snapshot": "chrome",
+  "instruction": "Could you assist me in turning off the dark mode feature in Google Chrome? I've noticed that while dark mode is great for reducing glare, it actually makes it more challenging for me to read text clearly, especially with my astigmatism.",
+  "source": "https://superuser.com/questions/1417973/how-to-disable-google-chrome-dark-mode",
+  "config": [
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "chrome"
+  ],
+  "evaluator": {
+    "func": "infeasible",
+    "result": {
+    },
+    "expected": {
+    }
+  }
+}

evaluation_examples/examples/chrome/ae78f875-5b98-4907-bbb5-9c737fc68c03.json

@@ -4,40 +4,16 @@
   "instruction": "Could you please change the number of search results displayed on one page to 50? I find that having more results visible at once significantly enhances my research efficiency, as it reduces the need to constantly click through multiple pages. ",
   "source": "https://support.google.com/chrome/thread/219988391/increase-search-results-per-page?hl=en",
   "config": [
-    {
-      "type": "launch",
-      "parameters": {
-        "command": [
-          "google-chrome",
-          "--remote-debugging-port=1337"
-        ]
-      }
-    },
-    {
-      "type": "launch",
-      "parameters": {
-        "command": [
-          "socat",
-          "tcp-listen:9222,fork",
-          "tcp:localhost:1337"
-        ]
-      }
-    }
   ],
   "trajectory": "trajectories/",
   "related_apps": [
     "chrome"
   ],
   "evaluator": {
-    "func": "exact_match",
+    "func": "infeasible",
     "result": {
-      "type": "number_of_search_results"
     },
     "expected": {
-      "type": "rule",
-      "rules": {
-        "expected": 50
-      }
     }
   }
 }

evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json

@@ -4,78 +4,16 @@
   "instruction": "Make sparkline chart line by line",
   "source": "https://www.youtube.com/shorts/L3Z-F1QTQFY",
   "config": [
-    {
-      "type": "download",
-      "parameters": {
-        "files": [
-          {
-            "url": "https://drive.usercontent.google.com/download?id=1uywX5XWMvesnb4-8LPKEzr2HFU7HmoIu&export=download&authuser=0&confirm=t&uuid=267bfe49-a861-4272-ae7c-39c95df35e84&at=APZUnTUbs-FF06hSMv3yWfdXc02l:1703508870351",
-            "path": "/home/user/OrderId_Month_Chart.xlsx"
-          }
-        ]
-      }
-    },
-    {
-      "type": "open",
-      "parameters": {
-        "path": "/home/user/OrderId_Month_Chart.xlsx"
-      }
-    }
   ],
   "trajectory": "trajectories/2bd59342-0664-4ccb-ba87-79379096cc08",
   "related_apps": [
-    "libreoffice calc"
+    "libreoffice_calc"
   ],
   "evaluator": {
-    "postconfig": [
+    "func": "infeasible",
-      {
-        "type": "activate_window",
-        "parameters": {
-          "window_name": "OrderId_Month_Chart.xlsx - LibreOffice Calc",
-          "strict": true
-        }
-      },
-      {
-        "type": "sleep",
-        "parameters": {
-          "seconds": 0.5
-        }
-      },
-      {
-        "type": "execute",
-        "parameters": {
-          "command": [
-            "python",
-            "-c",
-            "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
-          ]
-        }
-      }
-    ],
-    "func": "compare_table",
     "expected": {
-      "type": "cloud_file",
-      "path": "https://drive.usercontent.google.com/download?id=1KQJJLVPGtTL_7ArEWvwwbFbJSiA3cgSE&export=download&authuser=0&confirm=t&uuid=6b11c721-caad-439a-b369-4c13c7a485df&at=APZUnTV5-1isKrDKSHV9NeJ6TDeS:1703509054094",
-      "dest": "OrderId_Month_Chart_gold.xlsx"
     },
     "result": {
-      "type": "vm_file",
-      "path": "/home/user/OrderId_Month_Chart.xlsx",
-      "dest": "OrderId_Month_Chart.xlsx"
-    },
-    "options": {
-      "rules": [
-        {
-          "type": "sparkline",
-          "sheet_idx0": 0,
-          "sheet_idx1": "EI0"
-        },
-        {
-          "type": "sheet_data",
-          "sheet_idx0": 0,
-          "sheet_idx1": "EI0"
-        }
-      ]
     }
   }
 }

evaluation_examples/examples/libreoffice_calc/7b802dad-6e0f-4204-9815-d4e3f57627d8.json

@@ -0,0 +1,19 @@
+{
+  "id": "7b802dad-6e0f-4204-9815-d4e3f57627d8",
+  "snapshot": "libreoffice_calc",
+  "instruction": "I would like to sort this table based on cell color, placing all the rows marked with pink at the beginning, while keeping their order among themselves unchanged.",
+  "source": "https://www.youtube.com/shorts/Of-lzeP1usE",
+  "config": [
+  ],
+  "trajectory": "trajectories/7b802dad-6e0f-4204-9815-d4e3f57627d8",
+  "related_apps": [
+    "libreoffice_calc"
+  ],
+  "evaluator": {
+    "func": "infeasible",
+    "expected": {
+    },
+    "result": {
+    }
+  }
+}

evaluation_examples/examples/libreoffice_impress/358aa0a7-6677-453f-ae35-e440f004c31e.json

@@ -54,7 +54,7 @@
       {
         "type": "sleep",
         "parameters": {
-          "seconds": 0.5
+          "seconds": 15
         }
       }
     ],

evaluation_examples/examples/libreoffice_impress/a669ef01-ded5-4099-9ea9-25e99b569840.json

@@ -9,7 +9,7 @@
       "parameters": {
         "files": [
           {
-            "url": "https://drive.usercontent.google.com/download?id=1C0u-Qvvwa6UbJVTzzQHfdNIgp2i051xA&export=download&authuser=0&confirm=t&uuid=5551a43c-3ff7-424f-b82c-50a5c96b5809&at=APZUnTViShb8pJUviOkmVtn7Pums:1707299959829",
+            "url": "https://drive.usercontent.google.com/download?id=1hr2flq5iSyMYSps6Jd-3pDOEfZoHFCbb&export=download&authuser=0&confirm=t&uuid=02746987-6ea8-4fbb-8817-8051dab152e7&at=APZUnTWaH071WARB_12CQDvjfg6b:1707314868059",
             "path": "/home/user/Desktop/Writing-Outlines.pptx"
           }
         ]
@@ -71,16 +71,13 @@
     "func": "compare_pptx_files",
     "expected": {
       "type": "cloud_file",
-      "path": "https://drive.usercontent.google.com/download?id=1d4WJwm7KDhDIOJ9r9vKhRTrt_bF3PSln&export=download&authuser=0&confirm=t&uuid=a8ec87de-96cf-49f0-98e4-faa1218354fe&at=APZUnTWiOoiHFjyM4jTunLP4t5wE:1707299961717",
+      "path": "https://drive.usercontent.google.com/download?id=15mnwwGTDlelIf27C1HdJOiMgfEWNIoLl&export=download&authuser=0&confirm=t&uuid=b53e5fbb-565b-4498-9dc5-071eded307e0&at=APZUnTUdfXCDVFEQPCTYckB-H2Fn:1707314644205",
       "dest": "Writing-Outlines_Gold.pptx"
     },
     "result": {
       "type": "vm_file",
       "path": "/home/user/Desktop/Writing-Outlines.pptx",
       "dest": "Writing-Outlines.pptx"
-    },
-    "options": {
-      "examine_shape": false
     }
   }
 }

evaluation_examples/examples/os/b3d4a89c-53f2-4d6b-8b6a-541fb5d205fa.json

@@ -9,7 +9,7 @@
     "os"
   ],
   "evaluator": {
-    "func": "",
+    "func": "infeasible",
     "result": {
     },
     "expected": {

evaluation_examples/examples/os/e2eb4bf1-aa93-4192-b55d-03e2fb6dfd15.json

@@ -8,7 +8,7 @@
       "os"
     ],
     "evaluator": {
-      "func": "",
+      "func": "infeasible",
       "result": {
       },
       "expected": {

evaluation_examples/examples/os/fe41f596-a71b-4c2f-9b2f-9dcd40b568c3.json

@@ -5,32 +5,15 @@
   "source": "https://help.ubuntu.com/lts/ubuntu-help/power-percentage.html.en",
   "trajectory": "trajectories/",
   "config": [
-    {
-      "type": "execute",
-      "parameters": {
-        "command": [
-          "python",
-          "-c",
-          "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
-        ]
-      }
-    }
   ],
   "related_apps": [
     "os"
   ],
   "evaluator": {
-    "func": "exact_match",
+    "func": "infeasible",
     "result": {
-      "type": "vm_command_line",
-      "command": "gsettings get org.gnome.desktop.interface show-battery-percentage",
-      "shell": true
     },
     "expected": {
-      "type": "rule",
-      "rules": {
-        "expected": "true\n"
-      }
     }
   }
 }