Merge main

desktop_env/controllers/python.py

@@ -11,7 +11,7 @@ logger = logging.getLogger("desktopenv.pycontroller")
 
 
 class PythonController:
-    def __init__(self, vm_ip: str, pkgs_prefix: str = "import pyautogui; {command}"):
+    def __init__(self, vm_ip: str, pkgs_prefix: str = "import pyautogui; import time; {command}"):
         self.vm_ip = vm_ip
         self.http_server = f"http://{vm_ip}:5000"
         self.pkgs_prefix = pkgs_prefix  # fixme: this is a hacky way to execute python commands. fix it and combine it with installation of packages

desktop_env/envs/desktop_env.py

@@ -76,6 +76,8 @@ class DesktopEnv(gym.Env):
         self.cache_dir_base: str = cache_dir
         self.vm_screen_size = screen_size
 
+        os.makedirs(self.tmp_dir_base, exist_ok=True)
+
         # task-aware stuffs
         # todo: handling the logic of snapshot directory
         self._set_task_info(task_config)

desktop_env/evaluators/getters/__init__.py

@@ -2,7 +2,7 @@ from .chrome import get_default_search_engine, get_cookie_data, get_bookmarks, g
     get_shortcuts_on_desktop, get_history, get_enabled_experiments, get_chrome_language, get_chrome_font_size, \
     get_profile_name, get_number_of_search_results, get_googledrive_file, get_active_tab_info
 from .file import get_cloud_file, get_vm_file, get_cache_file
-from .general import get_vm_command_line
+from .general import get_vm_command_line, get_vm_terminal_output
 from .impress import get_audio_in_slide
 from .info import get_vm_screen_size, get_vm_window_size, get_vm_wallpaper, get_list_directory
 from .misc import get_rule, get_accessibility_tree

desktop_env/evaluators/getters/general.py

@@ -18,3 +18,7 @@ def get_vm_command_line(env, config: Dict[str, str]):
     else:
         logger.error("Failed to get vm command line. Status code: %d", response.status_code)
         return None
+
+
+def get_vm_terminal_output(env, config: Dict[str, str]):
+    return env.controller.get_terminal_output()

desktop_env/evaluators/metrics/__init__.py

@@ -10,15 +10,17 @@ from .general import exact_match, fuzzy_match
 from .docs import find_default_font, contains_page_break, compare_docx_files, compare_docx_tables, compare_line_spacing, \
     compare_insert_equation, compare_highlighted_text
 from .docs import is_first_line_centered, check_file_exists, compare_contains_image
-from .general import check_csv, check_accessibility_tree, run_sqlite3, check_json
+from .general import check_csv, check_accessibility_tree, run_sqlite3, check_json, check_list
 from .general import exact_match, fuzzy_match, check_include_exclude
 from .gimp import increase_saturation, decrease_brightness, check_file_exists, compare_triangle_positions
+from .gimp import compare_images
 from .libreoffice import check_libre_locale
 from .pdf import check_pdf_pages
 from .slides import check_presenter_console_disable, check_image_stretch_and_center, check_slide_numbers_color, \
     compare_pptx_files, check_strikethrough, \
-    check_slide_orientation_Portrait, evaluate_presentation_fill_to_rgb_distance, check_left_panel
-from .table import compare_table
+    check_slide_orientation_Portrait, evaluate_presentation_fill_to_rgb_distance, check_left_panel, check_transition
+# from .table import check_sheet_list, check_xlsx_freeze, check_xlsx_zoom, check_data_validations
+from .table import compare_table, compare_csv
 from .thunderbird import check_thunderbird_prefs, check_thunderbird_filter
 from .gimp import (
     check_brightness_decrease_and_structure_sim,
@@ -40,4 +42,4 @@ from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, co
     check_qt_slider_colours, check_global_key_play_pause
 from .vscode import compare_text_file, compare_config, compare_answer, is_extension_installed, check_json_settings, \
     check_json_keybindings
-from .os import check_gnome_favorite_apps, is_utc_0, check_text_enlarged, check_moved_jpgs
+from .basic_os import check_gnome_favorite_apps, is_utc_0, check_text_enlarged, check_moved_jpgs

desktop_env/evaluators/metrics/docs.py

@@ -1,6 +1,8 @@
 import logging
 import os
 import xml.etree.ElementTree as ET
+import zipfile
+import re
 from typing import List, Dict, Any
 
 from docx import Document
@@ -48,24 +50,30 @@ def contains_page_break(docx_file):
     return 0
 
 
-def compare_docx_files(file1, file2):
+def compare_docx_files(file1, file2, ignore_blanks=True):
     doc1 = Document(file1)
     doc2 = Document(file2)
 
     doc1_paragraphs = [p.text for p in doc1.paragraphs]
     doc2_paragraphs = [p.text for p in doc2.paragraphs]
 
-    if len(doc1_paragraphs) != len(doc2_paragraphs):
-        # print(len(doc1_paragraphs))
-        # print(len(doc2_paragraphs))
-        return 0
-
-    # Compare each paragraph
-    for p1, p2 in zip(doc1_paragraphs, doc2_paragraphs):
-        if p1 != p2:
-            # print(p1)
-            # print(p2)
+    if ignore_blanks:
+        text1 = re.sub(r'\s+', ' ', '\n'.join(doc1_paragraphs)).strip()
+        text2 = re.sub(r'\s+', ' ', '\n'.join(doc2_paragraphs)).strip()
+        if text1 != text2:
             return 0
+    else:
+        if len(doc1_paragraphs) != len(doc2_paragraphs):
+            # print(len(doc1_paragraphs))
+            # print(len(doc2_paragraphs))
+            return 0
+
+        # Compare each paragraph
+        for p1, p2 in zip(doc1_paragraphs, doc2_paragraphs):
+            if p1 != p2:
+                # print(p1)
+                # print(p2)
+                return 0
 
     return 1
 
@@ -245,16 +253,26 @@ def evaluate_colored_words_in_tables(file_path1, file_path2):
 
 
 def check_highlighted_words(file_path1, file_path2):
-    if not compare_docx_files(file_path1, file_path2):
+    # if not compare_docx_files(file_path1, file_path2):
+    #     return 0
+        
+    # Extract content.xml from the .odt file
+    extract_dir = file_path1 + "_extracted"
+    with zipfile.ZipFile(file_path1, 'r') as zip_ref:
+        zip_ref.extractall(extract_dir)
+    content_xml_path = os.path.join(extract_dir, 'content.xml')
+    with open(content_xml_path, 'r', encoding="utf-8") as file:
+        content_xml = file.read()
+
+    # Check for yellow highlights in the content.xml
+    yellow_highlight_pattern = re.compile(r'(.{0,50}background-color="#ffff00"[^>]*>.{0,50})')
+    yellow_highlight_matches = yellow_highlight_pattern.findall(content_xml)
+
+    # Return True if yellow highlights are NOT found, otherwise True
+    if yellow_highlight_matches:
         return 0
-    document = Document(file_path1)
-
-    for paragraph in document.paragraphs:
-        for run in paragraph.runs:
-            if run.font.highlight_color is not None:
-                return 0  # Highlighted words found
-
-    return 1  # No highlighted words found
+    else:
+        return 1
 
 
 def evaluate_strike_through_last_paragraph(file_path1, file_path2):
@@ -415,4 +433,4 @@ def compare_highlighted_text(file1, file2):
     doc2_highlighted = extract_highlighted_text(Document(file2))
 
     # Compare the sets of highlighted text to check if they are the same
-    return set(doc1_highlighted) == set(doc2_highlighted)
+    return set(doc1_highlighted) == set(doc2_highlighted)

desktop_env/evaluators/metrics/gimp.py

@@ -1,6 +1,23 @@
 import os
-from PIL import Image, ImageStat, ImageChops
+from typing import List, Union
 from skimage.metrics import structural_similarity as ssim
+from PIL import Image, ImageChops, ImageStat
+
+
+def compare_images(pred_img_path_list: Union[str, List[str]],
+                   gold_img_path_list: Union[str, List[str]]) -> float:
+    """ Compare two image lists, only if all images are the same, return 1.0, otherwise return 0.0
+    """
+    if type(pred_img_path_list) != list:
+        pred_img_path_list = [pred_img_path_list]
+        gold_img_path_list = [gold_img_path_list]
+    for pred_img_path, gold_img_path in zip(pred_img_path_list, gold_img_path_list):
+        pred_img = Image.open(pred_img_path)
+        gold_img = Image.open(gold_img_path)
+        diff = ImageChops.difference(pred_img, gold_img)
+        if diff.getbbox():
+            return 0.0
+    return 1.0
 
 
 def get_gimp_export_path():

desktop_env/evaluators/metrics/pdf.py

@@ -6,6 +6,8 @@ from pypdf import PdfReader
 
 
 def check_pdf_pages(pdf_file: str, rules: Dict[str, Any]) -> float:
+    if pdf_file is None:
+        return 0.0
     reader = PdfReader(pdf_file)
     nb_pages: int = len(reader.pages)
     return float(getattr(operator, rules["relation"])(nb_pages, rules["ref_value"]))

desktop_env/evaluators/metrics/slides.py

@@ -1,5 +1,6 @@
 import logging
 import xml.etree.ElementTree as ET
+import zipfile
 from math import sqrt
 
 from pptx import Presentation
@@ -57,9 +58,9 @@ def check_image_stretch_and_center(modified_ppt, original_ppt):
             abs(the_modified_image.height - original_pres.slide_height) > Inches(0.1) or
             abs(the_modified_image.left - (original_pres.slide_width - the_modified_image.width) / 2) > Inches(0.1) or
             abs(the_modified_image.top - (original_pres.slide_height - the_modified_image.height) / 2) > Inches(0.1)):
-        return False
+        return 0.
 
-    return True
+    return 1.
 
 
 def is_red_color(color):
@@ -97,6 +98,39 @@ def check_slide_numbers_color(pptx_file_path):
                     return 1 if font_color is not None and is_red_color(font_color) else 0
 
 
+# import numpy as np
+# from PIL import Image
+# from skimage.metrics import structural_similarity as ssim
+
+# def compare_images(image1_path, image2_path):
+#     # You would call this function with the paths to the two images you want to compare:
+#     # score = compare_images('path_to_image1', 'path_to_image2')
+#     # print("Similarity score:", score)
+
+#     if not image1_path or not image2_path:
+#         return 0
+
+#     # Open the images and convert to grayscale
+#     image1 = Image.open(image1_path).convert('L')
+#     image2 = Image.open(image2_path).convert('L')
+
+#     # Resize images to the smaller one's size for comparison
+#     image1_size = image1.size
+#     image2_size = image2.size
+#     new_size = min(image1_size, image2_size)
+
+#     image1 = image1.resize(new_size, Image.Resampling.LANCZOS)
+#     image2 = image2.resize(new_size, Image.Resampling.LANCZOS)
+
+#     # Convert images to numpy arrays
+#     image1_array = np.array(image1)
+#     image2_array = np.array(image2)
+
+#     # Calculate SSIM between two images
+#     similarity_index = ssim(image1_array, image2_array)
+
+#     return similarity_index
+
 def compare_pptx_files(file1_path, file2_path, **options):
     # todo: not strictly match since not all information is compared because we cannot get the info through pptx
     prs1 = Presentation(file1_path)
@@ -111,52 +145,58 @@ def compare_pptx_files(file1_path, file2_path, **options):
     examine_font_italic = options.get("examine_font_italic", True)
     examine_color_rgb = options.get("examine_color_rgb", True)
     examine_font_underline = options.get("examine_font_underline", True)
+    examine_strike_through = options.get("examine_strike_through", True)
 
     # compare the number of slides
     if len(prs1.slides) != len(prs2.slides) and examine_number_of_slides:
-        return False
+        return 0
 
     # compare the content of each slide
     for slide1, slide2 in zip(prs1.slides, prs2.slides):
+
         # check if the shapes are the same
         for shape1, shape2 in zip(slide1.shapes, slide2.shapes):
             if (
                     shape1.left != shape2.left or shape1.top != shape2.top or shape1.width != shape2.width or shape1.height != shape2.height) and examine_shape:
-                return False
+                return 0
 
             if hasattr(shape1, "text") and hasattr(shape2, "text"):
                 if shape1.text != shape2.text and examine_text:
-                    return False
+                    return 0
 
                 # check if the paragraphs are the same
                 for para1, para2 in zip(shape1.text_frame.paragraphs, shape2.text_frame.paragraphs):
                     # check if the runs are the same
                     for run1, run2 in zip(para1.runs, para2.runs):
                         if run1.text != run2.text and examine_text:
-                            return False
+                            return 0
 
                         # check if the font properties are the same
                         if run1.font.name != run2.font.name and examine_font_name:
-                            return False
+                            return 0
 
                         if run1.font.size != run2.font.size and examine_font_size:
-                            return False
+                            return 0
 
                         if run1.font.bold != run2.font.bold and examine_font_bold:
-                            return False
+                            return 0
 
                         if run1.font.italic != run2.font.italic and examine_font_italic:
-                            return False
+                            return 0
 
                         if run1.font.color.rgb != run2.font.color.rgb and examine_color_rgb:
-                            return False
+                            return 0
 
                         if run1.font.underline != run2.font.underline and examine_font_underline:
-                            return False
+                            return 0
+
+                        if ('strike' in run1.font._element.attrib) != (
+                                'strike' in run2.font._element.attrib) and examine_strike_through:
+                            return 0
 
                         # fixme: Actually there are more properties to be compared, but we cannot get them through pptx
 
-    return True
+    return 1
 
 
 def check_strikethrough(pptx_path, rules):
@@ -167,21 +207,27 @@ def check_strikethrough(pptx_path, rules):
     shape_index_s = rules["shape_index_s"]
     paragraph_index_s = rules["paragraph_index_s"]
 
-    for slide_index in slide_index_s:
-        # Get the slide
-        slide = presentation.slides[slide_index]
+    try:
+        for slide_index in slide_index_s:
+            # Get the slide
+            slide = presentation.slides[slide_index]
 
-        for shape_index in shape_index_s:
-            # Get the text box
-            paragraphs = slide.shapes[shape_index].text_frame.paragraphs
+            for shape_index in shape_index_s:
+                # Get the text box
+                paragraphs = slide.shapes[shape_index].text_frame.paragraphs
 
-            for paragraph_index in paragraph_index_s:
-                paragraph = paragraphs[paragraph_index]
-                run = paragraph.runs[0]
-                if 'strike' not in run.font._element.attrib:
-                    return False
+                for paragraph_index in paragraph_index_s:
+                    paragraph = paragraphs[paragraph_index]
+                    run = paragraph.runs[0]
+                    if 'strike' not in run.font._element.attrib:
+                        return 0
 
-    return True
+
+    except Exception as e:
+        logger.error(f"Error: {e}")
+        return 0
+
+    return 1
 
 
 def check_slide_orientation_Portrait(pptx_path):
@@ -221,14 +267,54 @@ def check_left_panel(accessibility_tree):
     root = ET.fromstring(accessibility_tree)
 
     for root_pane in root.iter('root-pane'):
-        for panel in root_pane.iter('panel'):
-            for split_pane in panel.iter('split-pane'):
-                # Get the left panel
-                if split_pane.attrib.get("{{{}}}parentcoord".format(namespaces['cp'])) == "(0, 0)":
-                    # Get the visible attribute
-                    visible = split_pane.attrib.get("{{{}}}visible".format(namespaces['st']))
-                    if visible:
-                        # decide if it is left panel
-                        return 1.
+        for split_pane in root_pane.iter('split-pane'):
+            for panel in split_pane.iter('panel'):
+                for scroll_panel in panel.iter('scroll-pane'):
+                    for document_frame in scroll_panel.iter('document-frame'):
+                        # Get the left panel
+                        panel_name = document_frame.get("name")
+                        # visible = scroll_bar.attrib.get(f"{{{namespaces['st']}}}visible")
+                        if panel_name == "Slides View":
+                            # Left panel is open
+                            return 1.0
 
-    return 0.
+    # Left panel is not open
+    return 0.0
+
+
+def check_transition(pptx_file, rules):
+    slide_idx = rules['slide_idx']
+    transition_type = rules['transition_type']
+
+    # Use the zipfile module to open the .pptx file
+    with zipfile.ZipFile(pptx_file, 'r') as zip_ref:
+        # Get the slide XML file
+        slide_name = 'ppt/slides/slide{}.xml'.format(slide_idx + 1)
+        try:
+            zip_ref.getinfo(slide_name)
+        except KeyError:
+            # Slide does not exist
+            return 0.
+
+        with zip_ref.open(slide_name) as slide_file:
+            # 解析XML
+            tree = ET.parse(slide_file)
+            root = tree.getroot()
+
+            # XML namespace
+            namespaces = {
+                'a': 'http://schemas.openxmlformats.org/drawingml/2006/main',
+                'p': 'http://schemas.openxmlformats.org/presentationml/2006/main',
+            }
+
+            # Search for the transition element
+            transition = root.find('.//p:transition', namespaces)
+            if transition is not None:
+                # Check if the transition is an expected transition
+                dissolve = transition.find('.//p:{}'.format(transition_type), namespaces)
+                if dissolve is not None:
+                    return 1.
+                else:
+                    return 0.
+            else:
+                return 0.

desktop_env/evaluators/metrics/table.py

@@ -1,32 +1,35 @@
+import functools
+import itertools
 import logging
-#import operator
+import os.path
+# import operator
 from numbers import Number
 from typing import Any, Union, cast, Callable, Iterable
 from typing import Dict, List, Tuple
-import os.path
-import itertools
-import functools
 
 import openpyxl
 import pandas as pd
 from openpyxl import Workbook
-from openpyxl.worksheet.worksheet import Worksheet
-#from openpyxl.worksheet.cell_range import MultiCellRange
-from openpyxl.worksheet.datavalidation import DataValidation
 from openpyxl.cell.cell import Cell
-#from openpyxl.utils import coordinate_to_tuple
+# from openpyxl.worksheet.cell_range import MultiCellRange
+from openpyxl.worksheet.datavalidation import DataValidation
+from openpyxl.worksheet.worksheet import Worksheet
 
-from .utils import load_charts, load_sparklines, load_rows_or_cols, load_xlsx_styles
 from .utils import _match_value_to_rule, _read_cell_style, read_cell_value
+from .utils import load_charts, load_sparklines, load_rows_or_cols, load_xlsx_styles
+
+# from openpyxl.utils import coordinate_to_tuple
 
 logger = logging.getLogger("desktopenv.metric.table")
 
 BOOK = Union[pd.ExcelFile, Workbook, str]
-def _parse_sheet_idx( sheet_idx: Union[int, str]
-                    , result: BOOK, expected: BOOK
-                    , result_sheet_names: List[str]
-                    , expected_sheet_names: List[str]
-                    ) -> Tuple[BOOK, str]:
+
+
+def _parse_sheet_idx(sheet_idx: Union[int, str]
+                     , result: BOOK, expected: BOOK
+                     , result_sheet_names: List[str]
+                     , expected_sheet_names: List[str]
+                     ) -> Tuple[BOOK, str]:
     #  function _parse_sheet_idx {{{ # 
     if isinstance(sheet_idx, int):
         index: str = result_sheet_names[sheet_idx]
@@ -49,7 +52,10 @@ def _parse_sheet_idx( sheet_idx: Union[int, str]
     return book, index
     #  }}} function _parse_sheet_idx # 
 
+
 SHEET = Union[pd.DataFrame, Worksheet, List[str]]
+
+
 def _load_sheet(book: BOOK, index: str) -> SHEET:
     #  function _load_sheet {{{ # 
     if isinstance(book, str):
@@ -57,12 +63,12 @@ def _load_sheet(book: BOOK, index: str) -> SHEET:
         csv_name: str = "{:}-{:}.csv".format(os.path.splitext(book)[0], index)
 
         with open(csv_name) as f:
-            csv_lines: List[str] = list( itertools.dropwhile( lambda l: len(l)==0
-                                                            , map( lambda l: l.strip()
-                                                                 , reversed(f.read().splitlines())
-                                                                 )
+            csv_lines: List[str] = list(itertools.dropwhile(lambda l: len(l) == 0
+                                                            , map(lambda l: l.strip()
+                                                                  , reversed(f.read().splitlines())
+                                                                  )
                                                             )
-                                       )
+                                        )
         return csv_lines
     if isinstance(book, pd.ExcelFile):
         return pd.read_excel(book, index)
@@ -72,7 +78,8 @@ def _load_sheet(book: BOOK, index: str) -> SHEET:
     raise NotImplementedError("Not supported workbook format")
     #  }}} function _load_sheet # 
 
-def compare_table(result: str, expected: str, **options) -> float:
+
+def compare_table(result: str, expected: str = None, **options) -> float:
     #  function compare_table {{{ # 
     """
     Args:
@@ -92,25 +99,35 @@ def compare_table(result: str, expected: str, **options) -> float:
     if result is None:
         return 0.
 
-    xlworkbookr: Workbook = openpyxl.load_workbook(filename=result)
-    pdworkbookr = pd.ExcelFile(result)
+    try:
+        xlworkbookr: Workbook = openpyxl.load_workbook(filename=result)
+        pdworkbookr = pd.ExcelFile(result)
+    except:
+        return 0.
     worksheetr_names: List[str] = pdworkbookr.sheet_names
 
-    xlworkbooke: Workbook = openpyxl.load_workbook(filename=expected)
-    pdworkbooke = pd.ExcelFile(expected)
-    worksheete_names: List[str] = pdworkbooke.sheet_names
+    if expected is not None:
 
-    parse_idx: Callable[[Union[str, int], BOOK, BOOK], BOOK] =\
-            functools.partial( _parse_sheet_idx
-                             , result_sheet_names=worksheetr_names
-                             , expected_sheet_names=worksheete_names
-                             )
+        xlworkbooke: Workbook = openpyxl.load_workbook(filename=expected)
+        pdworkbooke = pd.ExcelFile(expected)
+        worksheete_names: List[str] = pdworkbooke.sheet_names
+    else:
+        xlworkbooke: Workbook = None
+        pdworkbooke = None
+        worksheete_names: List[str] = None
+
+    parse_idx: Callable[[Union[str, int], BOOK, BOOK], BOOK] = \
+        functools.partial(
+            _parse_sheet_idx,
+            result_sheet_names=worksheetr_names,
+            expected_sheet_names=worksheete_names
+        )
 
     passes = True
     for r in options["rules"]:
         if r["type"] == "sheet_name":
             #  Compare Sheet Names {{{ # 
-            metric: bool = worksheetr_names==worksheete_names
+            metric: bool = worksheetr_names == worksheete_names
             logger.debug("Assertion: %s.sheet_names == %s.sheet_names - %s", result, expected, metric)
             #  }}} Compare Sheet Names # 
 
@@ -171,8 +188,8 @@ def compare_table(result: str, expected: str, **options) -> float:
 
             styles1: Dict[str, List[Any]] = load_xlsx_styles(*parse_idx(r["sheet_idx0"], xlworkbookr, xlworkbooke), **r)
             styles2: Dict[str, List[Any]] = load_xlsx_styles(*parse_idx(r["sheet_idx1"], xlworkbookr, xlworkbooke), **r)
-            #number_formats1: List[str] = [c.number_format.lower() for col in sheet1.iter_cols() for c in col if c.value is not None and c.data_type=="n"]
-            #number_formats2: List[str] = [c.number_format.lower() for col in sheet2.iter_cols() for c in col if c.value is not None and c.data_type=="n"]
+            # number_formats1: List[str] = [c.number_format.lower() for col in sheet1.iter_cols() for c in col if c.value is not None and c.data_type=="n"]
+            # number_formats2: List[str] = [c.number_format.lower() for col in sheet2.iter_cols() for c in col if c.value is not None and c.data_type=="n"]
             metric: bool = styles1 == styles2
             logger.debug("Assertion: %s.style == %s.style - %s", r["sheet_idx0"], r["sheet_idx1"], metric)
             #  }}} Compare Style (Also Conditional Formatiing) # 
@@ -185,11 +202,11 @@ def compare_table(result: str, expected: str, **options) -> float:
             sheet1: Worksheet = _load_sheet(*parse_idx(r["sheet_idx0"], xlworkbookr, xlworkbooke))
             sheet2: Worksheet = _load_sheet(*parse_idx(r["sheet_idx1"], xlworkbookr, xlworkbooke))
             metric: bool = sheet1.freeze_panes == sheet2.freeze_panes
-            logger.debug( "Assertion: %s.freeze(%s) == %s.freeze(%s) - %s"
-                        , r["sheet_idx0"], sheet1.freeze_panes
-                        , r["sheet_idx1"], sheet2.freeze_panes
-                        , metric
-                        )
+            logger.debug("Assertion: %s.freeze(%s) == %s.freeze(%s) - %s"
+                         , r["sheet_idx0"], sheet1.freeze_panes
+                         , r["sheet_idx1"], sheet2.freeze_panes
+                         , metric
+                         )
             #  }}} Compare Freezing # 
 
         elif r["type"] == "zoom":
@@ -201,7 +218,8 @@ def compare_table(result: str, expected: str, **options) -> float:
             sheet: Worksheet = _load_sheet(*parse_idx(r["sheet_idx"], xlworkbookr, xlworkbooke))
             zoom_scale: Number = sheet.sheet_view.zoomScale or 100.
             metric: bool = _match_value_to_rule(zoom_scale, r)
-            logger.debug("Assertion: %s.zoom(%.1f) %s %.1f - %s", r["sheet_idx"], zoom_scale, r["method"], r["ref"], metric)
+            logger.debug("Assertion: %s.zoom(%.1f) %s %.1f - %s", r["sheet_idx"], zoom_scale, r["method"], r["ref"],
+                         metric)
             #  }}} Check Zooming # 
 
         elif r["type"] == "data_validation":
@@ -228,15 +246,15 @@ def compare_table(result: str, expected: str, **options) -> float:
             sheet: Worksheet = _load_sheet(*parse_idx(r["sheet_idx"], xlworkbookr, xlworkbooke))
             data_validators: List[DataValidation] = sheet.data_validations.dataValidation
 
-            total_metric = len(data_validators)>=len(r["dv_props"])
+            total_metric = len(data_validators) >= len(r["dv_props"])
             for dat_vldt in data_validators:
                 metric = False
                 for prpt in r["dv_props"]:
-                    metric = metric or all( _match_value_to_rule( getattr(dat_vldt, attrbt)
+                    metric = metric or all(_match_value_to_rule(getattr(dat_vldt, attrbt)
                                                                 , mr
-                                                                )\
-                                        for attrbt, mr in prpt.items()
-                                          )
+                                                                ) \
+                                           for attrbt, mr in prpt.items()
+                                           )
                     if metric:
                         break
                 total_metric = total_metric and metric
@@ -253,14 +271,14 @@ def compare_table(result: str, expected: str, **options) -> float:
             # sheet_idx1: as sheet_idx0
             # props: list of str, see utils.load_rows_or_cols
 
-            rows1: Dict[str, Any] = load_rows_or_cols( *parse_idx(r["sheet_idx0"], xlworkbookr, xlworkbooke)
-                                                     , obj="row"
-                                                     , **r
-                                                     )
-            rows2: Dict[str, Any] = load_rows_or_cols( *parse_idx(r["sheet_idx1"], xlworkbookr, xlworkbooke)
-                                                     , obj="row"
-                                                     , **r
-                                                     )
+            rows1: Dict[str, Any] = load_rows_or_cols(*parse_idx(r["sheet_idx0"], xlworkbookr, xlworkbooke)
+                                                      , obj="row"
+                                                      , **r
+                                                      )
+            rows2: Dict[str, Any] = load_rows_or_cols(*parse_idx(r["sheet_idx1"], xlworkbookr, xlworkbooke)
+                                                      , obj="row"
+                                                      , **r
+                                                      )
             logger.debug("Rows1: %s", repr(rows1))
             logger.debug("Rows2: %s", repr(rows2))
             metric: bool = rows1 == rows2
@@ -273,14 +291,14 @@ def compare_table(result: str, expected: str, **options) -> float:
             # sheet_idx1: as sheet_idx0
             # props: list of str, see utils.load_rows_or_cols
 
-            cols1: Dict[str, Any] = load_rows_or_cols( *parse_idx(r["sheet_idx0"], xlworkbookr, xlworkbooke)
-                                                     , obj="column"
-                                                     , **r
-                                                     )
-            cols2: Dict[str, Any] = load_rows_or_cols( *parse_idx(r["sheet_idx1"], xlworkbookr, xlworkbooke)
-                                                     , obj="column"
-                                                     , **r
-                                                     )
+            cols1: Dict[str, Any] = load_rows_or_cols(*parse_idx(r["sheet_idx0"], xlworkbookr, xlworkbooke)
+                                                      , obj="column"
+                                                      , **r
+                                                      )
+            cols2: Dict[str, Any] = load_rows_or_cols(*parse_idx(r["sheet_idx1"], xlworkbookr, xlworkbooke)
+                                                      , obj="column"
+                                                      , **r
+                                                      )
             metric: bool = cols1 == cols2
             logger.debug("Assertion: %s[cols] == %s[cols] - %s", r["sheet_idx0"], r["sheet_idx1"], metric)
             #  }}} Check Row Properties # 
@@ -293,21 +311,21 @@ def compare_table(result: str, expected: str, **options) -> float:
             #   supported attributes: value & those supported by utils._read_cell_style
 
             sheet: Worksheet = _load_sheet(*parse_idx(r["sheet_idx"], xlworkbookr, xlworkbooke))
-            #data_frame: pd.DataFrame = _load_sheet(*parse_idx(r["sheet_idx"], pdworkbookr, pdworkbooke))
+            # data_frame: pd.DataFrame = _load_sheet(*parse_idx(r["sheet_idx"], pdworkbookr, pdworkbooke))
             cell: Cell = sheet[r["coordinate"]]
             metric: bool = True
             for prpt, rule in r["props"].items():
-                if prpt=="value":
+                if prpt == "value":
                     val = read_cell_value(*parse_idx(r["sheet_idx"], result, expected), r["coordinate"])
                 else:
                     val = _read_cell_style(prpt, cell)
 
                 metric = metric and _match_value_to_rule(val, rule)
 
-            logger.debug( "Assertion: %s[%s] :%s - %s"
-                        , r["sheet_idx"], r["coordinate"]
-                        , repr(r["props"]), metric
-                        )
+            logger.debug("Assertion: %s[%s] :%s - %s"
+                         , r["sheet_idx"], r["coordinate"]
+                         , repr(r["props"]), metric
+                         )
             #  }}} Check Cell Properties # 
 
         else:
@@ -320,6 +338,7 @@ def compare_table(result: str, expected: str, **options) -> float:
     return float(passes)
     #  }}} function compare_table # 
 
+
 def compare_csv(result: str, expected: str, **options) -> float:
     if result is None:
         return 0.
@@ -335,9 +354,10 @@ def compare_csv(result: str, expected: str, **options) -> float:
         result_lines = map(str.lower, result_lines)
         expected_lines = map(str.lower, expected_lines)
 
-    metric: bool = list(result_lines)==list(expected_lines)
+    metric: bool = list(result_lines) == list(expected_lines)
     return float(metric)
 
+
 if __name__ == '__main__':
     import datetime
     import sys
@@ -357,7 +377,8 @@ if __name__ == '__main__':
     stdout_handler.setLevel(logging.INFO)
     sdebug_handler.setLevel(logging.DEBUG)
 
-    formatter = logging.Formatter(fmt="\x1b[1;33m[%(asctime)s \x1b[31m%(levelname)s \x1b[32m%(module)s/%(lineno)d-%(processName)s\x1b[1;33m] \x1b[0m%(message)s")
+    formatter = logging.Formatter(
+        fmt="\x1b[1;33m[%(asctime)s \x1b[31m%(levelname)s \x1b[32m%(module)s/%(lineno)d-%(processName)s\x1b[1;33m] \x1b[0m%(message)s")
     file_handler.setFormatter(formatter)
     debug_handler.setFormatter(formatter)
     stdout_handler.setFormatter(formatter)
@@ -373,49 +394,49 @@ if __name__ == '__main__':
 
     path1 = "../../任务数据/LibreOffice Calc/Multiply_Time_Number.xlsx"
     path2 = "../../任务数据/LibreOffice Calc/Multiply_Time_Number_gold.xlsx"
-    rules = [ { "type": "check_cell"
-              , "sheet_idx": 0
-              , "coordinate": "E3"
-              , "props": { "value": { "method": "approx:0.001"
-                                    , "ref": 191.6667
-                                    }
-                         }
+    rules = [{"type": "check_cell"
+                 , "sheet_idx": 0
+                 , "coordinate": "E3"
+                 , "props": {"value": {"method": "approx:0.001"
+            , "ref": 191.6667
+                                       }
+                             }
               }
-            ]
-    print( compare_table( path1, path2
+             ]
+    print(compare_table(path1, path2
                         , rules=rules
                         )
-         )
-    print( compare_table( path2, path2
+          )
+    print(compare_table(path2, path2
                         , rules=rules
                         )
-         )
+          )
 
     # Row Properties
-    #path1 = "../../任务数据/LibreOffice Calc/Date_Budget_Variance_HideNA.xlsx"
-    #path2 = "../../任务数据/LibreOffice Calc/Date_Budget_Variance_HideNA_gold.xlsx"
-    #workbook: Workbook = openpyxl.load_workbook(filename=path1)
-    #worksheet: Worksheet = workbook.active
-    #for r_no, dms in worksheet.column_dimensions.items():
-        #print(r_no, type(r_no), type(dms), dms.hidden)
+    # path1 = "../../任务数据/LibreOffice Calc/Date_Budget_Variance_HideNA.xlsx"
+    # path2 = "../../任务数据/LibreOffice Calc/Date_Budget_Variance_HideNA_gold.xlsx"
+    # workbook: Workbook = openpyxl.load_workbook(filename=path1)
+    # worksheet: Worksheet = workbook.active
+    # for r_no, dms in worksheet.column_dimensions.items():
+    # print(r_no, type(r_no), type(dms), dms.hidden)
 
     # Conditional Formats
-    #import formulas
-    #path1 = "../../任务数据/LibreOffice Calc/Calendar_Highlight_Weekend_Days.xlsx"
-    #path2 = "../../任务数据/LibreOffice Calc/Calendar_Highlight_Weekend_Days_gold.xlsx"
-    #path3 = "../../任务数据/LibreOffice Calc/Calendar_Highlight_Weekend_Days_gold_test.xlsx"
-    #workbook: Workbook = openpyxl.load_workbook(filename=path2)
-    #worksheet: Worksheet = workbook.active
-    #print(worksheet.conditional_formatting)
-    #for itm in worksheet.conditional_formatting:
-        #print(itm.cells)
-        #for r in itm.rules:
-            #print( r.type, r.formula, r.dxf.font.color.rgb
-                 #, r.dxf.fill.fgColor.rgb, r.dxf.fill.bgColor.rgb
-                 #)
-            #condition = formulas.Parser().ast("=" + r.formula[0])[1].compile()
-            ##print(r.type, r.operator, r.dxfId, r.dxf)
-            #for r in itm.cells:
-                #for c in r.cells:
-                    #value = worksheet.cell(row=c[0], column=c[1]).value
-                    #print(value, condition(str(value)))
+    # import formulas
+    # path1 = "../../任务数据/LibreOffice Calc/Calendar_Highlight_Weekend_Days.xlsx"
+    # path2 = "../../任务数据/LibreOffice Calc/Calendar_Highlight_Weekend_Days_gold.xlsx"
+    # path3 = "../../任务数据/LibreOffice Calc/Calendar_Highlight_Weekend_Days_gold_test.xlsx"
+    # workbook: Workbook = openpyxl.load_workbook(filename=path2)
+    # worksheet: Worksheet = workbook.active
+    # print(worksheet.conditional_formatting)
+    # for itm in worksheet.conditional_formatting:
+    # print(itm.cells)
+    # for r in itm.rules:
+    # print( r.type, r.formula, r.dxf.font.color.rgb
+    # , r.dxf.fill.fgColor.rgb, r.dxf.fill.bgColor.rgb
+    # )
+    # condition = formulas.Parser().ast("=" + r.formula[0])[1].compile()
+    ##print(r.type, r.operator, r.dxfId, r.dxf)
+    # for r in itm.cells:
+    # for c in r.cells:
+    # value = worksheet.cell(row=c[0], column=c[1]).value
+    # print(value, condition(str(value)))

desktop_env/evaluators/metrics/utils.py

@@ -52,14 +52,17 @@ def load_sparklines(xlsx_file: str, sheet_name: str) -> Dict[str, str]:
     """
 
     # read xlsx
-    with zipfile.ZipFile(xlsx_file, "r") as z_f:
-        with z_f.open("xl/workbook.xml") as f:
-            workbook_database: _Element = lxml.etree.fromstring(f.read())
-            sheets: List[_Element] = _sheet_name_selector(workbook_database)
-            sheet_names: Dict[str, str] = {sh.get("name"): sh.get("sheetId") for sh in sheets}
-        with z_f.open("xl/worksheets/sheet{:}.xml".format(sheet_names[sheet_name])) as f:
-            sheet: _Element = lxml.etree.fromstring(f.read())
-            sparklines: List[_Element] = _sparklines_selector(sheet)
+    try:
+        with zipfile.ZipFile(xlsx_file, "r") as z_f:
+            with z_f.open("xl/workbook.xml") as f:
+                workbook_database: _Element = lxml.etree.fromstring(f.read())
+                sheets: List[_Element] = _sheet_name_selector(workbook_database)
+                sheet_names: Dict[str, str] = {sh.get("name"): sh.get("sheetId") for sh in sheets}
+            with z_f.open("xl/worksheets/sheet{:}.xml".format(sheet_names[sheet_name])) as f:
+                sheet: _Element = lxml.etree.fromstring(f.read())
+                sparklines: List[_Element] = _sparklines_selector(sheet)
+    except zipfile.BadZipFile:
+        return {}
 
     sparklines_dict: Dict[str, str] = {}
     for sp_l in sparklines:
@@ -158,29 +161,32 @@ def load_charts(xlsx_file: Workbook, sheet_name: str, **options) -> Dict[str, An
 _shared_str_selector = lxml.cssselect.CSSSelector("oo|sst>oo|si>oo|t", namespaces=_xlsx_ns_mapping)
 def read_cell_value(xlsx_file: str, sheet_name: str, coordinate: str) -> Any:
     #  read_cell_value {{{ # 
-    with zipfile.ZipFile(xlsx_file, "r") as z_f:
-        try:
-            with z_f.open("xl/sharedStrings.xml") as f:
-                shared_str_xml: _Element = lxml.etree.fromstring(f.read())
-                str_elements: List[_Element] = _shared_str_selector(shared_str_xml)
-                shared_strs: List[str] = [elm.text for elm in str_elements]
-        except:
-            logger.debug("Read shared strings error: %s", xlsx_file)
+    try:
+        with zipfile.ZipFile(xlsx_file, "r") as z_f:
+            try:
+                with z_f.open("xl/sharedStrings.xml") as f:
+                    shared_str_xml: _Element = lxml.etree.fromstring(f.read())
+                    str_elements: List[_Element] = _shared_str_selector(shared_str_xml)
+                    shared_strs: List[str] = [elm.text for elm in str_elements]
+            except:
+                logger.debug("Read shared strings error: %s", xlsx_file)
 
-        with z_f.open("xl/workbook.xml") as f:
-            workbook_database: _Element = lxml.etree.fromstring(f.read())
-            sheets: List[_Element] = _sheet_name_selector(workbook_database)
-            sheet_names: Dict[str, str] = {sh.get("name"): sh.get("sheetId") for sh in sheets}
+            with z_f.open("xl/workbook.xml") as f:
+                workbook_database: _Element = lxml.etree.fromstring(f.read())
+                sheets: List[_Element] = _sheet_name_selector(workbook_database)
+                sheet_names: Dict[str, str] = {sh.get("name"): sh.get("sheetId") for sh in sheets}
 
-        with z_f.open("xl/worksheets/sheet{:}.xml".format(sheet_names[sheet_name])) as f:
-            sheet: _Element = lxml.etree.fromstring(f.read())
-            cells: List[_Element] =\
-                    lxml.cssselect.CSSSelector( 'oo|row>oo|c[r="{:}"]'.format(coordinate)
-                                              , namespaces=_xlsx_ns_mapping
-                                              )(sheet)
-            if len(cells)==0:
-                return None
-            cell: _Element = cells[0]
+            with z_f.open("xl/worksheets/sheet{:}.xml".format(sheet_names[sheet_name])) as f:
+                sheet: _Element = lxml.etree.fromstring(f.read())
+                cells: List[_Element] =\
+                        lxml.cssselect.CSSSelector( 'oo|row>oo|c[r="{:}"]'.format(coordinate)
+                                                  , namespaces=_xlsx_ns_mapping
+                                                  )(sheet)
+                if len(cells)==0:
+                    return None
+                cell: _Element = cells[0]
+    except zipfile.BadZipFile:
+        return None
 
     cell: Dict[str, str] = xmltodict.parse( lxml.etree.tostring(cell, encoding="unicode")
                                           , process_namespaces=True

desktop_env/server/main.py

@@ -5,7 +5,7 @@ import shlex
 import subprocess
 from pathlib import Path
 from typing import Any, Optional
-from typing import List, Dict
+from typing import List, Dict, Tuple
 
 import Xlib
 import lxml.etree
@@ -16,7 +16,7 @@ from PIL import Image
 from Xlib import display, X
 from flask import Flask, request, jsonify, send_file, abort  # , send_from_directory
 from lxml.etree import _Element
-from pyatspi import Accessible, StateType
+from pyatspi import Accessible, StateType, STATE_SHOWING
 from pyatspi import Action as ATAction
 from pyatspi import Component, Document
 from pyatspi import Text as ATText
@@ -182,6 +182,11 @@ _accessibility_ns_map = {"st": "uri:deskat:state.at-spi.gnome.org"
 
 
 def _create_atspi_node(node: Accessible, depth: int, flag: Optional[str] = None) -> _Element:
+    if node.getRoleName() == "document spreadsheet":
+        flag = "calc"
+    if node.getRoleName() == "application" and node.name=="Thunderbird":
+        flag = "thunderbird"
+
     attribute_dict: Dict[str, Any] = {"name": node.name}
 
     #  States {{{ # 
@@ -247,6 +252,12 @@ def _create_atspi_node(node: Accessible, depth: int, flag: Optional[str] = None)
         # only text shown on current screen is available
         # attribute_dict["txt:text"] = text_obj.getText(0, text_obj.characterCount)
         text: str = text_obj.getText(0, text_obj.characterCount)
+        if flag=="thunderbird":
+            # appeard in thunderbird (uFFFC), "Object Replacement Character" in
+            # Unicode, "used as placeholder in text for an otherwise
+            # unspecified object; uFFFD is another "Replacement Character",
+            # just in case
+            text = text.replace("\ufffc", "").replace("\ufffd", "")
     #  }}} Text # 
 
     #  Selection {{{ # 
@@ -264,10 +275,22 @@ def _create_atspi_node(node: Accessible, depth: int, flag: Optional[str] = None)
     except NotImplementedError:
         pass
     else:
-        attribute_dict["{{{:}}}value".format(_accessibility_ns_map["val"])] = str(value.currentValue)
-        attribute_dict["{{{:}}}min".format(_accessibility_ns_map["val"])] = str(value.minimumValue)
-        attribute_dict["{{{:}}}max".format(_accessibility_ns_map["val"])] = str(value.maximumValue)
-        attribute_dict["{{{:}}}step".format(_accessibility_ns_map["val"])] = str(value.minimumIncrement)
+        try:
+            attribute_dict["{{{:}}}value".format(_accessibility_ns_map["val"])] = str(value.currentValue)
+        except:
+            pass
+        try:
+            attribute_dict["{{{:}}}min".format(_accessibility_ns_map["val"])] = str(value.minimumValue)
+        except:
+            pass
+        try:
+            attribute_dict["{{{:}}}max".format(_accessibility_ns_map["val"])] = str(value.maximumValue)
+        except:
+            pass
+        try:
+            attribute_dict["{{{:}}}step".format(_accessibility_ns_map["val"])] = str(value.minimumIncrement)
+        except:
+            pass
     #  }}} Value # 
 
     #  Action {{{ # 
@@ -303,18 +326,11 @@ def _create_atspi_node(node: Accessible, depth: int, flag: Optional[str] = None)
     if "text" in locals() and len(text) > 0:
         xml_node.text = text
 
-    # HACK: libreoffice has a problem -> billions of children for parent with RoleName "document spreadsheet"
-    #if node.getRoleName() == "document spreadsheet":
-        #return xml_node
     # HYPERPARAMETER
     if depth==50:
         logger.warning("Max depth reached")
         return xml_node
 
-    # HACK: libreoffice has a problem -> billions of children for parent with RoleName "document spreadsheet"
-    if node.getRoleName() == "document spreadsheet":
-        flag = "calc"
-
     if flag=="calc" and node_role_name=="table":
         # Maximum column: 1024 if ver<=7.3 else 16384
         # Maximum row: 104 8576

evaluation_examples/examples/dbt/0aa56709-3293-5849-ad47-e377f49fd3a0.json

@@ -1,119 +0,0 @@
-{
-    "id": "0aa56709-3293-5849-ad47-e377f49fd3a0",
-    "snapshot": "dbt",
-    "instruction": "Update models/schema.yml file to include some description fields. 1) The description fields include what each table is about. 2) For the primary key column of each table, add the description \"Primary key\". 3) Also insert one description for column customers.first_order_date that \"NULL when a customer has not yet placed an order.\". Then, use dbt docs command to generate the documentation for this dbt project and launch the documentation in a local website with port 8020.",
-    "source": [
-      "https://docs.getdbt.com/guides/manual-install?step=13"
-  ],
-    "config": [
-      {
-        "type": "download",
-        "parameters": {
-            "files": [
-                {
-                    "url": "https://drive.usercontent.google.com/download?id=1TZmmW7wYnWUQVMCH5JOBCxxdgm_QN-vz&export=download&authuser=0&confirm=t&uuid=43113538-c0e4-4e23-8a18-12b727b9f890&at=APZUnTWgSRXjNNZZIt8ni7rsTxoy:1705910001969",
-                    "path": "/home/user/projects/jaffle_shop.zip"
-                }
-            ]
-        }
-      },
-      {
-        "type": "execute",
-        "parameters": {
-            "command": ["/bin/bash", "-c", "unzip -oq /home/user/projects/jaffle_shop.zip -d /home/user/projects/ && rm /home/user/projects/jaffle_shop.zip && mkdir -p /home/user/.dbt"]
-        }
-      },
-      {
-        "type": "download",
-        "parameters": {
-            "files": [
-                {
-                    "url": "https://drive.usercontent.google.com/download?id=1xkXjiFhRNoiX_-0zikfdwzJ8UvdXEVAt&export=download&authuser=0&confirm=t&uuid=bc8d0eb8-99b3-4be7-b110-c0a56246d2d2&at=APZUnTUaPD_Z6ov6uMWDNf5rSy3-:1705978221799",
-                    "path": "/home/user/.dbt/profiles.yml"
-                }
-            ]
-        }
-      }
-    ],
-    "trajectory": "trajectories/",
-    "related_apps": [
-      "dbt",
-      "duckdb"
-    ],
-    "evaluator": {
-      "func": [
-        "check_yaml_file",
-        "check_dbt_command",
-        "check_dbt_command",
-        "check_dbt_command",
-        "check_dbt_command"
-      ],
-      "conj": "and",
-      "result": [
-        {
-          "type": "vm_file",
-          "path": "/home/user/projects/jaffle_shop/models/schema.yml",
-          "dest": "schema.yml"
-        },
-        {
-          "type": "vm_command_line",
-          "command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; if [ -s target/catalog.json ] && [ -s target/index.html ] ; then echo \"dbt docs generate succeed\" ; else echo \"dbt docs generate failed\" ; fi"]
-        },
-        {
-          "type": "vm_command_line",
-          "command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; curl -o index.html http://localhost:8020; if [ -s index.html ] ; then echo \"dbt docs serve succeed\" ; else echo \"dbt docs serve failed\" ; fi"]
-        },
-        {
-          "type": "vm_command_line",
-          "command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; diff index.html target/index.html > diff.log; if [ -s diff.log ] ; then echo \"dbt docs serve failed\" ; else echo \"dbt docs serve succeed\" ; fi ; rm -rf diff.log index.html"]
-        },
-        {
-          "type": "vm_command_line",
-          "command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; dbt docs generate"]
-        }
-      ],
-      "expected": [
-        {
-          "type": "rule",
-          "rules": [
-            ["not_null", ["models", ["name", "customers"], "description"], ""],
-            ["not_null", ["models", ["name", "stg_customers"], "description"], ""],
-            ["not_null", ["models", ["name", "stg_orders"], "description"], ""],
-            ["match", ["models", ["name", "customers"], "columns", ["name", "customer_id"], "description"], "Primary key"],
-            ["match", ["models", ["name", "stg_customers"], "columns", ["name", "customer_id"], "description"], "Primary key"],
-            ["match", ["models", ["name", "stg_orders"], "columns", ["name", "order_id"], "description"], "Primary key"],
-            ["match", ["models", ["name", "customers"], "columns", ["name", "first_order_date"], "description"], "NULL when a customer has not yet placed an order."]
-          ]
-        },
-        {
-          "type": "rule",
-          "rules": [
-            ["contains", "dbt docs generate succeed", ""],
-            ["excludes", "dbt docs generate failed", ""]
-          ]
-        },
-        {
-          "type": "rule",
-          "rules": [
-            ["contains", "dbt docs serve succeed", ""],
-            ["excludes", "dbt docs serve failed", ""]
-          ]
-        },
-        {
-          "type": "rule",
-          "rules": [
-            ["contains", "dbt docs serve succeed", ""],
-            ["excludes", "dbt docs serve failed", ""]
-          ]
-        },
-        {
-          "type": "rule",
-          "rules": [
-            ["excludes", "error", ""],
-            ["excludes", "Error", ""],
-            ["contains", "Catalog written to", ""]
-          ]
-        }
-      ]
-    }
-}

evaluation_examples/examples/dbt/492c2c87-b04a-544a-b5dd-eb808036bf85.json

@@ -1,89 +0,0 @@
-{
-    "id": "492c2c87-b04a-544a-b5dd-eb808036bf85",
-    "snapshot": "dbt",
-    "instruction": "Separate the single dbt model customers into separate ones and use the ``ref`` function to build models on top of others. 1) Create a new SQL file, models/stg_customers.sql, with the SQL from the customers common table expressions (CTE) in the original query. 2) Create a second new SQL file, models/stg_orders.sql, with the SQL from the orders CTE in the original query. These two stg models are materialized as view. 3) Edit the SQL in models/customers.sql file to refer two these two staged models and execute dbt run. Please ensure the running succeeds.",
-    "source": [
-      "https://docs.getdbt.com/guides/manual-install?step=11"
-  ],
-    "config": [
-        {
-          "type": "download",
-          "parameters": {
-              "files": [
-                  {
-                      "url": "https://drive.usercontent.google.com/download?id=1-Nn0leqKdVnA1gQ6s50wkGSa2xZauEUe&export=download&authuser=0&confirm=t&uuid=d7596b15-ae39-43cb-a940-03bee8a0c961&at=APZUnTXwp5xMjHpi5AYVF3Z_bfw2:1705582080467",
-                      "path": "/home/user/projects/jaffle_shop.zip"
-                  }
-              ]
-          }
-        },
-        {
-          "type": "execute",
-          "parameters": {
-              "command": ["/bin/bash", "-c", "unzip -oq /home/user/projects/jaffle_shop.zip -d /home/user/projects/ && rm /home/user/projects/jaffle_shop.zip && mkdir -p /home/user/.dbt"]
-          }
-        },
-        {
-          "type": "download",
-          "parameters": {
-              "files": [
-                  {
-                      "url": "https://drive.usercontent.google.com/download?id=1xkXjiFhRNoiX_-0zikfdwzJ8UvdXEVAt&export=download&authuser=0&confirm=t&uuid=bc8d0eb8-99b3-4be7-b110-c0a56246d2d2&at=APZUnTUaPD_Z6ov6uMWDNf5rSy3-:1705978221799",
-                      "path": "/home/user/.dbt/profiles.yml"
-                  }
-              ]
-          }
-        }
-    ],
-    "trajectory": "trajectories/",
-    "related_apps": [
-      "dbt",
-      "duckdb"
-    ],
-    "evaluator": {
-      "func": [
-        "check_database",
-        "check_dbt_command"
-      ],
-      "conj": "and",
-      "result": [
-        {
-          "type": "vm_file",
-          "path": "/home/user/projects/jaffle_shop/jaffle_shop.duckdb",
-          "dest": "jaffle_shop.duckdb"
-        },
-        {
-          "type": "vm_command_line",
-          "command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; dbt run"]
-        }
-      ],
-      "expected": [
-        {
-          "type": "cloud_file",
-          "path": "https://drive.usercontent.google.com/download?id=1rtRXtgKiwyWSVPCbDVkjYXh8AMni7t1W&export=download&authuser=0&confirm=t&uuid=6e4331f7-4e46-4894-910f-9af77450534d&at=APZUnTXySypK2OcEdOb8FVHQnbd7:1705583620817",
-          "dest": "gold_jaffle_shop.duckdb"
-        },
-        {
-          "type": "rule",
-          "rules": [
-            ["contains", "of 3 OK created sql view model .*\\.stg_customers", ""],
-            ["contains", "of 3 OK created sql view model .*\\.stg_orders", ""],
-            ["contains", "3 of 3 OK created sql table model .*\\.customers", ""],
-            ["contains", "Completed successfully", ""],
-            ["contains", "PASS=3", ""],
-            ["contains", "TOTAL=3", ""]
-          ]
-        }
-      ],
-      "options": [
-        {
-          "db_type": "duckdb", 
-          "check_type": [
-            "table-schema-content",
-            "view-schema-content"
-          ]
-        },
-        {}
-      ]
-    }
-}

evaluation_examples/examples/dbt/49f981ee-f793-5e27-9a53-083d66934ea1.json

@@ -1,126 +0,0 @@
-{
-    "id": "49f981ee-f793-5e27-9a53-083d66934ea1",
-    "snapshot": "dbt",
-    "instruction": "Add dbt test for the three models in jaffle_shop project. 1) Create a YAML file in the models directory, named models/schema.yml. 2) Ensure that customer_id and order_id are unique and not empty in related tables or views. Column status must be value from placed, shipped, completed, return_pending, returned. And customer_id in stg_orders is not null and is a foreign key in the referenced stg_customers. 3) Run dbt test, and confirm that all tests passed.",
-    "source": [
-      "https://docs.getdbt.com/guides/manual-install?step=12"
-  ],
-    "config": [
-      {
-        "type": "download",
-        "parameters": {
-            "files": [
-                {
-                    "url": "https://drive.usercontent.google.com/download?id=1QhPSdctnfYk0O5Tuo2NzldMDw0cKZi16&export=download&authuser=0&confirm=t&uuid=d68dcd34-304d-4e66-a215-99abd3954c24&at=APZUnTXihMvZPodIOVVxg3S0tUs4:1705587177507",
-                    "path": "/home/user/projects/jaffle_shop.zip"
-                }
-            ]
-        }
-      },
-      {
-        "type": "execute",
-        "parameters": {
-            "command": ["/bin/bash", "-c", "unzip -oq /home/user/projects/jaffle_shop.zip -d /home/user/projects/ && rm /home/user/projects/jaffle_shop.zip && mkdir -p /home/user/.dbt"]
-        }
-      },
-      {
-        "type": "download",
-        "parameters": {
-            "files": [
-                {
-                    "url": "https://drive.usercontent.google.com/download?id=1xkXjiFhRNoiX_-0zikfdwzJ8UvdXEVAt&export=download&authuser=0&confirm=t&uuid=bc8d0eb8-99b3-4be7-b110-c0a56246d2d2&at=APZUnTUaPD_Z6ov6uMWDNf5rSy3-:1705978221799",
-                    "path": "/home/user/.dbt/profiles.yml"
-                }
-            ]
-        }
-      }
-    ],
-    "trajectory": "trajectories/",
-    "related_apps": [
-      "dbt",
-      "duckdb"
-    ],
-    "evaluator": {
-      "func": "check_dbt_command",
-      "result": {
-        "type": "dbt_test_result",
-        "pre-processing": ["mv", "/home/user/projects/jaffle_shop/jaffle_shop.duckdb", "/home/user/Desktop/jaffle_shop.duckdb"],
-        "path": [
-          "https://drive.usercontent.google.com/download?id=1AZ6hCtbyN8Ypzf0e2nkkxivxdAmKNHZQ&export=download&authuser=0&confirm=t&uuid=dffccf1e-a42b-45e7-966d-72d289a1062e&at=APZUnTUsGPahiUmvgXxji9x9Ii7o:1705668446168",
-          "https://drive.usercontent.google.com/download?id=1z0hApNSqvs2oUwJiBQmFXrsomAxa1RhJ&export=download&authuser=0&confirm=t&uuid=0a2ba1be-1e15-4e8d-8458-0a661eaeef6f&at=APZUnTVZMe3y1OpU6ipsm0U6Ryb6:1705668535249",
-          "https://drive.usercontent.google.com/download?id=1OYsLQSYAdaAyu0sa6Y8IstmWz7wXneUN&export=download&authuser=0&confirm=t&uuid=33818f3c-a125-44d4-b9ba-7c5465976250&at=APZUnTXWFrNUtydf460ZWA-2jJrg:1705668651799",
-          "https://drive.usercontent.google.com/download?id=1KgGuJMeCXpIG2_TAKIJsT-YoIbSWswYp&export=download&authuser=0&confirm=t&uuid=4f2aadaa-2bc0-4ecf-b725-19c73375e370&at=APZUnTVt5cFNMzCsRxekJUaBPfhw:1705668748132",
-          "https://drive.usercontent.google.com/download?id=1OJ0xgAF1KqhovIvkimp3J5ZM-crfZU6g&export=download&authuser=0&confirm=t&uuid=48bcfac9-d40c-40c5-ac07-da9288a4ebea&at=APZUnTVUl1FaKtXy4oVpkYnGKIdt:1705669212860",
-          "https://drive.usercontent.google.com/download?id=10mr1jhdA52_bKOgoDgG0mAal1nzXkG21&export=download&authuser=0&confirm=t&uuid=5adde4ce-59d5-4da0-ada8-9c7df3e7434c&at=APZUnTVwJ4WDHSnfUv8yB08jr1b7:1705669280449",
-          "https://drive.usercontent.google.com/download?id=1_Dh9lwVDo8TfB0jg8QsdemRZ14r9O-uY&export=download&authuser=0&confirm=t&uuid=a8445889-8d13-4b66-88e7-4b675a943bec&at=APZUnTWrfFiAMJVLNFuteOctpjzs:1705669328524",
-          "https://drive.usercontent.google.com/download?id=1n4fLzFWj9dEqRdLOSwSB5SB8O3kHyEBZ&export=download&authuser=0&confirm=t&uuid=78cd4358-cc9e-4d2c-b74f-93b7760de4d3&at=APZUnTXbEj7FEQL5DEU5g4wsDFHs:1705669396969",
-          "https://drive.usercontent.google.com/download?id=1sjwB1aoi5UzbHQI-mb0hph2qd4qRKZR_&export=download&authuser=0&confirm=t&uuid=2fe93a52-d820-462e-9cce-36daa3a5ce30&at=APZUnTXPsOgN81L1RkenHg__oXUB:1705669474430",
-          "https://drive.usercontent.google.com/download?id=1ujsyfS7F3A6YCgZJ7N9ixcRlgNdAAdXJ&export=download&authuser=0&confirm=t&uuid=7e1b73c9-a14b-4d8b-800c-77f16ac5f897&at=APZUnTWBVbznnKO-zYj7UI6qk-qz:1705669501449",
-          "https://drive.usercontent.google.com/download?id=1tujswM-r4GKav5CpmciF-H3zYAdEq-uL&export=download&authuser=0&confirm=t&uuid=32253ad7-d343-4961-bc14-1e4cf4fee244&at=APZUnTWiRpT1pHF6Qp17y1VqcT7Z:1705676183100"
-        ],
-        "dest": "/home/user/projects/jaffle_shop/jaffle_shop.duckdb",
-        "command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; dbt test"],
-        "post-processing": ["/bin/bash", "-c", "mv /home/user/Desktop/jaffle_shop.duckdb /home/user/projects/jaffle_shop/jaffle_shop.duckdb"]
-      },
-      "expected": {
-          "type": "rule",
-          "rules": [
-            [
-              ["excludes", "Nothing to do", ""],
-              ["contains", "Completed successfully", ""],
-              ["contains", "ERROR=0", ""]
-            ],
-            [
-              ["excludes", "Nothing to do", ""],
-              ["excludes", "Completed successfully", ""],
-              ["excludes", "ERROR=0", ""]
-            ],
-            [
-              ["excludes", "Nothing to do", ""],
-              ["excludes", "Completed successfully", ""],
-              ["excludes", "ERROR=0", ""]
-            ],
-            [
-              ["excludes", "Nothing to do", ""],
-              ["excludes", "Completed successfully", ""],
-              ["excludes", "ERROR=0", ""]
-            ],
-            [
-              ["excludes", "Nothing to do", ""],
-              ["excludes", "Completed successfully", ""],
-              ["excludes", "ERROR=0", ""]
-            ],
-            [
-              ["excludes", "Nothing to do", ""],
-              ["excludes", "Completed successfully", ""],
-              ["excludes", "ERROR=0", ""]
-            ],
-            [
-              ["excludes", "Nothing to do", ""],
-              ["excludes", "Completed successfully", ""],
-              ["excludes", "ERROR=0", ""]
-            ],
-            [
-              ["excludes", "Nothing to do", ""],
-              ["excludes", "Completed successfully", ""],
-              ["excludes", "ERROR=0", ""]
-            ],
-            [
-              ["excludes", "Nothing to do", ""],
-              ["excludes", "Completed successfully", ""],
-              ["excludes", "ERROR=0", ""]
-            ],
-            [
-              ["excludes", "Nothing to do", ""],
-              ["excludes", "Completed successfully", ""],
-              ["excludes", "ERROR=0", ""]
-            ],
-            [
-              ["excludes", "Nothing to do", ""],
-              ["contains", "Completed successfully", ""],
-              ["contains", "ERROR=0", ""]
-            ]
-          ]
-        }
-    }
-}

evaluation_examples/examples/dbt/8aa9e870-b0c9-5417-be80-03154e83c7a3.json

@@ -1,102 +0,0 @@
-{
-    "id": "8aa9e870-b0c9-5417-be80-03154e83c7a3",
-    "snapshot": "dbt",
-    "instruction": "Create a dbt project called jaffle_shop under ~/projects and configure the connection to duckdb with path jaffle_shop/jaffle_shop.duckdb for the target dev and prod. Load data from jaffle-shop-data.zip into that database using dbt.",
-    "source": [
-      "https://docs.getdbt.com/docs/core/connect-data-platform/duckdb-setup#",
-      "https://docs.getdbt.com/guides/manual-install?step=3",
-      "https://docs.getdbt.com/reference/commands/seed"
-  ],
-    "config": [
-      {
-        "type": "download",
-        "parameters": {
-            "files": [
-                {
-                    "url": "https://drive.usercontent.google.com/download?id=18pbzQOnAZEZ2psFLWJJQIyLlQBSNRxWs&export=download&authuser=0&confirm=t&uuid=d8045faa-0f74-4c34-ab98-55ab41bcc2c9&at=APZUnTUoi3u-jMpwO8osMEll4gDr:1705548224569",
-                    "path": "/home/user/projects/jaffle-shop-data.zip"
-                }
-            ]
-        }
-      }
-    ],
-    "trajectory": "trajectories/",
-    "related_apps": [
-      "dbt",
-      "duckdb"
-    ],
-    "evaluator": {
-      "func": [
-        "check_yaml_file",
-        "check_dbt_command",
-        "check_database",
-        "check_dbt_command"
-      ],
-      "conj": "and",
-      "result": [
-        {
-          "type": "dbt_profiles",
-          "paths": [
-            "/home/user/.dbt/profiles.yml",
-            "/home/user/projects/jaffle_shop/profiles.yml",
-            "$DBT_PROFILES_DIR"
-          ],
-          "dest": "profiles.yml"
-        },
-        {
-          "type": "vm_command_line",
-          "command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; dbt debug"]
-        },
-        {
-          "type": "vm_file",
-          "path": "/home/user/projects/jaffle_shop/jaffle_shop.duckdb",
-          "dest": "jaffle_shop.duckdb"
-        },
-        {
-          "type": "vm_command_line",
-          "command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; dbt seed"]
-        }
-      ],
-      "expected": [
-        {
-          "type": "rule",
-          "rules": [
-            ["match", ["jaffle_shop", "outputs", "dev", "type"], "duckdb"],
-            ["match", ["jaffle_shop", "outputs", "prod", "type"], "duckdb"],
-            ["in", ["jaffle_shop", "outputs", "dev", "path"], ["jaffle_shop.duckdb", "/home/user/projects/jaffle_shop/jaffle_shop.duckdb"]],
-            ["in", ["jaffle_shop", "outputs", "prod", "path"], ["jaffle_shop.duckdb", "/home/user/projects/jaffle_shop/jaffle_shop.duckdb"]]
-          ]
-        },
-        {
-          "type": "rule",
-          "rules": [
-            ["contains", "OK connection ok", ""],
-            ["contains", "All checks passed", ""]
-          ]
-        },
-        {
-          "type": "cloud_file",
-          "path": "https://drive.usercontent.google.com/download?id=12xJuEcBxGqPHjJriUPnkuGRkw9HsZniJ&export=download&authuser=0&confirm=t&uuid=f2b7147f-5114-461d-a939-486750c19029&at=APZUnTVBf4QJMyDPk5BmfZzTecU5:1705548962476",
-          "dest": "gold_jaffle_shop.duckdb"
-        },
-        {
-          "type": "rule",
-          "rules": [
-            ["excludes", "Nothing to do", ""],
-            ["contains", "Completed successfully", ""],
-            ["contains", "PASS=3", ""],
-            ["contains", "TOTAL=3", ""]
-          ]
-        }
-      ],
-      "options": [
-        {},
-        {},
-        {
-          "db_type": "duckdb",
-          "check_type": ["table-schema-content"]
-        },
-        {}
-      ]
-    }
-  }

evaluation_examples/examples/dbt/8ff98608-8e0e-526e-9413-d744554ba708.json

@@ -1,86 +0,0 @@
-{
-    "id": "8ff98608-8e0e-526e-9413-d744554ba708",
-    "snapshot": "dbt",
-    "instruction": "Delete the dbt example models and write one dbt model \"customers\" from existing tables raw_customers and raw_orders. It contains columns customer_ids, first_name, last_name, first_order_date, most_recent_order_date and number_of_orders (default to 0). Set the materialization method as table and build it with dbt run to make it work.",
-    "source": [
-      "https://docs.getdbt.com/guides/manual-install?step=10",
-      "https://docs.getdbt.com/guides/manual-install?step=8",
-      "https://docs.getdbt.com/guides/manual-install?step=9"
-  ],
-    "config": [
-        {
-          "type": "download",
-          "parameters": {
-              "files": [
-                  {
-                      "url": "https://drive.usercontent.google.com/download?id=1iqimXoJUa2fd16I_qCZGHbVMHABJAh7D&export=download&authuser=0&confirm=t&uuid=65b73e29-0b30-415d-9928-d1ccf3be9880&at=APZUnTWCBNMro4gX8whSahDtx05S:1705563480661",
-                      "path": "/home/user/projects/jaffle_shop.zip"
-                  }
-              ]
-          }
-        },
-        {
-          "type": "execute",
-          "parameters": {
-              "command": ["/bin/bash", "-c", "unzip -oq /home/user/projects/jaffle_shop.zip -d /home/user/projects/ && rm /home/user/projects/jaffle_shop.zip && mkdir -p /home/user/.dbt"]
-          }
-        },
-        {
-          "type": "download",
-          "parameters": {
-              "files": [
-                  {
-                      "url": "https://drive.usercontent.google.com/download?id=1xkXjiFhRNoiX_-0zikfdwzJ8UvdXEVAt&export=download&authuser=0&confirm=t&uuid=bc8d0eb8-99b3-4be7-b110-c0a56246d2d2&at=APZUnTUaPD_Z6ov6uMWDNf5rSy3-:1705978221799",
-                      "path": "/home/user/.dbt/profiles.yml"
-                  }
-              ]
-          }
-        }
-    ],
-    "trajectory": "trajectories/",
-    "related_apps": [
-      "dbt",
-      "duckdb"
-    ],
-    "evaluator": {
-      "func": [
-        "check_database",
-        "check_dbt_command"
-      ],
-      "conj": "and",
-      "result": [
-        {
-          "type": "vm_file",
-          "path": "/home/user/projects/jaffle_shop/jaffle_shop.duckdb",
-          "dest": "jaffle_shop.duckdb"
-        },
-        {
-          "type": "vm_command_line",
-          "command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; dbt run"]
-        }
-      ],
-      "expected": [
-        {
-          "type": "cloud_file",
-          "path": "https://drive.usercontent.google.com/download?id=1BnTQOiDIKriWGhLRj7C8MPgfk4JH-uf9&export=download&authuser=0&confirm=t&uuid=cee74971-d66b-4da0-8cdd-cb9f1b9da8e7&at=APZUnTWI7GqSBLnQ_g-NNsS7kDxq:1705567576749",
-          "dest": "gold_jaffle_shop.duckdb"
-        },
-        {
-          "type": "rule",
-          "rules": [
-            ["contains", "1 of 1 OK created sql table model .*\\.customers", ""],
-            ["contains", "Completed successfully", ""],
-            ["contains", "PASS=1", ""],
-            ["contains", "TOTAL=1", ""]
-          ]
-        }
-      ],
-      "options": [
-        {
-          "db_type": "duckdb",
-          "check_type": ["table-schema-content"]
-        },
-        {}
-      ]
-    }
-  }

evaluation_examples/examples/libreoffice_calc/01b269ae-2111-4a07-81fd-3fcd711993b0.json

@@ -47,7 +47,7 @@
 					"command": [
 						"python",
 						"-c",
-						"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
+						"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
 					]
 				}
 			}

evaluation_examples/examples/libreoffice_calc/0bf05a7d-b28b-44d2-955a-50b41e24012a.json

@@ -7,7 +7,7 @@
     {
       "type": "download",
       "parameters": {
-        "file": [
+        "files": [
           {
             "url": "https://drive.usercontent.google.com/download?id=1DqGy5JRKOuZMRJ8O76d4Cds4WaRyz8V1&export=download&authuser=0&confirm=t&uuid=fa0694d1-2a77-4fd2-89d3-d9b854317823&at=APZUnTU9BxqG7E8tLZ104c0E8BEL:1705501029016",
             "path": "/home/user/Customers_New_7digit_Id.xlsx"
@@ -21,55 +21,55 @@
         "path": "/home/user/Customers_New_7digit_Id.xlsx"
       }
     }
-  },
+  ],
   "trajectory": "trajectories/0bf05a7d-b28b-44d2-955a-50b41e24012a",
   "related_apps": [
     "libreoffice calc"
   ],
   "evaluator": {
     "postconfig": [
-			{
-				"type": "activate_window",
-				"parameters": {
-					"window_name": "Customers_New_7digit_Id.xlsx - LibreOffice Calc",
-					"strict": true
-				}
-			},
-			{
-				"type": "sleep",
-				"parameters": {
-					"seconds": 0.5
-				}
-			},
-			{
-				"type": "execute",
-				"parameters": {
-					"command": [
-						"python",
-						"-c",
-						"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
-					]
-				}
-			},
-			{
-				"type": "sleep",
-				"parameters": {
-					"seconds": 0.5
-				}
-			},
-			{
-				"type": "execute",
-				"parameters": {
-					"command": [
-						"libreoffice",
-						"--convert-to",
-						"csv:Text - txt - csv (StarCalc):44,34,UTF-8,,,,false,true,true,false,false,1",
-						"--outdir",
-						"/home/user",
-						"/home/user/Customers_New_7digit_Id.xlsx"
-					]
-				}
-			}
+      {
+        "type": "activate_window",
+        "parameters": {
+          "window_name": "Customers_New_7digit_Id.xlsx - LibreOffice Calc",
+          "strict": true
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python",
+            "-c",
+            "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
+          ]
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "libreoffice",
+            "--convert-to",
+            "csv:Text - txt - csv (StarCalc):44,34,UTF-8,,,,false,true,true,false,false,1",
+            "--outdir",
+            "/home/user",
+            "/home/user/Customers_New_7digit_Id.xlsx"
+          ]
+        }
+      }
     ],
     "func": "compare_table",
     "result": {

evaluation_examples/examples/libreoffice_calc/0cecd4f3-74de-457b-ba94-29ad6b5dafb6.json

@@ -1,7 +1,7 @@
 {
   "id": "0cecd4f3-74de-457b-ba94-29ad6b5dafb6",
   "snapshot": "libreoffice_calc",
-  "instruction": "Copy Sheet1 and insert it before Sheet2 and rename it as Sheet1.bak",
+  "instruction": "Rename \"Sheet 1\" to \"LARS Resources\". Then make a copy of it. Place the copy before \"Sheet 2\" and rename it by appending a suffix \"(Backup)\", concatenated by a which space. And Also rename \"Sheet2\" by appending the suffix \"(Offline)\".",
   "source": "https://www.libreofficehelp.com/add-insert-delete-copy-move-rename-a-worksheet-in-libreoffice-calc/",
   "config": [
     {
@@ -9,7 +9,7 @@
       "parameters": {
         "files": [
           {
-            "url": "https://drive.usercontent.google.com/download?id=1ejNXBNOZtn64ugmvXot21pOEjx5xa-I5&export=download&authuser=0&confirm=t&uuid=61aa93e2-03f7-4b28-8e4a-cdff16a642f7&at=APZUnTVgPAHHfXaEjfKau5CDY1_K:1703509323791",
+            "url": "https://drive.usercontent.google.com/download?id=1emE6hfKYJpsKNoJ7fmSKCug9pZ9Rv0fx&export=download&authuser=0&confirm=t&uuid=b84ee976-0dab-4a32-a99d-56873f016eec&at=APZUnTVs-ZH77mNQ9-YCLqj2H5xh:1706261078154",
             "path": "/home/user/copy_sheet_insert.xlsx"
           }
         ]
@@ -47,7 +47,7 @@
           "command": [
             "python",
             "-c",
-            "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
+            "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
           ]
         }
       }
@@ -60,7 +60,7 @@
     },
     "expected": {
       "type": "cloud_file",
-      "path": "https://drive.usercontent.google.com/download?id=15SjZNQtdt55SW6FIpi2JmWmVOpoZGjEB&export=download&authuser=0&confirm=t&uuid=182ac3dc-e0e9-4d30-b800-b842d5fcd665&at=APZUnTXVlvE4vv1-QO7HKqQOrikJ:1705850636082",
+      "path": "https://drive.usercontent.google.com/download?id=1u0mYkDejNIFWfSOvQCi2z6nXTinUqhGG&export=download&authuser=0&confirm=t&uuid=d9036b53-dc50-4451-9e81-a5327b846c29&at=APZUnTU6PV42c83Ug6CGUQ6zx0mA:1706261109913",
       "dest": "copy_sheet_insert_gold.xlsx"
     },
     "options": {
@@ -77,6 +77,11 @@
           "type": "sheet_data",
           "sheet_idx0": 1,
           "sheet_idx1": "EI1"
+        },
+        {
+          "type": "sheet_data",
+          "sheet_idx0": 2,
+          "sheet_idx1": "EI2"
         }
       ]
     }

evaluation_examples/examples/libreoffice_calc/1334ca3e-f9e3-4db8-9ca7-b4c653be7d17.json

@@ -7,7 +7,7 @@
 		{
 			"type": "download",
 			"parameters": {
-				"file": [
+				"files": [
 					{
 						"url": "https://drive.usercontent.google.com/download?id=1Wkepf_vic9o7CZFiosZ4jZT_Hy2WbRPZ&export=download&authuser=0&confirm=t&uuid=bc2ce901-a6bb-433f-bcce-cbe42d813f18&at=APZUnTVQcGTcXjwqenmtSH6IMFkM:1703858853235",
 						"path": "/home/user/Zoom_Out_Oversized_Cells.xlsx"
@@ -47,7 +47,7 @@
 					"command": [
 						"python",
 						"-c",
-						"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
+						"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
 					]
 				}
 			}

evaluation_examples/examples/libreoffice_calc/21df9241-f8d7-4509-b7f1-37e501a823f7.json

@@ -7,7 +7,7 @@
 		{
 			"type": "download",
 			"parameters": {
-				"file": [
+				"files": [
 					{
 						"url": "https://drive.usercontent.google.com/download?id=16PowrQA4E71xUoJmpXPHy0dr9HBcTRmo&export=download&authuser=0&confirm=t&uuid=9a6265f7-585c-4cf8-b321-3b859aec1e68&at=APZUnTWzzOw85wws0ojXNPsIwnjE:1703858126178",
 						"path": "/home/user/Represent_in_millions_billions.xlsx"
@@ -47,7 +47,7 @@
 					"command": [
 						"python",
 						"-c",
-						"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
+						"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
 					]
 				}
 			},

evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json

@@ -47,7 +47,7 @@
           "command": [
             "python",
             "-c",
-            "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
+            "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
           ]
         }
       }

evaluation_examples/examples/libreoffice_calc/347ef137-7eeb-4c80-a3bb-0951f26a8aff.json

@@ -47,7 +47,7 @@
           "command": [
             "python",
             "-c",
-            "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
+            "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
           ]
         }
       }

evaluation_examples/examples/libreoffice_calc/357ef137-7eeb-4c80-a3bb-0951f26a8aff.json

@@ -47,7 +47,7 @@
 					"command": [
 						"python",
 						"-c",
-						"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
+						"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
 					]
 				}
 			}

evaluation_examples/examples/libreoffice_calc/37608790-6147-45d0-9f20-1137bb35703d.json

@@ -47,7 +47,7 @@
           "command": [
             "python",
             "-c",
-            "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
+            "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
           ]
         }
       }

evaluation_examples/examples/libreoffice_calc/39aa4e37-dc91-482e-99af-132a612d40f3.json

@@ -47,7 +47,7 @@
                     "command": [
                         "python",
                         "-c",
-                        "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
+                        "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
                     ]
                 }
             }

evaluation_examples/examples/libreoffice_calc/3aaa4e37-dc91-482e-99af-132a612d40f3.json

@@ -7,7 +7,7 @@
 		{
 			"type": "download",
 			"parameters": {
-				"file": [
+				"files": [
 					{
 						"url": "https://drive.usercontent.google.com/download?id=17sNGNFpZtmwuz74Pid2WwWL_rgTkgTg0&export=download&authuser=0&confirm=t&uuid=18d3601d-e329-4525-bd11-633c678601d6&at=APZUnTVY9kvNoMeETP6HVRynDHqq:1706012082827",
 						"path": "/home/user/Export_Calc_to_CSV.xlsx"

evaluation_examples/examples/libreoffice_calc/4188d3a4-077d-46b7-9c86-23e1a036f6c1.json

@@ -26,7 +26,7 @@
 	"related_apps": [
 		"libreoffice_calc"
 	],
-	"evaluators": {
+	"evaluator": {
     "postconfig": [
       {
         "type": "activate_window",
@@ -47,7 +47,7 @@
           "command": [
             "python",
             "-c",
-            "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
+            "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
           ]
         }
       }

evaluation_examples/examples/libreoffice_calc/4e6fcf72-daf3-439f-a232-c434ce416af6.json

@@ -47,7 +47,7 @@
 					"command": [
 						"python",
 						"-c",
-						"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
+						"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
 					]
 				}
 			}

evaluation_examples/examples/libreoffice_calc/4f07fbe9-70de-4927-a4d5-bb28bc12c52c.json

@@ -7,7 +7,7 @@
 		{
 			"type": "download",
 			"parameters": {
-				"file": [
+				"files": [
 					{
 						"url": "https://drive.usercontent.google.com/download?id=1LHxVvEpLI7kp0Iiy8K74gwkoGiwcLeYP&export=download&authuser=0&confirm=t&uuid=690287ee-d413-46e7-9b01-c56c12e445ff&at=APZUnTVCSd_ajhMGWpEgLHiExfbf:1704199487820",
 						"path": "/home/user/Padding_Decimals_In_Formular.xlsx"
@@ -47,7 +47,7 @@
 					"command": [
 						"python",
 						"-c",
-						"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
+						"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
 					]
 				}
 			},
@@ -92,8 +92,9 @@
 			],
 			"dest": [
 				"Padding_Decimals_In_Formular_gold.xlsx",
-				"Padding_Decimals_In_Formular_gold-Sheet1.xlsx"
-			]
+				"Padding_Decimals_In_Formular_gold-Sheet1.csv"
+			],
+			"multi": true
 		},
 		"options": {
 			"rules": [

evaluation_examples/examples/libreoffice_calc/51b11269-2ca8-4b2a-9163-f21758420e78.json

@@ -47,7 +47,7 @@
 					"command": [
 						"python",
 						"-c",
-						"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
+						"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
 					]
 				}
 			}

evaluation_examples/examples/libreoffice_calc/6054afcb-5bab-4702-90a0-b259b5d3217c.json

@@ -47,7 +47,7 @@
 					"command": [
 						"python",
 						"-c",
-						"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
+						"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
 					]
 				}
 			}

evaluation_examples/examples/libreoffice_calc/6e99a1ad-07d2-4b66-a1ce-ece6d99c20a5.json

@@ -47,7 +47,7 @@
 					"command": [
 						"python",
 						"-c",
-						"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
+						"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
 					]
 				}
 			},
@@ -88,7 +88,7 @@
 			"type": "vm_file",
 			"path": [
 				"/home/user/Keep_Two_decimal_points.xlsx",
-				"/home/user/Keep_Two_decimal_points-Sheet1.xlsx"
+				"/home/user/Keep_Two_decimal_points-Sheet1.csv"
 			],
 			"dest": [
 				"Keep_Two_decimal_points.xlsx",

evaluation_examples/examples/libreoffice_calc/7a4e4bc8-922c-4c84-865c-25ba34136be1.json

@@ -47,7 +47,7 @@
           "command": [
             "python",
             "-c",
-            "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
+            "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
           ]
         }
       }

evaluation_examples/examples/libreoffice_calc/7e429b8d-a3f0-4ed0-9b58-08957d00b127.json

@@ -47,7 +47,7 @@
 					"command": [
 						"python",
 						"-c",
-						"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
+						"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
 					]
 				}
 			}

evaluation_examples/examples/libreoffice_calc/7efeb4b1-3d19-4762-b163-63328d66303b.json

@@ -47,7 +47,7 @@
           "command": [
             "python",
             "-c",
-            "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
+            "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
           ]
         }
       }

evaluation_examples/examples/libreoffice_calc/8b1ce5f2-59d2-4dcc-b0b0-666a714b9a14.json

@@ -47,7 +47,7 @@
 					"command": [
 						"python",
 						"-c",
-						"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
+						"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
 					]
 				}
 			}
@@ -74,7 +74,7 @@
 					"type": "style",
 					"sheet_idx0": 0,
 					"sheet_idx1": "EI0",
-					"props": "bgcolor"
+					"props": ["bgcolor"]
 				}
 			]
 		}

evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json

@@ -7,7 +7,7 @@
 		{
 			"type": "download",
 			"parameters": {
-				"file": [
+				"files": [
 					{
 						"url": "https://drive.usercontent.google.com/download?id=1uT0axjo9lwkKu6hYVnsAL2FCrdH0DLUv&export=download&authuser=0&confirm=t&uuid=e7da6304-9c7a-4862-8a30-9f2284b843da&at=APZUnTVNHThpAZJmF6IuPckFvslw:1704187618838",
 						"path": "/home/user/Set_Decimal_Separator_Dot.xlsx"
@@ -47,7 +47,7 @@
 					"command": [
 						"python",
 						"-c",
-						"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
+						"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
 					]
 				}
 			},

evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json.json

@@ -1,85 +0,0 @@
-{
-	"id": "a01fbce3-2793-461f-ab86-43680ccbae25",
-	"snapshot": "libreoffice_calc",
-	"instruction": "I need to set the decimal separator as a comma (,) for localized data representation and clarity in visualization. Can you assist with this?",
-	"source": "https://superuser.com/questions/1250677/how-to-set-decimal-separator-in-libre-office-calc",
-	"config": [],
-	"trajectory": "trajectories/a01fbce3-2793-461f-ab86-43680ccbae25",
-	"related_apps": [
-		"libreoffice_calc"
-	],
-	"evaluator": {
-		"postconfig": [
-			{
-				"type": "activate_window",
-				"parameters": {
-					"window_name": "Set_Decimal_Separator_Dot.xlsx - LibreOffice Calc",
-					"strict": true
-				}
-			},
-			{
-				"type": "sleep",
-				"parameters": {
-					"seconds": 0.5
-				}
-			},
-			{
-				"type": "execute",
-				"parameters": {
-					"command": [
-						"python",
-						"-c",
-						"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
-					]
-				}
-			},
-			{
-				"type": "sleep",
-				"parameters": {
-					"seconds": 0.5
-				}
-			},
-			{
-				"type": "execute",
-				"parameters": {
-					"command": [
-						"libreoffice",
-						"--convert-to",
-						"csv:Text - txt - csv (StarCalc):44,34,UTF-8,,,,false,true,true",
-						"--outdir",
-						"/home/user",
-						"/home/user/Set_Decimal_Separator_Dot.xlsx"
-					]
-				}
-			}
-		],
-		"func": "compare_table",
-		"result": {
-			"type": "vm_file",
-			"path": [
-				"/home/user/Set_Decimal_Separator_Dot.xlsx",
-				"/home/user/Set_Decimal_Separator_Dot.csv"
-			],
-			"dest": [
-				"Set_Decimal_Separator_Dot.xlsx",
-				"Set_Decimal_Separator_Dot.csv"
-			],
-			"multi": true
-		},
-		"expected": {
-			"type": "cloud_file",
-			"path": [
-				"https://drive.usercontent.google.com/download?id=15O0l5fxVi1JX_12KOLfbxWPHjXPZPon5&export=download&authuser=0&confirm=t&uuid=395e6c57-11a7-4b33-af4c-98ff2390742b&at=APZUnTVKcrUGrjRfBEwT_AD53Cmn:1705497822975",
-				"https://drive.usercontent.google.com/download?id=1rKDWcovxw4Qtd3RHs7M5p_QqryI0SQO3&export=download&authuser=0&confirm=t&uuid=eb6ffb6d-f7c2-44d8-ad77-db6c0aaf5cc7&at=APZUnTWr2VxrJPiiKVMdFd0IykrR:1705497846507"
-			],
-			"dest": [
-				"Set_Decimal_Separator_Dot_gold.xlsx",
-				"Set_Decimal_Separator_Dot_gold.csv"
-			],
-			"multi": true
-		},
-		"options": {
-			"as_shown": true
-		}
-	}
-}

evaluation_examples/examples/libreoffice_calc/a9f325aa-8c05-4e4f-8341-9e4358565f4f.json

@@ -47,7 +47,7 @@
           "command": [
             "python",
             "-c",
-            "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
+            "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
           ]
         }
       }

evaluation_examples/examples/libreoffice_calc/aa3a8974-2e85-438b-b29e-a64df44deb4b.json

@@ -7,7 +7,7 @@
 		{
 			"type": "download",
 			"parameters": {
-				"file": [
+				"files": [
 					{
 						"url": "https://drive.usercontent.google.com/download?id=1O4bw7jEsVdFGeGeSX8hDjsvIbozV38sd&export=download&authuser=0&confirm=t&uuid=b6ceade0-e9c3-47bf-8c40-fef77b5ea1f1&at=APZUnTUUYaEx0Y_lAESeK1DfQZw6:1704179724348",
 						"path": "/home/user/Resize_Cells_Fit_Page.xlsx"

evaluation_examples/examples/libreoffice_calc/abed40dc-063f-4598-8ba5-9fe749c0615d.json

@@ -47,7 +47,7 @@
 					"command": [
 						"python",
 						"-c",
-						"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
+						"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
 					]
 				}
 			}

evaluation_examples/examples/libreoffice_calc/d681960f-7bc3-4286-9913-a8812ba3261a.json

@@ -47,7 +47,7 @@
           "command": [
             "python",
             "-c",
-            "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
+            "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
           ]
         }
       }

evaluation_examples/examples/libreoffice_calc/eb03d19a-b88d-4de4-8a64-ca0ac66f426b.json

@@ -47,7 +47,7 @@
           "command": [
             "python",
             "-c",
-            "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
+            "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
           ]
         }
       }

evaluation_examples/examples/libreoffice_calc/ecb0df7a-4e8d-4a03-b162-053391d3afaf.json

@@ -47,7 +47,7 @@
           "command": [
             "python",
             "-c",
-            "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
+            "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
           ]
         }
       }

evaluation_examples/examples/libreoffice_calc/f9584479-3d0d-4c79-affa-9ad7afdd8850.json

@@ -47,7 +47,7 @@
           "command": [
             "python",
             "-c",
-            "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
+            "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
           ]
         }
       }

evaluation_examples/examples/libreoffice_impress/0f84bef9-9790-432e-92b7-eece357603fb.json

@@ -3,7 +3,25 @@
   "snapshot": "libreoffice_impress",
   "instruction": "On it Whenever I launch a LibreOffice Impress, it uses both screens, one for current slide and next slide and another for actual presentation. What I want is to use only one monitor which shows presentation. I dont want the screen with Current slide and Next slide so that it can be used for other purposes. How should I achieve this?",
   "source": "https://stackoverflow.com/questions/29036788/how-to-disable-libreoffice-impress-to-use-multiple-display",
-  "config": [],
+  "config": [
+    {
+      "type": "download",
+      "parameters": {
+        "files": [
+          {
+            "url": "https://drive.usercontent.google.com/download?id=1qKOdf1Wx9nGtk_3l7hjZ9gXWFzWgsyoH&export=download&authuser=0&confirm=t&uuid=0bceb604-af00-4940-a137-8dd00512d060&at=APZUnTUlTutATfe49vsbBrobLPAG:1706370599333",
+            "path": "Desktop/multimedia_classroom_podium-2020.pptx"
+          }
+        ]
+      }
+    },
+    {
+      "type": "open",
+      "parameters": {
+        "path": "Desktop/multimedia_classroom_podium-2020.pptx"
+      }
+    }
+  ],
   "trajectory": "trajectories/",
   "related_apps": [
     "libreoffice_impress"
@@ -16,4 +34,4 @@
       "dest": "registrymodifications.xcu"
     }
   }
-}
+}

evaluation_examples/examples/libreoffice_impress/21760ecb-8f62-40d2-8d85-0cee5725cb72.json

@@ -3,10 +3,89 @@
   "snapshot": "libreoffice_impress",
   "instruction": "Could you help me add silde transition \"dissolve\" to my first page?",
   "source": "https://www.libreofficehelp.com/add-animations-transitions-libreoffice-impress-slides/",
-  "config": [],
+  "config": [
+    {
+      "type": "download",
+      "parameters": {
+        "files": [
+          {
+            "url": "https://drive.usercontent.google.com/download?id=1A9nNj3hWBB0Bnr2Cf7zHs3WFiTKdEeNt&export=download&authuser=0&confirm=t&uuid=7c0f29fd-1f1a-4d48-b717-5c677373e615&at=APZUnTX9YQH_-UbVyFRLblwZ3rAn:1706530787330",
+            "path": "Desktop/Ch4 Video Effect.pptx"
+          }
+        ]
+      }
+    },
+    {
+      "type": "open",
+      "parameters": {
+        "path": "Desktop/Ch4 Video Effect.pptx"
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 0.5
+      }
+    },
+    {
+      "type": "execute",
+      "parameters": {
+        "command": [
+          "python",
+          "-c",
+          "import pyautogui; import time;  time.sleep(4); pyautogui.doubleClick(x=960, y=540); time.sleep(0.5);pyautogui.mouseDown(); pyautogui.mouseUp(); time.sleep(0.5);"
+        ]
+      }
+    }
+  ],
   "trajectory": "trajectories/",
   "related_apps": [
-    ""
+    "libreoffice_impress"
   ],
-  "evaluator": "evaluation_dir"
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "activate_window",
+        "parameters": {
+          "window_name": "Ch4 Video Effect.pptx - LibreOffice Impress",
+          "strict": true
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python",
+            "-c",
+            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);"
+          ]
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      }
+    ],
+    "func": "check_transition",
+    "expected": {
+      "type": "rule",
+      "rules": {
+        "slide_idx": 0,
+        "transition_type": "dissolve"
+      }
+    },
+    "result": {
+      "type": "vm_file",
+      "path": "Desktop/Ch4 Video Effect.pptx",
+      "dest": "Ch4 Video Effect.pptx"
+    }
+  }
 }

evaluation_examples/examples/libreoffice_impress/3b27600c-3668-4abd-8f84-7bcdebbccbdb.json

@@ -28,35 +28,35 @@
   ],
   "evaluator": {
     "postconfig": [
-			{
-				"type": "activate_window",
-				"parameters": {
-					"window_name": "lec17-gui-events.pptx - LibreOffice Impress",
-					"strict": true
-				}
-			},
-			{
-				"type": "sleep",
-				"parameters": {
-					"seconds": 0.5
-				}
-			},
-			{
-				"type": "execute",
-				"parameters": {
-					"command": [
-						"python",
-						"-c",
-						"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
-					]
-				}
-			},
-			{
-				"type": "sleep",
-				"parameters": {
-					"seconds": 0.5
-				}
-			}
+      {
+        "type": "activate_window",
+        "parameters": {
+          "window_name": "lec17-gui-events.pptx - LibreOffice Impress",
+          "strict": true
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python",
+            "-c",
+            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);"
+          ]
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      }
     ],
     "func": "evaluate_presentation_fill_to_rgb_distance",
     "expected": {

evaluation_examples/examples/libreoffice_impress/455d3c66-7dc6-4537-a39a-36d3e9119df7.json

@@ -27,6 +27,37 @@
     "libreoffice_impress"
   ],
   "evaluator": {
+    "postconfig": [
+      {
+        "type": "activate_window",
+        "parameters": {
+          "window_name": "wssf-project-plan-on-a-page.pptx - LibreOffice Impress",
+          "strict": true
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python",
+            "-c",
+            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);"
+          ]
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      }
+    ],
     "func": "compare_images",
     "expected": {
       "type": "cloud_file",

evaluation_examples/examples/libreoffice_impress/550ce7e7-747b-495f-b122-acdc4d0b8e54.json

@@ -9,7 +9,7 @@
       "parameters": {
         "files": [
           {
-            "url": "https://drive.usercontent.google.com/download?id=1fw0baVZ15s0r1WGEBftgED2H0ljZgYtu&export=download&authuser=0&confirm=t&uuid=df03788a-81ef-4e55-b33a-2fba7ab28cb8&at=APZUnTXPb-sm88KNwmNeugbhPrzx:17052529805399",
+            "url": "https://drive.usercontent.google.com/download?id=126TZ3vOBN2XAfdEmvJVBaa4qs_MZlgN7&export=download&authuser=0&confirm=t&uuid=acea6223-64ff-44cf-9d83-fe1a1640d374&at=APZUnTVBY9Kkm1Xo3ZU-Fe8hoOqC:1706521734182",
             "path": "Desktop/New_Club_Spring_2018_Training.pptx"
           }
         ]
@@ -20,6 +20,22 @@
       "parameters": {
         "path": "Desktop/New_Club_Spring_2018_Training.pptx"
       }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 0.5
+      }
+    },
+    {
+      "type": "execute",
+      "parameters": {
+        "command": [
+          "python",
+          "-c",
+          "import pyautogui; import time;  time.sleep(4); pyautogui.doubleClick(x=200, y=650); time.sleep(0.5);pyautogui.mouseDown(); pyautogui.mouseUp(); time.sleep(0.5);"
+        ]
+      }
     }
   ],
   "trajectory": "trajectories/",
@@ -28,49 +44,49 @@
   ],
   "evaluator": {
     "postconfig": [
-			{
-				"type": "activate_window",
-				"parameters": {
-					"window_name": "New_Club_Spring_2018_Training.pptx - LibreOffice Impress",
-					"strict": true
-				}
-			},
-			{
-				"type": "sleep",
-				"parameters": {
-					"seconds": 0.5
-				}
-			},
-			{
-				"type": "execute",
-				"parameters": {
-					"command": [
-						"python",
-						"-c",
-						"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
-					]
-				}
-			},
-			{
-				"type": "sleep",
-				"parameters": {
-					"seconds": 0.5
-				}
-			}
-    ],
-    "func": "check_strikethrough",
-    "expected": {
-      "type": "rule",
-      "rules": {
-        "slide_index_s": [4],
-        "shape_index_s": [1],
-        "paragraph_index_s": [1, 2]
+      {
+        "type": "activate_window",
+        "parameters": {
+          "window_name": "New_Club_Spring_2018_Training.pptx - LibreOffice Impress",
+          "strict": true
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python",
+            "-c",
+            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);"
+          ]
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
       }
+    ],
+    "func": "compare_pptx_files",
+    "expected": {
+      "type": "cloud_file",
+      "path": "https://drive.usercontent.google.com/download?id=1FNoJ8nGSNbsf9rhj9He1Lw17RCe2LJHD&export=download&authuser=0&confirm=t&uuid=8b876496-91c1-4209-be53-34437901b613&at=APZUnTUpOTXFXdPMATBQXSgJHsk7:1706521731233",
+      "dest": "New_Club_Spring_2018_Training_Gold.pptx"
     },
     "result": {
       "type": "vm_file",
       "path": "Desktop/New_Club_Spring_2018_Training.pptx",
       "dest": "New_Club_Spring_2018_Training.pptx"
-    }
+    },
+    "options": {
+            "examine_shape": false
+        }
   }
 }

evaluation_examples/examples/libreoffice_impress/5d901039-a89c-4bfb-967b-bf66f4df075e.json

@@ -28,35 +28,35 @@
   ],
   "evaluator": {
     "postconfig": [
-			{
-				"type": "activate_window",
-				"parameters": {
-					"window_name": "CPD_Background_Investigation_Process.pptx - LibreOffice Impress",
-					"strict": true
-				}
-			},
-			{
-				"type": "sleep",
-				"parameters": {
-					"seconds": 0.5
-				}
-			},
-			{
-				"type": "execute",
-				"parameters": {
-					"command": [
-						"python",
-						"-c",
-						"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
-					]
-				}
-			},
-			{
-				"type": "sleep",
-				"parameters": {
-					"seconds": 0.5
-				}
-			}
+      {
+        "type": "activate_window",
+        "parameters": {
+          "window_name": "CPD_Background_Investigation_Process.pptx - LibreOffice Impress",
+          "strict": true
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python",
+            "-c",
+            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);"
+          ]
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      }
     ],
     "func": "check_image_stretch_and_center",
     "expected": {

evaluation_examples/examples/libreoffice_impress/9ec204e4-f0a3-42f8-8458-b772a6797cab.json

@@ -28,35 +28,35 @@
   ],
   "evaluator": {
     "postconfig": [
-			{
-				"type": "activate_window",
-				"parameters": {
-					"window_name": "MLA_Workshop_061X_Works_Cited.pptx - LibreOffice Impress",
-					"strict": true
-				}
-			},
-			{
-				"type": "sleep",
-				"parameters": {
-					"seconds": 0.5
-				}
-			},
-			{
-				"type": "execute",
-				"parameters": {
-					"command": [
-						"python",
-						"-c",
-						"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
-					]
-				}
-			},
-			{
-				"type": "sleep",
-				"parameters": {
-					"seconds": 0.5
-				}
-			}
+      {
+        "type": "activate_window",
+        "parameters": {
+          "window_name": "MLA_Workshop_061X_Works_Cited.pptx - LibreOffice Impress",
+          "strict": true
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python",
+            "-c",
+            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);"
+          ]
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      }
     ],
     "func": "compare_pptx_files",
     "expected": {
@@ -66,7 +66,7 @@
     },
     "result": {
       "type": "vm_file",
-      "path": "/home/user/Desktop/MLA_Workshop_061X_Works_Cited.pptx",
+      "path": "Desktop/MLA_Workshop_061X_Works_Cited.pptx",
       "dest": "MLA_Workshop_061X_Works_Cited.pptx"
     }
   }

evaluation_examples/examples/libreoffice_impress/af23762e-2bfd-4a1d-aada-20fa8de9ce07.json

@@ -28,35 +28,35 @@
   ],
   "evaluator": {
     "postconfig": [
-			{
-				"type": "activate_window",
-				"parameters": {
-					"window_name": "Forests.pptx - LibreOffice Impress",
-					"strict": true
-				}
-			},
-			{
-				"type": "sleep",
-				"parameters": {
-					"seconds": 0.5
-				}
-			},
-			{
-				"type": "execute",
-				"parameters": {
-					"command": [
-						"python",
-						"-c",
-						"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
-					]
-				}
-			},
-			{
-				"type": "sleep",
-				"parameters": {
-					"seconds": 0.5
-				}
-			}
+      {
+        "type": "activate_window",
+        "parameters": {
+          "window_name": "Forests.pptx - LibreOffice Impress",
+          "strict": true
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python",
+            "-c",
+            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);"
+          ]
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      }
     ],
     "func": "compare_pptx_files",
     "expected": {

evaluation_examples/examples/libreoffice_impress/bf4e9888-f10f-47af-8dba-76413038b73c.json

@@ -0,0 +1,142 @@
+{
+    "id": "bf4e9888-f10f-47af-8dba-76413038b73c",
+    "snapshot": "libreoffice_impress",
+    "instruction": "I have a series of .png images, saved as pic1.png, pic2.png,..., pic6.png on the Desktop. Now I want to insert these pictures into the current blank presentation to create a document suitable to run continuously in a kiosk or multimedia show. Could you help me?",
+    "source": "https://help.libreoffice.org/6.4/en-US/text/simpress/guide/photo_album.html?DbPAR=IMPRESS",
+    "config": [
+        {
+            "type": "download",
+            "parameters": {
+                "files": [
+                    {
+                        "url": "https://drive.usercontent.google.com/download?id=1o8BOn9Q5NpQYYcXxeCHL_znkG11ni7ws&export=download&authuser=0&confirm=t&uuid=21a62a00-fd30-4405-ae87-00c3a165e381&at=APZUnTWlhaHv92g3wMYQJn80Vdpo:1706366778535",
+                        "path": "Desktop/4.3-Template_4.29.2016.pptx"
+                    }
+                ]
+            }
+        },
+        {
+            "type": "open",
+            "parameters": {
+                "path": "Desktop/4.3-Template_4.29.2016.pptx"
+            }
+        },
+        {
+            "type": "download",
+            "parameters": {
+                "files": [
+                    {
+                        "url": "https://drive.usercontent.google.com/download?id=15mklOcjIUJkzU3GhFhZM69h5gG0WOFVa&export=download&authuser=0&confirm=t&uuid=f7f61d48-5354-4eed-b743-ef620f308d79&at=APZUnTWzBs107NBYI1OvLHId8ff5:1706365135272",
+                        "path": "Desktop/pic1.png"
+                    }
+                ]
+            }
+        },
+        {
+            "type": "download",
+            "parameters": {
+                "files": [
+                    {
+                        "url": "https://drive.usercontent.google.com/download?id=13-ATVSrmAIUNAzGlE65JkkHpa4d87tqj&export=download&authuser=0&confirm=t&uuid=5ebcaa42-805b-4176-9262-1a320e668666&at=APZUnTW0H_QRSJYAFPe4OppuwE5o:1706365136587",
+                        "path": "Desktop/pic2.png"
+                    }
+                ]
+            }
+        },
+        {
+            "type": "download",
+            "parameters": {
+                "files": [
+                    {
+                        "url": "https://drive.usercontent.google.com/download?id=1-K1XgMJj3yj7bz3BWm9Zddk4_lvYO4IP&export=download&authuser=0&confirm=t&uuid=6c268649-b627-49aa-a1aa-c97510b96e69&at=APZUnTV630QYwgh3nx-jtBp_uGAc:1706365137855",
+                        "path": "Desktop/pic3.png"
+                    }
+                ]
+            }
+        },
+        {
+            "type": "download",
+            "parameters": {
+                "files": [
+                    {
+                        "url": "https://drive.usercontent.google.com/download?id=1Bk8esN_UFxOU50AMxJoSY8C-mQgX3uY3&export=download&authuser=0&confirm=t&uuid=c5a6ae81-426e-40e2-ac1b-16ba161e347d&at=APZUnTWD9bHDiTsGWBE1tJfcR6Zp:1706365140032",
+                        "path": "Desktop/pic4.png"
+                    }
+                ]
+            }
+        },
+        {
+            "type": "download",
+            "parameters": {
+                "files": [
+                    {
+                        "url": "https://drive.usercontent.google.com/download?id=1DZapoWlmXPYsFdFGppCj2F-MZvpGZXca&export=download&authuser=0&confirm=t&uuid=7afc4de2-cdee-4c9e-9e4d-305509054627&at=APZUnTXX-2uDuQdsmA5kQRzDGRs7:1706365140910",
+                        "path": "Desktop/pic5.png"
+                    }
+                ]
+            }
+        },
+        {
+            "type": "download",
+            "parameters": {
+                "files": [
+                    {
+                        "url": "https://drive.usercontent.google.com/download?id=1jwBqiulZbICiCn3c9W6avo0-Mu7yeIAV&export=download&authuser=0&confirm=t&uuid=64f0e1d1-ec71-453a-887e-b81218e4e756&at=APZUnTWQeckP8zn8b8grHfMtQYr2:1706365141943",
+                        "path": "Desktop/pic6.png"
+                    }
+                ]
+            }
+        }
+    ],
+    "trajectory": "trajectories/",
+    "related_apps": [
+        "libreoffice_impress"
+    ],
+    "evaluator": {
+        "postconfig": [
+            {
+                "type": "activate_window",
+                "parameters": {
+                    "window_name": "4.3-Template_4.29.2016.pptx - LibreOffice Impress",
+                    "strict": true
+                }
+            },
+            {
+                "type": "sleep",
+                "parameters": {
+                    "seconds": 0.5
+                }
+            },
+            {
+                "type": "execute",
+                "parameters": {
+                    "command": [
+                        "python",
+                        "-c",
+                        "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);"
+                    ]
+                }
+            },
+            {
+                "type": "sleep",
+                "parameters": {
+                    "seconds": 0.5
+                }
+            }
+        ],
+        "func": "compare_pptx_files",
+        "expected": {
+            "type": "cloud_file",
+            "path": "https://drive.usercontent.google.com/download?id=1WeqZYQ0ZZvcFowo8rK15XHEIjoFCMVTN&export=download&authuser=0&confirm=t&uuid=e3bb39f8-0606-4bdf-808c-5b26f158dd88&at=APZUnTU_aAkyljmX3R_fkT5geNPv:1706366780602",
+            "dest": "4.3-Template_4.29.2016_Gold.pptx"
+        },
+        "result": {
+            "type": "vm_file",
+            "path": "Desktop/4.3-Template_4.29.2016.pptx",
+            "dest": "4.3-Template_4.29.2016.pptx"
+        },
+        "options": {
+            "examine_shape": false
+        }
+    }
+}

evaluation_examples/examples/libreoffice_impress/c59742c0-4323-4b9d-8a02-723c251deaa0.json

@@ -39,35 +39,35 @@
   ],
   "evaluator": {
     "postconfig": [
-			{
-				"type": "activate_window",
-				"parameters": {
-					"window_name": "Mady_and_Mia_Baseball.pptx - LibreOffice Impress",
-					"strict": true
-				}
-			},
-			{
-				"type": "sleep",
-				"parameters": {
-					"seconds": 0.5
-				}
-			},
-			{
-				"type": "execute",
-				"parameters": {
-					"command": [
-						"python",
-						"-c",
-						"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
-					]
-				}
-			},
-			{
-				"type": "sleep",
-				"parameters": {
-					"seconds": 0.5
-				}
-			}
+      {
+        "type": "activate_window",
+        "parameters": {
+          "window_name": "Mady_and_Mia_Baseball.pptx - LibreOffice Impress",
+          "strict": true
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python",
+            "-c",
+            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);"
+          ]
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      }
     ],
     "func": "compare_audios",
     "result": {

evaluation_examples/examples/libreoffice_impress/ce88f674-ab7a-43da-9201-468d38539e4a.json

@@ -28,35 +28,35 @@
   ],
   "evaluator": {
     "postconfig": [
-			{
-				"type": "activate_window",
-				"parameters": {
-					"window_name": "AM_Last_Page_Template.pptx - LibreOffice Impress",
-					"strict": true
-				}
-			},
-			{
-				"type": "sleep",
-				"parameters": {
-					"seconds": 0.5
-				}
-			},
-			{
-				"type": "execute",
-				"parameters": {
-					"command": [
-						"python",
-						"-c",
-						"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
-					]
-				}
-			},
-			{
-				"type": "sleep",
-				"parameters": {
-					"seconds": 0.5
-				}
-			}
+      {
+        "type": "activate_window",
+        "parameters": {
+          "window_name": "AM_Last_Page_Template.pptx - LibreOffice Impress",
+          "strict": true
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python",
+            "-c",
+            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);"
+          ]
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      }
     ],
     "func": "check_slide_orientation_Portrait",
     "result": {

evaluation_examples/examples/libreoffice_impress/ef9d12bd-bcee-4ba0-a40e-918400f43ddf.json

@@ -12,7 +12,17 @@
           "--impress"
         ]
       }
-    }
+    },
+    {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python",
+            "-c",
+            "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5); pyautogui.press('esc'); time.sleep(0.3); pyautogui.press('f10'); time.sleep(0.3); pyautogui.press('right', presses=2, interval=0.1); time.sleep(0.3); pyautogui.press('down', presses=11, interval=0.1); pyautogui.press('enter')"
+          ]
+        }
+      }
   ],
   "trajectory": "trajectories/",
   "related_apps": [

evaluation_examples/examples/libreoffice_impress/f0a334af-f91b-4c03-b578-aac9bec2b543.json

@@ -58,7 +58,7 @@
 					"command": [
 						"python",
 						"-c",
-						"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
+						"import pyautogui; pyautogui.hotkey('ctrl', 's');"
 					]
 				}
 			},

evaluation_examples/examples/libreoffice_writer/0810415c-bde4-4443-9047-d5f70165a697.json

@@ -27,6 +27,31 @@
     "libreoffice_writer"
   ],
   "evaluator": {
+    "postconfig": [
+      {
+        "type": "activate_window",
+        "parameters": {
+          "window_name": "Novels_Intro_Packet.docx - LibreOffice Writer",
+          "strict": true
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python",
+            "-c",
+            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
+          ]
+        }
+      }
+    ],
     "func": "compare_line_spacing",
     "expected": {
       "type": "cloud_file",

evaluation_examples/examples/libreoffice_writer/0a0faba3-5580-44df-965d-f562a99b291c.json

@@ -47,7 +47,7 @@
           "command": [
             "python",
             "-c",
-            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
+            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
           ]
         }
       }

evaluation_examples/examples/libreoffice_writer/0b17a146-2934-46c7-8727-73ff6b6483e8.json

@@ -27,6 +27,31 @@
     "libreoffice_writer"
   ],
   "evaluator": {
+    "postconfig": [
+      {
+        "type": "activate_window",
+        "parameters": {
+          "window_name": "H2O_Factsheet_WA.docx - LibreOffice Writer",
+          "strict": true
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python",
+            "-c",
+            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
+          ]
+        }
+      }
+    ],
     "func": "compare_docx_files",
     "result": {
       "type": "vm_file",

evaluation_examples/examples/libreoffice_writer/0e47de2a-32e0-456c-a366-8c607ef7a9d2.json

@@ -27,6 +27,31 @@
     "libreoffice_writer"
   ],
   "evaluator": {
+    "postconfig": [
+      {
+        "type": "activate_window",
+        "parameters": {
+          "window_name": "LibreOffice_Open_Source_Word_Processing.docx - LibreOffice Writer",
+          "strict": true
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python",
+            "-c",
+            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
+          ]
+        }
+      }
+    ],
     "func": "has_page_numbers_in_footers",
     "result": {
       "type": "vm_file",

evaluation_examples/examples/libreoffice_writer/0e763496-b6bb-4508-a427-fad0b6c3e195.json

@@ -27,6 +27,31 @@
     "libreoffice_writer"
   ],
   "evaluator": {
+    "postconfig": [
+      {
+        "type": "activate_window",
+        "parameters": {
+          "window_name": "Dublin_Zoo_Intro.docx - LibreOffice Writer",
+          "strict": true
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python",
+            "-c",
+            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
+          ]
+        }
+      }
+    ],
     "func": "compare_font_names",
     "expected": {
       "type": "rule",

evaluation_examples/examples/libreoffice_writer/3ef2b351-8a84-4ff2-8724-d86eae9b842e.json

@@ -27,6 +27,31 @@
     "libreoffice_writer"
   ],
   "evaluator": {
+    "postconfig": [
+      {
+        "type": "activate_window",
+        "parameters": {
+          "window_name": "Constitution_Template_With_Guidelines.docx - LibreOffice Writer",
+          "strict": true
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python",
+            "-c",
+            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
+          ]
+        }
+      }
+    ],
     "func": "is_first_line_centered",
     "result": {
       "type": "vm_file",

evaluation_examples/examples/libreoffice_writer/41c621f7-3544-49e1-af8d-dafd0f834f75.json

@@ -18,7 +18,7 @@
     {
       "type": "open",
       "parameters": {
-        "path": "How_to_read_a_scientific_article.docx"
+        "path": "Desktop/How_to_read_a_scientific_article.docx"
       }
     }
   ],
@@ -27,6 +27,31 @@
     "libreoffice_writer"
   ],
   "evaluator": {
+    "postconfig": [
+      {
+        "type": "activate_window",
+        "parameters": {
+          "window_name": "How_to_read_a_scientific_article.docx - LibreOffice Writer",
+          "strict": true
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python",
+            "-c",
+            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
+          ]
+        }
+      }
+    ],
     "func": [
       "compare_highlighted_text",
       "compare_docx_files"

evaluation_examples/examples/libreoffice_writer/45d61a06-6545-4422-97b7-bc76cfa964c1.json

@@ -27,6 +27,31 @@
     "libreoffice_writer"
   ],
   "evaluator": {
+    "postconfig": [
+      {
+        "type": "activate_window",
+        "parameters": {
+          "window_name": "NOVEL_Submission_Guidelines.docx - LibreOffice Writer",
+          "strict": true
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python",
+            "-c",
+            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
+          ]
+        }
+      }
+    ],
     "func": "compare_line_spacing",
     "expected": {
       "type": "cloud_file",

evaluation_examples/examples/libreoffice_writer/4bcb1253-a636-4df4-8cb0-a35c04dfef31.json

@@ -27,8 +27,16 @@
     "libreoffice_writer"
   ],
   "evaluator": {
-    "func": "check_file_exists",
-    "file_name": "View_Person_Organizational_Summary.pdf",
-    "directory": "/home/user/Downloads/"
+    "func": "compare_pdfs",
+    "expected": {
+      "type": "cloud_file",
+      "path": "https://drive.usercontent.google.com/download?id=1Spn-Gw7D-bRvV4udNQoGNEOViUqf6bL0&export=download&authuser=0&confirm=t&uuid=dcc0eb01-89ed-4852-a7cb-d0400d977ac8&at=APZUnTX57XnHwmb-y3m4JdNkvu6z:1706328786805",
+      "dest": "Constitution_Template_With_Guidelines_Gold.pdf"
+    },
+    "result": {
+      "type": "vm_file",
+      "path": "Desktop/View_Person_Organizational_Summary.pdf",
+      "dest": "Constitution_Template_With_Guidelines.pdf"
+    }
   }
 }

evaluation_examples/examples/libreoffice_writer/663876c7-3471-43db-ba51-f410b13d9d7d.json

@@ -27,6 +27,31 @@
     "libreoffice_writer"
   ],
   "evaluator": {
+    "postconfig": [
+      {
+        "type": "activate_window",
+        "parameters": {
+          "window_name": "Factoring_Perfect_Square_Trinomials.docx - LibreOffice Writer",
+          "strict": true
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python",
+            "-c",
+            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
+          ]
+        }
+      }
+    ],
     "func": "compare_insert_equation",
     "expected": {
       "type": "cloud_file",

evaluation_examples/examples/libreoffice_writer/66399b0d-8fda-4618-95c4-bfc6191617e9.json

@@ -9,7 +9,7 @@
       "parameters": {
         "files": [
           {
-            "url": "https://drive.usercontent.google.com/download?id=1kXBP0jMxTeVahzFLYbYHJtjjgmuzrA8R&export=download&authuser=0&confirm=t&uuid=f8b9bad3-415d-4d39-a4fb-05a4cf881cf0&at=APZUnTXaohwzl8_2RDF_tgUsP9cH:1704181463579",
+            "url": "https://drive.usercontent.google.com/download?id=1yeD0YIFCSEAdi6MPiiqLe6F4a19I5wZP&export=download&authuser=0&confirm=t&uuid=b6693633-00d7-4a8b-b35e-77ec452dd6a3&at=APZUnTVHRdoysZFBTmGmX6rCJLHK:1706330570587",
             "path": "Desktop/Table_Of_Work_Effort_Instructions.docx"
           }
         ]
@@ -20,6 +20,16 @@
       "parameters": {
         "path": "Desktop/Table_Of_Work_Effort_Instructions.docx"
       }
+    },
+    {
+      "type": "execute",
+      "parameters": {
+        "command": [
+          "python",
+          "-c",
+          "import pyautogui; import time; time.sleep(1); pyautogui.press(\"down\", presses=40, interval=0.1); time.sleep(1); pyautogui.scroll(-2)"
+        ]
+      }
     }
   ],
   "trajectory": "trajectories/",
@@ -27,6 +37,31 @@
     "libreoffice_writer"
   ],
   "evaluator": {
+    "postconfig": [
+      {
+        "type": "activate_window",
+        "parameters": {
+          "window_name": "Table_Of_Work_Effort_Instructions.docx - LibreOffice Writer",
+          "strict": true
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python",
+            "-c",
+            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
+          ]
+        }
+      }
+    ],
     "func": "compare_docx_tables",
     "expected": {
       "type": "cloud_file",

evaluation_examples/examples/libreoffice_writer/6a33f9b9-0a56-4844-9c3f-96ec3ffb3ba2.json

@@ -9,8 +9,8 @@
       "parameters": {
         "files": [
           {
-            "url": "https://drive.usercontent.google.com/download?id=10hgB73d_DoQXQVgUjvgXFUCP1Hd9YxDb&export=download&authuser=0&confirm=t&uuid=845f9616-2fb7-476a-abab-8b620d482ac2&at=APZUnTXB71PxHF7Dq9TC2OL_cRLm:1706199789147",
-            "path": "Desktop/sample-recruitment-phone-script.docx"
+            "url": "https://drive.google.com/uc?id=1Ul4mtQ4SpUNLVDlaiJThPprBe6WTGZ2L&export=download",
+            "path": "Desktop/sample-recruitment-phone-script.odt"
           }
         ]
       }
@@ -18,7 +18,7 @@
     {
       "type": "open",
       "parameters": {
-        "path": "Desktop/sample-recruitment-phone-script.docx"
+        "path": "Desktop/sample-recruitment-phone-script.odt"
       }
     }
   ],
@@ -31,7 +31,7 @@
       {
         "type": "activate_window",
         "parameters": {
-          "window_name": "sample-recruitment-phone-script.docx - LibreOffice Writer",
+          "window_name": "sample-recruitment-phone-script.odt - LibreOffice Writer",
           "strict": true
         }
       },
@@ -47,7 +47,7 @@
           "command": [
             "python",
             "-c",
-            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
+            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);"
           ]
         }
       }
@@ -55,13 +55,13 @@
     "func": "check_highlighted_words",
     "expected": {
       "type": "cloud_file",
-      "path": "https://drive.usercontent.google.com/download?id=1s9Dsy66-zxbCAgeTyCh0P7AT7P4jF6o3&export=download&authuser=0&confirm=t&uuid=1239f2a1-8c86-45a4-8e7d-36388ac22a69&at=APZUnTVZQzXQAMNsKKQzOw5ppT8A:1706017721589",
-      "dest": "sample-recruitment-phone-script_Gold.docx"
+      "path": "https://drive.google.com/uc?id=12iMkgCYuUyhKUXux96kANLIeud0Wz9ct&export=download",
+      "dest": "sample-recruitment-phone-script_Gold.odt"
     },
     "result": {
       "type": "vm_file",
-      "path": "Desktop/sample-recruitment-phone-script.docx",
-      "dest": "sample-recruitment-phone-script.docx"
+      "path": "Desktop/sample-recruitment-phone-script.odt",
+      "dest": "sample-recruitment-phone-script.odt"
     }
   }
 }

evaluation_examples/examples/libreoffice_writer/6ada715d-3aae-4a32-a6a7-429b2e43fb93.json

@@ -31,6 +31,16 @@
       "parameters": {
         "path": "Desktop/Viewing_Your_Class_Schedule_and_Textbooks.docx"
       }
+    },
+    {
+      "type": "execute",
+      "parameters": {
+        "command": [
+          "python",
+          "-c",
+          "import pyautogui; import time; time.sleep(1); pyautogui.press(\"down\", presses=8); time.sleep(1); pyautogui.scroll(-2)"
+        ]
+      }
     }
   ],
   "trajectory": "trajectories/",
@@ -38,6 +48,31 @@
     "libreoffice_writer"
   ],
   "evaluator": {
+    "postconfig": [
+      {
+        "type": "activate_window",
+        "parameters": {
+          "window_name": "Viewing_Your_Class_Schedule_and_Textbooks.docx - LibreOffice Writer",
+          "strict": true
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python",
+            "-c",
+            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
+          ]
+        }
+      }
+    ],
     "func": "compare_contains_image",
     "result": {
       "type": "vm_file",

evaluation_examples/examples/libreoffice_writer/6f81754e-285d-4ce0-b59e-af7edb02d108.json

@@ -47,7 +47,7 @@
           "command": [
             "python",
             "-c",
-            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
+            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
           ]
         }
       }

evaluation_examples/examples/libreoffice_writer/72b810ef-4156-4d09-8f08-a0cf57e7cefe.json

@@ -47,7 +47,7 @@
           "command": [
             "python",
             "-c",
-            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
+            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
           ]
         }
       }

evaluation_examples/examples/libreoffice_writer/8472fece-c7dd-4241-8d65-9b3cd1a0b568.json

@@ -47,7 +47,7 @@
           "command": [
             "python",
             "-c",
-            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
+            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
           ]
         }
       }

evaluation_examples/examples/libreoffice_writer/88fe4b2d-3040-4c70-9a70-546a47764b48.json

@@ -47,7 +47,7 @@
           "command": [
             "python",
             "-c",
-            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
+            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
           ]
         }
       }

evaluation_examples/examples/libreoffice_writer/936321ce-5236-426a-9a20-e0e3c5dc536f.json

@@ -27,6 +27,31 @@
     "libreoffice_writer"
   ],
   "evaluator": {
+    "postconfig": [
+      {
+        "type": "activate_window",
+        "parameters": {
+          "window_name": "Graphemes_Sound_Letter_Patterns_Gold.docx - LibreOffice Writer",
+          "strict": true
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python",
+            "-c",
+            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
+          ]
+        }
+      }
+    ],
     "func": "compare_docx_tables",
     "expected": {
       "type": "cloud_file",

evaluation_examples/examples/libreoffice_writer/adf5e2c3-64c7-4644-b7b6-d2f0167927e7.json

@@ -1,7 +1,7 @@
 {
   "id": "adf5e2c3-64c7-4644-b7b6-d2f0167927e7",
   "snapshot": "libreoffice_writer",
-  "instruction": "Help me adding \"Steinberg, F. M., Bearden, M. M., & Keen, C. L. (2003). Cocoa and chocolate flavonoids: Implications for cardiovascular health. Journal of the American Dietetic Association, 103(2), 215-223. doi: 10.1053/jada.2003.50028\" to my reference list, and add a cross reference where my cursor is located (in the fourth paragraph).",
+  "instruction": "Help me adding \"Steinberg, F. M., Bearden, M. M., & Keen, C. L. (2003). Cocoa and chocolate flavonoids: Implications for cardiovascular health. Journal of the American Dietetic Association, 103(2), 215-223. doi: 10.1053/jada.2003.50028\" to my reference list, and add a cross reference in the fourth paragraph where I marked \"<add here>\".",
   "source": "https://seekstar.github.io/2022/04/11/libreoffice%E5%BC%95%E7%94%A8%E6%96%87%E7%8C%AE/",
   "config": [
     {
@@ -9,7 +9,7 @@
       "parameters": {
         "files": [
           {
-            "url": "https://drive.usercontent.google.com/download?id=1ShGL4gWSV7nzamAb0V2KqjoCOhyodcKU&export=download&authuser=0&confirm=t&uuid=5f67edb8-cbbf-4a83-b46e-f193ad55e1e8&at=APZUnTVRJenYCM--vETagQ5ACTT5:1704979226579",
+            "url": "https://drive.usercontent.google.com/download?id=1ShGL4gWSV7nzamAb0V2KqjoCOhyodcKU&export=download&authuser=0&confirm=t&uuid=17f2a63f-df71-4ea7-85a0-4b364afa336c&at=APZUnTXij39N124BO91KxN6yFR7Y:1706357955122",
             "path": "Desktop/Essay_Writing_English_for_uni.docx"
           }
         ]
@@ -27,6 +27,31 @@
     "libreoffice_writer"
   ],
   "evaluator": {
+    "postconfig": [
+      {
+        "type": "activate_window",
+        "parameters": {
+          "window_name": "Essay_Writing_English_for_uni.docx - LibreOffice Writer",
+          "strict": true
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python",
+            "-c",
+            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
+          ]
+        }
+      }
+    ],
     "func": "compare_docx_files",
     "expected": {
       "type": "cloud_file",

evaluation_examples/examples/libreoffice_writer/b21acd93-60fd-4127-8a43-2f5178f4a830.json

@@ -1,7 +1,7 @@
 {
   "id": "b21acd93-60fd-4127-8a43-2f5178f4a830",
   "snapshot": "libreoffice_writer",
-  "instruction": "I have been praciticing professional writing lately. Now I am writing essay which requires one paragraph each for introduction, body and conclusion with single-space for introduction, double-space for body then one-and-a-half-space for conclusion. The font size of this essay is 12. Could you help me on this?",
+  "instruction": "I have been practicing professional writing lately. Now I am writing essay which requires one paragraph each for introduction, body and conclusion with single-space for introduction, double-space for body then one-and-a-half-space for conclusion. The font size of this essay is 12. Could you help me on this?",
   "source": "https://superuser.com/questions/1097199/how-can-i-double-space-a-document-in-libreoffice?rq=1",
   "config": [
     {
@@ -47,7 +47,7 @@
           "command": [
             "python",
             "-c",
-            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
+            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
           ]
         }
       }

evaluation_examples/examples/libreoffice_writer/d53ff5ee-3b1a-431e-b2be-30ed2673079b.json

@@ -48,7 +48,7 @@
           "command": [
             "python",
             "-c",
-            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
+            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
           ]
         }
       }

evaluation_examples/examples/libreoffice_writer/e246f6d8-78d7-44ac-b668-fcf47946cb50.json

@@ -47,7 +47,7 @@
           "command": [
             "python",
             "-c",
-            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
+            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
           ]
         }
       }

evaluation_examples/examples/libreoffice_writer/e528b65e-1107-4b8c-8988-490e4fece599.json

@@ -10,7 +10,7 @@
         "files": [
           {
             "url": "https://drive.usercontent.google.com/download?id=181hyG_NZUJaUp7kUdeFoUpNGpOcjzYFV&export=download&authuser=0&confirm=t&uuid=fdb8aee4-ea1f-43c1-bc66-9d44099512e1&at=APZUnTVUrypBeVnTfkJsriv7S2GO:1704969486759",
-            "path": "Geography_And_Magical_Realism.docx"
+            "path": "Desktop/Geography_And_Magical_Realism.docx"
           }
         ]
       }
@@ -27,6 +27,31 @@
     "libreoffice_writer"
   ],
   "evaluator": {
+    "postconfig": [
+      {
+        "type": "activate_window",
+        "parameters": {
+          "window_name": "Geography_And_Magical_Realism.docx - LibreOffice Writer",
+          "strict": true
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python",
+            "-c",
+            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
+          ]
+        }
+      }
+    ],
     "func": "compare_docx_files",
     "expected": {
       "type": "cloud_file",

evaluation_examples/examples/libreoffice_writer/ecc2413d-8a48-416e-a3a2-d30106ca36cb.json

@@ -1,7 +1,7 @@
 {
   "id": "ecc2413d-8a48-416e-a3a2-d30106ca36cb",
   "snapshot": "libreoffice_writer",
-  "instruction": "Help me insert a blank page where my cursor is located.",
+  "instruction": "Hey, can you throw in a blank page right after this one?",
   "source": "https://www.quora.com/How-can-I-insert-a-blank-page-on-libreoffice",
   "config": [
     {
@@ -27,6 +27,31 @@
     "libreoffice_writer"
   ],
   "evaluator": {
+    "postconfig": [
+      {
+        "type": "activate_window",
+        "parameters": {
+          "window_name": "Sample_Statutory_Declaration.docx - LibreOffice Writer",
+          "strict": true
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python",
+            "-c",
+            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
+          ]
+        }
+      }
+    ],
     "func": "contains_page_break",
     "result": {
       "type": "vm_file",

evaluation_examples/examples/multi_apps/2b9493d7-49b8-493a-a71b-56cd1f4d6908.json

@@ -0,0 +1,101 @@
+{
+    "id": "2b9493d7-49b8-493a-a71b-56cd1f4d6908",
+    "snapshot": "libreoffice_writer",
+    "instruction": "Hey, my LibreOffice Writer seems to have frozen and I can't get it to close normally. Can you help me force quit the application from the command line? I'm on Ubuntu and I don't want to restart my computer or lose any other work I have open.",
+    "source": "https://devicetests.com/kill-libreoffice-writer-command-line-ubuntu",
+    "config": [
+        {
+            "type": "download",
+            "parameters": {
+              "files": [
+                {
+                  "url": "https://drive.usercontent.google.com/download?id=104pg3yochKyH2Uvlp3BdvKmHgYmSIESu&export=download&authuser=0&confirm=t&uuid=d1926366-4e54-4a44-8dcd-fc49ed6524d7&at=APZUnTXcBFV9kcacsA0toU83lMKJ:1706505549057d",
+                  "path": "/home/user/Desktop/15-MB-docx-file-download.docx"
+                }
+              ]
+            }
+        },
+        {
+            "type": "open",
+            "parameters": {
+                "path": "/home/user/Desktop/15-MB-docx-file-download.docx"
+            }
+        },
+        {
+            "type": "execute",
+            "parameters": {
+                "command": [
+                    "/bin/bash",
+                    "-c",
+                    "history -c && echo > ~/.bash_history && sleep 3"
+                ]
+            }
+        },
+        {
+            "type": "launch",
+            "parameters": {
+                "command": [
+                    "gnome-terminal",
+                    "--maximize"
+                ]
+            }
+        }
+    ],
+    "trajectory": "trajectories/",
+    "related_apps": [
+        "libreoffice_writer",
+        "terminal"
+    ],
+    "evaluator": {
+        "postconfig": [
+            {
+              "type": "execute",
+              "parameters": {
+                "command": [
+                    "/bin/bash",
+                    "-c",
+                    "killall gnome-terminal-server"
+                ]
+              }
+            }
+        ],
+        "func": ["check_include_exclude", "check_include_exclude"],
+        "conj": "and",
+        "result": [
+            {
+                "type": "vm_command_line",
+                "command": [
+                    "/bin/bash",
+                    "-c",
+                    "output=$(ps aux | grep \"[s]office\"]); if [ -z \"$output\" ]; then echo \"true\"; else echo \"false\"; fi"
+                ]
+            },
+            {
+                "type": "vm_command_line",
+                "command": [
+                    "/bin/bash",
+                    "-c",
+                    "output=$(cat ~/.bash_history | grep \"[k]ill\"); if [ -z \"$output\" ]; then echo \"false\"; else echo \"true\"; fi"
+                ]
+            }
+        ],
+        "expected": [
+            {
+                "type": "rule",
+                "rules": {
+                    "include": [
+                        "true\n"
+                    ]
+                }
+            },
+            {
+                "type": "rule",
+                "rules": {
+                    "include": [
+                        "true\n"
+                    ]
+                }
+            }
+        ]
+    }
+}

evaluation_examples/examples/multi_apps/2c9fc0de-3ee7-45e1-a5df-c86206ad78b5.json

@@ -0,0 +1,63 @@
+{
+    "id": "2c9fc0de-3ee7-45e1-a5df-c86206ad78b5",
+    "snapshot": "os",
+    "instruction": "Could you help me push the changes from commandline in current project to origin main, with the commit message \"daily update\"?",
+    "source": "https://nikki-ricks.medium.com/how-to-use-git-add-commit-and-push-in-vs-code-and-command-line-35c0e8c47b62",
+    "config": [
+        {
+            "type": "download",
+            "path": "",
+        },
+        {
+            "type": "execute",
+            "parameters": {
+                "command": [
+                    "/bin/bash",
+                    "-c",
+                    "git config --global user.name \"xlang\" && git config --global user.email \"xlang2024anonym@gmail.com\" && mkdir -p /home/user/projects/remote_project && cd /home/user/projects/remote_project && git init --initial-branch=main --bare .git"
+                ]
+            }
+        },
+        {
+            "type": "execute",
+            "parameters": {
+                "command": [
+                    "/bin/bash",
+                    "-c",
+                    "mkdir -p /home/user/projects/hello_world && cd /home/user/projects/hello_world && git init --initial-branch main && git remote add origin /home/user/projects/remote_project && echo \"Hello World!\" > README.md"
+                ]
+            }
+        },
+        {
+            "type": "launch",
+            "parameters": {
+                "command": [
+                    "gnome-terminal",
+                    "--maximize",
+                    "--working-directory=/home/user/projects/hello_world"
+                ]
+            }
+        }
+    ],
+    "trajectory": "trajectories/",
+    "related_apps": [
+        "os",
+        "terminal"
+    ],
+    "evaluator": {
+        "func": "compare_docx_files",
+        "result": {
+            "type": "vm_file",
+            "path": "/home/user/Desktop/notes.docx",
+            "dest": "notes.docx"
+        },
+        "expected": {
+            "type": "cloud_file",
+            "path": "https://drive.usercontent.google.com/download?id=1Xl6tgQ0K5qA1BDA2fKTK2xFLzXwbtkZ6&export=download",
+            "dest": "notes_gold.docx"
+        },
+        "options": {
+            "ignore_blanks": true
+        }
+    }
+}

evaluation_examples/examples/multi_apps/46407397-a7d5-4c6b-92c6-dbe038b1457b.json

@@ -1,7 +1,7 @@
 {
     "id": "46407397-a7d5-4c6b-92c6-dbe038b1457b",
     "snapshot": "chrome",
-    "instruction": "Help me export charts, graph or other images from docx files received in email xxx in Thunderbird and upload them in the figures/ folder in Google Drive for later use (use pure numbers to name them).",
+    "instruction": "Help me export charts, graph or other images from docx files received in email xxx in Thunderbird and upload these png files to the figures/ folder in Google Drive for later use (use numbers to name them).",
     "source": "https://marketplace.uipath.com/listings/merge-pdfs-from-gmail-email-attachments-and-upload-to-gogle-drive",
     "config": [
         {
@@ -93,7 +93,7 @@
         "chrome"
     ],
     "evaluator": {
-        "func": "compare_figures",
+        "func": "compare_images",
         "result": {
             "type": "googledrive_file",
             "settings_file": "evaluation_examples/settings/googledrive/settings.yml",
@@ -120,9 +120,9 @@
         "expected": {
             "type": "cloud_file",
             "path": [
-                "file1",
-                "file2",
-                "file3"
+                "https://drive.usercontent.google.com/download?id=19J5tzWjx9hdo-n0MC3upzAntVMa8WUgk&export=download&authuser=0&confirm=t&uuid=be790579-8db9-4bd2-a757-beb27af386af&at=APZUnTVM2PjNDXhlwFZ6WAFdNVsD:1706497547717",
+                "https://drive.usercontent.google.com/download?id=1S04RpR5dk80LylIYGvA4e3sAUBd6wdlQ&export=download&authuser=0&confirm=t&uuid=b302de03-04f7-455c-ab0c-b3cbbeb6929a&at=APZUnTVD8zMZGO1_GWaFUm1cNXul:1706497555463",
+                "https://drive.usercontent.google.com/download?id=11NRLh93RTzEd0Cy-cYwMyNJSFG7-vP9c&export=download&authuser=0&confirm=t&uuid=02500115-dea3-481a-af4f-a723d9a62169&at=APZUnTW9-gENlsyfdIPA4PTA0emh:1706497560874"
             ],
             "dest": [
                 "1_gold.png",

evaluation_examples/examples/multi_apps/51f5801c-18b3-4f25-b0c3-02f85507a078.json

@@ -0,0 +1,46 @@
+{
+    "id": "51f5801c-18b3-4f25-b0c3-02f85507a078",
+    "snapshot": "libreoffice_impress",
+    "instruction": "I've been working on this presentation in LibreOffice Impress and I've added a bunch of speaker notes for my upcoming talk. I'd like to have those notes handy in a separate document when I rehearse. Could you assist me in extracting all the presenter notes from the Impress file and saving them as a Word document? Just keep the text of the notes, do not add any formatting or page number information. I'd like the file to be named 'notes.docx' and placed on my Desktop for easy access.",
+    "source": "https://github.com/danielrcollins1/ImpressExtractNotes",
+    "config": [
+        {
+            "type": "download",
+            "parameters": {
+              "files": [
+                {
+                  "url": "https://drive.usercontent.google.com/download?id=1e12nL_V7bffaLSocQ86EiGCdygzggWeu&export=download",
+                  "path": "/home/user/Desktop/Dickinson_Slides.pptx"
+                }
+              ]
+            }
+        },
+        {
+            "type": "open",
+            "parameters": {
+                "path": "/home/user/Desktop/Dickinson_Slides.pptx"
+            }
+        }
+    ],
+    "trajectory": "trajectories/",
+    "related_apps": [
+        "libreoffice_impress",
+        "libreoffice_writer"
+    ],
+    "evaluator": {
+        "func": "compare_docx_files",
+        "result": {
+            "type": "vm_file",
+            "path": "/home/user/Desktop/notes.docx",
+            "dest": "notes.docx"
+        },
+        "expected": {
+            "type": "cloud_file",
+            "path": "https://drive.usercontent.google.com/download?id=1Xl6tgQ0K5qA1BDA2fKTK2xFLzXwbtkZ6&export=download",
+            "dest": "notes_gold.docx"
+        },
+        "options": {
+            "ignore_blanks": true
+        }
+    }
+}

evaluation_examples/examples/os/13584542-872b-42d8-b299-866967b5c3ef.json

@@ -1,17 +1,56 @@
 {
-    "id": "13584542-872b-42d8-b299-866967b5c3ef",
-    "snapshot": "os",
-    "instruction": "Set the default terminal size and screen x,y coordinates in order to make the terminal window always open at the same location and with the same size.",
-    "source": "https://superuser.com/questions/72176/linux-set-default-terminal-size-and-screen-position",
-    "trajectory": "trajectories/",
-    "related_apps": [
-      "os"
-    ],
-    "evaluator": {
-      "func": "",
-      "result": {
-      },
-      "expected": {
+  "id": "13584542-872b-42d8-b299-866967b5c3ef",
+  "snapshot": "os",
+  "instruction": "I click in terminal: terminal->132x43 to change terminal size but after each reboot terminal size is set to default setting and I have to change it again. Help me set it permanently",
+  "source": "https://superuser.com/questions/72176/linux-set-default-terminal-size-and-screen-position",
+  "trajectory": "trajectories/",
+  "config": [
+    {
+      "type": "execute",
+      "parameters": {
+        "command": [
+          "python",
+          "-c",
+          "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
+        ]
       }
     }
-  }
+  ],
+  "related_apps": [
+    "os"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 1
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python",
+            "-c",
+            "import pyautogui; import time; time.sleep(0.5); pyautogui.hotkey('ctrl', 'alt', 't'); time.sleep(0.5); pyautogui.write('stty size'); time.sleep(0.5); pyautogui.press('enter')"
+          ]
+        }
+      }
+    ],
+    "func": "check_include_exclude",
+    "result": {
+      "type": "vm_terminal_output"
+    },
+    "expected": {
+      "type": "rule",
+      "rules": {
+        "include": [
+          "43 132"
+        ],
+        "exclude": [
+        ]
+      }
+    }
+  }
+}

evaluation_examples/examples/os/23393935-50c7-4a86-aeea-2b78fd089c5c.json

@@ -80,6 +80,16 @@
           }
         ]
       }
+    },
+    {
+      "type": "execute",
+      "parameters": {
+        "command": [
+          "python",
+          "-c",
+          "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
+        ]
+      }
     }
   ],
   "related_apps": [

evaluation_examples/examples/os/28cc3b7e-b194-4bc9-8353-d04c0f4d56d2.json

@@ -4,7 +4,18 @@
   "instruction": "The volume of my system is too small. Can you help me turn up to the max volume?",
   "source": "https://help.ubuntu.com/lts/ubuntu-help/sound-volume.html.en",
   "trajectory": "trajectories/",
-  "config": [],
+  "config": [
+    {
+      "type": "execute",
+      "parameters": {
+        "command": [
+          "python",
+          "-c",
+          "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
+        ]
+      }
+    }
+  ],
   "related_apps": [
     "os"
   ],

evaluation_examples/examples/os/3ce045a0-877b-42aa-8d2c-b4a863336ab8.json

@@ -4,7 +4,18 @@
   "instruction": "My glasses are broken, and I'm having trouble seeing small things clearly. Could you help me enlarge the text on my screen so it's easier to read?",
   "source": "https://help.ubuntu.com/lts/ubuntu-help/a11y-font-size.html.en",
   "trajectory": "trajectories/",
-  "config": [],
+  "config": [
+    {
+      "type": "execute",
+      "parameters": {
+        "command": [
+          "python",
+          "-c",
+          "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
+        ]
+      }
+    }
+  ],
   "related_apps": [
     "os"
   ],

evaluation_examples/examples/os/43c2d64c-bab5-4dcb-a30c-b888321c319a.json

@@ -3,7 +3,18 @@
     "snapshot": "os",
     "instruction": "Could you please help me create a dir named 'test' in the root directory of current computer?",
     "source": "https://ubuntu.com/tutorials/command-line-for-beginners#4-creating-folders-and-files",
-    "config": [],
+    "config": [
+      {
+      "type": "execute",
+      "parameters": {
+        "command": [
+          "python",
+          "-c",
+          "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
+        ]
+      }
+    }
+    ],
     "trajectory": "trajectories/",
     "related_apps": [
       "os"
@@ -12,7 +23,7 @@
       "func": "exact_match",
       "result": {
         "type": "vm_command_line",
-        "command": "[ -d '/home/user/Desktop/test' ] && echo 'Directory exists.' || echo 'Directory does not exist.'",
+        "command": "[ -d '/test' ] && echo 'Directory exists.' || echo 'Directory does not exist.'",
         "shell": true
       },
       "expected": {

evaluation_examples/examples/os/5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57.json

@@ -11,7 +11,7 @@
         "files": [
           {
             "url": "https://drive.usercontent.google.com/download?id=1XaTnC_lLbR_tGTz8tcN2Tp6cNrMlNW3R&export=download&authuser=0&confirm=t&uuid=89e69a23-43cf-4316-833a-fb9d3e281460&at=APZUnTWn5zZTH4GlClO6lV1i4WwP:1706184669922",
-            "path": "poster_party_night.webp"
+            "path": "/home/user/Desktop/poster_party_night.webp"
           }
         ]
       }
@@ -19,9 +19,19 @@
     {
       "type": "execute",
       "parameters": {
-        "command": "mv ~/poster_party_night.webp ~/.local/share/Trash/files/",
+        "command": "gio trash /home/user/Desktop/poster_party_night.webp",
         "shell": true
       }
+    },
+    {
+      "type": "execute",
+      "parameters": {
+        "command": [
+          "python",
+          "-c",
+          "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
+        ]
+      }
     }
   ],
   "related_apps": [

evaluation_examples/examples/os/765d2b74-88a7-4d50-bf51-34e4106fd24a.json

@@ -3,7 +3,18 @@
   "snapshot": "os",
   "instruction": "Can you help me delete the \"test\" file on my desktop?",
   "source": "https://help.ubuntu.com/lts/ubuntu-help/files-delete.html.en",
-  "config": [],
+  "config": [
+    {
+      "type": "execute",
+      "parameters": {
+        "command": [
+          "python",
+          "-c",
+          "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
+        ]
+      }
+    }
+  ],
   "trajectory": "trajectories/",
   "related_apps": [
     "os"

evaluation_examples/examples/os/7688b85f-87a4-4e4a-b2f8-f3d6c3f29b82.json

@@ -17,7 +17,17 @@
           "command": "echo 'password' | sudo -S chmod 777 ~/Desktop/todo.txt && sudo chmod 777 ~/Desktop/done",
           "shell": true
         }
+      },
+      {
+      "type": "execute",
+      "parameters": {
+        "command": [
+          "python",
+          "-c",
+          "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
+        ]
       }
+    }
     ],
     "trajectory": "trajectories/",
     "related_apps": [

evaluation_examples/examples/os/94d95f96-9699-4208-98ba-3c3119edf9c2.json

@@ -4,7 +4,18 @@
   "instruction": "I want to install Spotify on my current system. Could you please help me?",
   "source": "https://help.ubuntu.com/lts/ubuntu-help/addremove-install.html.en",
   "trajectory": "trajectories/",
-  "config": [],
+  "config": [
+    {
+      "type": "execute",
+      "parameters": {
+        "command": [
+          "python",
+          "-c",
+          "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
+        ]
+      }
+    }
+  ],
   "related_apps": [
     "os"
   ],

evaluation_examples/examples/os/a462a795-fdc7-4b23-b689-e8b6df786b78.json

@@ -10,6 +10,16 @@
         "command": "echo password | sudo -S su - charles",
         "shell": true
       }
+    },
+    {
+      "type": "execute",
+      "parameters": {
+        "command": [
+          "python",
+          "-c",
+          "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
+        ]
+      }
     }
   ],
   "trajectory": "trajectories/",

evaluation_examples/examples/os/a4d98375-215b-4a4d-aee9-3d4370fccc41.json

@@ -4,6 +4,18 @@
   "instruction": "I want to have my computer automatically locked after I leaved. Can you help me?",
   "source": "https://help.ubuntu.com/lts/ubuntu-help/privacy-screen-lock.html.en",
   "trajectory": "trajectories/",
+  "config": [
+    {
+      "type": "execute",
+      "parameters": {
+        "command": [
+          "python",
+          "-c",
+          "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
+        ]
+      }
+    }
+  ],
   "related_apps": [
     "os"
   ],