Merge branch 'main' into zhoujun/gimp

desktop_env/evaluators/getters/__init__.py

@@ -1,13 +1,25 @@
-from .chrome import get_default_search_engine, get_cookie_data, get_bookmarks, get_open_tabs_info, get_pdf_from_url, \
+from .chrome import (
-    get_shortcuts_on_desktop, get_history, get_enabled_experiments, get_chrome_language, get_chrome_font_size, \
+    get_default_search_engine,
-    get_profile_name, get_number_of_search_results, get_googledrive_file, get_active_tab_info
+    get_cookie_data,
+    get_bookmarks,
+    get_open_tabs_info,
+    get_pdf_from_url,
+    get_shortcuts_on_desktop,
+    get_history,
+    get_enabled_experiments,
+    get_chrome_language,
+    get_chrome_font_size,
+    get_profile_name,
+    get_number_of_search_results,
+    get_googledrive_file,
+    get_active_tab_info
+)
 from .file import get_cloud_file, get_vm_file, get_cache_file
 from .general import get_vm_command_line, get_vm_terminal_output
+from .gimp import get_gimp_config_file
 from .impress import get_audio_in_slide
 from .info import get_vm_screen_size, get_vm_window_size, get_vm_wallpaper, get_list_directory
 from .misc import get_rule, get_accessibility_tree
 from .replay import get_replay
-from .vlc import get_vlc_playing_info, get_vlc_config
+from .vlc import get_vlc_playing_info, get_vlc_config, get_default_video_player
 from .vscode import get_vscode_config
-from .impress import get_audio_in_slide
-from .gimp import get_gimp_config_file

desktop_env/evaluators/getters/vlc.py

@@ -1,7 +1,8 @@
 import logging
 import os
 from typing import Dict
-
+from collections import Counter
+from .general import get_vm_command_line
 import requests
 
 logger = logging.getLogger("desktopenv.getters.vlc")
@@ -58,3 +59,28 @@ def get_vlc_config(env, config: Dict[str, str]):
         f.write(content)
 
     return _path
+
+
+def get_default_video_player(env, config: dict):
+    """ Gets the default application for a category or file extension.
+    """
+
+    os_type = env.vm_platform
+
+    if os_type == "Linux":
+        extensions = ['3gp', '3gp', '3gpp', '3gpp', '3gpp2', '3gpp2', 'avi', 'avi', 'divx', 'divx', 'dv', 'dv', 'fli', 'fli', 'flv', 'flv', 'mp2t', 'mp2t', 'mp4', 'mp4', 'mp4v-es', 'mp4v-es', 'mpeg', 'mpeg', 'mpeg-system', 'mpeg-system', 'msvideo', 'msvideo', 'ogg', 'ogg', 'quicktime', 'quicktime', 'vnd.divx', 'vnd.divx', 'vnd.mpegurl', 'vnd.mpegurl', 'vnd.rn-realvideo', 'vnd.rn-realvideo', 'webm', 'webm', 'x-anim', 'x-anim', 'x-avi', 'x-avi', 'x-flc', 'x-flc', 'x-fli', 'x-fli', 'x-flv', 'x-flv', 'x-m4v', 'x-m4v', 'x-matroska', 'x-matroska', 'x-mpeg', 'x-mpeg', 'x-mpeg-system', 'x-mpeg-system', 'x-mpeg2', 'x-mpeg2', 'x-ms-asf', 'x-ms-asf', 'x-ms-asf-plugin', 'x-ms-asf-plugin', 'x-ms-asx', 'x-ms-asx', 'x-ms-wm', 'x-ms-wm', 'x-ms-wmv', 'x-ms-wmv', 'x-ms-wmx', 'x-ms-wmx', 'x-ms-wvx', 'x-ms-wvx', 'x-msvideo', 'x-msvideo', 'x-nsv', 'x-nsv', 'x-ogm', 'x-ogm', 'x-ogm+ogg', 'x-theora', 'x-theora', 'x-theora+ogg', 'x-theora+ogg']
+        apps = []
+        for ext in extensions:
+            app = get_vm_command_line(env, {"command": ["xdg-mime", "query", "default", f"video/{ext}"]})
+            if app:
+                apps.append(app)
+        if len(apps) == 0:
+            return 'unknown'
+        else:
+            return Counter(apps).most_common(1)[0][0]
+    elif os_type == "Darwin":
+        raise Exception("Unsupported operating system", os_type)
+    elif os_type == "Windows":
+        raise Exception("Unsupported operating system", os_type)
+    else:
+        raise Exception("Unsupported operating system", os_type)

desktop_env/evaluators/getters/vscode.py

@@ -1,6 +1,6 @@
 import logging
 from typing import Any, Dict
-
+import time
 from .file import get_vm_file
 from .replay import get_replay
 
@@ -27,6 +27,7 @@ def get_vscode_config(env, config: Dict[str, Any]) -> str:
         ]
 
     get_replay(env, trajectory)
+    time.sleep(1.0)
 
     return get_vm_file(env, {
         "path": config["path"],

desktop_env/evaluators/metrics/__init__.py

@@ -1,27 +1,58 @@
-from .chrome import is_expected_tabs, is_expected_bookmarks, compare_pdfs, is_cookie_deleted, is_shortcut_on_desktop, \
+from .basic_os import (
-    check_font_size, \
+    check_gnome_favorite_apps,
-    check_enabled_experiments, check_history_deleted, is_expected_search_query
+    is_utc_0,
-from .docs import compare_font_names, compare_subscript_contains, has_page_numbers_in_footers, compare_docx_lines
+    check_text_enlarged,
-from .docs import evaluate_colored_words_in_tables, check_highlighted_words, evaluate_strike_through_last_paragraph, \
+    check_moved_jpgs
-    evaluate_conversion, evaluate_spacing, check_italic_font_size_14, evaluate_alignment, get_unique_train_ids, \
+)
-    check_no_duplicates, compare_init_lines
+from .chrome import (
-from .general import exact_match, fuzzy_match
+    is_expected_tabs,
-
+    is_expected_bookmarks,
-from .docs import find_default_font, contains_page_break, compare_docx_files, compare_docx_tables, compare_line_spacing, \
+    compare_pdfs,
-    compare_insert_equation, compare_highlighted_text
+    is_cookie_deleted,
-from .docs import is_first_line_centered, check_file_exists, compare_contains_image
+    is_shortcut_on_desktop,
-from .general import check_csv, check_accessibility_tree, run_sqlite3, check_json, check_list
+    check_font_size,
-from .general import exact_match, fuzzy_match, check_include_exclude
+    check_enabled_experiments,
-from .gimp import increase_saturation, decrease_brightness, check_file_exists, compare_triangle_positions
+    check_history_deleted,
-from .gimp import compare_images
+    is_expected_search_query
-from .libreoffice import check_libre_locale
+)
-from .pdf import check_pdf_pages
+from .docs import (
-from .slides import check_presenter_console_disable, check_image_stretch_and_center, check_slide_numbers_color, \
+    compare_font_names,
-    compare_pptx_files, check_strikethrough, \
+    compare_subscript_contains,
-    check_slide_orientation_Portrait, evaluate_presentation_fill_to_rgb_distance, check_left_panel, check_transition
+    has_page_numbers_in_footers,
-# from .table import check_sheet_list, check_xlsx_freeze, check_xlsx_zoom, check_data_validations
+    compare_docx_lines,
-from .table import compare_table, compare_csv
+    evaluate_colored_words_in_tables,
-from .thunderbird import check_thunderbird_prefs, check_thunderbird_filter
+    check_highlighted_words,
+    evaluate_strike_through_last_paragraph,
+    evaluate_conversion,
+    evaluate_spacing,
+    check_italic_font_size_14,
+    evaluate_alignment,
+    get_unique_train_ids,
+    check_no_duplicates,
+    compare_init_lines
+)
+from .docs import (
+    find_default_font,
+    contains_page_break,
+    compare_docx_files,
+    compare_docx_tables,
+    compare_line_spacing,
+    compare_insert_equation,
+    compare_highlighted_text,
+    is_first_line_centered,
+    check_file_exists,
+    compare_contains_image
+)
+from .general import (
+    check_csv,
+    check_accessibility_tree,
+    run_sqlite3,
+    check_json,
+    check_list,
+    exact_match,
+    fuzzy_match,
+    check_include_exclude
+)
 from .gimp import (
     check_brightness_decrease_and_structure_sim,
     check_contrast_increase_and_structure_sim,
@@ -35,11 +66,53 @@ from .gimp import (
     check_triangle_position,
     check_structure_sim,
     check_config_status,
+    compare_images,
+    increase_saturation,
+    decrease_brightness,
+    check_file_exists,
+    compare_triangle_positions
 )
-from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, compare_images, compare_audios, \
+from .libreoffice import check_libre_locale
-    compare_videos, check_qt_bgcone, check_one_instance_when_started_from_file, check_qt_minimal_view, \
+from .pdf import check_pdf_pages
-    check_qt_max_volume, \
+from .slides import (
-    check_qt_slider_colours, check_global_key_play_pause
+    check_presenter_console_disable,
-from .vscode import compare_text_file, compare_config, compare_answer, is_extension_installed, check_json_settings, \
+    check_image_stretch_and_center,
+    check_slide_numbers_color,
+    compare_pptx_files,
+    check_strikethrough,
+    check_slide_orientation_Portrait,
+    evaluate_presentation_fill_to_rgb_distance,
+    check_left_panel,
+    check_transition,
+    check_page_number_colors
+)
+from .table import (
+    compare_table,
+    compare_csv
+)
+from .thunderbird import (
+    check_thunderbird_prefs,
+    check_thunderbird_filter
+)
+from .vlc import (
+    is_vlc_playing,
+    is_vlc_recordings_folder,
+    is_vlc_fullscreen,
+    compare_images,
+    compare_audios,
+    compare_videos,
+    check_qt_bgcone,
+    check_one_instance_when_started_from_file,
+    check_qt_minimal_view,
+    check_qt_max_volume,
+    check_qt_slider_colours,
+    check_global_key_play_pause
+)
+from .vscode import (
+    compare_text_file,
+    compare_config,
+    compare_answer,
+    is_extension_installed,
+    check_json_settings,
     check_json_keybindings
-from .basic_os import check_gnome_favorite_apps, is_utc_0, check_text_enlarged, check_moved_jpgs
+)

desktop_env/evaluators/metrics/docs.py

@@ -1,13 +1,16 @@
 import logging
 import os
+import re
 import xml.etree.ElementTree as ET
 import zipfile
-import re
 from typing import List, Dict, Any
 
 from docx import Document
 from docx.enum.text import WD_PARAGRAPH_ALIGNMENT
 from docx.shared import RGBColor
+from odf.opendocument import load
+from odf.text import P
+from odf.text import Span
 
 logger = logging.getLogger("desktopenv.metric.docs")
 
@@ -51,12 +54,39 @@ def contains_page_break(docx_file):
 
 
 def compare_docx_files(file1, file2, ignore_blanks=True):
-    doc1 = Document(file1)
+    def get_paragraph_texts_odt(document):
-    doc2 = Document(file2)
+        paragraphs = document.getElementsByType(P)
+        paragraph_texts = []
+        for paragraph in paragraphs:
+            text_parts = []
+            for node in paragraph.childNodes:
+                if node.nodeType == node.TEXT_NODE:
+                    text_parts.append(node.data)
+                elif node.nodeType == node.ELEMENT_NODE and node.tagName == 'text:span':
+                    # Assuming direct text content in <text:span>, for simplicity
+                    for child in node.childNodes:
+                        if child.nodeType == child.TEXT_NODE:
+                            text_parts.append(child.data)
+            paragraph_texts.append(''.join(text_parts))
+        return paragraph_texts
 
-    doc1_paragraphs = [p.text for p in doc1.paragraphs]
+    # Determine file types and load documents
-    doc2_paragraphs = [p.text for p in doc2.paragraphs]
+    if file1.endswith('.docx') and file2.endswith('.docx'):
+        doc1 = Document(file1)
+        doc2 = Document(file2)
+        doc1_paragraphs = [p.text for p in doc1.paragraphs]
+        doc2_paragraphs = [p.text for p in doc2.paragraphs]
+    elif file1.endswith('.odt') and file2.endswith('.odt'):
+        doc1 = load(file1)
+        doc2 = load(file2)
+        doc1_paragraphs = get_paragraph_texts_odt(doc1)
+        doc2_paragraphs = get_paragraph_texts_odt(doc2)
+    else:
+        # Unsupported file types or mismatch
+        print("Unsupported file types or mismatch between file types.")
+        return 0
 
+    # Process and compare documents
     if ignore_blanks:
         text1 = re.sub(r'\s+', ' ', '\n'.join(doc1_paragraphs)).strip()
         text2 = re.sub(r'\s+', ' ', '\n'.join(doc2_paragraphs)).strip()
@@ -64,19 +94,16 @@ def compare_docx_files(file1, file2, ignore_blanks=True):
             return 0
     else:
         if len(doc1_paragraphs) != len(doc2_paragraphs):
-            # print(len(doc1_paragraphs))
-            # print(len(doc2_paragraphs))
             return 0
 
         # Compare each paragraph
         for p1, p2 in zip(doc1_paragraphs, doc2_paragraphs):
             if p1 != p2:
-                # print(p1)
-                # print(p2)
                 return 0
 
     return 1
 
+
 def compare_init_lines(file1, file2):
     doc1 = Document(file1)
     doc2 = Document(file2)
@@ -93,6 +120,7 @@ def compare_init_lines(file1, file2):
 
     return 1
 
+
 def compare_docx_tables(docx_file1, docx_file2):
     doc1 = Document(docx_file1)
     doc2 = Document(docx_file2)
@@ -217,6 +245,7 @@ def compare_contains_image(docx_file1, docx_file2):
                 return 0
     return 1
 
+
 # file1 = 'path/to/file1.docx'
 # file2 = 'path/to/file2.docx'
 
@@ -253,26 +282,25 @@ def evaluate_colored_words_in_tables(file_path1, file_path2):
 
 
 def check_highlighted_words(file_path1, file_path2):
-    # if not compare_docx_files(file_path1, file_path2):
+    if not compare_docx_files(file_path1, file_path2):
-    #     return 0
-        
-    # Extract content.xml from the .odt file
-    extract_dir = file_path1 + "_extracted"
-    with zipfile.ZipFile(file_path1, 'r') as zip_ref:
-        zip_ref.extractall(extract_dir)
-    content_xml_path = os.path.join(extract_dir, 'content.xml')
-    with open(content_xml_path, 'r', encoding="utf-8") as file:
-        content_xml = file.read()
-
-    # Check for yellow highlights in the content.xml
-    yellow_highlight_pattern = re.compile(r'(.{0,50}background-color="#ffff00"[^>]*>.{0,50})')
-    yellow_highlight_matches = yellow_highlight_pattern.findall(content_xml)
-
-    # Return True if yellow highlights are NOT found, otherwise True
-    if yellow_highlight_matches:
         return 0
-    else:
+
-        return 1
+    doc = load(file_path1)
+    highlighted = False
+
+    for span in doc.getElementsByType(Span):
+        style_name = span.getAttribute('stylename')
+        if style_name:
+            for automatic_style in doc.automaticstyles.childNodes:
+                if automatic_style.getAttribute('name') == style_name:
+                    for property in automatic_style.childNodes:
+                        if property.getAttribute('backgroundcolor') == '#ffff00':
+                            highlighted = True
+                            break
+            if highlighted:
+                break
+
+    return 0 if highlighted else 1
 
 
 def evaluate_strike_through_last_paragraph(file_path1, file_path2):
@@ -318,9 +346,9 @@ def evaluate_spacing(file_path):
     body_spacing = document.paragraphs[1].paragraph_format.line_spacing
     conclusion_spacing = document.paragraphs[2].paragraph_format.line_spacing
     if (introduction_spacing == 1.0 and body_spacing == 2.0 and conclusion_spacing == 1.5):
-      return 1
+        return 1
     else:
-      return 0
+        return 0
 
 
 def check_italic_font_size_14(path1, path2):
@@ -358,13 +386,14 @@ def evaluate_alignment(docx_path):
             second_part = ' '.join(words[3:])
 
             # Check if the sentence structure matches the pattern: first part + large space/tab + second part
-            if not (first_part in sentence and second_part in sentence and sentence.find(first_part) < sentence.find(second_part)):
+            if not (first_part in sentence and second_part in sentence and sentence.find(first_part) < sentence.find(
+                    second_part)):
                 return 0  # The sentence does not meet the alignment criteria
 
     return 1  # All sentences meet the alignment criteria
 
 
-def get_unique_train_ids(initial_file): #fixed standard
+def get_unique_train_ids(initial_file):  # fixed standard
     doc = Document(initial_file)
     train_ids = set()
     processed_lines = 0
@@ -374,11 +403,12 @@ def get_unique_train_ids(initial_file): #fixed standard
         if len(line_parts) == 4:
             train_id = line_parts[1].strip()
             if train_id not in train_ids:
-              train_ids.add(train_id)
+                train_ids.add(train_id)
-              processed_lines += 1
+                processed_lines += 1
 
     return train_ids, processed_lines
 
+
 def check_no_duplicates(initial_file, processed_file):
     # Open the document
     train_ids_ini, ini_lines = get_unique_train_ids(initial_file)
@@ -405,6 +435,7 @@ def check_no_duplicates(initial_file, processed_file):
     # No duplicates found and at least one valid line was processed
     return 1
 
+
 def compare_docx_lines(file1, file2):
     # Read the text of the document, line by line
     doc1 = Document(file1)
@@ -416,21 +447,46 @@ def compare_docx_lines(file1, file2):
     # print(doc2_lines)
 
     # Convert the list of lines to sets and compare
-    return set(doc1_lines) == set(doc2_lines)
+    if set(doc1_lines) == set(doc2_lines):
+        return 1
+    else:
+        return 0
 
+
+# Docx file saved in the ubuntu cannot use this function to compare highlight, don't know why, deprecated
 def compare_highlighted_text(file1, file2):
-    def extract_highlighted_text(doc):
+    def extract_highlighted_text(file_path):
-        highlighted_text = []
+        highlighted_texts = []
-        # Iterate through each run in each paragraph to check for highlight
+
-        for paragraph in doc.paragraphs:
+        # Open the .docx file as a zip file and read the document.xml
-            for run in paragraph.runs:
+        with zipfile.ZipFile(file_path, 'r') as docx:
-                if run.font.highlight_color:  # Checks if the run is highlighted
+            with docx.open('word/document.xml') as document_xml:
-                    highlighted_text.append(run.text.strip())
+                tree = ET.parse(document_xml)
-        return highlighted_text
+                root = tree.getroot()
+
+        # Define the namespaces
+        namespaces = {
+            'w': 'http://schemas.openxmlformats.org/wordprocessingml/2006/main',
+        }
+
+        # Find all runs with highlight property
+        for run in root.findall('.//w:r', namespaces):
+            highlight = run.find('.//w:highlight', namespaces)
+            if highlight is not None and highlight.get(
+                    '{http://schemas.openxmlformats.org/wordprocessingml/2006/main}val') != 'none':
+                text = run.find('.//w:t', namespaces)
+                if text is not None:
+                    highlighted_texts.append(text.text)
+
+        return highlighted_texts
 
     # Read the highlighted text from both documents
-    doc1_highlighted = extract_highlighted_text(Document(file1))
+    doc1_highlighted = extract_highlighted_text(file1)
-    doc2_highlighted = extract_highlighted_text(Document(file2))
+    doc2_highlighted = extract_highlighted_text(file2)
 
     # Compare the sets of highlighted text to check if they are the same
-    return set(doc1_highlighted) == set(doc2_highlighted)
+    if set(doc1_highlighted) == set(doc2_highlighted):
+        return 1
+    else:
+        return 0
+

desktop_env/evaluators/metrics/gimp.py

@@ -211,7 +211,10 @@ def check_brightness_decrease_and_structure_sim(src_path, tgt_path):
     img_tgt_normalized = normalize_brightness(img_tgt, target_brightness)
 
     structure_same = structure_check_by_mse(img_src_normalized, img_tgt_normalized)
-    return brightness_reduced and structure_same
+    if brightness_reduced and structure_same:
+        return 1.
+    else:
+        return 0.
 
 
 def check_saturation_increase_and_structure_sim(src_path, tgt_path):
@@ -240,7 +243,10 @@ def check_saturation_increase_and_structure_sim(src_path, tgt_path):
     else:
         structure_same = False
 
-    return saturation_increased and structure_same
+    if saturation_increased and structure_same:
+        return 1.
+    else:
+        return 0.
 
 
 def check_file_exists_and_structure_sim(src_path, tgt_path):
@@ -294,7 +300,10 @@ def check_triangle_position(tgt_path):
     tolerance = 0.05 * np.array(img_array.shape[:2])
     middle = np.all(np.abs(centroid - image_center) < tolerance)
 
-    return middle
+    if bool(middle):
+        return 1.
+    else:
+        return 0.
 
 
 def check_structure_sim(src_path, tgt_path):
@@ -325,7 +334,10 @@ def check_contrast_increase_and_structure_sim(src_path, tgt_path):
     # Check structure
     structure_same = structure_check_by_ssim(source_image, target_image, threshold=0.65)
 
-    return higher_contrast and structure_same
+    if higher_contrast and structure_same:
+        return 1.
+    else:
+        return 0.
 
 
 def check_config_status(actual_config_path, rule):
@@ -341,13 +353,13 @@ def check_config_status(actual_config_path, rule):
         items = line.strip().lstrip('(').rstrip(')\n').split()
         if isinstance(rule["key"], str):
             if items[0] == rule["key"] and items[-1] == rule["value"]:
-                return True
+                return 1.
         elif isinstance(rule["key"], list) and len(rule["key"]) == 2:
             if items[0] == rule["key"][0] \
                     and items[1] == rule["key"][1] \
                     and items[-1] == rule["value"]:
-                return True
+                return 1.
-    return False
+    return 0.
 
 
 def check_image_size_and_structure_sim(src_path, tgt_path, height=512, width=None):
@@ -373,7 +385,10 @@ def check_image_size_and_structure_sim(src_path, tgt_path, height=512, width=Non
     resized_target_image = target_image.resize(source_image.size)
     structure_same = structure_check_by_ssim(source_image, resized_target_image)
 
-    return width_same and height_same and structure_same
+    if width_same and height_same and structure_same:
+        return 1.
+    else:
+        return 0.
 
 
 def check_palette_and_structure_sim(src_path, tgt_path):
@@ -389,7 +404,10 @@ def check_palette_and_structure_sim(src_path, tgt_path):
     target_image = Image.open(tgt_path)
     source_image = source_image.convert('RGB')
     structure_same = structure_check_by_ssim(source_image, target_image)
-    return palette_based and structure_same
+    if palette_based and structure_same:
+        return 1.
+    else:
+        return 0.
 
 
 def check_textbox_on_leftside(src_path):
@@ -411,7 +429,10 @@ def check_textbox_on_leftside(src_path):
                 break  # Stop after finding the first dark pixel in this row
 
     # Here we define "almost" on the left side as being within the left 5% of the image
-    return left_most_dark_pixel < width * 0.05
+    if left_most_dark_pixel < width * 0.05:
+        return 1.
+    else:
+        return 0.
 
 
 def check_image_mirror(src_path, tgt_path):

desktop_env/evaluators/metrics/slides.py

@@ -318,3 +318,50 @@ def check_transition(pptx_file, rules):
                     return 0.
             else:
                 return 0.
+
+
+def check_page_number_colors(pptx_file, rules):
+    color = rules["color"]
+
+    def is_red(rgb_str, threshold=50):
+        r, g, b = int(rgb_str[1:3], 16), int(rgb_str[3:5], 16), int(rgb_str[5:7], 16)
+        return r > g + threshold and r > b + threshold
+
+    def is_blue(rgb_str, threshold=50):
+        r, g, b = int(rgb_str[1:3], 16), int(rgb_str[3:5], 16), int(rgb_str[5:7], 16)
+        return b > g + threshold and b > r + threshold
+
+    def is_green(rgb_str, threshold=50):
+        r, g, b = int(rgb_str[1:3], 16), int(rgb_str[3:5], 16), int(rgb_str[5:7], 16)
+        return g > r + threshold and g > b + threshold
+
+    def is_black(rgb_str, threshold=50):
+        r, g, b = int(rgb_str[1:3], 16), int(rgb_str[3:5], 16), int(rgb_str[5:7], 16)
+        return r < threshold and g < threshold and b < threshold
+
+    with zipfile.ZipFile(pptx_file, 'r') as zip_ref:
+        slide_master_name = 'ppt/slideMasters/slideMaster1.xml'
+        with zip_ref.open(slide_master_name) as slide_master_file:
+            tree = ET.parse(slide_master_file)
+            root = tree.getroot()
+
+            namespaces = {
+                'a': 'http://schemas.openxmlformats.org/drawingml/2006/main',
+                'p': 'http://schemas.openxmlformats.org/presentationml/2006/main',
+            }
+
+            color_elems = root.findall('.//a:solidFill//a:srgbClr', namespaces)
+            slides_color_val = color_elems[-2].get('val')
+
+    if slides_color_val is None:
+        return 0
+    elif color == "red" and not is_red(slides_color_val):
+        return 0
+    elif color == "blue" and not is_blue(slides_color_val):
+        return 0
+    elif color == "green" and not is_green(slides_color_val):
+        return 0
+    elif color == "black" and not is_black(slides_color_val):
+        return 0
+
+    return 1

evaluation_examples/examples/gimp/06ca5602-62ca-47f6-ad4f-da151cde54cc.json

@@ -10,7 +10,7 @@
         "files": [
           {
             "url": "https://drive.google.com/uc?export=download&id=1fxR-hAaF5BoONWxmzJTrIxXFw9EHTg_A",
-            "path": "~/Desktop/computer.png"
+            "path": "Desktop/computer.png"
           }
         ]
       }
@@ -20,7 +20,7 @@
       "parameters": {
         "command": [
           "gimp",
-          "~/Desktop/computer.png"
+          "Desktop/computer.png"
         ]
       }
     }
@@ -83,12 +83,12 @@
     "func": "check_palette_and_structure_sim",
     "expected": {
       "type": "vm_file",
-      "path": "~/Desktop/computer.png",
+      "path": "Desktop/computer.png",
       "dest": "computer.png"
     },
     "result": {
       "type": "vm_file",
-      "path": "~/Desktop/palette_computer.png",
+      "path": "Desktop/palette_computer.png",
       "dest": "palette_computer.png"
     }
   }

evaluation_examples/examples/gimp/2a729ded-3296-423d-aec4-7dd55ed5fbb3.json

@@ -10,11 +10,11 @@
                 "files": [
                     {
                         "url": "https://drive.google.com/uc?export=download&id=1TOtPi1CQsWblGUtQ6AqayfjsPZ_THBJo",
-                        "path": "~/Desktop/dog_with_background.png"
+                        "path": "Desktop/dog_with_background.png"
                     },
                     {
                         "url": "https://drive.google.com/uc?export=download&id=15YWmeOyUaA7vMX9Ts7-qyh82T8mHeepx",
-                        "path": "~/Desktop/dog_cutout_gold.png"
+                        "path": "Desktop/dog_cutout_gold.png"
                     }
                 ]
             }
@@ -24,7 +24,7 @@
             "parameters": {
                 "command": [
                     "gimp",
-                    "~/Desktop/dog_with_background.png"
+                    "Desktop/dog_with_background.png"
                 ]
             }
         }
@@ -87,12 +87,12 @@
         "func": "check_structure_sim",
         "expected": {
             "type": "vm_file",
-            "path": "~/Desktop/dog_cutout_gold.png",
+            "path": "Desktop/dog_cutout_gold.png",
             "dest": "dog_cutout_gold.png"
         },
         "result": {
             "type": "vm_file",
-            "path": "~/Desktop/dog_without_background.png",
+            "path": "Desktop/dog_without_background.png",
             "dest": "dog_without_background.png"
         }
     }

evaluation_examples/examples/gimp/554785e9-4523-4e7a-b8e1-8016f565f56a.json

@@ -10,7 +10,7 @@
                 "files": [
                     {
                         "url": "https://drive.usercontent.google.com/download?id=1-3cc3qqjlJ_3d9IpwsuvSuYpyJ2LxrIA&export=download&authuser=0&confirm=t&uuid=6b08e902-ed25-47d1-97d9-7794f9508288&at=APZUnTW74TY2hHtBbWDa0N7-cE58:1705060487948",
-                        "path": "~/Desktop/woman_sitting_by_the_tree2.png"
+                        "path": "Desktop/woman_sitting_by_the_tree2.png"
                     }
                 ]
             }
@@ -20,7 +20,7 @@
             "parameters": {
                 "command": [
                     "gimp",
-                    "~/Desktop/woman_sitting_by_the_tree2.png"
+                    "Desktop/woman_sitting_by_the_tree2.png"
                 ]
             }
         }
@@ -83,12 +83,12 @@
         "func": "check_saturation_increase_and_structure_sim",
         "expected": {
             "type": "vm_file",
-            "path": "~/Desktop/woman_sitting_by_the_tree2.png",
+            "path": "Desktop/woman_sitting_by_the_tree2.png",
             "dest": "woman_sitting_by_the_tree2.png"
         },
         "result": {
             "type": "vm_file",
-            "path": "~/Desktop/edited_colorful.png",
+            "path": "Desktop/edited_colorful.png",
             "dest": "edited_colorful.png"
         }
     }

evaluation_examples/examples/gimp/72f83cdc-bf76-4531-9a1b-eb893a13f8aa.json

@@ -10,7 +10,7 @@
         "files": [
           {
             "url": "https://drive.google.com/uc?export=download&id=1bS4xGh8bhQVSuRCwFscYZO_xzymAJgM8",
-            "path": "~/Desktop/berry.png"
+            "path": "Desktop/berry.png"
           }
         ]
       }
@@ -20,7 +20,7 @@
       "parameters": {
         "command": [
           "gimp",
-          "~/Desktop/berry.png"
+          "Desktop/berry.png"
         ]
       }
     }
@@ -83,12 +83,12 @@
     "func": "check_image_mirror",
     "expected": {
       "type": "vm_file",
-      "path": "~/Desktop/berry.png",
+      "path": "Desktop/berry.png",
       "dest": "berry.png"
     },
     "result": {
       "type": "vm_file",
-      "path": "~/Desktop/berry_mirror.png",
+      "path": "Desktop/berry_mirror.png",
       "dest": "berry_mirror.png"
     }
   }

evaluation_examples/examples/gimp/734d6579-c07d-47a8-9ae2-13339795476b.json

@@ -10,11 +10,11 @@
         "files": [
           {
             "url": "https://drive.google.com/uc?export=download&id=18r4e9FqlMqqD1ltiHRixZkE_Jc865wgb",
-            "path": "~/Desktop/white_background_with_object.xcf"
+            "path": "Desktop/white_background_with_object.xcf"
           },
           {
             "url": "https://drive.google.com/uc?export=download&id=1D9rLa3XDEk5warlI0jVxcbixyr8SQm_C",
-            "path": "~/Desktop/white_background_with_object.png"
+            "path": "Desktop/white_background_with_object.png"
           }
         ]
       }
@@ -24,7 +24,7 @@
       "parameters": {
         "command": [
           "gimp",
-          "~/Desktop/white_background_with_object.xcf"
+          "Desktop/white_background_with_object.xcf"
         ]
       }
     }
@@ -87,12 +87,12 @@
     "func": "check_green_background",
     "expected": {
       "type": "vm_file",
-      "path": "~/Desktop/white_background_with_object.png",
+      "path": "Desktop/white_background_with_object.png",
       "dest": "white_background_with_object.png"
     },
     "result": {
       "type": "vm_file",
-      "path": "~/Desktop/green_background_with_object.png",
+      "path": "Desktop/green_background_with_object.png",
       "dest": "green_background_with_object.png"
     }
   }

evaluation_examples/examples/gimp/77b8ab4d-994f-43ac-8930-8ca087d7c4b4.json

@@ -10,7 +10,7 @@
                 "files": [
                     {
                         "url": "https://drive.usercontent.google.com/download?id=13ECFsPxznuoCANto21ijj9OzP0APukIH&export=download&authuser=0&confirm=t&uuid=d8f2dd03-8992-4646-be62-3a3cf89583f2&at=APZUnTVsR0xmbXvpFIpXLzCcLrMa:1705062951627",
-                        "path": "~/Desktop/The_Lost_River_Of_Dreams.jpg"
+                        "path": "Desktop/The_Lost_River_Of_Dreams.jpg"
                     }
                 ]
             }
@@ -20,7 +20,7 @@
             "parameters": {
                 "command": [
                     "gimp",
-                    "~/Desktop/The_Lost_River_Of_Dreams.jpg"
+                    "Desktop/The_Lost_River_Of_Dreams.jpg"
                 ]
             }
         }
@@ -33,12 +33,12 @@
         "func": "check_file_exists_and_structure_sim",
         "expected": {
             "type": "vm_file",
-            "path": "~/Desktop/The_Lost_River_Of_Dreams.jpg",
+            "path": "Desktop/The_Lost_River_Of_Dreams.jpg",
             "dest": "The_Lost_River_Of_Dreams.jpg"
         },
         "result": {
             "type": "vm_file",
-            "path": "~/Desktop/export.jpg",
+            "path": "Desktop/export.jpg",
             "dest": "export.jpg"
         }
     }

evaluation_examples/examples/gimp/7a4deb26-d57d-4ea9-9a73-630f66a7b568.json

@@ -10,7 +10,7 @@
                 "files": [
                     {
                         "url": "https://drive.usercontent.google.com/download?id=1SIvX9Wimyw6i2UvnoLTNDHIObvDLAsIM&export=download&authuser=0&confirm=t&uuid=a48447ab-13a2-421f-9662-6ffff8f6f6d5&at=APZUnTVRxofs822XxgEv33WwYCkb:1705046264363",
-                        "path": "~/Desktop/woman_sitting_by_the_tree.png"
+                        "path": "Desktop/woman_sitting_by_the_tree.png"
                     }
                 ]
             }
@@ -20,7 +20,7 @@
             "parameters": {
                 "command": [
                     "gimp",
-                    "~/Desktop/woman_sitting_by_the_tree.png"
+                    "Desktop/woman_sitting_by_the_tree.png"
                 ]
             }
         }
@@ -83,12 +83,12 @@
         "func": "check_brightness_decrease_and_structure_sim",
         "expected": {
             "type": "vm_file",
-            "path": "~/Desktop/woman_sitting_by_the_tree.png",
+            "path": "Desktop/woman_sitting_by_the_tree.png",
             "dest": "woman_sitting_by_the_tree.png"
         },
         "result": {
             "type": "vm_file",
-            "path": "~/Desktop/edited_darker.png",
+            "path": "Desktop/edited_darker.png",
             "dest": "edited_darker.png"
         }
     }

evaluation_examples/examples/gimp/a746add2-cab0-4740-ac36-c3769d9bfb46.json

@@ -10,7 +10,7 @@
                 "files": [
                     {
                         "url": "https://drive.google.com/uc?export=download&id=1IQPiNT6Vmd0YO0OytaB8lb_Dvo1mCGnP",
-                        "path": "~/Desktop/dog_with_background.png"
+                        "path": "Desktop/dog_with_background.png"
                     }
                 ]
             }
@@ -20,7 +20,7 @@
             "parameters": {
                 "command": [
                     "gimp",
-                    "~/Desktop/dog_with_background.png"
+                    "Desktop/dog_with_background.png"
                 ]
             }
         }

evaluation_examples/examples/gimp/b148e375-fe0b-4bec-90e7-38632b0d73c2.json

@@ -10,7 +10,7 @@
                 "files": [
                     {
                         "url": "https://drive.google.com/uc?export=download&id=1CqcNw_8O0VeLuD0VkVTgDHOoUMFQNOXN",
-                        "path": "~/Desktop/white_background.xcf"
+                        "path": "Desktop/white_background.xcf"
                     }
                 ]
             }
@@ -20,7 +20,7 @@
             "parameters": {
                 "command": [
                     "gimp",
-                    "~/Desktop/white_background.xcf"
+                    "Desktop/white_background.xcf"
                 ]
             }
         }

evaluation_examples/examples/gimp/d16c99dc-2a1e-46f2-b350-d97c86c85c15.json

@@ -10,11 +10,11 @@
         "files": [
           {
             "url": "https://drive.google.com/uc?export=download&id=14ZEnUNHrIVDvcyF6A6EFgma6LcgPWj1X",
-            "path": "~/Desktop/dog_with_background.png"
+            "path": "Desktop/dog_with_background.png"
           },
           {
             "url": "https://drive.google.com/uc?export=download&id=1wq2sjBX8X-SGIEj1GKLGAuQ8BNVYqa0-",
-            "path": "~/Desktop/dog_with_background_two_layers.xcf"
+            "path": "Desktop/dog_with_background_two_layers.xcf"
           }
         ]
       }
@@ -24,7 +24,7 @@
       "parameters": {
         "command": [
           "gimp",
-          "~/Desktop/dog_with_background_two_layers.xcf"
+          "Desktop/dog_with_background_two_layers.xcf"
         ]
       }
     }
@@ -87,12 +87,12 @@
     "func": "check_image_size_and_structure_sim",
     "expected":{
         "type": "vm_file",
-        "path": "~/Desktop/dog_with_background.png",
+        "path": "Desktop/dog_with_background.png",
         "dest": "dog_with_background.png"
       },
     "result": {
       "type": "vm_file",
-      "path": "~/Desktop/resized.png",
+      "path": "Desktop/resized.png",
       "dest": "resized.png"
     }
   }

evaluation_examples/examples/gimp/e2dd0213-26db-4349-abe5-d5667bfd725c.json

@@ -10,7 +10,7 @@
         "files": [
           {
             "url": "https://drive.google.com/uc?export=download&id=1KMHuScgF8Eq9OPR40SCntxO3Q1DUmbKV",
-            "path": "~/Desktop/orange_background.xcf"
+            "path": "Desktop/orange_background.xcf"
           }
         ]
       }
@@ -20,7 +20,7 @@
       "parameters": {
         "command": [
           "gimp",
-          "~/Desktop/orange_background.xcf"
+          "Desktop/orange_background.xcf"
         ]
       }
     }
@@ -83,7 +83,7 @@
     "func": "check_textbox_on_leftside",
     "result": {
       "type": "vm_file",
-      "path": "~/Desktop/leftside_textbox.png",
+      "path": "Desktop/leftside_textbox.png",
       "dest": "leftside_textbox.png"
     }
   }

evaluation_examples/examples/gimp/f4aec372-4fb0-4df5-a52b-79e0e2a5d6ce.json

@@ -10,11 +10,11 @@
                 "files": [
                     {
                         "url": "https://drive.google.com/uc?export=download&id=1p0m5meh7NIT4v6hNdqJqK3GCcpZiMapB",
-                        "path": "~/Desktop/Triangle_On_The_Side.png"
+                        "path": "Desktop/Triangle_On_The_Side.png"
                     },
                     {
                         "url": "https://drive.google.com/uc?export=download&id=1w7f2fW2mWkUbteQvQd-lepHzvBC9GaG_",
-                        "path": "~/Desktop/Triangle_On_The_Side.xcf"
+                        "path": "Desktop/Triangle_On_The_Side.xcf"
                     }
                 ]
             }
@@ -24,7 +24,7 @@
             "parameters": {
                 "command": [
                     "gimp",
-                    "~/Desktop/Triangle_On_The_Side.xcf"
+                    "Desktop/Triangle_On_The_Side.xcf"
                 ]
             }
         }
@@ -87,7 +87,7 @@
         "func": "check_triangle_position",
         "result": {
             "type": "vm_file",
-            "path": "~/Desktop/Triangle_In_The_Middle.png",
+            "path": "Desktop/Triangle_In_The_Middle.png",
             "dest": "Triangle_In_The_Middle.png"
         }
     }

evaluation_examples/examples/gimp/f723c744-e62c-4ae6-98d1-750d3cd7d79d.json

@@ -10,7 +10,7 @@
         "files": [
           {
             "url": "https://drive.google.com/uc?export=download&id=1X42_kOanL74vu_p6QdcZuiyzDQi3kA7F",
-            "path": "~/Desktop/berries.png"
+            "path": "Desktop/berries.png"
           }
         ]
       }
@@ -20,7 +20,7 @@
       "parameters": {
         "command": [
           "gimp",
-          "~/Desktop/berries.png"
+          "Desktop/berries.png"
         ]
       }
     }
@@ -83,12 +83,12 @@
     "func": "check_contrast_increase_and_structure_sim",
     "expected": {
       "type": "vm_file",
-      "path": "~/Desktop/berries.png",
+      "path": "Desktop/berries.png",
       "dest": "berries.png"
     },
     "result": {
       "type": "vm_file",
-      "path": "~/Desktop/berries_contrast.png",
+      "path": "Desktop/berries_contrast.png",
       "dest": "berries_contrast.png"
     }
   }

evaluation_examples/examples/libreoffice_impress/ac9bb6cb-1888-43ab-81e4-a98a547918cd.json

@@ -3,10 +3,66 @@
   "snapshot": "libreoffice_impress",
   "instruction": "I am preparing a PPT in Libreoffice impress. The slide number is barely visible to me. Please help me change the color of the slide number to red?",
   "source": "https://superuser.com/questions/1674211/how-to-change-colour-of-slide-number-in-libre-office",
-  "config": [],
+  "config": [
+    {
+      "type": "download",
+      "parameters": {
+        "files": [
+          {
+            "url": "https://drive.usercontent.google.com/download?id=1_AmuPiTAyCSrquI2ERV0DnUtt1yaxtcj&export=download&authuser=0&confirm=t&uuid=d9c63ec2-45e6-4198-8519-c04c497ea1bb&at=APZUnTVtpEJatGaAex2OPSHDWgGM:1706533369423",
+            "path": "Desktop/saa-format-guide.pptx"
+          }
+        ]
+      }
+    },
+    {
+      "type": "open",
+      "parameters": {
+        "path": "Desktop/saa-format-guide.pptx"
+      }
+    }
+  ],
   "trajectory": "trajectories/",
   "related_apps": [
     "libreoffice_impress"
   ],
-  "evaluator": "evaluation_dir"
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "activate_window",
+        "parameters": {
+          "window_name": "saa-format-guide.pptx - LibreOffice Impress",
+          "strict": true
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python",
+            "-c",
+            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);"
+          ]
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      }
+    ],
+    "func": "check_page_number_colors",
+    "result": {
+      "type": "vm_file",
+      "path": "/home/user/Desktop/saa-format-guide.pptx",
+      "dest": "saa-format-guide.pptx"
+    }
+  }
 }

evaluation_examples/examples/libreoffice_writer/41c621f7-3544-49e1-af8d-dafd0f834f75.json

@@ -1,7 +1,7 @@
 {
   "id": "41c621f7-3544-49e1-af8d-dafd0f834f75",
   "snapshot": "libreoffice_writer",
-  "instruction": "I am adding comments which begin with \"#\" right beside sentences my students have written. I want to make my comments to stand out more, so I am highlighting my comments to yellow. Could you help me on this? It is hard for me to color comments one by one.",
+  "instruction": "I added annotations starting with a \"#\" next to the sentences that my students wrote. I want my annotations to stand out more, so I highlighted them in yellow. Can you help me with this favor? I'm having a hard time coloring the annotations one by one. By the way, remember to remove the # sign after highlighting the text part. Thanks!",
   "source": "https://superuser.com/questions/1668018/how-to-auto-format-lines-in-libre-office-writer",
   "config": [
     {
@@ -52,10 +52,7 @@
         }
       }
     ],
-    "func": [
+    "func": "compare_highlighted_text",
-      "compare_highlighted_text",
-      "compare_docx_files"
-    ],
     "expected": {
       "type": "cloud_file",
       "path": "https://drive.usercontent.google.com/download?id=1x2gplRDN1AOnRWiqfqmJqN9JTQ_x4sPu&export=download&authuser=0&confirm=t&uuid=ea6b0a61-fd06-4823-b253-05473c77c192&at=APZUnTUxWWFB_TOKLR9chabErm7b:1706018376493",

evaluation_examples/examples/libreoffice_writer/6a33f9b9-0a56-4844-9c3f-96ec3ffb3ba2.json

@@ -9,7 +9,7 @@
       "parameters": {
         "files": [
           {
-            "url": "https://drive.google.com/uc?id=1Ul4mtQ4SpUNLVDlaiJThPprBe6WTGZ2L&export=download",
+            "url": "https://drive.usercontent.google.com/download?id=1_1uItoxJsp8PgBqaED0U3bOfKxIweNkJ&export=download&authuser=0&confirm=t&uuid=facec3ed-2fd6-49f7-aeda-95feaa38f45c&at=APZUnTU6B0xoLG-kI0V2WZ74oSI6:1706544439742",
             "path": "Desktop/sample-recruitment-phone-script.odt"
           }
         ]
@@ -55,7 +55,7 @@
     "func": "check_highlighted_words",
     "expected": {
       "type": "cloud_file",
-      "path": "https://drive.google.com/uc?id=12iMkgCYuUyhKUXux96kANLIeud0Wz9ct&export=download",
+      "path": "https://drive.usercontent.google.com/download?id=1Qxz3EjHuiLa8yZaCr1DJHwEOSd6U-cjY&export=download&authuser=0&confirm=t&uuid=c502680e-51f9-4e4f-90d0-58b123a09b3d&at=APZUnTXN2UX9EuvB1PZoJ5koJwmQ:1706544440969",
       "dest": "sample-recruitment-phone-script_Gold.odt"
     },
     "result": {

evaluation_examples/examples/multi_apps/2b9493d7-49b8-493a-a71b-56cd1f4d6908.json

@@ -36,7 +36,8 @@
             "parameters": {
                 "command": [
                     "gnome-terminal",
-                    "--maximize"
+                    "--maximize",
+                    "--working-directory=/home/user/"
                 ]
             }
         }
@@ -59,43 +60,27 @@
               }
             }
         ],
-        "func": ["check_include_exclude", "check_include_exclude"],
+        "func": "check_include_exclude",
-        "conj": "and",
+        "result": {
-        "result": [
+            "type": "vm_command_line",
-            {
+            "command": [
-                "type": "vm_command_line",
+                "/bin/bash",
-                "command": [
+                "-c",
-                    "/bin/bash",
+                "output=$(ps aux | grep \"[s]office\"]); if [ -z \"$output\" ]; then echo \"no libreoffice is running\"; else echo \"libreoffice is still running\"; fi; output=$(cat ~/.bash_history | grep \"[k]ill\"); if [ -z \"$output\" ]; then echo \"not killed from terminal\"; else echo \"killed from terminal\"; fi"
-                    "-c",
+            ]
-                    "output=$(ps aux | grep \"[s]office\"]); if [ -z \"$output\" ]; then echo \"true\"; else echo \"false\"; fi"
+        },
-                ]
+        "expected": {
-            },
+            "type": "rule",
-            {
+            "rules": {
-                "type": "vm_command_line",
+                "include": [
-                "command": [
+                    "no libreoffice is running",
-                    "/bin/bash",
+                    "killed from terminal"
-                    "-c",
+                ],
-                    "output=$(cat ~/.bash_history | grep \"[k]ill\"); if [ -z \"$output\" ]; then echo \"false\"; else echo \"true\"; fi"
+                "exclude": [
+                    "libreoffice is still running",
+                    "not killed from terminal"
                 ]
             }
-        ],
+        }
-        "expected": [
-            {
-                "type": "rule",
-                "rules": {
-                    "include": [
-                        "true\n"
-                    ]
-                }
-            },
-            {
-                "type": "rule",
-                "rules": {
-                    "include": [
-                        "true\n"
-                    ]
-                }
-            }
-        ]
     }
 }

evaluation_examples/examples/multi_apps/2c9fc0de-3ee7-45e1-a5df-c86206ad78b5.json

@@ -6,15 +6,12 @@
     "config": [
         {
             "type": "download",
-            "path": "",
-        },
-        {
-            "type": "execute",
             "parameters": {
-                "command": [
+                "files": [
-                    "/bin/bash",
+                    {
-                    "-c",
+                        "url": "https://drive.usercontent.google.com/download?id=1aANBCEHT6K8GmHDMEYtL1LQXdApqVoGv&export=download",
-                    "git config --global user.name \"xlang\" && git config --global user.email \"xlang2024anonym@gmail.com\" && mkdir -p /home/user/projects/remote_project && cd /home/user/projects/remote_project && git init --initial-branch=main --bare .git"
+                        "path": "/home/user/Downloads/binder.zip"
+                    }
                 ]
             }
         },
@@ -24,7 +21,17 @@
                 "command": [
                     "/bin/bash",
                     "-c",
-                    "mkdir -p /home/user/projects/hello_world && cd /home/user/projects/hello_world && git init --initial-branch main && git remote add origin /home/user/projects/remote_project && echo \"Hello World!\" > README.md"
+                    "git config --global user.name \"xlang\" && git config --global user.email \"xlang2024anonym@gmail.com\" && mkdir -p /home/user/projects/remote_project && cd /home/user/projects/remote_project && git init --initial-branch=main && git config receive.denyCurrentBranch ignore "
+                ]
+            }
+        },
+        {
+            "type": "execute",
+            "parameters": {
+                "command": [
+                    "/bin/bash",
+                    "-c",
+                    "unzip -q /home/user/Downloads/binder.zip -d /home/user/projects/ && cd /home/user/projects/binder && git init --initial-branch main && git remote add origin /home/user/projects/remote_project"
                 ]
             }
         },
@@ -34,7 +41,7 @@
                 "command": [
                     "gnome-terminal",
                     "--maximize",
-                    "--working-directory=/home/user/projects/hello_world"
+                    "--working-directory=/home/user/projects/binder"
                 ]
             }
         }
@@ -45,19 +52,23 @@
         "terminal"
     ],
     "evaluator": {
-        "func": "compare_docx_files",
+        "func": "check_include_exclude",
         "result": {
-            "type": "vm_file",
+            "type": "vm_command_line",
-            "path": "/home/user/Desktop/notes.docx",
+            "command": [
-            "dest": "notes.docx"
+                "/bin/bash",
+                "-c",
+                "cd /home/user/projects/remote_project; git log --oneline | head -n 1 | awk '{for (i=2; i<=NF; i++) {printf \"%s%s\", $i, (i==NF ? \"\" : \" \")}; print \"\"}'; remote_id=$(git log --oneline | head -n 1 | awk '{print $1}'); cd /home/user/projects/binder; local_id=$(git log --oneline | head -n 1 | awk '{print $1}'); if [ \"${local_id}\" = \"${remote_id}\" ]; then echo \"repo is synchronous\"; else echo \"repo is not synchronous\"; fi"
+            ]
         },
         "expected": {
-            "type": "cloud_file",
+            "type": "rule",
-            "path": "https://drive.usercontent.google.com/download?id=1Xl6tgQ0K5qA1BDA2fKTK2xFLzXwbtkZ6&export=download",
+            "rules": {
-            "dest": "notes_gold.docx"
+                "include": [
-        },
+                    "daily update",
-        "options": {
+                    "repo is synchronous"
-            "ignore_blanks": true
+                ]
+            }
         }
     }
 }

evaluation_examples/examples/multi_apps/510f64c8-9bcc-4be1-8d30-638705850618.json

@@ -0,0 +1,116 @@
+{
+    "id": "510f64c8-9bcc-4be1-8d30-638705850618",
+    "snapshot": "vscode",
+    "instruction": "Could you start VS Code in folder ~/Desktop/project from the terminal?",
+    "source": "https://www.geeksforgeeks.org/how-to-start-vs-code-from-the-terminal-command-line/",
+    "config": [
+        {
+            "type": "download",
+            "parameters": {
+                "files": [
+                    {
+                        "url": "https://drive.usercontent.google.com/download?id=1PnC-zxTtntYvuGlnIV2_05jiHfkX_1E-&export=download",
+                        "path": "/home/user/Downloads/vscodeEvalExtension.zip"
+                    }
+                ]
+            }
+        },
+        {
+            "type": "execute",
+            "parameters": {
+                "command": [
+                    "/bin/bash",
+                    "-c",
+                    "cd /home/user/Downloads && unzip -q vscodeEvalExtension.zip && code --install-extension vscodeEvalExtension/eval-0.0.1.vsix && rm -rf vscodeEvalExtension vscodeEvalExtension.zip && mkdir -p ~/Desktop/project/.vscode && history -c && echo > ~/.bash_history"
+                ]
+            }
+        },
+        {
+            "type": "download",
+            "parameters": {
+                "files": [
+                    {
+                        "url": "https://drive.usercontent.google.com/download?id=1akdsiRVdq6CUtT-FX8Dpf8ruPTq6DcFn&export=download&authuser=0&confirm=t&uuid=ce2fa96a-454e-43d9-bbe3-98553b7eed0d&at=APZUnTVw_YQ1URTvP34vrmKcw0b4:1705222451052",
+                        "path": "/home/user/Desktop/project/main.py"
+                    },
+                    {
+                        "url": "https://drive.usercontent.google.com/download?id=1BkwtqtAzv_K2CrTbJZ0HbMHBffzdD9vc&export=download&authuser=0&confirm=t&uuid=28f77090-deef-49a1-b156-91317881e75e&at=APZUnTXuaR6i_3t3Prslk535GaO5:1705222457290",
+                        "path": "/home/user/Desktop/project/README.md"
+                    },
+                    {
+                        "url": "https://drive.usercontent.google.com/download?id=1ea_zF2tbcXOB8w9neBV-U5xI2nnPzIw_&export=download&authuser=0&confirm=t&uuid=9cf8c5bb-a880-475c-b80b-967a0c4fbea4&at=APZUnTUdjIj80F3Mbgi72eZDTZLO:1705222462443",
+                        "path": "/home/user/Desktop/project/.vscode/settings.json"
+                    }
+                ]
+            }
+        },
+        {
+            "type": "launch",
+            "parameters": {
+                "command": [
+                    "gnome-terminal",
+                    "--maximize",
+                    "--working-directory=/home/user/"
+                ]
+            }
+        }
+    ],
+    "trajectory": "trajectories/",
+    "related_apps": [
+        "vscode",
+        "terminal"
+    ],
+    "evaluator": {
+        "postconfig": [
+            {
+              "type": "execute",
+              "parameters": {
+                "command": [
+                    "/bin/bash",
+                    "-c",
+                    "killall gnome-terminal-server"
+                ]
+              }
+            },
+            {
+                "type": "activate_window",
+                "parameters": {
+                    "window_name": "Visual Studio Code"
+                }
+            }
+        ],
+        "func": ["check_include_exclude", "compare_config"],
+        "result": [
+            {
+                "type": "vm_command_line",
+                "command": [
+                    "/bin/bash",
+                    "-c",
+                    "use_terminal=$(cat ~/.bash_history | grep \"[c]ode \"); if [ -z \"$use_terminal\" ]; then echo \"false\"; else echo \"true\"; fi"
+                ]
+            },
+            {
+                "type": "vscode_config",
+                "vscode_extension_command": "OpenProject",
+                "path": "/home/user/OpenProject.txt",
+                "dest": "OpenProject.txt"
+            }
+        ],
+        "expected": [
+            {
+                "type": "rule",
+                "rules": {
+                    "include": [
+                        "true"
+                    ]
+                }
+            },
+            {
+                "type": "rule",
+                "rules": {
+                    "expect": "project"
+                }
+            }
+        ]
+    }
+}

evaluation_examples/examples/multi_apps/937087b6-f668-4ba6-9110-60682ee33441.json

@@ -0,0 +1,26 @@
+{
+    "id": "937087b6-f668-4ba6-9110-60682ee33441",
+    "snapshot": "vlc",
+    "instruction": "I am currently using a ubuntu system. Could you help me set the default video player as VLC?",
+    "source": "https://superuser.com/questions/187440/set-default-ubuntu-video-player-as-vlc",
+    "config": [],
+    "trajectory": "trajectories/",
+    "related_apps": [
+        "vlc",
+        "os"
+    ],
+    "evaluator": {
+        "func": "check_include_exclude",
+        "result": {
+            "type": "default_video_player"
+        },
+        "expected": {
+            "type": "rule",
+            "rules": {
+                "include": [
+                    "vlc.desktop"
+                ]
+            }
+        }
+    }
+}

evaluation_examples/examples/vs_code/53ad5833-3455-407b-bbc6-45b4c79ab8fb.json

@@ -4,6 +4,27 @@
   "instruction": "Please help me use VS Code to open the \"project\" in the \"user\" folder under \"home\".",
   "source": "https://www.youtube.com/watch?v=VqCgcpAypFQ",
   "config": [
+    {
+      "type": "download",
+      "parameters": {
+        "files": [
+          {
+            "url": "https://drive.usercontent.google.com/download?id=1PnC-zxTtntYvuGlnIV2_05jiHfkX_1E-&export=download",
+            "path": "/home/user/Downloads/vscodeEvalExtension.zip"
+          }
+        ]
+      }
+    },
+    {
+      "type": "execute",
+      "parameters": {
+        "command": [
+          "/bin/bash",
+          "-c",
+          "cd /home/user/Downloads && unzip -q vscodeEvalExtension.zip && code --install-extension vscodeEvalExtension/eval-0.0.1.vsix && rm -rf vscodeEvalExtension vscodeEvalExtension.zip"
+        ]
+      }
+    },
     {
       "type": "launch",
       "parameters": {

mm_agents/accessibility_tree_wrap/heuristic_retrieve.py

@@ -2,6 +2,7 @@ import xml.etree.ElementTree as ET
 
 from PIL import Image, ImageDraw, ImageFont
 
+from typing import Tuple
 
 def find_leaf_nodes(xlm_file_str):
     if not xlm_file_str:
@@ -105,15 +106,20 @@ def draw_bounding_boxes(nodes, image_file_path, output_image_file_path):
 
                 # Draw index number at the bottom left of the bounding box with black background
                 text_position = (coords[0], bottom_right[1])  # Adjust Y to be above the bottom right
-                draw.rectangle([text_position, (text_position[0] + 25, text_position[1] + 18)], fill='black')
+                text_bbox: Tuple[int, int ,int ,int] = draw.textbbox(text_position, str(index), font=font, anchor="lb")
-                draw.text(text_position, str(index), font=font, fill="white")
+                #offset: int = bottom_right[1]-text_bbox[3]
+                #text_bbox = (text_bbox[0], text_bbox[1]+offset, text_bbox[2], text_bbox[3]+offset)
+
+                #draw.rectangle([text_position, (text_position[0] + 25, text_position[1] + 18)], fill='black')
+                draw.rectangle(text_bbox, fill='black')
+                draw.text(text_position, str(index), font=font, anchor="lb", fill="white")
                 index += 1
 
                 # each mark is an x, y, w, h tuple
                 marks.append([coords[0], coords[1], size[0], size[1]])
                 drew_nodes.append(_node)
 
-            except ValueError as e:
+            except ValueError:
                 pass
 
     # Save the result

requirements.txt

@@ -36,5 +36,6 @@ backoff
 formulas
 pydrive
 fastdtw
-
+odfpy
 openai
+func-timeout