lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into zhoujun/gimp

Browse Source
This commit is contained in:
BlankCheng
2024-01-30 14:36:21 +08:00
parent 8db2eca8d0 2df7de25ab
commit b9ec1c5362
30 changed files with 669 additions and 214 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
desktop_env/evaluators/getters/__init__.py
View File

@@ -1,13 +1,25 @@
from .chrome import get_default_search_engine, get_cookie_data, get_bookmarks, get_open_tabs_info, get_pdf_from_url, \
get_shortcuts_on_desktop, get_history, get_enabled_experiments, get_chrome_language, get_chrome_font_size, \
get_profile_name, get_number_of_search_results, get_googledrive_file, get_active_tab_info
from .chrome import (
get_default_search_engine,
get_cookie_data,
get_bookmarks,
get_open_tabs_info,
get_pdf_from_url,
get_shortcuts_on_desktop,
get_history,
get_enabled_experiments,
get_chrome_language,
get_chrome_font_size,
get_profile_name,
get_number_of_search_results,
get_googledrive_file,
get_active_tab_info
)
from .file import get_cloud_file, get_vm_file, get_cache_file
from .general import get_vm_command_line, get_vm_terminal_output
from .gimp import get_gimp_config_file
from .impress import get_audio_in_slide
from .info import get_vm_screen_size, get_vm_window_size, get_vm_wallpaper, get_list_directory
from .misc import get_rule, get_accessibility_tree
from .replay import get_replay
from .vlc import get_vlc_playing_info, get_vlc_config
from .vlc import get_vlc_playing_info, get_vlc_config, get_default_video_player
from .vscode import get_vscode_config
from .impress import get_audio_in_slide
from .gimp import get_gimp_config_file

28
desktop_env/evaluators/getters/vlc.py
View File

@@ -1,7 +1,8 @@
import logging
import os
from typing import Dict
from collections import Counter
from .general import get_vm_command_line
import requests
logger = logging.getLogger("desktopenv.getters.vlc")
@@ -58,3 +59,28 @@ def get_vlc_config(env, config: Dict[str, str]):
f.write(content)
return _path
def get_default_video_player(env, config: dict):
""" Gets the default application for a category or file extension.
"""
os_type = env.vm_platform
if os_type == "Linux":
extensions = ['3gp', '3gp', '3gpp', '3gpp', '3gpp2', '3gpp2', 'avi', 'avi', 'divx', 'divx', 'dv', 'dv', 'fli', 'fli', 'flv', 'flv', 'mp2t', 'mp2t', 'mp4', 'mp4', 'mp4v-es', 'mp4v-es', 'mpeg', 'mpeg', 'mpeg-system', 'mpeg-system', 'msvideo', 'msvideo', 'ogg', 'ogg', 'quicktime', 'quicktime', 'vnd.divx', 'vnd.divx', 'vnd.mpegurl', 'vnd.mpegurl', 'vnd.rn-realvideo', 'vnd.rn-realvideo', 'webm', 'webm', 'x-anim', 'x-anim', 'x-avi', 'x-avi', 'x-flc', 'x-flc', 'x-fli', 'x-fli', 'x-flv', 'x-flv', 'x-m4v', 'x-m4v', 'x-matroska', 'x-matroska', 'x-mpeg', 'x-mpeg', 'x-mpeg-system', 'x-mpeg-system', 'x-mpeg2', 'x-mpeg2', 'x-ms-asf', 'x-ms-asf', 'x-ms-asf-plugin', 'x-ms-asf-plugin', 'x-ms-asx', 'x-ms-asx', 'x-ms-wm', 'x-ms-wm', 'x-ms-wmv', 'x-ms-wmv', 'x-ms-wmx', 'x-ms-wmx', 'x-ms-wvx', 'x-ms-wvx', 'x-msvideo', 'x-msvideo', 'x-nsv', 'x-nsv', 'x-ogm', 'x-ogm', 'x-ogm+ogg', 'x-theora', 'x-theora', 'x-theora+ogg', 'x-theora+ogg']
apps = []
for ext in extensions:
app = get_vm_command_line(env, {"command": ["xdg-mime", "query", "default", f"video/{ext}"]})
if app:
apps.append(app)
if len(apps) == 0:
return 'unknown'
else:
return Counter(apps).most_common(1)[0][0]
elif os_type == "Darwin":
raise Exception("Unsupported operating system", os_type)
elif os_type == "Windows":
raise Exception("Unsupported operating system", os_type)
else:
raise Exception("Unsupported operating system", os_type)

3
desktop_env/evaluators/getters/vscode.py
View File

@@ -1,6 +1,6 @@
import logging
from typing import Any, Dict
import time
from .file import get_vm_file
from .replay import get_replay
@@ -27,6 +27,7 @@ def get_vscode_config(env, config: Dict[str, Any]) -> str:
]
get_replay(env, trajectory)
time.sleep(1.0)
return get_vm_file(env, {
"path": config["path"],

133
desktop_env/evaluators/metrics/__init__.py
View File

@@ -1,27 +1,58 @@
from .chrome import is_expected_tabs, is_expected_bookmarks, compare_pdfs, is_cookie_deleted, is_shortcut_on_desktop, \
check_font_size, \
check_enabled_experiments, check_history_deleted, is_expected_search_query
from .docs import compare_font_names, compare_subscript_contains, has_page_numbers_in_footers, compare_docx_lines
from .docs import evaluate_colored_words_in_tables, check_highlighted_words, evaluate_strike_through_last_paragraph, \
evaluate_conversion, evaluate_spacing, check_italic_font_size_14, evaluate_alignment, get_unique_train_ids, \
check_no_duplicates, compare_init_lines
from .general import exact_match, fuzzy_match
from .docs import find_default_font, contains_page_break, compare_docx_files, compare_docx_tables, compare_line_spacing, \
compare_insert_equation, compare_highlighted_text
from .docs import is_first_line_centered, check_file_exists, compare_contains_image
from .general import check_csv, check_accessibility_tree, run_sqlite3, check_json, check_list
from .general import exact_match, fuzzy_match, check_include_exclude
from .gimp import increase_saturation, decrease_brightness, check_file_exists, compare_triangle_positions
from .gimp import compare_images
from .libreoffice import check_libre_locale
from .pdf import check_pdf_pages
from .slides import check_presenter_console_disable, check_image_stretch_and_center, check_slide_numbers_color, \
compare_pptx_files, check_strikethrough, \
check_slide_orientation_Portrait, evaluate_presentation_fill_to_rgb_distance, check_left_panel, check_transition
# from .table import check_sheet_list, check_xlsx_freeze, check_xlsx_zoom, check_data_validations
from .table import compare_table, compare_csv
from .thunderbird import check_thunderbird_prefs, check_thunderbird_filter
from .basic_os import (
check_gnome_favorite_apps,
is_utc_0,
check_text_enlarged,
check_moved_jpgs
)
from .chrome import (
is_expected_tabs,
is_expected_bookmarks,
compare_pdfs,
is_cookie_deleted,
is_shortcut_on_desktop,
check_font_size,
check_enabled_experiments,
check_history_deleted,
is_expected_search_query
)
from .docs import (
compare_font_names,
compare_subscript_contains,
has_page_numbers_in_footers,
compare_docx_lines,
evaluate_colored_words_in_tables,
check_highlighted_words,
evaluate_strike_through_last_paragraph,
evaluate_conversion,
evaluate_spacing,
check_italic_font_size_14,
evaluate_alignment,
get_unique_train_ids,
check_no_duplicates,
compare_init_lines
)
from .docs import (
find_default_font,
contains_page_break,
compare_docx_files,
compare_docx_tables,
compare_line_spacing,
compare_insert_equation,
compare_highlighted_text,
is_first_line_centered,
check_file_exists,
compare_contains_image
)
from .general import (
check_csv,
check_accessibility_tree,
run_sqlite3,
check_json,
check_list,
exact_match,
fuzzy_match,
check_include_exclude
)
from .gimp import (
check_brightness_decrease_and_structure_sim,
check_contrast_increase_and_structure_sim,
@@ -35,11 +66,53 @@ from .gimp import (
check_triangle_position,
check_structure_sim,
check_config_status,
compare_images,
increase_saturation,
decrease_brightness,
check_file_exists,
compare_triangle_positions
)
from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, compare_images, compare_audios, \
compare_videos, check_qt_bgcone, check_one_instance_when_started_from_file, check_qt_minimal_view, \
check_qt_max_volume, \
check_qt_slider_colours, check_global_key_play_pause
from .vscode import compare_text_file, compare_config, compare_answer, is_extension_installed, check_json_settings, \
from .libreoffice import check_libre_locale
from .pdf import check_pdf_pages
from .slides import (
check_presenter_console_disable,
check_image_stretch_and_center,
check_slide_numbers_color,
compare_pptx_files,
check_strikethrough,
check_slide_orientation_Portrait,
evaluate_presentation_fill_to_rgb_distance,
check_left_panel,
check_transition,
check_page_number_colors
)
from .table import (
compare_table,
compare_csv
)
from .thunderbird import (
check_thunderbird_prefs,
check_thunderbird_filter
)
from .vlc import (
is_vlc_playing,
is_vlc_recordings_folder,
is_vlc_fullscreen,
compare_images,
compare_audios,
compare_videos,
check_qt_bgcone,
check_one_instance_when_started_from_file,
check_qt_minimal_view,
check_qt_max_volume,
check_qt_slider_colours,
check_global_key_play_pause
)
from .vscode import (
compare_text_file,
compare_config,
compare_answer,
is_extension_installed,
check_json_settings,
check_json_keybindings
from .basic_os import check_gnome_favorite_apps, is_utc_0, check_text_enlarged, check_moved_jpgs
)

148
desktop_env/evaluators/metrics/docs.py
View File

@@ -1,13 +1,16 @@
import logging
import os
import re
import xml.etree.ElementTree as ET
import zipfile
import re
from typing import List, Dict, Any
from docx import Document
from docx.enum.text import WD_PARAGRAPH_ALIGNMENT
from docx.shared import RGBColor
from odf.opendocument import load
from odf.text import P
from odf.text import Span
logger = logging.getLogger("desktopenv.metric.docs")
@@ -51,12 +54,39 @@ def contains_page_break(docx_file):
def compare_docx_files(file1, file2, ignore_blanks=True):
doc1 = Document(file1)
doc2 = Document(file2)
def get_paragraph_texts_odt(document):
paragraphs = document.getElementsByType(P)
paragraph_texts = []
for paragraph in paragraphs:
text_parts = []
for node in paragraph.childNodes:
if node.nodeType == node.TEXT_NODE:
text_parts.append(node.data)
elif node.nodeType == node.ELEMENT_NODE and node.tagName == 'text:span':
# Assuming direct text content in <text:span>, for simplicity
for child in node.childNodes:
if child.nodeType == child.TEXT_NODE:
text_parts.append(child.data)
paragraph_texts.append(''.join(text_parts))
return paragraph_texts
doc1_paragraphs = [p.text for p in doc1.paragraphs]
doc2_paragraphs = [p.text for p in doc2.paragraphs]
# Determine file types and load documents
if file1.endswith('.docx') and file2.endswith('.docx'):
doc1 = Document(file1)
doc2 = Document(file2)
doc1_paragraphs = [p.text for p in doc1.paragraphs]
doc2_paragraphs = [p.text for p in doc2.paragraphs]
elif file1.endswith('.odt') and file2.endswith('.odt'):
doc1 = load(file1)
doc2 = load(file2)
doc1_paragraphs = get_paragraph_texts_odt(doc1)
doc2_paragraphs = get_paragraph_texts_odt(doc2)
else:
# Unsupported file types or mismatch
print("Unsupported file types or mismatch between file types.")
return 0
# Process and compare documents
if ignore_blanks:
text1 = re.sub(r'\s+', ' ', '\n'.join(doc1_paragraphs)).strip()
text2 = re.sub(r'\s+', ' ', '\n'.join(doc2_paragraphs)).strip()
@@ -64,19 +94,16 @@ def compare_docx_files(file1, file2, ignore_blanks=True):
return 0
else:
if len(doc1_paragraphs) != len(doc2_paragraphs):
# print(len(doc1_paragraphs))
# print(len(doc2_paragraphs))
return 0
# Compare each paragraph
for p1, p2 in zip(doc1_paragraphs, doc2_paragraphs):
if p1 != p2:
# print(p1)
# print(p2)
return 0
return 1
def compare_init_lines(file1, file2):
doc1 = Document(file1)
doc2 = Document(file2)
@@ -93,6 +120,7 @@ def compare_init_lines(file1, file2):
return 1
def compare_docx_tables(docx_file1, docx_file2):
doc1 = Document(docx_file1)
doc2 = Document(docx_file2)
@@ -217,6 +245,7 @@ def compare_contains_image(docx_file1, docx_file2):
return 0
return 1
# file1 = 'path/to/file1.docx'
# file2 = 'path/to/file2.docx'
@@ -253,26 +282,25 @@ def evaluate_colored_words_in_tables(file_path1, file_path2):
def check_highlighted_words(file_path1, file_path2):
# if not compare_docx_files(file_path1, file_path2):
# return 0
# Extract content.xml from the .odt file
extract_dir = file_path1 + "_extracted"
with zipfile.ZipFile(file_path1, 'r') as zip_ref:
zip_ref.extractall(extract_dir)
content_xml_path = os.path.join(extract_dir, 'content.xml')
with open(content_xml_path, 'r', encoding="utf-8") as file:
content_xml = file.read()
# Check for yellow highlights in the content.xml
yellow_highlight_pattern = re.compile(r'(.{0,50}background-color="#ffff00"[^>]*>.{0,50})')
yellow_highlight_matches = yellow_highlight_pattern.findall(content_xml)
# Return True if yellow highlights are NOT found, otherwise True
if yellow_highlight_matches:
if not compare_docx_files(file_path1, file_path2):
return 0
else:
return 1
doc = load(file_path1)
highlighted = False
for span in doc.getElementsByType(Span):
style_name = span.getAttribute('stylename')
if style_name:
for automatic_style in doc.automaticstyles.childNodes:
if automatic_style.getAttribute('name') == style_name:
for property in automatic_style.childNodes:
if property.getAttribute('backgroundcolor') == '#ffff00':
highlighted = True
break
if highlighted:
break
return 0 if highlighted else 1
def evaluate_strike_through_last_paragraph(file_path1, file_path2):
@@ -318,9 +346,9 @@ def evaluate_spacing(file_path):
body_spacing = document.paragraphs[1].paragraph_format.line_spacing
conclusion_spacing = document.paragraphs[2].paragraph_format.line_spacing
if (introduction_spacing == 1.0 and body_spacing == 2.0 and conclusion_spacing == 1.5):
return 1
return 1
else:
return 0
return 0
def check_italic_font_size_14(path1, path2):
@@ -358,13 +386,14 @@ def evaluate_alignment(docx_path):
second_part = ' '.join(words[3:])
# Check if the sentence structure matches the pattern: first part + large space/tab + second part
if not (first_part in sentence and second_part in sentence and sentence.find(first_part) < sentence.find(second_part)):
if not (first_part in sentence and second_part in sentence and sentence.find(first_part) < sentence.find(
second_part)):
return 0 # The sentence does not meet the alignment criteria
return 1 # All sentences meet the alignment criteria
def get_unique_train_ids(initial_file): #fixed standard
def get_unique_train_ids(initial_file): # fixed standard
doc = Document(initial_file)
train_ids = set()
processed_lines = 0
@@ -374,11 +403,12 @@ def get_unique_train_ids(initial_file): #fixed standard
if len(line_parts) == 4:
train_id = line_parts[1].strip()
if train_id not in train_ids:
train_ids.add(train_id)
processed_lines += 1
train_ids.add(train_id)
processed_lines += 1
return train_ids, processed_lines
def check_no_duplicates(initial_file, processed_file):
# Open the document
train_ids_ini, ini_lines = get_unique_train_ids(initial_file)
@@ -405,6 +435,7 @@ def check_no_duplicates(initial_file, processed_file):
# No duplicates found and at least one valid line was processed
return 1
def compare_docx_lines(file1, file2):
# Read the text of the document, line by line
doc1 = Document(file1)
@@ -416,21 +447,46 @@ def compare_docx_lines(file1, file2):
# print(doc2_lines)
# Convert the list of lines to sets and compare
return set(doc1_lines) == set(doc2_lines)
if set(doc1_lines) == set(doc2_lines):
return 1
else:
return 0
# Docx file saved in the ubuntu cannot use this function to compare highlight, don't know why, deprecated
def compare_highlighted_text(file1, file2):
def extract_highlighted_text(doc):
highlighted_text = []
# Iterate through each run in each paragraph to check for highlight
for paragraph in doc.paragraphs:
for run in paragraph.runs:
if run.font.highlight_color: # Checks if the run is highlighted
highlighted_text.append(run.text.strip())
return highlighted_text
def extract_highlighted_text(file_path):
highlighted_texts = []
# Open the .docx file as a zip file and read the document.xml
with zipfile.ZipFile(file_path, 'r') as docx:
with docx.open('word/document.xml') as document_xml:
tree = ET.parse(document_xml)
root = tree.getroot()
# Define the namespaces
namespaces = {
'w': 'http://schemas.openxmlformats.org/wordprocessingml/2006/main',
}
# Find all runs with highlight property
for run in root.findall('.//w:r', namespaces):
highlight = run.find('.//w:highlight', namespaces)
if highlight is not None and highlight.get(
'{http://schemas.openxmlformats.org/wordprocessingml/2006/main}val') != 'none':
text = run.find('.//w:t', namespaces)
if text is not None:
highlighted_texts.append(text.text)
return highlighted_texts
# Read the highlighted text from both documents
doc1_highlighted = extract_highlighted_text(Document(file1))
doc2_highlighted = extract_highlighted_text(Document(file2))
doc1_highlighted = extract_highlighted_text(file1)
doc2_highlighted = extract_highlighted_text(file2)
# Compare the sets of highlighted text to check if they are the same
return set(doc1_highlighted) == set(doc2_highlighted)
if set(doc1_highlighted) == set(doc2_highlighted):
return 1
else:
return 0

41
desktop_env/evaluators/metrics/gimp.py
View File

@@ -211,7 +211,10 @@ def check_brightness_decrease_and_structure_sim(src_path, tgt_path):
img_tgt_normalized = normalize_brightness(img_tgt, target_brightness)
structure_same = structure_check_by_mse(img_src_normalized, img_tgt_normalized)
return brightness_reduced and structure_same
if brightness_reduced and structure_same:
return 1.
else:
return 0.
def check_saturation_increase_and_structure_sim(src_path, tgt_path):
@@ -240,7 +243,10 @@ def check_saturation_increase_and_structure_sim(src_path, tgt_path):
else:
structure_same = False
return saturation_increased and structure_same
if saturation_increased and structure_same:
return 1.
else:
return 0.
def check_file_exists_and_structure_sim(src_path, tgt_path):
@@ -294,7 +300,10 @@ def check_triangle_position(tgt_path):
tolerance = 0.05 * np.array(img_array.shape[:2])
middle = np.all(np.abs(centroid - image_center) < tolerance)
return middle
if bool(middle):
return 1.
else:
return 0.
def check_structure_sim(src_path, tgt_path):
@@ -325,7 +334,10 @@ def check_contrast_increase_and_structure_sim(src_path, tgt_path):
# Check structure
structure_same = structure_check_by_ssim(source_image, target_image, threshold=0.65)
return higher_contrast and structure_same
if higher_contrast and structure_same:
return 1.
else:
return 0.
def check_config_status(actual_config_path, rule):
@@ -341,13 +353,13 @@ def check_config_status(actual_config_path, rule):
items = line.strip().lstrip('(').rstrip(')\n').split()
if isinstance(rule["key"], str):
if items[0] == rule["key"] and items[-1] == rule["value"]:
return True
return 1.
elif isinstance(rule["key"], list) and len(rule["key"]) == 2:
if items[0] == rule["key"][0] \
and items[1] == rule["key"][1] \
and items[-1] == rule["value"]:
return True
return False
return 1.
return 0.
def check_image_size_and_structure_sim(src_path, tgt_path, height=512, width=None):
@@ -373,7 +385,10 @@ def check_image_size_and_structure_sim(src_path, tgt_path, height=512, width=Non
resized_target_image = target_image.resize(source_image.size)
structure_same = structure_check_by_ssim(source_image, resized_target_image)
return width_same and height_same and structure_same
if width_same and height_same and structure_same:
return 1.
else:
return 0.
def check_palette_and_structure_sim(src_path, tgt_path):
@@ -389,7 +404,10 @@ def check_palette_and_structure_sim(src_path, tgt_path):
target_image = Image.open(tgt_path)
source_image = source_image.convert('RGB')
structure_same = structure_check_by_ssim(source_image, target_image)
return palette_based and structure_same
if palette_based and structure_same:
return 1.
else:
return 0.
def check_textbox_on_leftside(src_path):
@@ -411,7 +429,10 @@ def check_textbox_on_leftside(src_path):
break # Stop after finding the first dark pixel in this row
# Here we define "almost" on the left side as being within the left 5% of the image
return left_most_dark_pixel < width * 0.05
if left_most_dark_pixel < width * 0.05:
return 1.
else:
return 0.
def check_image_mirror(src_path, tgt_path):

47
desktop_env/evaluators/metrics/slides.py
View File

@@ -318,3 +318,50 @@ def check_transition(pptx_file, rules):
return 0.
else:
return 0.
def check_page_number_colors(pptx_file, rules):
color = rules["color"]
def is_red(rgb_str, threshold=50):
r, g, b = int(rgb_str[1:3], 16), int(rgb_str[3:5], 16), int(rgb_str[5:7], 16)
return r > g + threshold and r > b + threshold
def is_blue(rgb_str, threshold=50):
r, g, b = int(rgb_str[1:3], 16), int(rgb_str[3:5], 16), int(rgb_str[5:7], 16)
return b > g + threshold and b > r + threshold
def is_green(rgb_str, threshold=50):
r, g, b = int(rgb_str[1:3], 16), int(rgb_str[3:5], 16), int(rgb_str[5:7], 16)
return g > r + threshold and g > b + threshold
def is_black(rgb_str, threshold=50):
r, g, b = int(rgb_str[1:3], 16), int(rgb_str[3:5], 16), int(rgb_str[5:7], 16)
return r < threshold and g < threshold and b < threshold
with zipfile.ZipFile(pptx_file, 'r') as zip_ref:
slide_master_name = 'ppt/slideMasters/slideMaster1.xml'
with zip_ref.open(slide_master_name) as slide_master_file:
tree = ET.parse(slide_master_file)
root = tree.getroot()
namespaces = {
'a': 'http://schemas.openxmlformats.org/drawingml/2006/main',
'p': 'http://schemas.openxmlformats.org/presentationml/2006/main',
}
color_elems = root.findall('.//a:solidFill//a:srgbClr', namespaces)
slides_color_val = color_elems[-2].get('val')
if slides_color_val is None:
return 0
elif color == "red" and not is_red(slides_color_val):
return 0
elif color == "blue" and not is_blue(slides_color_val):
return 0
elif color == "green" and not is_green(slides_color_val):
return 0
elif color == "black" and not is_black(slides_color_val):
return 0
return 1