From 61b145ab13d83c31a8c8226de86f8abd4dd0f23b Mon Sep 17 00:00:00 2001 From: thomasshin Date: Mon, 22 Jan 2024 23:22:44 +0800 Subject: [PATCH] add writer evals 8 examples --- desktop_env/evaluators/metrics/__init__.py | 3 + desktop_env/evaluators/metrics/docs.py | 160 ++++++++++++++++++ .../0a0faba3-5580-44df-965d-f562a99b291c.json | 42 +++++ .../6a33f9b9-0a56-4844-9c3f-96ec3ffb3ba2.json | 42 +++++ .../6f81754e-285d-4ce0-b59e-af7edb02d108.json | 42 +++++ .../72b810ef-4156-4d09-8f08-a0cf57e7cefe.json | 42 +++++ .../8472fece-c7dd-4241-8d65-9b3cd1a0b568.json | 42 +++++ .../b21acd93-60fd-4127-8a43-2f5178f4a830.json | 42 +++++ .../d53ff5ee-3b1a-431e-b2be-30ed2673079b.json | 42 +++++ .../e246f6d8-78d7-44ac-b668-fcf47946cb50.json | 42 +++++ 10 files changed, 499 insertions(+) create mode 100644 evaluation_examples/examples/libreoffice_writer/0a0faba3-5580-44df-965d-f562a99b291c.json create mode 100644 evaluation_examples/examples/libreoffice_writer/6a33f9b9-0a56-4844-9c3f-96ec3ffb3ba2.json create mode 100644 evaluation_examples/examples/libreoffice_writer/6f81754e-285d-4ce0-b59e-af7edb02d108.json create mode 100644 evaluation_examples/examples/libreoffice_writer/72b810ef-4156-4d09-8f08-a0cf57e7cefe.json create mode 100644 evaluation_examples/examples/libreoffice_writer/8472fece-c7dd-4241-8d65-9b3cd1a0b568.json create mode 100644 evaluation_examples/examples/libreoffice_writer/b21acd93-60fd-4127-8a43-2f5178f4a830.json create mode 100644 evaluation_examples/examples/libreoffice_writer/d53ff5ee-3b1a-431e-b2be-30ed2673079b.json create mode 100644 evaluation_examples/examples/libreoffice_writer/e246f6d8-78d7-44ac-b668-fcf47946cb50.json diff --git a/desktop_env/evaluators/metrics/__init__.py b/desktop_env/evaluators/metrics/__init__.py index 7a1fa94..b07f71d 100644 --- a/desktop_env/evaluators/metrics/__init__.py +++ b/desktop_env/evaluators/metrics/__init__.py @@ -3,6 +3,9 @@ from .docs import compare_font_names, compare_subscript_contains, has_page_numbe from .docs import find_default_font, contains_page_break, compare_docx_files, compare_docx_tables, compare_line_spacing, \ compare_insert_equation from .docs import is_first_line_centered, check_file_exists, compare_contains_image +from .docs import evaluate_colored_words_in_tables, check_highlighted_words, evaluate_strike_through_last_paragraph, \ + evaluate_conversion, evaluate_spacing, check_italic_font_size_14, evaluate_alignment, get_unique_train_ids, \ + check_no_duplicates from .general import exact_match, fuzzy_match, check_csv, check_accessibility_tree, check_list from .libreoffice import check_libre_locale from .pdf import check_pdf_pages diff --git a/desktop_env/evaluators/metrics/docs.py b/desktop_env/evaluators/metrics/docs.py index 9f2d940..d0108c1 100644 --- a/desktop_env/evaluators/metrics/docs.py +++ b/desktop_env/evaluators/metrics/docs.py @@ -5,6 +5,7 @@ from typing import List, Dict, Any from docx import Document from docx.enum.text import WD_PARAGRAPH_ALIGNMENT +from docx.shared import RGBColor logger = logging.getLogger("desktopenv.metric.docs") @@ -197,3 +198,162 @@ def compare_contains_image(docx_file1, docx_file2): # config_path = "/home/[username]/.config/libreoffice/4/user/registrymodifications.xcu" # print(find_default_font("Ani", config_path)) + + +def evaluate_colored_words_in_tables(file_path): + document = Document(file_path) + + for table in document.tables: + # Iterate through rows and cells in the table + for row in table.rows: + for cell in row.cells: + for paragraph in cell.paragraphs: + for run in paragraph.runs: + word = run.text + if word: + first_letter = word[0].lower() + + if first_letter in 'aeiou' and run.font.color.rgb != RGBColor(255, 0, 0): + return 0 # Vowel-colored words should be red + elif first_letter not in 'aeiou' and run.font.color.rgb != RGBColor(0, 0, 255): + return 0 # Non-vowel-colored words should be blue + + return 1 # All words in tables are correctly colored + + +def check_highlighted_words(file_path): + document = Document(file_path) + + for paragraph in document.paragraphs: + for run in paragraph.runs: + if run.font.highlight_color is not None: + return 0 # Highlighted words found + + return 1 # No highlighted words found + + +def evaluate_strike_through_last_paragraph(file_path): + document = Document(file_path) + + # Get the last paragraph + last_paragraph = document.paragraphs[-1] + + # Check if any run in the last paragraph has strike-through formatting + for run in last_paragraph.runs: + if not run.font.strike: + return 0 # At least one word does not have strike-through formatting + + return 1 # All words in the last paragraph have strike-through formatting + + +def evaluate_conversion(file_path): + document = Document(file_path) + + for table in document.tables: + for row in table.rows: + for cell in row.cells: + for paragraph in cell.paragraphs: + for run in paragraph.runs: + if run.text.isupper(): + return 0 # Uppercase text should be converted to lowercase + + for paragraph in document.paragraphs: + for run in paragraph.runs: + if run.text.isupper(): + return 0 # Uppercase text should be converted to lowercase + + return 1 # All uppercase text has been successfully converted + + +def evaluate_spacing(file_path): + document = Document(file_path) + + # Check line spacing for introduction, body, and conclusion + introduction_spacing = document.paragraphs[0].paragraph_format.line_spacing + body_spacing = document.paragraphs[1].paragraph_format.line_spacing + conclusion_spacing = document.paragraphs[2].paragraph_format.line_spacing + if (introduction_spacing == 1.0 and body_spacing == 2.0 and conclusion_spacing == 1.5): + return 1 + else: + return 0 + + +def check_italic_font_size_14(path): + document = Document(path) + for paragraph in document.paragraphs: + for run in paragraph.runs: + if run.italic: + # Check if font size is 14 + if run.font.size is None or run.font.size.pt != 14: + return 0 + return 1 + + +def evaluate_alignment(docx_path): + # Load the document + doc = Document(docx_path) + + # Iterate through each paragraph in the document + for para in doc.paragraphs: + # Split the paragraph into individual sentences + sentences = para.text.split('.') + + for sentence in sentences: + # Split the sentence into words + words = sentence.strip().split() + + # Check if the sentence has at least three words + if len(words) < 3: + continue # Skip sentences with less than three words + + # The first three words should be separated from the rest + first_part = ' '.join(words[:3]) + second_part = ' '.join(words[3:]) + + # Check if the sentence structure matches the pattern: first part + large space/tab + second part + if not (first_part in sentence and second_part in sentence and sentence.find(first_part) < sentence.find(second_part)): + return 0 # The sentence does not meet the alignment criteria + + return 1 # All sentences meet the alignment criteria + + +def get_unique_train_ids(initial_file): #fixed standard + doc = Document(initial_file) + train_ids = set() + processed_lines = 0 + + for para in doc.paragraphs: + line_parts = para.text.split(',') + if len(line_parts) == 4: + train_id = line_parts[1].strip() + if train_id not in train_ids: + train_ids.add(train_id) + processed_lines += 1 + + return train_ids, processed_lines + +def check_no_duplicates(initial_file, processed_file): + # Open the document + train_ids_ini, ini_lines = get_unique_train_ids(initial_file) + doc_processed = Document(processed_file) + train_ids_pro = set() + processed_lines = 0 # Counter for valid lines processed + + # processed + for para in doc_processed.paragraphs: + # Each line has the format: time_HH:MM:SS, train_id, station_id, platform_no + line_parts = para.text.split(',') + # Ensure the line has the correct format + if len(line_parts) == 4: + train_id = line_parts[1].strip() + # If train_id is already in the set, it's a duplicate + if train_id in train_ids_pro: + return 0 # Duplicate found + train_ids_pro.add(train_id) + processed_lines += 1 # Increment valid lines counter + + if train_ids_pro != train_ids_ini or processed_lines != ini_lines: + return 0 + + # No duplicates found and at least one valid line was processed + return 1 diff --git a/evaluation_examples/examples/libreoffice_writer/0a0faba3-5580-44df-965d-f562a99b291c.json b/evaluation_examples/examples/libreoffice_writer/0a0faba3-5580-44df-965d-f562a99b291c.json new file mode 100644 index 0000000..94b55b9 --- /dev/null +++ b/evaluation_examples/examples/libreoffice_writer/0a0faba3-5580-44df-965d-f562a99b291c.json @@ -0,0 +1,42 @@ +{ + "id": "0a0faba3-5580-44df-965d-f562a99b291c", + "snapshot": "libreoffice_writer", + "instruction": "I would like to make the first three words of the sentence left-aligned and the rest right-aligned. I basically want to have some empty space in the middle to add some photos. Assume that every sentence will have at least three words. Could you help me on alignment for me?", + "source": "https://stackoverflow.com/questions/64528055/how-to-make-part-of-my-sentence-left-aligned-and-rest-as-right-aligned", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?id=1Wrjxsf184Go70TcRGM4Tohczh29Q9B_U&export=download", + "path": "Desktop/04 CHIN9505 EBook Purchasing info 2021 Jan.docx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "Desktop/04 CHIN9505 EBook Purchasing info 2021 Jan.docx" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "libreoffice_writer" + ], + "evaluator": { + "func": "evaluate_alignment", + "expected": { + "type": "cloud_file", + "path": "https://drive.google.com/uc?id=1yyHGj8KUHDMsZmc1QeJ1KkvSEGy83jMR&export=download", + "dest": "04 CHIN9505 EBook Purchasing info 2021 Jan_Gold.docx" + }, + "result": { + "type": "vm_file", + "path": "Desktop/04 CHIN9505 EBook Purchasing info 2021 Jan.docx", + "dest": "04 CHIN9505 EBook Purchasing info 2021 Jan.docx" + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/libreoffice_writer/6a33f9b9-0a56-4844-9c3f-96ec3ffb3ba2.json b/evaluation_examples/examples/libreoffice_writer/6a33f9b9-0a56-4844-9c3f-96ec3ffb3ba2.json new file mode 100644 index 0000000..737bf91 --- /dev/null +++ b/evaluation_examples/examples/libreoffice_writer/6a33f9b9-0a56-4844-9c3f-96ec3ffb3ba2.json @@ -0,0 +1,42 @@ +{ + "id": "6a33f9b9-0a56-4844-9c3f-96ec3ffb3ba2", + "snapshot": "libreoffice_writer", + "instruction": "I have been editing my document and some words that needed to be rewritten are highlighted in yellow. As I fixed those words, I removed highlight. Now I want to make sure that there is no highlight word. Could you help me on finding if there is no highlighted words in the file?", + "source": "https://superuser.com/questions/762500/how-do-i-find-all-highlighted-text-in-libreoffice-writer", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?id=1-ygC5pClvU1vxPQ5SGxl3teQAbxCVm8s&export=download", + "path": "Desktop/DG75-DrawGuide.docx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "Desktop/DG75-DrawGuide.docx" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "libreoffice_writer" + ], + "evaluator": { + "func": "check_highlighted_words", + "expected": { + "type": "cloud_file", + "path": "https://drive.google.com/uc?id=1Z5WkW0YH5tWh-D2YuU5zR7QLNef-37ya&export=download", + "dest": "DG75-DrawGuide_Gold.docx" + }, + "result": { + "type": "vm_file", + "path": "Desktop/DG75-DrawGuide.docx", + "dest": "DG75-DrawGuide.docx" + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/libreoffice_writer/6f81754e-285d-4ce0-b59e-af7edb02d108.json b/evaluation_examples/examples/libreoffice_writer/6f81754e-285d-4ce0-b59e-af7edb02d108.json new file mode 100644 index 0000000..21f5719 --- /dev/null +++ b/evaluation_examples/examples/libreoffice_writer/6f81754e-285d-4ce0-b59e-af7edb02d108.json @@ -0,0 +1,42 @@ +{ + "id": "6f81754e-285d-4ce0-b59e-af7edb02d108", + "snapshot": "libreoffice_writer", + "instruction": "A certain railway company in Hong Kong uses a signaling system to keep track of trains in its railway system. Each line in the docx file represents a train calling at a station from 0600 to 1200 on 2022-09-22, and has the following format: time_HH:MM:SS, train_id, station_id, platform_no.. I want to remove duplicated train ids in order to know how many different trains are running from 0600 to 1200. Could you help me on this? I am doing it manually and it is very inefficient.", + "source": "https://superuser.com/questions/789473/remove-duplicate-lines-in-libreoffice-openoffice-writer", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?id=1cK1AMt_qKVAPp6EndSFG8y8r7KOPsqC1&export=download", + "path": "Desktop/HK train record.docx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "Desktop/HK train record.docx" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "libreoffice_writer" + ], + "evaluator": { + "func": ["get_unique_train_ids", "check_no_duplicates"], + "result": { + "type": "vm_file", + "path": "Desktop/HK train record.docx", + "dest": "HK train record.docx" + }, + "expected": { + "type": "cloud_file", + "path": "https://drive.google.com/uc?id=1wZ5CKxCD3biB4mFFlrBInZO-bzo36vVG&export=download", + "dest": "HK train record_Gold.docx" + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/libreoffice_writer/72b810ef-4156-4d09-8f08-a0cf57e7cefe.json b/evaluation_examples/examples/libreoffice_writer/72b810ef-4156-4d09-8f08-a0cf57e7cefe.json new file mode 100644 index 0000000..cf10a2c --- /dev/null +++ b/evaluation_examples/examples/libreoffice_writer/72b810ef-4156-4d09-8f08-a0cf57e7cefe.json @@ -0,0 +1,42 @@ +{ + "id": "72b810ef-4156-4d09-8f08-a0cf57e7cefe", + "snapshot": "libreoffice_writer", + "instruction": "I am peer-reviewing my friend's course outline. I think the last paragraph is redundant so I want to add strike-through on words in the last paragraph. Can you do this for me?", + "source": "https://superuser.com/questions/657792/libreoffice-writer-how-to-apply-strikethrough-text-formatting?rq=1", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?id=1Uqgr9Y_kjoMoDoUwt80hv1EtFaisyztU&export=download", + "path": "Desktop/GEOG2169_Course_Outline_2022-23.docx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "GEOG2169_Course_Outline_2022-23.docx" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "libreoffice_writer" + ], + "evaluator": { + "func": "evaluate_strike_through_last_paragraph", + "expected": { + "type": "cloud_file", + "path": "https://drive.google.com/uc?id=1IpAnQRYo1whrnzIGyo8UldZf4Tli-yVT&export=download", + "dest": "GEOG2169_Course_Outline_2022-23_Gold.docx" + }, + "result": { + "type": "vm_file", + "path": "Desktop/GEOG2169_Course_Outline_2022-23.docx", + "dest": "GEOG2169_Course_Outline_2022-23.docx" + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/libreoffice_writer/8472fece-c7dd-4241-8d65-9b3cd1a0b568.json b/evaluation_examples/examples/libreoffice_writer/8472fece-c7dd-4241-8d65-9b3cd1a0b568.json new file mode 100644 index 0000000..e050e90 --- /dev/null +++ b/evaluation_examples/examples/libreoffice_writer/8472fece-c7dd-4241-8d65-9b3cd1a0b568.json @@ -0,0 +1,42 @@ +{ + "id": "8472fece-c7dd-4241-8d65-9b3cd1a0b568", + "snapshot": "libreoffice_writer", + "instruction": "I am writing a word list for a dyslexic kid. To ease things for him, I want to use red for words start with vowels and blue for those start with non-vowels. Can you do this for me? I'm doing it manually, and it is a pain.", + "source": "https://stackoverflow.com/questions/37259827/libreoffice-writer-how-to-set-different-colors-to-each-letter", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?id=1QHk3fVFSlvYu2k013_7ahEkVQl_o1GTU&export=download", + "path": "Desktop/Dolch Sight Words Primer.docx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "Desktop/Dolch Sight Words Primer.docx" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "libreoffice_writer" + ], + "evaluator": { + "func": "evaluate_colored_words_in_tables", + "expected": { + "type": "cloud_file", + "path": "https://drive.google.com/uc?id=1ksn444K17lFOdm5pELrQYvuZHkOsKq69&export=download", + "dest": "Dolch Sight Words Primer_Gold.docx" + }, + "result": { + "type": "vm_file", + "path": "Desktop/Dolch Sight Words Primer.docx", + "dest": "Dolch Sight Words Primer.docx" + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/libreoffice_writer/b21acd93-60fd-4127-8a43-2f5178f4a830.json b/evaluation_examples/examples/libreoffice_writer/b21acd93-60fd-4127-8a43-2f5178f4a830.json new file mode 100644 index 0000000..4412193 --- /dev/null +++ b/evaluation_examples/examples/libreoffice_writer/b21acd93-60fd-4127-8a43-2f5178f4a830.json @@ -0,0 +1,42 @@ +{ + "id": "b21acd93-60fd-4127-8a43-2f5178f4a830", + "snapshot": "libreoffice_writer", + "instruction": "I have been praciticing professional writing lately. Now I am writing essay which requires one paragraph each for introduction, body and conclusion with single-space for introduction, double-space for body then one-and-a-half-space for conclusion. The font size of this essay is 12. Could you help me on this?", + "source": "https://superuser.com/questions/1097199/how-can-i-double-space-a-document-in-libreoffice?rq=1", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?id=1akFeAURJiqnK9wGNlRgPoPuQ6vRmnUPe&export=download", + "path": "Desktop/CCHU9045 Course Outline 2019-20.docx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "CCHU9045 Course Outline 2019-20.docx" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "libreoffice_writer" + ], + "evaluator": { + "func": "evaluate_spacing", + "expected": { + "type": "cloud_file", + "path": "https://drive.google.com/uc?id=16LN7uYSSXk_xwgc4IZXnN2Z1nCmPJfLm&export=download", + "dest": "CCHU9045 Course Outline 2019-20_Gold.docx" + }, + "result": { + "type": "vm_file", + "path": "Desktop/CCHU9045 Course Outline 2019-20.docx", + "dest": "CCHU9045 Course Outline 2019-20.docx" + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/libreoffice_writer/d53ff5ee-3b1a-431e-b2be-30ed2673079b.json b/evaluation_examples/examples/libreoffice_writer/d53ff5ee-3b1a-431e-b2be-30ed2673079b.json new file mode 100644 index 0000000..d4b3719 --- /dev/null +++ b/evaluation_examples/examples/libreoffice_writer/d53ff5ee-3b1a-431e-b2be-30ed2673079b.json @@ -0,0 +1,42 @@ +{ + "id": "d53ff5ee-3b1a-431e-b2be-30ed2673079b", + "snapshot": "libreoffice_writer", + "instruction": "I am currently engaged in text processing and require assistance in converting all uppercase text to lowercase within my document. This precision is critical for maintaining a uniform and polished presentation. Could you help me on this?", + "source": "https://ask.libreoffice.org/t/how-to-convert-all-uppercase-to-lowercase/53341", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?id=1j6Gx6KCxA9Cp-TE1uZ5lKcTSKVRPW-CB&export=download", + "path": "Desktop/presentation instruction 2023 Feb.docx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "presentation instruction 2023 Feb.docx" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "libreoffice_writer" + ], + "evaluator": { + "func": "evaluate_conversion", + "expected": { + "type": "cloud_file", + "path": "https://drive.google.com/uc?id=1bB1N2TWN0puZ6DwUFS_TDjvRWchaGP9T&export=download", + "dest": "presentation instruction 2023 Feb_Gold.docx" + }, + "result": { + "type": "vm_file", + "path": "Desktop/presentation instruction 2023 Feb.docx", + "dest": "presentation instruction 2023 Feb.docx" + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/libreoffice_writer/e246f6d8-78d7-44ac-b668-fcf47946cb50.json b/evaluation_examples/examples/libreoffice_writer/e246f6d8-78d7-44ac-b668-fcf47946cb50.json new file mode 100644 index 0000000..e5468e6 --- /dev/null +++ b/evaluation_examples/examples/libreoffice_writer/e246f6d8-78d7-44ac-b668-fcf47946cb50.json @@ -0,0 +1,42 @@ +{ + "id": "e246f6d8-78d7-44ac-b668-fcf47946cb50", + "snapshot": "libreoffice_writer", + "instruction": "I found Italic font very hard to discern from the normal text for me, as it is also dark black with the same size. Current font size is 12 and I want to change the font size of italicized words to 14 to make it more discernible. Can you help me on this?", + "source": "https://ask.libreoffice.org/t/how-to-change-text-size-color-of-italic-font/77712", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?id=1b8mPpEDlBrTLcOpf0ZcjdUV4vLAwxH1r&export=download", + "path": "Desktop/Y22-2119-assign4.docx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "Desktop/Y22-2119-assign4.docx" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "libreoffice_writer" + ], + "evaluator": { + "func": "check_italic_font_size_14", + "expected": { + "type": "cloud_file", + "path": "https://drive.google.com/uc?id=1GTZ-DkMxpdYx38z_s0ab85Ejgxv3qfEp&export=download", + "dest": "Y22-2119-assign4.docx_Gold.docx" + }, + "result": { + "type": "vm_file", + "path": "Desktop/Y22-2119-assign4.docx", + "dest": "Y22-2119-assign4.docx" + } + } +} \ No newline at end of file