lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

load libreoffice writer eval -batch 2

Browse Source
This commit is contained in:
tsuky_chen
2024-01-26 02:07:26 +08:00
parent d757b39985
commit 3e7cfa8699
18 changed files with 362 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
desktop_env/controllers/setup.py
View File

@@ -105,7 +105,6 @@ class SetupController:
cache_path: str = os.path.join(self.cache_dir, "{:}_{:}".format(
uuid.uuid5(uuid.NAMESPACE_URL, url),
os.path.basename(path)))
if not url or not path:
raise Exception(f"Setup Download - Invalid URL ({url}) or path ({path}).")

4
desktop_env/envs/desktop_env.py
View File

@@ -47,8 +47,8 @@ class DesktopEnv(gym.Env):
path_to_vm: str,
action_space: str = "computer_13",
task_config: Dict[str, Any] = None,
tmp_dir: str = "tmp",
cache_dir: str = "cache",
tmp_dir: str = "D:\\NJU\\HKUNLP\\Desktop-Env\\tmp",
cache_dir: str = "D:\\NJU\\HKUNLP\\Desktop-Env\\cache",
screen_size: Tuple[int] = (1920, 1080)
):
"""

2
desktop_env/evaluators/getters/__init__.py
View File

@@ -7,4 +7,4 @@ from .misc import get_rule, get_accessibility_tree
from .replay import get_replay
from .vlc import get_vlc_playing_info, get_vlc_config
from .vscode import get_vscode_config
from .impress import get_audio_in_slide
# from .impress import get_audio_in_slide

6
desktop_env/evaluators/metrics/__init__.py
View File

@@ -5,7 +5,7 @@ from .docs import find_default_font, contains_page_break, compare_docx_files, co
from .docs import is_first_line_centered, check_file_exists, compare_contains_image
from .docs import evaluate_colored_words_in_tables, check_highlighted_words, evaluate_strike_through_last_paragraph, \
evaluate_conversion, evaluate_spacing, check_italic_font_size_14, evaluate_alignment, get_unique_train_ids, \
check_no_duplicates
check_no_duplicates, compare_init_lines
from .general import exact_match, fuzzy_match
from .general import check_csv, check_accessibility_tree, run_sqlite3, check_json
from .gimp import increase_saturation, decrease_brightness, check_file_exists, compare_triangle_positions
@@ -16,7 +16,7 @@ from .pdf import check_pdf_pages
#from .table import check_sheet_list, check_xlsx_freeze, check_xlsx_zoom, check_data_validations
from .table import compare_table
from .thunderbird import check_thunderbird_prefs, check_thunderbird_filter
from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, compare_images, compare_audios, \
compare_videos
# from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, compare_images, compare_audios, \
# compare_videos
from .thunderbird import check_thunderbird_prefs, check_thunderbird_filter
from .vscode import compare_text_file, compare_config, compare_answer, is_extension_installed, check_json_settings, check_json_keybindings

47
desktop_env/evaluators/metrics/docs.py
View File

@@ -56,15 +56,34 @@ def compare_docx_files(file1, file2):
doc2_paragraphs = [p.text for p in doc2.paragraphs]
if len(doc1_paragraphs) != len(doc2_paragraphs):
# print(len(doc1_paragraphs))
# print(len(doc2_paragraphs))
return 0
# Compare each paragraph
for p1, p2 in zip(doc1_paragraphs, doc2_paragraphs):
if p1 != p2:
# print(p1)
# print(p2)
return 0
return 1
def compare_init_lines(file1, file2):
doc1 = Document(file1)
doc2 = Document(file2)
doc1_paragraphs = [p.text for p in doc1.paragraphs]
doc2_paragraphs = [p.text for p in doc2.paragraphs]
# Compare each paragraph
for p1, p2 in zip(doc1_paragraphs, doc2_paragraphs):
if p1 != p2:
# print(p1)
# print(p2)
return 0
return 1
def compare_docx_tables(docx_file1, docx_file2):
doc1 = Document(docx_file1)
@@ -93,6 +112,8 @@ def compare_docx_tables(docx_file1, docx_file2):
def compare_line_spacing(docx_file1, docx_file2):
if not compare_docx_files(docx_file1, docx_file2):
return 0
doc1 = Document(docx_file1)
doc2 = Document(docx_file2)
@@ -200,8 +221,10 @@ def compare_contains_image(docx_file1, docx_file2):
# print(find_default_font("Ani", config_path))
def evaluate_colored_words_in_tables(file_path):
document = Document(file_path)
def evaluate_colored_words_in_tables(file_path1, file_path2):
if not compare_docx_files(file_path1, file_path2):
return 0
document = Document(file_path1)
for table in document.tables:
# Iterate through rows and cells in the table
@@ -221,8 +244,10 @@ def evaluate_colored_words_in_tables(file_path):
return 1 # All words in tables are correctly colored
def check_highlighted_words(file_path):
document = Document(file_path)
def check_highlighted_words(file_path1, file_path2):
if not compare_docx_files(file_path1, file_path2):
return 0
document = Document(file_path1)
for paragraph in document.paragraphs:
for run in paragraph.runs:
@@ -232,8 +257,10 @@ def check_highlighted_words(file_path):
return 1 # No highlighted words found
def evaluate_strike_through_last_paragraph(file_path):
document = Document(file_path)
def evaluate_strike_through_last_paragraph(file_path1, file_path2):
if not compare_docx_files(file_path1, file_path2):
return 0
document = Document(file_path1)
# Get the last paragraph
last_paragraph = document.paragraphs[-1]
@@ -278,8 +305,10 @@ def evaluate_spacing(file_path):
return 0
def check_italic_font_size_14(path):
document = Document(path)
def check_italic_font_size_14(path1, path2):
if not compare_docx_files(path1, path2):
return 0
document = Document(path1)
for paragraph in document.paragraphs:
for run in paragraph.runs:
if run.italic:
@@ -365,6 +394,8 @@ def compare_docx_lines(file1, file2):
doc2 = Document(file2)
doc2_lines = [p.text.strip() for p in doc2.paragraphs if p.text.strip()]
# print(doc1_lines)
# print(doc2_lines)
# Convert the list of lines to sets and compare
return set(doc1_lines) == set(doc2_lines)

7
evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json
View File

@@ -19,8 +19,9 @@
"type": "open",
"parameters": {
"path": "/home/user/OrderId_Month_Chart.xlsx"
]
},
}
}
],
"trajectory": "trajectories/2bd59342-0664-4ccb-ba87-79379096cc08",
"related_apps": [
"libreoffice calc"
@@ -77,4 +78,4 @@
]
}
}
}
}

79
evaluation_examples/examples/libreoffice_calc/39aa4e37-dc91-482e-99af-132a612d40f3.json Normal file
View File

@@ -0,0 +1,79 @@
{
"id": "39aa4e37-dc91-482e-99af-132a612d40f3",
"snapshot": "libreoffice_calc",
"instruction": "Help me rename sheet1 \"LARS_Science_Assessment\"",
"source": "https://www.libreofficehelp.com/add-insert-delete-copy-move-rename-a-worksheet-in-libreoffice-calc/",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1Y9KUHsACajMU1kWz_XjW0VBO3RleDBXw&export=download&authuser=0&confirm=t&uuid=73fe859e-7fe1-449b-83cd-2ddaab067f90&at=APZUnTW-Gp9kn8VhJVCXi5VWxjMT:1706099595303",
"path": "/home/user/LARS_Science_Assessment_Resource_List_7-30-21.xlsx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/LARS_Science_Assessment_Resource_List_7-30-21.xlsx"
}
}
],
"trajectory": "trajectories/0cecd4f3-74de-457b-ba94-29ad6b5dafb6",
"related_apps": [
"libreoffice_calc"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "LARS_Science_Assessment_Resource_List_7-30-21.xlsx - LibreOffice Calc",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
]
}
}
],
"func": "compare_table",
"result": {
"type": "vm_file",
"path": "/home/user/LARS_Science_Assessment_Resource_List_7-30-21.xlsx",
"dest": "LARS_Science_Assessment_Resource_List_7-30-21.xlsx"
},
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1JC8iKt5_XjhksBDSePi0THwrfM5oF-5A&export=download&authuser=0&confirm=t&uuid=6124298b-1487-4a85-b7d9-d9ba133c26e0&at=APZUnTVfW7o4XruPUft-Dph7um6J:1706099593830",
"dest": "LARS_Science_Assessment_Resource_List_7-30-21_gold.xlsx"
},
"options": {
"rules": [
{
"type": "sheet_name"
},
{
"type": "sheet_data",
"sheet_idx0": 0,
"sheet_idx1": "EI0"
}
]
}
}
}

0
evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json.nosetup → evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json.json
View File

27
evaluation_examples/examples/libreoffice_writer/0a0faba3-5580-44df-965d-f562a99b291c.json
View File

@@ -27,7 +27,32 @@
"libreoffice_writer"
],
"evaluator": {
"func": "compare_docx_files",
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "04 CHIN9505 EBook Purchasing info 2021 Jan.docx - LibreOffice Writer",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
]
}
}
],
"func": "compare_init_lines",
"expected": {
"type": "cloud_file",
"path": "https://drive.google.com/uc?id=1yyHGj8KUHDMsZmc1QeJ1KkvSEGy83jMR&export=download",

30
evaluation_examples/examples/libreoffice_writer/6a33f9b9-0a56-4844-9c3f-96ec3ffb3ba2.json
View File

@@ -9,7 +9,7 @@
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=10hgB73d_DoQXQVgUjvgXFUCP1Hd9YxDb&export=download&authuser=0&confirm=t&uuid=df2bb3c3-e75e-4cbc-a6bc-24120512a4e1&at=APZUnTUak54ZfgQDNxbt_PqBvNQu:1706017722797",
"url": "https://drive.usercontent.google.com/download?id=10hgB73d_DoQXQVgUjvgXFUCP1Hd9YxDb&export=download&authuser=0&confirm=t&uuid=845f9616-2fb7-476a-abab-8b620d482ac2&at=APZUnTXB71PxHF7Dq9TC2OL_cRLm:1706199789147",
"path": "Desktop/sample-recruitment-phone-script.docx"
}
]
@@ -27,10 +27,32 @@
"libreoffice_writer"
],
"evaluator": {
"func": [
"compare_docx_files",
"check_highlighted_words"
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "sample-recruitment-phone-script.docx - LibreOffice Writer",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
]
}
}
],
"func": "check_highlighted_words",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1s9Dsy66-zxbCAgeTyCh0P7AT7P4jF6o3&export=download&authuser=0&confirm=t&uuid=1239f2a1-8c86-45a4-8e7d-36388ac22a69&at=APZUnTVZQzXQAMNsKKQzOw5ppT8A:1706017721589",

25
evaluation_examples/examples/libreoffice_writer/6f81754e-285d-4ce0-b59e-af7edb02d108.json
View File

@@ -27,6 +27,31 @@
"libreoffice_writer"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "HK_train_record.docx - LibreOffice Writer",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
]
}
}
],
"func": "compare_docx_lines",
"result": {
"type": "vm_file",

30
evaluation_examples/examples/libreoffice_writer/72b810ef-4156-4d09-8f08-a0cf57e7cefe.json
View File

@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
"path": "GEOG2169_Course_Outline_2022-23.docx"
"path": "Desktop/GEOG2169_Course_Outline_2022-23.docx"
}
}
],
@@ -27,10 +27,32 @@
"libreoffice_writer"
],
"evaluator": {
"func": [
"evaluate_strike_through_last_paragraph",
"compare_docx_files"
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "GEOG2169_Course_Outline_2022-23.docx - LibreOffice Writer",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
]
}
}
],
"func": "evaluate_strike_through_last_paragraph",
"expected": {
"type": "cloud_file",
"path": "https://drive.google.com/uc?id=1IpAnQRYo1whrnzIGyo8UldZf4Tli-yVT&export=download",

28
evaluation_examples/examples/libreoffice_writer/8472fece-c7dd-4241-8d65-9b3cd1a0b568.json
View File

@@ -27,10 +27,32 @@
"libreoffice_writer"
],
"evaluator": {
"func": [
"evaluate_colored_words_in_tables",
"compare_docx_files"
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "Dolch_Sight_Words_Primer.docx - LibreOffice Writer",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
]
}
}
],
"func": "evaluate_colored_words_in_tables",
"expected": {
"type": "cloud_file",
"path": "https://drive.google.com/uc?id=1ksn444K17lFOdm5pELrQYvuZHkOsKq69&export=download",

27
evaluation_examples/examples/libreoffice_writer/88fe4b2d-3040-4c70-9a70-546a47764b48.json
View File

@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
"path": "CCCH9003_Tutorial_guidelines.docx"
"path": "Desktop/CCCH9003_Tutorial_guidelines.docx"
}
}
],
@@ -27,6 +27,31 @@
"libreoffice_writer"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "CCCH9003_Tutorial_guidelines.docx - LibreOffice Writer",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
]
}
}
],
"func": "compare_docx_files",
"expected": {
"type": "cloud_file",

30
evaluation_examples/examples/libreoffice_writer/b21acd93-60fd-4127-8a43-2f5178f4a830.json
View File

@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
"path": "CCHU9045_Course_Outline_2019-20.docx"
"path": "Desktop/CCHU9045_Course_Outline_2019-20.docx"
}
}
],
@@ -27,10 +27,32 @@
"libreoffice_writer"
],
"evaluator": {
"func": [
"compare_line_spacing",
"compare_docx_files"
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "CCHU9045_Course_Outline_2019-20.docx - LibreOffice Writer",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
]
}
}
],
"func": "compare_line_spacing",
"expected": {
"type": "cloud_file",
"path": "https://drive.google.com/uc?id=16LN7uYSSXk_xwgc4IZXnN2Z1nCmPJfLm&export=download",

27
evaluation_examples/examples/libreoffice_writer/d53ff5ee-3b1a-431e-b2be-30ed2673079b.json
View File

@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
"path": "presentation_instruction_2023_Feb.docx"
"path": "Desktop/presentation_instruction_2023_Feb.docx"
}
}
],
@@ -28,6 +28,31 @@
],
"evaluator": {
"func": "compare_docx_files",
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "presentation_instruction_2023_Feb.docx - LibreOffice Writer",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
]
}
}
],
"expected": {
"type": "cloud_file",
"path": "https://drive.google.com/uc?id=1bB1N2TWN0puZ6DwUFS_TDjvRWchaGP9T&export=download",

28
evaluation_examples/examples/libreoffice_writer/e246f6d8-78d7-44ac-b668-fcf47946cb50.json
View File

@@ -27,10 +27,32 @@
"libreoffice_writer"
],
"evaluator": {
"func": [
"check_italic_font_size_14",
"compare_docx_files"
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "Y22-2119-assign4.docx - LibreOffice Writer",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
]
}
}
],
"func": "check_italic_font_size_14",
"expected": {
"type": "cloud_file",
"path": "https://drive.google.com/uc?id=1GTZ-DkMxpdYx38z_s0ab85Ejgxv3qfEp&export=download",

6
main.py
View File

@@ -46,12 +46,12 @@ def human_agent():
Runs the Gym environment with human input.
"""
with open("evaluation_examples/examples/vs_code/59ed65c7-e9a6-43db-833f-76d6730c0004.json", "r") as f:
with open("evaluation_examples/examples/libreoffice_writer/72b810ef-4156-4d09-8f08-a0cf57e7cefe.json", "r") as f:
example = json.load(f)
example["snapshot"] = "vscode_setup"
example["snapshot"] = "base18"
env = DesktopEnv(
path_to_vm=r"C:\Users\tianbaox\Documents\Virtual Machines\Ubuntu\Ubuntu.vmx",
path_to_vm=r"D:\Ubuntu\Ubuntu\Ubuntu.vmx",
action_space="computer_13",
task_config=example
)