diff --git a/desktop_env/evaluators/metrics/slides.py b/desktop_env/evaluators/metrics/slides.py index d3b8542..a025933 100644 --- a/desktop_env/evaluators/metrics/slides.py +++ b/desktop_env/evaluators/metrics/slides.py @@ -146,6 +146,8 @@ def compare_pptx_files(file1_path, file2_path, **options): examine_color_rgb = options.get("examine_color_rgb", True) examine_font_underline = options.get("examine_font_underline", True) examine_strike_through = options.get("examine_strike_through", True) + examine_alignment = options.get("examine_alignment", True) + examine_bottom_position = options.get("examine_bottom_position", False) # compare the number of slides if len(prs1.slides) != len(prs2.slides) and examine_number_of_slides: @@ -153,19 +155,24 @@ def compare_pptx_files(file1_path, file2_path, **options): # compare the content of each slide for slide1, slide2 in zip(prs1.slides, prs2.slides): - # check if the shapes are the same for shape1, shape2 in zip(slide1.shapes, slide2.shapes): - if ( - shape1.left != shape2.left or shape1.top != shape2.top or shape1.width != shape2.width or shape1.height != shape2.height) and examine_shape: + if examine_bottom_position and shape1.top != shape2.top: + if hasattr(shape1, "text") and hasattr(shape2, "text") and shape1.text == shape2.text and shape1.text == "Product Comparison": + if shape1.top >= shape2.top: + return 0 + + if (shape1.left != shape2.left or shape1.top != shape2.top or shape1.width != shape2.width or shape1.height != shape2.height) and examine_shape: return 0 if hasattr(shape1, "text") and hasattr(shape2, "text"): if shape1.text != shape2.text and examine_text: - return 0 - + return 0 + # check if the paragraphs are the same for para1, para2 in zip(shape1.text_frame.paragraphs, shape2.text_frame.paragraphs): + if para1.alignment != para2.alignment and examine_alignment: + return 0 # check if the runs are the same for run1, run2 in zip(para1.runs, para2.runs): @@ -182,8 +189,9 @@ def compare_pptx_files(file1_path, file2_path, **options): if run1.font.italic != run2.font.italic and examine_font_italic: return 0 - if run1.font.color.rgb != run2.font.color.rgb and examine_color_rgb: - return 0 + if hasattr(run1.font.color, "rgb") and hasattr(run2.font.color, "rgb"): + if run1.font.color.rgb != run2.font.color.rgb and examine_color_rgb: + return 0 if run1.font.underline != run2.font.underline and examine_font_underline: return 0 diff --git a/evaluation_examples/examples/libreoffice_impress/08aced46-45a2-48d7-993b-ed3fb5b32302.json b/evaluation_examples/examples/libreoffice_impress/08aced46-45a2-48d7-993b-ed3fb5b32302.json new file mode 100644 index 0000000..a3696be --- /dev/null +++ b/evaluation_examples/examples/libreoffice_impress/08aced46-45a2-48d7-993b-ed3fb5b32302.json @@ -0,0 +1,79 @@ +{ + "id": "08aced46-45a2-48d7-993b-ed3fb5b32302", + "snapshot": "libreoffice_impress", + "instruction": "Give the slide the right aligned title, \"Note\"", + "source": "https://arxiv.org/pdf/2311.01767.pdf", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=15OOk9xJzM4WBJRy5rDT2iMrEmm7b-VXj&export=download&authuser=0&confirm=t&uuid=501abcf6-9d70-4ef7-b1ed-0c6e0a47ae7c&at=APZUnTUn3aTxAlhaMSjmljHvPfrr:1707741990590", + "path": "/home/user/Desktop/22_6.pptx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/Desktop/22_6.pptx" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "libreoffice_impress" + ], + "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "22_6.pptx - LibreOffice Impress", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);pyautogui.press('down');time.sleep(0.5);pyautogui.press('enter');time.sleep(0.5);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + } + ], + "func": "compare_pptx_files", + "expected": { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=1lIdDVe95j3Hpscn9eXEuaawhSL2QmhEl&export=download&authuser=0&confirm=t&uuid=63a092e2-a4c1-458d-9904-b288f5f8422b&at=APZUnTXEYGXubO9J-0Ybvi5TXOfb:1707741962330", + "dest": "22_6_Gold.pptx" + }, + "result": { + "type": "vm_file", + "path": "/home/user/Desktop/22_6.pptx", + "dest": "22_6.pptx" + }, + "options": { + "examine_shape": false, + "examine_font_size": false, + "examine_font_bold": false, + "examine_font_name": false + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/libreoffice_impress/15aece23-a215-4579-91b4-69eec72e18da.json b/evaluation_examples/examples/libreoffice_impress/15aece23-a215-4579-91b4-69eec72e18da.json new file mode 100644 index 0000000..bbb1a7f --- /dev/null +++ b/evaluation_examples/examples/libreoffice_impress/15aece23-a215-4579-91b4-69eec72e18da.json @@ -0,0 +1,81 @@ +{ + "id": "15aece23-a215-4579-91b4-69eec72e18da", + "snapshot": "libreoffice_impress", + "instruction": "Move the title to the bottom of the slide. ## 1, title, slide, bottom corner", + "source": "https://arxiv.org/pdf/2311.01767.pdf", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1SCyQzGU05PF-BPrXJ7GsFrnyuRcKcbBb&export=download&authuser=0&confirm=t&uuid=a3b90508-2288-48a4-95d5-c8e7bc11d5fd&at=APZUnTUt44ntxFBUaL3I6oqRhfjy:1707917097401", + "path": "/home/user/Desktop/134_2.pptx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/Desktop/134_2.pptx" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "libreoffice_impress" + ], + "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "134_2.pptx - LibreOffice Impress", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);pyautogui.press('down');time.sleep(0.5);pyautogui.press('enter');time.sleep(0.5);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + } + ], + "func": "compare_pptx_files", + "expected": { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=1OqpF8JpJjaXwHvDANB0E7nJR2m4ssXJO&export=download&authuser=0&confirm=t&uuid=d9eb02df-4b20-4f54-afe1-53437820946a&at=APZUnTUiporlBynLvGUB1-zDS9xm:1707916188451", + "dest": "134_2_Gold.pptx" + }, + "result": { + "type": "vm_file", + "path": "/home/user/Desktop/134_2.pptx", + "dest": "134_2.pptx" + }, + "options": { + "examine_shape": false, + "examine_font_size": false, + "examine_font_bold": false, + "examine_font_name": false, + "examine_alignment": false, + "examine_bottom_position": true + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/libreoffice_impress/550ce7e7-747b-495f-b122-acdc4d0b8e54.json b/evaluation_examples/examples/libreoffice_impress/550ce7e7-747b-495f-b122-acdc4d0b8e54.json index 4e447b3..8dd61a7 100644 --- a/evaluation_examples/examples/libreoffice_impress/550ce7e7-747b-495f-b122-acdc4d0b8e54.json +++ b/evaluation_examples/examples/libreoffice_impress/550ce7e7-747b-495f-b122-acdc4d0b8e54.json @@ -84,9 +84,6 @@ "type": "vm_file", "path": "/home/user/Desktop/New_Club_Spring_2018_Training.pptx", "dest": "New_Club_Spring_2018_Training.pptx" - }, - "options": { - "examine_shape": false - } + } } } \ No newline at end of file diff --git a/evaluation_examples/examples/libreoffice_impress/648de793-7368-4fb7-b16f-a15362da8003.json b/evaluation_examples/examples/libreoffice_impress/648de793-7368-4fb7-b16f-a15362da8003.json new file mode 100644 index 0000000..12583c8 --- /dev/null +++ b/evaluation_examples/examples/libreoffice_impress/648de793-7368-4fb7-b16f-a15362da8003.json @@ -0,0 +1,80 @@ +{ + "id": "648de793-7368-4fb7-b16f-a15362da8003", + "snapshot": "libreoffice_impress", + "instruction": "In the slide, uh, add a right arrow in the left side", + "source": "https://arxiv.org/pdf/2311.01767.pdf", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1kOQ_PlYlGOKhdm-etxB2D8VvLaA4mqNT&export=download&authuser=0&confirm=t&uuid=9be6111e-53cf-4724-bcef-2ae4e819003b&at=APZUnTXZJ2KPlfmSL9R4ZBU0jAPs:1707932348450", + "path": "/home/user/Desktop/19_0.pptx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/Desktop/19_0.pptx" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "libreoffice_impress" + ], + "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "19_0.pptx - LibreOffice Impress", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);pyautogui.press('down');time.sleep(0.5);pyautogui.press('enter');time.sleep(0.5);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + } + ], + "func": "compare_pptx_files", + "expected": { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=180atkxCNr-fZrT9YTKV_8qyhpMFqpEUq&export=download&authuser=0&confirm=t&uuid=e5ee127a-93a1-4069-b058-86a714de64ff&at=APZUnTVUuP0snmxIvS92-t4ObLf9:1707932394767", + "dest": "19_0_Gold.pptx" + }, + "result": { + "type": "vm_file", + "path": "/home/user/Desktop/19_0.pptx", + "dest": "19_0.pptx" + }, + "options": { + "examine_shape": false, + "examine_font_size": false, + "examine_font_bold": false, + "examine_font_name": false, + "examine_alignment": false + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/libreoffice_impress/73c99fb9-f828-43ce-b87a-01dc07faa224.json b/evaluation_examples/examples/libreoffice_impress/73c99fb9-f828-43ce-b87a-01dc07faa224.json new file mode 100644 index 0000000..cb25842 --- /dev/null +++ b/evaluation_examples/examples/libreoffice_impress/73c99fb9-f828-43ce-b87a-01dc07faa224.json @@ -0,0 +1,80 @@ +{ + "id": "73c99fb9-f828-43ce-b87a-01dc07faa224", + "snapshot": "libreoffice_impress", + "instruction": "Add \"Page 1\" into the slide.", + "source": "https://arxiv.org/pdf/2311.01767.pdf", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1KDDJCei7KE1yRWGNUSmUnFnpZBH2gDeE&export=download&authuser=0&confirm=t&uuid=f85aa168-2476-46ab-9ee1-4bb373904650&at=APZUnTVvlubece5OCv6g-ULVovsQ:1707912989623", + "path": "/home/user/Desktop/109_4.pptx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/Desktop/109_4.pptx" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "libreoffice_impress" + ], + "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "109_4.pptx - LibreOffice Impress", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);pyautogui.press('down');time.sleep(0.5);pyautogui.press('enter');time.sleep(0.5);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + } + ], + "func": "compare_pptx_files", + "expected": { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=1w6DoJnhq8SDJQ76iDB_L2r7mGZNtVFIO&export=download&authuser=0&confirm=t&uuid=1d12f276-0b9c-4f8b-84fa-71d4ae011224&at=APZUnTUJOXjccK3edFKFEyoU6VRA:1707913047283", + "dest": "109_4_Gold.pptx" + }, + "result": { + "type": "vm_file", + "path": "/home/user/Desktop/109_4.pptx", + "dest": "109_4.pptx" + }, + "options": { + "examine_shape": false, + "examine_font_size": false, + "examine_font_bold": false, + "examine_font_name": false, + "examine_alignment": false + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/libreoffice_impress/986fc832-6af2-417c-8845-9272b3a1528b.json b/evaluation_examples/examples/libreoffice_impress/986fc832-6af2-417c-8845-9272b3a1528b.json new file mode 100644 index 0000000..cc1e97c --- /dev/null +++ b/evaluation_examples/examples/libreoffice_impress/986fc832-6af2-417c-8845-9272b3a1528b.json @@ -0,0 +1,81 @@ +{ + "id": "986fc832-6af2-417c-8845-9272b3a1528b", + "snapshot": "libreoffice_impress", + "instruction": "underline the content and make the font color on this slide dark red 2.", + "source": "https://arxiv.org/pdf/2311.01767.pdf", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1WgEV25hWlzvOXIImI1ZaDZO1yKC72Qay&export=download&authuser=0&confirm=t&uuid=e94d9f07-feeb-4b7b-82f7-30a90bb15983&at=APZUnTUw8ObGQCHaMY5AZdIMvA-k:1707919959393", + "path": "/home/user/Desktop/154_3.pptx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/Desktop/154_3.pptx" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "libreoffice_impress" + ], + "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "154_3.pptx - LibreOffice Impress", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);pyautogui.press('down');time.sleep(0.5);pyautogui.press('enter');time.sleep(0.5);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + } + ], + "func": "compare_pptx_files", + "expected": { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=1trYqwEuj2ruF830UGUHF1M3I1at4nW11&export=download&authuser=0&confirm=t&uuid=1d215b22-0d02-46a8-9c4e-64f0c8ce0d0d&at=APZUnTUaJxIF7EgFitSO1Dppy1LT:1707919967130", + "dest": "154_3_Gold.pptx" + }, + "result": { + "type": "vm_file", + "path": "/home/user/Desktop/154_3.pptx", + "dest": "154_3.pptx" + }, + "options": { + "examine_shape": false, + "examine_font_size": false, + "examine_font_bold": false, + "examine_font_name": false, + "examine_alignment": false, + "examine_bottom_position": true + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/libreoffice_impress/edb61b14-a854-4bf5-a075-c8075c11293a.json b/evaluation_examples/examples/libreoffice_impress/edb61b14-a854-4bf5-a075-c8075c11293a.json new file mode 100644 index 0000000..685e13f --- /dev/null +++ b/evaluation_examples/examples/libreoffice_impress/edb61b14-a854-4bf5-a075-c8075c11293a.json @@ -0,0 +1,79 @@ +{ + "id": "edb61b14-a854-4bf5-a075-c8075c11293a", + "snapshot": "libreoffice_impress", + "instruction": "In the last slide, make the font style \"Times New Roman\"", + "source": "https://arxiv.org/pdf/2311.01767.pdf", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1SvG7uqJ4nd-YimLiQK1XsrI65iCxOMOw&export=download&authuser=0&confirm=t&uuid=20e98914-c490-4bb6-8a37-1c839e2226fe&at=APZUnTU-4RRKoHat4k60maslvMSZ:1707901107881", + "path": "/home/user/Desktop/24_8.pptx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/Desktop/24_8.pptx" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "libreoffice_impress" + ], + "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "24_8.pptx - LibreOffice Impress", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);pyautogui.press('down');time.sleep(0.5);pyautogui.press('enter');time.sleep(0.5);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + } + ], + "func": "compare_pptx_files", + "expected": { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=1a7sXQrGQBlPntn64owyl8KWE8eLUXJMv&export=download&authuser=0&confirm=t&uuid=cc02c1a4-7613-484e-9d13-478f57a795a0&at=APZUnTV_gcXjdY6Jz9AAWK-W55l6:1707900948910", + "dest": "24_8_Gold.pptx" + }, + "result": { + "type": "vm_file", + "path": "/home/user/Desktop/24_8.pptx", + "dest": "24_8.pptx" + }, + "options": { + "examine_shape": false, + "examine_font_size": false, + "examine_font_bold": false, + "examine_alignment": false + } + } +} \ No newline at end of file