From 4eb27fb8000bf6ac0ff97a8819a169855df2e758 Mon Sep 17 00:00:00 2001 From: Timothyxxx <384084775@qq.com> Date: Wed, 24 Jan 2024 11:34:38 +0800 Subject: [PATCH] (makeup) Finish Impress v2 loading, some pre-process left --- desktop_env/evaluators/getters/impress.py | 62 +++++++++++++++++++ .../35761501-3a99-48d1-8ec6-cea8242349b6.json | 12 ---- .../39478d4a-1049-456f-aa77-407811393add.json | 37 ----------- .../3b27600c-3668-4abd-8f84-7bcdebbccbdb.json | 31 ++++++++++ .../550ce7e7-747b-495f-b122-acdc4d0b8e54.json | 31 ++++++++++ .../5d901039-a89c-4bfb-967b-bf66f4df075e.json | 31 ++++++++++ .../9ec204e4-f0a3-42f8-8458-b772a6797cab.json | 31 ++++++++++ .../ac9bb6cb-1888-43ab-81e4-a98a547918cd.json | 2 +- .../af23762e-2bfd-4a1d-aada-20fa8de9ce07.json | 31 ++++++++++ .../c59742c0-4323-4b9d-8a02-723c251deaa0.json | 31 ++++++++++ .../ce88f674-ab7a-43da-9201-468d38539e4a.json | 31 ++++++++++ .../f0a334af-f91b-4c03-b578-aac9bec2b543.json | 31 ++++++++++ 12 files changed, 311 insertions(+), 50 deletions(-) create mode 100644 desktop_env/evaluators/getters/impress.py delete mode 100644 evaluation_examples/examples/libreoffice_impress/35761501-3a99-48d1-8ec6-cea8242349b6.json delete mode 100644 evaluation_examples/examples/libreoffice_impress/39478d4a-1049-456f-aa77-407811393add.json diff --git a/desktop_env/evaluators/getters/impress.py b/desktop_env/evaluators/getters/impress.py new file mode 100644 index 0000000..ec4a154 --- /dev/null +++ b/desktop_env/evaluators/getters/impress.py @@ -0,0 +1,62 @@ +import os +import tempfile +import xml.etree.ElementTree as ET +import zipfile +from typing import Dict + +from desktop_env.evaluators.getters.file import get_vm_file + + +def get_audio_in_slide(env, config: Dict[str, str]): + ppt_file_path, slide_index, dest = config["ppt_file_path"], int(config["slide_index"]), config["dest"] + + # Open the .pptx file as a zip file, fixme: now we assume there is only one audio file in the slides + audio_file_path = None + + ppt_file_localhost_path = get_vm_file(env, {"path": ppt_file_path, "dest": os.path.split(ppt_file_path)[-1]}) + + with zipfile.ZipFile(ppt_file_localhost_path, 'r') as myzip: + # Find the relationships XML file for the first slide + slide1_rels_file = 'ppt/slides/_rels/slide{}.xml.rels'.format(slide_index + 1) + if slide1_rels_file in myzip.namelist(): + with myzip.open(slide1_rels_file) as f: + # Parse the XML tree from the relationships file + tree = ET.parse(f) + root = tree.getroot() + # Define the namespace used in the relationships file + namespaces = {'r': 'http://schemas.openxmlformats.org/package/2006/relationships'} + # Look for all relationship elements that have a type attribute for audio + for rel in root.findall('r:Relationship', namespaces): + # Check if the relationship is for an audio file + if 'audio' in rel.attrib['Type']: + # The audio can be embedded inside the file or linked to an external file + # Get the target attribute which contains the audio file path + target = rel.attrib['Target'] + + if target.startswith('..'): + # Resolve the relative path to get the correct path within the zip file + audio_file_path = os.path.normpath(os.path.join('ppt/slides', target)) + # Replace backslashes with forward slashes for ZIP compatibility + audio_file_path = audio_file_path.replace('\\', '/') + + # Create a temporary directory to extract the audio file + with tempfile.TemporaryDirectory() as tmpdirname: + # Extract the audio file + myzip.extract(audio_file_path, tmpdirname) + # Get the full path of the extracted audio file + extracted_audio_path = os.path.join(tmpdirname, audio_file_path) + # Return the extracted audio file path + audio_file_path = extracted_audio_path + + else: + # the audio file is external to the .pptx file + # Return the audio file path + assert target.startswith("file://"), target + audio_file_path = target[7:] + + if audio_file_path is None: + return None + + else: + # Get the audio file from vm and return the file path in the host + return get_vm_file(env, {"path": audio_file_path, "dest": dest}) diff --git a/evaluation_examples/examples/libreoffice_impress/35761501-3a99-48d1-8ec6-cea8242349b6.json b/evaluation_examples/examples/libreoffice_impress/35761501-3a99-48d1-8ec6-cea8242349b6.json deleted file mode 100644 index 112bd78..0000000 --- a/evaluation_examples/examples/libreoffice_impress/35761501-3a99-48d1-8ec6-cea8242349b6.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "id": "35761501-3a99-48d1-8ec6-cea8242349b6", - "snapshot": "libreoffice_impress", - "instruction": "Help me compress the image I selected.", - "source": "https://www.quora.com/How-do-I-compress-a-selection-of-images-in-LibreOffice-Impress", - "config": [], - "trajectory": "trajectories/", - "related_apps": [ - "" - ], - "evaluator": "evaluation_dir" -} diff --git a/evaluation_examples/examples/libreoffice_impress/39478d4a-1049-456f-aa77-407811393add.json b/evaluation_examples/examples/libreoffice_impress/39478d4a-1049-456f-aa77-407811393add.json deleted file mode 100644 index 6ba438f..0000000 --- a/evaluation_examples/examples/libreoffice_impress/39478d4a-1049-456f-aa77-407811393add.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "id": "39478d4a-1049-456f-aa77-407811393add", - "snapshot": "libreoffice_impress", - "instruction": "Could you help me add a hat symbol to \"Y\" in text in page 14? I need to represent vector sign for my statistics class.", - "source": "https://www.reddit.com/r/libreoffice/comments/jul3o8/putting_cap_or_hat_or_carat_symbol_in_libre/", - "config": [ - { - "type": "download", - "parameters": { - "files": [ - { - "url": "https://drive.usercontent.google.com/download?id=1WT1-L0iiIlF2kuIK77IDxTfBaQ0X0BbX&export=download&authuser=0&confirm=t&uuid=3daac1dc-0f6e-449b-b6bc-09fd246697aa&at=APZUnTVgf_yEeeaARnUISIE4wr4E:1705768410739", - "path": "Desktop/Ch5.pptx" - } - ] - } - }, - { - "type": "open", - "parameters": { - "path": "Desktop/Ch5.pptx" - } - } - ], - "trajectory": "trajectories/", - "related_apps": [ - "libreoffice_impress" - ], - "evaluator": { - "func": "check_formula_shape", - "result": { - "type": "vm_file", - "path": "Desktop/Ch5.pptx", - "dest": "Ch5.pptx" - } - } -} \ No newline at end of file diff --git a/evaluation_examples/examples/libreoffice_impress/3b27600c-3668-4abd-8f84-7bcdebbccbdb.json b/evaluation_examples/examples/libreoffice_impress/3b27600c-3668-4abd-8f84-7bcdebbccbdb.json index 6180dd6..7a6d00c 100644 --- a/evaluation_examples/examples/libreoffice_impress/3b27600c-3668-4abd-8f84-7bcdebbccbdb.json +++ b/evaluation_examples/examples/libreoffice_impress/3b27600c-3668-4abd-8f84-7bcdebbccbdb.json @@ -27,6 +27,37 @@ "libreoffice_impress" ], "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "lec17-gui-events.pptx - LibreOffice Impress", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + } + ], "func": "evaluate_presentation_fill_to_rgb_distance", "expected": { "type": "rule", diff --git a/evaluation_examples/examples/libreoffice_impress/550ce7e7-747b-495f-b122-acdc4d0b8e54.json b/evaluation_examples/examples/libreoffice_impress/550ce7e7-747b-495f-b122-acdc4d0b8e54.json index 6c0e229..8ab1196 100644 --- a/evaluation_examples/examples/libreoffice_impress/550ce7e7-747b-495f-b122-acdc4d0b8e54.json +++ b/evaluation_examples/examples/libreoffice_impress/550ce7e7-747b-495f-b122-acdc4d0b8e54.json @@ -27,6 +27,37 @@ "libreoffice_impress" ], "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "New_Club_Spring_2018_Training.pptx - LibreOffice Impress", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + } + ], "func": "check_strikethrough", "expected": { "type": "rule", diff --git a/evaluation_examples/examples/libreoffice_impress/5d901039-a89c-4bfb-967b-bf66f4df075e.json b/evaluation_examples/examples/libreoffice_impress/5d901039-a89c-4bfb-967b-bf66f4df075e.json index 773ef8c..2654d8e 100644 --- a/evaluation_examples/examples/libreoffice_impress/5d901039-a89c-4bfb-967b-bf66f4df075e.json +++ b/evaluation_examples/examples/libreoffice_impress/5d901039-a89c-4bfb-967b-bf66f4df075e.json @@ -27,6 +27,37 @@ "libreoffice_impress" ], "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "CPD_Background_Investigation_Process.pptx - LibreOffice Impress", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + } + ], "func": "check_image_stretch_and_center", "expected": { "type": "cloud_file", diff --git a/evaluation_examples/examples/libreoffice_impress/9ec204e4-f0a3-42f8-8458-b772a6797cab.json b/evaluation_examples/examples/libreoffice_impress/9ec204e4-f0a3-42f8-8458-b772a6797cab.json index 7e232f9..fc9a9cb 100644 --- a/evaluation_examples/examples/libreoffice_impress/9ec204e4-f0a3-42f8-8458-b772a6797cab.json +++ b/evaluation_examples/examples/libreoffice_impress/9ec204e4-f0a3-42f8-8458-b772a6797cab.json @@ -27,6 +27,37 @@ "libreoffice_impress" ], "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "MLA_Workshop_061X_Works_Cited.pptx - LibreOffice Impress", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + } + ], "func": "compare_pptx_files", "expected": { "type": "cloud_file", diff --git a/evaluation_examples/examples/libreoffice_impress/ac9bb6cb-1888-43ab-81e4-a98a547918cd.json b/evaluation_examples/examples/libreoffice_impress/ac9bb6cb-1888-43ab-81e4-a98a547918cd.json index 8b5dc98..3f465a2 100644 --- a/evaluation_examples/examples/libreoffice_impress/ac9bb6cb-1888-43ab-81e4-a98a547918cd.json +++ b/evaluation_examples/examples/libreoffice_impress/ac9bb6cb-1888-43ab-81e4-a98a547918cd.json @@ -1,7 +1,7 @@ { "id": "ac9bb6cb-1888-43ab-81e4-a98a547918cd", "snapshot": "libreoffice_impress", - "instruction": "Please help me change the color of the slide number to blue", + "instruction": "I am preparing a PPT in Libreoffice impress. The slide number is barely visible to me. Please help me change the color of the slide number to red?", "source": "https://superuser.com/questions/1674211/how-to-change-colour-of-slide-number-in-libre-office", "config": [], "trajectory": "trajectories/", diff --git a/evaluation_examples/examples/libreoffice_impress/af23762e-2bfd-4a1d-aada-20fa8de9ce07.json b/evaluation_examples/examples/libreoffice_impress/af23762e-2bfd-4a1d-aada-20fa8de9ce07.json index a5213fe..4435b4d 100644 --- a/evaluation_examples/examples/libreoffice_impress/af23762e-2bfd-4a1d-aada-20fa8de9ce07.json +++ b/evaluation_examples/examples/libreoffice_impress/af23762e-2bfd-4a1d-aada-20fa8de9ce07.json @@ -27,6 +27,37 @@ "libreoffice_impress" ], "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "Forests.pptx - LibreOffice Impress", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + } + ], "func": "compare_pptx_files", "expected": { "type": "cloud_file", diff --git a/evaluation_examples/examples/libreoffice_impress/c59742c0-4323-4b9d-8a02-723c251deaa0.json b/evaluation_examples/examples/libreoffice_impress/c59742c0-4323-4b9d-8a02-723c251deaa0.json index fce805f..afbb92b 100644 --- a/evaluation_examples/examples/libreoffice_impress/c59742c0-4323-4b9d-8a02-723c251deaa0.json +++ b/evaluation_examples/examples/libreoffice_impress/c59742c0-4323-4b9d-8a02-723c251deaa0.json @@ -38,6 +38,37 @@ "libreoffice_impress" ], "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "Mady_and_Mia_Baseball.pptx - LibreOffice Impress", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + } + ], "func": "compare_audios", "result": { "type": "audio_in_slide", diff --git a/evaluation_examples/examples/libreoffice_impress/ce88f674-ab7a-43da-9201-468d38539e4a.json b/evaluation_examples/examples/libreoffice_impress/ce88f674-ab7a-43da-9201-468d38539e4a.json index 7f3b272..64d63f7 100644 --- a/evaluation_examples/examples/libreoffice_impress/ce88f674-ab7a-43da-9201-468d38539e4a.json +++ b/evaluation_examples/examples/libreoffice_impress/ce88f674-ab7a-43da-9201-468d38539e4a.json @@ -27,6 +27,37 @@ "libreoffice_impress" ], "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "AM_Last_Page_Template.pptx - LibreOffice Impress", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + } + ], "func": "check_slide_orientation_Portrait", "result": { "type": "vm_file", diff --git a/evaluation_examples/examples/libreoffice_impress/f0a334af-f91b-4c03-b578-aac9bec2b543.json b/evaluation_examples/examples/libreoffice_impress/f0a334af-f91b-4c03-b578-aac9bec2b543.json index 3411aa8..39a2ade 100644 --- a/evaluation_examples/examples/libreoffice_impress/f0a334af-f91b-4c03-b578-aac9bec2b543.json +++ b/evaluation_examples/examples/libreoffice_impress/f0a334af-f91b-4c03-b578-aac9bec2b543.json @@ -38,6 +38,37 @@ "libreoffice_impress" ], "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "Movie_countdown_2.pptx - LibreOffice Impress", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + } + ], "func": "contains_mp4_video", "result": { "type": "vm_file",