From 49ece15ac3d5c31d49f69a6370972de043d779a7 Mon Sep 17 00:00:00 2001 From: Timothyxxx <384084775@qq.com> Date: Wed, 10 Jan 2024 23:18:30 +0800 Subject: [PATCH] VLC v1 finished, improve on instructions, improve on infra --- desktop_env/controllers/python.py | 12 ++++ desktop_env/envs/desktop_env.py | 7 +- desktop_env/evaluators/getters/__init__.py | 1 + desktop_env/evaluators/metrics/README.md | 6 ++ desktop_env/evaluators/metrics/__init__.py | 2 +- desktop_env/evaluators/metrics/vlc.py | 72 +++++++++++++++++-- desktop_env/server/main.py | 3 +- .../59f21cfb-0120-4326-b255-a5b827b38967.json | 2 +- .../8f080098-ddb1-424c-b438-4e96e5e4786e.json | 24 ++++--- .../a1c3ab35-02de-4999-a7ed-2fd12c972c6e.json | 2 +- .../bba3381f-b5eb-4439-bd9e-80c22218d5a7.json | 2 +- .../efcf0d81-0835-4880-b2fd-d866e8bc2294.json | 33 ++++++--- .../fba2c100-79e8-42df-ae74-b592418d54f4.json | 42 ++++++++++- requirements.txt | 1 + 14 files changed, 174 insertions(+), 35 deletions(-) diff --git a/desktop_env/controllers/python.py b/desktop_env/controllers/python.py index 169bec7..7dd884f 100644 --- a/desktop_env/controllers/python.py +++ b/desktop_env/controllers/python.py @@ -241,6 +241,18 @@ class PythonController: logger.error("Failed to get window size. Status code: %d", response.status_code) return None + def get_vm_wallpaper(self): + """ + Gets the wallpaper of the vm. + """ + response = requests.post(self.http_server + "/wallpaper") + if response.status_code == 200: + logger.info("Wallpaper downloaded successfully") + return response.content + else: + logger.error("Failed to get wallpaper. Status code: %d", response.status_code) + return None + def get_vlc_status(self, host='localhost', port=8080, password='password'): url = f'http://{host}:{port}/requests/status.xml' diff --git a/desktop_env/envs/desktop_env.py b/desktop_env/envs/desktop_env.py index 613dd7b..91b99ca 100644 --- a/desktop_env/envs/desktop_env.py +++ b/desktop_env/envs/desktop_env.py @@ -225,7 +225,12 @@ class DesktopEnv(gym.Env): self.setup_controller.setup(self.evaluator["postconfig"]) if "postconfig" in self.evaluator else None - result_state = self.result_getter(self, self.evaluator["result"]) + try: + result_state = self.result_getter(self, self.evaluator["result"]) + except FileNotFoundError: + logger.error("File not found!") + return 0 + expected_state = self.expected_getter(self, self.evaluator["expected"]) if "expected" in self.evaluator \ else None diff --git a/desktop_env/evaluators/getters/__init__.py b/desktop_env/evaluators/getters/__init__.py index c0484d1..d91de89 100644 --- a/desktop_env/evaluators/getters/__init__.py +++ b/desktop_env/evaluators/getters/__init__.py @@ -1,3 +1,4 @@ from .file import get_cloud_file, get_vm_file, get_cache_file from .misc import get_rule +from .info import get_vm_screen_size, get_vm_window_size, get_vm_wallpaper from .vlc import get_vlc_playing_info, get_vlc_config diff --git a/desktop_env/evaluators/metrics/README.md b/desktop_env/evaluators/metrics/README.md index 7ca036c..eff47ac 100644 --- a/desktop_env/evaluators/metrics/README.md +++ b/desktop_env/evaluators/metrics/README.md @@ -95,6 +95,12 @@ with sync_playwright() as playwright: ## VLC Media Player + +### Bugs fix +One thing on Ubuntu need to do, enter into the `meida`>`convert/save`>select files>`convert/save` +Then enter the profile of `Audio - MP3`, change the profile for mp3, section audiocodec from "MP3" to "MPEG Audio" +Otherwise the mp3 file will be created but with 0 bytes. It's a bug of VLC. + ### Setting Up VLC's HTTP Interface To enable and use the HTTP interface in VLC Media Player for remote control and status checks, follow these steps: diff --git a/desktop_env/evaluators/metrics/__init__.py b/desktop_env/evaluators/metrics/__init__.py index 32bfb9f..95aea8d 100644 --- a/desktop_env/evaluators/metrics/__init__.py +++ b/desktop_env/evaluators/metrics/__init__.py @@ -5,5 +5,5 @@ from .docs import compare_font_names, compare_subscript_contains, has_page_numbe from .docs import is_first_line_centered, check_file_exists, compare_contains_image from .pdf import check_pdf_pages from .libreoffice import check_libre_locale -from .vlc import is_vlc_playing, is_vlc_recordings_folder +from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, compare_images, compare_audios, compare_videos from .general import check_csv diff --git a/desktop_env/evaluators/metrics/vlc.py b/desktop_env/evaluators/metrics/vlc.py index 21d7b08..2c381fe 100644 --- a/desktop_env/evaluators/metrics/vlc.py +++ b/desktop_env/evaluators/metrics/vlc.py @@ -7,7 +7,10 @@ from xml.etree import ElementTree import acoustid import cv2 import imagehash +from skimage.metrics import structural_similarity as ssim +import librosa from PIL import Image +import numpy as np logger = logging.getLogger("desktopenv.metrics.vlc") @@ -57,24 +60,79 @@ def is_vlc_recordings_folder(actual_config_path: str, rule: Dict[str, str]) -> f current_path = line.split('=')[-1].strip() # Compare with the Desktop path if current_path == expected_recording_file_path: - return True + return 1 else: - return False + return 0 # The configuration key was not found in the file - return False + return 0 except FileNotFoundError: logger.error("VLC configuration file not found.") - return False + return 0 except Exception as e: logger.error(f"An error occurred: {e}") - return False + return 0 def is_vlc_fullscreen(actual_window_size, screen_size): if actual_window_size['width'] == screen_size['width'] and actual_window_size['height'] == screen_size['height']: - return True + return 1 else: - return False + return 0 + + +def compare_images(image1_path, image2_path): + # You would call this function with the paths to the two images you want to compare: + # score = compare_images('path_to_image1', 'path_to_image2') + # print("Similarity score:", score) + + # Open the images and convert to grayscale + image1 = Image.open(image1_path).convert('L') + image2 = Image.open(image2_path).convert('L') + + # Resize images to the smaller one's size for comparison + image1_size = image1.size + image2_size = image2.size + new_size = min(image1_size, image2_size) + + image1 = image1.resize(new_size, Image.Resampling.LANCZOS) + image2 = image2.resize(new_size, Image.Resampling.LANCZOS) + + # Convert images to numpy arrays + image1_array = np.array(image1) + image2_array = np.array(image2) + + # Calculate SSIM between two images + similarity_index = ssim(image1_array, image2_array) + + return similarity_index + + +def compare_audios(audio_path_1, audio_path_2, max_distance=1000): + """ + Compare two audio files and return a similarity score in the range [0, 1]. + audio_path_1, audio_path_2: paths to the audio files to compare + max_distance: an empirically determined maximum expected DTW distance + """ + # Example Usage: + # similarity = compare_audios_simple('path_to_audio1.mp3', 'path_to_audio2.mp3') + # print(f'Similarity Score: {similarity}') + + # Convert to common format if necessary and load audio + y1, sr1 = librosa.load(audio_path_1) + y2, sr2 = librosa.load(audio_path_2) + + # Extract MFCC features + mfcc1 = librosa.feature.mfcc(y=y1, sr=sr1) + mfcc2 = librosa.feature.mfcc(y=y2, sr=sr2) + + # Compute Dynamic Time Warping distance + distance, path = librosa.sequence.dtw(mfcc1.T, mfcc2.T) + + # Normalize distance to get a similarity score + normalized_distance = np.mean(distance) / max_distance + similarity_score = 1 - min(normalized_distance, 1) # Ensure the score is within [0, 1] + + return similarity_score def compare_videos(video_path1, video_path2, max_frames_to_check=100, threshold=5): diff --git a/desktop_env/server/main.py b/desktop_env/server/main.py index 829e9d0..1b9e42c 100644 --- a/desktop_env/server/main.py +++ b/desktop_env/server/main.py @@ -213,8 +213,7 @@ def get_wallpaper(): if wallpaper_path: try: # Ensure the filename is secure - filename = secure_filename(os.path.basename(wallpaper_path)) - return send_file(wallpaper_path, attachment_filename=filename) + return send_file(wallpaper_path, mimetype='image/png') except Exception as e: app.logger.error(f"An error occurred while serving the wallpaper file: {e}") abort(500, description="Unable to serve the wallpaper file") diff --git a/evaluation_examples/examples/vlc/59f21cfb-0120-4326-b255-a5b827b38967.json b/evaluation_examples/examples/vlc/59f21cfb-0120-4326-b255-a5b827b38967.json index b65ddfb..32d7570 100644 --- a/evaluation_examples/examples/vlc/59f21cfb-0120-4326-b255-a5b827b38967.json +++ b/evaluation_examples/examples/vlc/59f21cfb-0120-4326-b255-a5b827b38967.json @@ -1,7 +1,7 @@ { "id": "59f21cfb-0120-4326-b255-a5b827b38967", "snapshot": "base_setup", - "instruction": "Play the music video on my desktop", + "instruction": "Could you play the music video that's saved on my desktop for me?", "source": "https://docs.videolan.me/vlc-user/desktop/3.0/en/basic/media.html#playing-a-file", "config": [ { diff --git a/evaluation_examples/examples/vlc/8f080098-ddb1-424c-b438-4e96e5e4786e.json b/evaluation_examples/examples/vlc/8f080098-ddb1-424c-b438-4e96e5e4786e.json index db15765..a39bd31 100644 --- a/evaluation_examples/examples/vlc/8f080098-ddb1-424c-b438-4e96e5e4786e.json +++ b/evaluation_examples/examples/vlc/8f080098-ddb1-424c-b438-4e96e5e4786e.json @@ -1,7 +1,7 @@ { "id": "8f080098-ddb1-424c-b438-4e96e5e4786e", "snapshot": "base_setup", - "instruction": "Could you help me extract MP3 Audio to AUDIO_PATH from Video at VIDEO_PATH using VLC Media Player?", + "instruction": "Could you download the song from this music video and save it as an MP3 file? I'd like to have it on my device to play whenever I want. Please title the file \"Baby Justin Bieber.mp3.\" I really appreciate your help!", "source": "https://medium.com/@jetscribe_ai/how-to-extract-mp3-audio-from-videos-using-vlc-media-player-beeef644ebfb", "config": [ { @@ -10,7 +10,7 @@ "files": [ { "url": "https://drive.usercontent.google.com/download?id=19jBiz8sb0M7KHHATO9qeTPr17aWm4me-&export=download&authuser=0&confirm=t&uuid=7a2261f4-3905-433f-b53f-a52dd0845651&at=APZUnTU1nmXSa1ObrA5NHYt8t1-p:1704710908141", - "path": "Baby Justin Bieber.mp4" + "path": "Desktop/Baby Justin Bieber.mp4" } ] } @@ -18,7 +18,11 @@ { "type": "launch", "parameters": { - "command": "vlc" + "command": [ + "vlc", + "--start-time=73", + "Desktop/Baby Justin Bieber.mp4" + ] } } ], @@ -27,16 +31,16 @@ "vlc" ], "evaluator": { - "func": "is_vlc_recordings_folder", + "func": "compare_audios", "expected": { - "type": "rule", - "rules": { - "recording_file_path": "/home/user/Desktop" - } + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=1eRuuiUAOmckyn2zQebx1nWQ40kpjSJ_N&export=download&authuser=0&confirm=t&uuid=8012703a-9e45-4d91-9bda-90c119e42254&at=APZUnTW0uyCSeg7FClSc50UJVRz-:1704898635455", + "dest": "baby_gold.mp3" }, "result": { - "type": "vlc_config", - "dest": "vlcrc" + "type": "vm_file", + "path": "Desktop/Baby Justin Bieber.mp3", + "dest": "baby.mp3" } } } diff --git a/evaluation_examples/examples/vlc/a1c3ab35-02de-4999-a7ed-2fd12c972c6e.json b/evaluation_examples/examples/vlc/a1c3ab35-02de-4999-a7ed-2fd12c972c6e.json index 676092a..b279cb5 100644 --- a/evaluation_examples/examples/vlc/a1c3ab35-02de-4999-a7ed-2fd12c972c6e.json +++ b/evaluation_examples/examples/vlc/a1c3ab35-02de-4999-a7ed-2fd12c972c6e.json @@ -1,7 +1,7 @@ { "id": "a1c3ab35-02de-4999-a7ed-2fd12c972c6e", "snapshot": "base_setup", - "instruction": "Could you help me compress the video to DIR_PATH?", + "instruction": "Could you help me compress the video to MPEG-4 format and save with name with prefix _?", "source": "https://www.quora.com/How-do-I-compress-a-video-with-VLC", "config": [], "trajectory": "trajectories/", diff --git a/evaluation_examples/examples/vlc/bba3381f-b5eb-4439-bd9e-80c22218d5a7.json b/evaluation_examples/examples/vlc/bba3381f-b5eb-4439-bd9e-80c22218d5a7.json index 670aa5a..a5cd48d 100644 --- a/evaluation_examples/examples/vlc/bba3381f-b5eb-4439-bd9e-80c22218d5a7.json +++ b/evaluation_examples/examples/vlc/bba3381f-b5eb-4439-bd9e-80c22218d5a7.json @@ -1,7 +1,7 @@ { "id": "bba3381f-b5eb-4439-bd9e-80c22218d5a7", "snapshot": "base_setup", - "instruction": "Help me play the online video at https://www.youtube.com/watch?v=pgBsyTKAwLw", + "instruction": "Can you start streaming the video from this link for me? https://www.youtube.com/watch?v=pgBsyTKAwLw", "source": "https://www.quora.com/How-do-I-play-online-videos-using-the-VLC-media-player", "config": [ { diff --git a/evaluation_examples/examples/vlc/efcf0d81-0835-4880-b2fd-d866e8bc2294.json b/evaluation_examples/examples/vlc/efcf0d81-0835-4880-b2fd-d866e8bc2294.json index 3238837..3b07f3c 100644 --- a/evaluation_examples/examples/vlc/efcf0d81-0835-4880-b2fd-d866e8bc2294.json +++ b/evaluation_examples/examples/vlc/efcf0d81-0835-4880-b2fd-d866e8bc2294.json @@ -1,30 +1,47 @@ { "id": "efcf0d81-0835-4880-b2fd-d866e8bc2294", "snapshot": "base_setup", - "instruction": "Set this frame of the current video as my wallpaper", - "source": "https://www.youtube.com/watch?v=XHprwDJ0-fU&t=436s", + "instruction": "Make this part of the video my computer's background picture", + "source": "https://www.youtube.com/watch?v=XHprwDJ0-fU&t=436s, https://help.ubuntu.com/stable/ubuntu-help/look-background.html.en", "config": [ { "type": "download", "parameters": { "files": [ { - "url": "", - "path": "" + "url": "https://drive.usercontent.google.com/download?id=1H9D0jSkzpzEWsJZG0HdNNNHDMi3fnGli&export=download&authuser=0&confirm=t&uuid=a6d03223-db7a-48bd-b5f8-50b51f4d3d68&at=APZUnTVi3cqgvRV49c6sluV-nMo_:1704879367756", + "path": "Desktop/Interstellar Movie - Official Trailer.mp4" } ] } }, { "type": "launch", - "parameters": { - "command": ["vlc", "/path/to/your/video.mp4", "--start-time=0", "--run-time=10", "vlc://quit", "&&", "vlc", "/path/to/your/video.mp4", "--start-time=10"] - } + "parameters": { + "command": [ + "vlc", + "--start-time=120.5", + "--stop-time=121", + "--play-and-pause", + "Desktop/Interstellar Movie - Official Trailer.mp4" + ] + } } ], "trajectory": "trajectories/", "related_apps": [ "vlc" ], - "evaluator": "evaluation_dir" + "evaluator": { + "func": "compare_images", + "expected": { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=10P8kLkWeYhtK3Gl15nB-gH7VLO4S7Xal&export=download&authuser=0&confirm=t&uuid=d5c5db29-435d-404f-8900-683bf56fbc66&at=APZUnTXm1ZHS7CbGiWaB04WySH-t:1704891263184", + "dest": "interstellar_wallpaper_gold.png" + }, + "result": { + "type": "vm_wallpaper", + "dest": "result_wallpaper.png" + } + } } diff --git a/evaluation_examples/examples/vlc/fba2c100-79e8-42df-ae74-b592418d54f4.json b/evaluation_examples/examples/vlc/fba2c100-79e8-42df-ae74-b592418d54f4.json index f564f73..0cd51d9 100644 --- a/evaluation_examples/examples/vlc/fba2c100-79e8-42df-ae74-b592418d54f4.json +++ b/evaluation_examples/examples/vlc/fba2c100-79e8-42df-ae74-b592418d54f4.json @@ -1,12 +1,48 @@ { "id": "fba2c100-79e8-42df-ae74-b592418d54f4", "snapshot": "base_setup", - "instruction": "Screenshot the current frame of the video", + "instruction": "Snap a photo of the current video scene, save it as 'interstellar.png', and put it on the Desktop, please.", "source": "https://www.youtube.com/watch?v=XHprwDJ0-fU&t=436s", - "config": [], + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1u7CFVr5bWti7OnhvyjtSdQolzy0lYicm&export=download&authuser=0&confirm=t&uuid=18498a7c-9b52-4c00-a2cc-2e8b521c4d9c&at=APZUnTX0R_TRccU_UmtJWXZ6On3x:1704890439561", + "path": "Desktop/Interstellar Movie - Official Trailer.mp4" + } + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "vlc", + "--start-time=120.5", + "--stop-time=121", + "--play-and-pause", + "Desktop/Interstellar Movie - Official Trailer.mp4" + ] + } + } + ], "trajectory": "trajectories/", "related_apps": [ "vlc" ], - "evaluator": "evaluation_dir" + "evaluator": { + "func": "compare_images", + "expected": { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=1shU1TQ3ao9QWfhmC63vbD10YFbhv7PYv&export=download&authuser=0&confirm=t&uuid=6a22a47a-6d35-4996-b02e-b24b33a1c449&at=APZUnTVlvDKpxW7QKmBBmdNq2jRw:1704891215314", + "dest": "interstellar_gold.png" + }, + "result": { + "type": "vm_file", + "path": "Desktop/interstellar.png", + "dest": "interstellar.png" + } + } } diff --git a/requirements.txt b/requirements.txt index d97aedd..50a95a2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,3 +23,4 @@ python-docx python-pptx pypdf PyGetWindow +scikit-image