lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

VLC v1 finished, improve on instructions, improve on infra

Browse Source
This commit is contained in:
Timothyxxx
2024-01-10 23:18:30 +08:00
parent abcafce750
commit 49ece15ac3
14 changed files with 174 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
desktop_env/controllers/python.py
View File

@@ -241,6 +241,18 @@ class PythonController:
logger.error("Failed to get window size. Status code: %d", response.status_code)
return None
def get_vm_wallpaper(self):
"""
Gets the wallpaper of the vm.
"""
response = requests.post(self.http_server + "/wallpaper")
if response.status_code == 200:
logger.info("Wallpaper downloaded successfully")
return response.content
else:
logger.error("Failed to get wallpaper. Status code: %d", response.status_code)
return None
def get_vlc_status(self, host='localhost', port=8080, password='password'):
url = f'http://{host}:{port}/requests/status.xml'

7
desktop_env/envs/desktop_env.py
View File

@@ -225,7 +225,12 @@ class DesktopEnv(gym.Env):
self.setup_controller.setup(self.evaluator["postconfig"]) if "postconfig" in self.evaluator else None
result_state = self.result_getter(self, self.evaluator["result"])
try:
result_state = self.result_getter(self, self.evaluator["result"])
except FileNotFoundError:
logger.error("File not found!")
return 0
expected_state = self.expected_getter(self, self.evaluator["expected"]) if "expected" in self.evaluator \
else None

1
desktop_env/evaluators/getters/__init__.py
View File

@@ -1,3 +1,4 @@
from .file import get_cloud_file, get_vm_file, get_cache_file
from .misc import get_rule
from .info import get_vm_screen_size, get_vm_window_size, get_vm_wallpaper
from .vlc import get_vlc_playing_info, get_vlc_config

6
desktop_env/evaluators/metrics/README.md
View File

@@ -95,6 +95,12 @@ with sync_playwright() as playwright:
## VLC Media Player
### Bugs fix
One thing on Ubuntu need to do, enter into the `meida`>`convert/save`>select files>`convert/save`
Then enter the profile of `Audio - MP3`, change the profile for mp3, section audiocodec from "MP3" to "MPEG Audio"
Otherwise the mp3 file will be created but with 0 bytes. It's a bug of VLC.
### Setting Up VLC's HTTP Interface
To enable and use the HTTP interface in VLC Media Player for remote control and status checks, follow these steps:

2
desktop_env/evaluators/metrics/__init__.py
View File

@@ -5,5 +5,5 @@ from .docs import compare_font_names, compare_subscript_contains, has_page_numbe
from .docs import is_first_line_centered, check_file_exists, compare_contains_image
from .pdf import check_pdf_pages
from .libreoffice import check_libre_locale
from .vlc import is_vlc_playing, is_vlc_recordings_folder
from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, compare_images, compare_audios, compare_videos
from .general import check_csv

72
desktop_env/evaluators/metrics/vlc.py
View File

@@ -7,7 +7,10 @@ from xml.etree import ElementTree
import acoustid
import cv2
import imagehash
from skimage.metrics import structural_similarity as ssim
import librosa
from PIL import Image
import numpy as np
logger = logging.getLogger("desktopenv.metrics.vlc")
@@ -57,24 +60,79 @@ def is_vlc_recordings_folder(actual_config_path: str, rule: Dict[str, str]) -> f
current_path = line.split('=')[-1].strip()
# Compare with the Desktop path
if current_path == expected_recording_file_path:
return True
return 1
else:
return False
return 0
# The configuration key was not found in the file
return False
return 0
except FileNotFoundError:
logger.error("VLC configuration file not found.")
return False
return 0
except Exception as e:
logger.error(f"An error occurred: {e}")
return False
return 0
def is_vlc_fullscreen(actual_window_size, screen_size):
if actual_window_size['width'] == screen_size['width'] and actual_window_size['height'] == screen_size['height']:
return True
return 1
else:
return False
return 0
def compare_images(image1_path, image2_path):
# You would call this function with the paths to the two images you want to compare:
# score = compare_images('path_to_image1', 'path_to_image2')
# print("Similarity score:", score)
# Open the images and convert to grayscale
image1 = Image.open(image1_path).convert('L')
image2 = Image.open(image2_path).convert('L')
# Resize images to the smaller one's size for comparison
image1_size = image1.size
image2_size = image2.size
new_size = min(image1_size, image2_size)
image1 = image1.resize(new_size, Image.Resampling.LANCZOS)
image2 = image2.resize(new_size, Image.Resampling.LANCZOS)
# Convert images to numpy arrays
image1_array = np.array(image1)
image2_array = np.array(image2)
# Calculate SSIM between two images
similarity_index = ssim(image1_array, image2_array)
return similarity_index
def compare_audios(audio_path_1, audio_path_2, max_distance=1000):
"""
Compare two audio files and return a similarity score in the range [0, 1].
audio_path_1, audio_path_2: paths to the audio files to compare
max_distance: an empirically determined maximum expected DTW distance
"""
# Example Usage:
# similarity = compare_audios_simple('path_to_audio1.mp3', 'path_to_audio2.mp3')
# print(f'Similarity Score: {similarity}')
# Convert to common format if necessary and load audio
y1, sr1 = librosa.load(audio_path_1)
y2, sr2 = librosa.load(audio_path_2)
# Extract MFCC features
mfcc1 = librosa.feature.mfcc(y=y1, sr=sr1)
mfcc2 = librosa.feature.mfcc(y=y2, sr=sr2)
# Compute Dynamic Time Warping distance
distance, path = librosa.sequence.dtw(mfcc1.T, mfcc2.T)
# Normalize distance to get a similarity score
normalized_distance = np.mean(distance) / max_distance
similarity_score = 1 - min(normalized_distance, 1) # Ensure the score is within [0, 1]
return similarity_score
def compare_videos(video_path1, video_path2, max_frames_to_check=100, threshold=5):

3
desktop_env/server/main.py
View File

@@ -213,8 +213,7 @@ def get_wallpaper():
if wallpaper_path:
try:
# Ensure the filename is secure
filename = secure_filename(os.path.basename(wallpaper_path))
return send_file(wallpaper_path, attachment_filename=filename)
return send_file(wallpaper_path, mimetype='image/png')
except Exception as e:
app.logger.error(f"An error occurred while serving the wallpaper file: {e}")
abort(500, description="Unable to serve the wallpaper file")

2
evaluation_examples/examples/vlc/59f21cfb-0120-4326-b255-a5b827b38967.json
View File

@@ -1,7 +1,7 @@
{
"id": "59f21cfb-0120-4326-b255-a5b827b38967",
"snapshot": "base_setup",
"instruction": "Play the music video on my desktop",
"instruction": "Could you play the music video that's saved on my desktop for me?",
"source": "https://docs.videolan.me/vlc-user/desktop/3.0/en/basic/media.html#playing-a-file",
"config": [
{

24
evaluation_examples/examples/vlc/8f080098-ddb1-424c-b438-4e96e5e4786e.json
View File

@@ -1,7 +1,7 @@
{
"id": "8f080098-ddb1-424c-b438-4e96e5e4786e",
"snapshot": "base_setup",
"instruction": "Could you help me extract MP3 Audio to AUDIO_PATH from Video at VIDEO_PATH using VLC Media Player?",
"instruction": "Could you download the song from this music video and save it as an MP3 file? I'd like to have it on my device to play whenever I want. Please title the file \"Baby Justin Bieber.mp3.\" I really appreciate your help!",
"source": "https://medium.com/@jetscribe_ai/how-to-extract-mp3-audio-from-videos-using-vlc-media-player-beeef644ebfb",
"config": [
{
@@ -10,7 +10,7 @@
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=19jBiz8sb0M7KHHATO9qeTPr17aWm4me-&export=download&authuser=0&confirm=t&uuid=7a2261f4-3905-433f-b53f-a52dd0845651&at=APZUnTU1nmXSa1ObrA5NHYt8t1-p:1704710908141",
"path": "Baby Justin Bieber.mp4"
"path": "Desktop/Baby Justin Bieber.mp4"
}
]
}
@@ -18,7 +18,11 @@
{
"type": "launch",
"parameters": {
"command": "vlc"
"command": [
"vlc",
"--start-time=73",
"Desktop/Baby Justin Bieber.mp4"
]
}
}
],
@@ -27,16 +31,16 @@
"vlc"
],
"evaluator": {
"func": "is_vlc_recordings_folder",
"func": "compare_audios",
"expected": {
"type": "rule",
"rules": {
"recording_file_path": "/home/user/Desktop"
}
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1eRuuiUAOmckyn2zQebx1nWQ40kpjSJ_N&export=download&authuser=0&confirm=t&uuid=8012703a-9e45-4d91-9bda-90c119e42254&at=APZUnTW0uyCSeg7FClSc50UJVRz-:1704898635455",
"dest": "baby_gold.mp3"
},
"result": {
"type": "vlc_config",
"dest": "vlcrc"
"type": "vm_file",
"path": "Desktop/Baby Justin Bieber.mp3",
"dest": "baby.mp3"
}
}
}

2
evaluation_examples/examples/vlc/a1c3ab35-02de-4999-a7ed-2fd12c972c6e.json
View File

@@ -1,7 +1,7 @@
{
"id": "a1c3ab35-02de-4999-a7ed-2fd12c972c6e",
"snapshot": "base_setup",
"instruction": "Could you help me compress the video to DIR_PATH?",
"instruction": "Could you help me compress the video to MPEG-4 format and save with name with prefix _?",
"source": "https://www.quora.com/How-do-I-compress-a-video-with-VLC",
"config": [],
"trajectory": "trajectories/",

2
evaluation_examples/examples/vlc/bba3381f-b5eb-4439-bd9e-80c22218d5a7.json
View File

@@ -1,7 +1,7 @@
{
"id": "bba3381f-b5eb-4439-bd9e-80c22218d5a7",
"snapshot": "base_setup",
"instruction": "Help me play the online video at https://www.youtube.com/watch?v=pgBsyTKAwLw",
"instruction": "Can you start streaming the video from this link for me? https://www.youtube.com/watch?v=pgBsyTKAwLw",
"source": "https://www.quora.com/How-do-I-play-online-videos-using-the-VLC-media-player",
"config": [
{

33
evaluation_examples/examples/vlc/efcf0d81-0835-4880-b2fd-d866e8bc2294.json
View File

@@ -1,30 +1,47 @@
{
"id": "efcf0d81-0835-4880-b2fd-d866e8bc2294",
"snapshot": "base_setup",
"instruction": "Set this frame of the current video as my wallpaper",
"source": "https://www.youtube.com/watch?v=XHprwDJ0-fU&t=436s",
"instruction": "Make this part of the video my computer's background picture",
"source": "https://www.youtube.com/watch?v=XHprwDJ0-fU&t=436s, https://help.ubuntu.com/stable/ubuntu-help/look-background.html.en",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "",
"path": ""
"url": "https://drive.usercontent.google.com/download?id=1H9D0jSkzpzEWsJZG0HdNNNHDMi3fnGli&export=download&authuser=0&confirm=t&uuid=a6d03223-db7a-48bd-b5f8-50b51f4d3d68&at=APZUnTVi3cqgvRV49c6sluV-nMo_:1704879367756",
"path": "Desktop/Interstellar Movie - Official Trailer.mp4"
}
]
}
},
{
"type": "launch",
"parameters": {
"command": ["vlc", "/path/to/your/video.mp4", "--start-time=0", "--run-time=10", "vlc://quit", "&&", "vlc", "/path/to/your/video.mp4", "--start-time=10"]
}
"parameters": {
"command": [
"vlc",
"--start-time=120.5",
"--stop-time=121",
"--play-and-pause",
"Desktop/Interstellar Movie - Official Trailer.mp4"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"vlc"
],
"evaluator": "evaluation_dir"
"evaluator": {
"func": "compare_images",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=10P8kLkWeYhtK3Gl15nB-gH7VLO4S7Xal&export=download&authuser=0&confirm=t&uuid=d5c5db29-435d-404f-8900-683bf56fbc66&at=APZUnTXm1ZHS7CbGiWaB04WySH-t:1704891263184",
"dest": "interstellar_wallpaper_gold.png"
},
"result": {
"type": "vm_wallpaper",
"dest": "result_wallpaper.png"
}
}
}

42
evaluation_examples/examples/vlc/fba2c100-79e8-42df-ae74-b592418d54f4.json
View File

@@ -1,12 +1,48 @@
{
"id": "fba2c100-79e8-42df-ae74-b592418d54f4",
"snapshot": "base_setup",
"instruction": "Screenshot the current frame of the video",
"instruction": "Snap a photo of the current video scene, save it as 'interstellar.png', and put it on the Desktop, please.",
"source": "https://www.youtube.com/watch?v=XHprwDJ0-fU&t=436s",
"config": [],
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1u7CFVr5bWti7OnhvyjtSdQolzy0lYicm&export=download&authuser=0&confirm=t&uuid=18498a7c-9b52-4c00-a2cc-2e8b521c4d9c&at=APZUnTX0R_TRccU_UmtJWXZ6On3x:1704890439561",
"path": "Desktop/Interstellar Movie - Official Trailer.mp4"
}
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"vlc",
"--start-time=120.5",
"--stop-time=121",
"--play-and-pause",
"Desktop/Interstellar Movie - Official Trailer.mp4"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"vlc"
],
"evaluator": "evaluation_dir"
"evaluator": {
"func": "compare_images",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1shU1TQ3ao9QWfhmC63vbD10YFbhv7PYv&export=download&authuser=0&confirm=t&uuid=6a22a47a-6d35-4996-b02e-b24b33a1c449&at=APZUnTVlvDKpxW7QKmBBmdNq2jRw:1704891215314",
"dest": "interstellar_gold.png"
},
"result": {
"type": "vm_file",
"path": "Desktop/interstellar.png",
"dest": "interstellar.png"
}
}
}

1
requirements.txt
View File

@@ -23,3 +23,4 @@ python-docx
python-pptx
pypdf
PyGetWindow
scikit-image