diff --git a/desktop_env/controllers/python.py b/desktop_env/controllers/python.py index 6d8f7d1..39afb33 100644 --- a/desktop_env/controllers/python.py +++ b/desktop_env/controllers/python.py @@ -228,6 +228,13 @@ class PythonController: else: raise Exception(f"Unknown action type: {action_type}") + # Additional info + def get_vm_platform(self): + """ + Gets the size of the vm screen. + """ + return self.execute_python_command("import platform; print(platform.system())")['output'].strip() + def get_vm_screen_size(self): """ Gets the size of the vm screen. @@ -250,6 +257,19 @@ class PythonController: logger.error("Failed to get window size. Status code: %d", response.status_code) return None + def get_vm_wallpaper(self): + """ + Gets the wallpaper of the vm. + """ + response = requests.post(self.http_server + "/wallpaper") + if response.status_code == 200: + logger.info("Wallpaper downloaded successfully") + return response.content + else: + logger.error("Failed to get wallpaper. Status code: %d", response.status_code) + return None + + # VLC def get_vlc_status(self, host='localhost', port=8080, password='password'): url = f'http://{host}:{port}/requests/status.xml' diff --git a/desktop_env/envs/desktop_env.py b/desktop_env/envs/desktop_env.py index e256f7c..a309489 100644 --- a/desktop_env/envs/desktop_env.py +++ b/desktop_env/envs/desktop_env.py @@ -35,7 +35,10 @@ def _execute_command(command: List[str]) -> None: class DesktopEnv(gym.Env): - """DesktopEnv with OpenAI Gym interface.""" + """ + DesktopEnv with OpenAI Gym interface. + Fixme: refactor the logic when implementing the multi-process version + """ def __init__( self, @@ -81,6 +84,10 @@ class DesktopEnv(gym.Env): self.controller = PythonController(http_server=self.host) self.setup_controller = SetupController(http_server=self.host, cache_dir=self.cache_dir) + # Meta info of the VM + self.vm_platform = self.controller.get_vm_platform() + self.vm_screen_size = self.controller.get_vm_screen_size() + # mode: human or machine assert action_space in ["computer_13", "pyautogui"] self.action_space = action_space @@ -225,7 +232,12 @@ class DesktopEnv(gym.Env): self.setup_controller.setup(self.evaluator.get("postconfig", [])) - result_state = self.result_getter(self, self.evaluator["result"]) + try: + result_state = self.result_getter(self, self.evaluator["result"]) + except FileNotFoundError: + logger.error("File not found!") + return 0 + expected_state = self.expected_getter(self, self.evaluator["expected"]) if "expected" in self.evaluator \ else None diff --git a/desktop_env/evaluators/metrics/README.md b/desktop_env/evaluators/README.md similarity index 93% rename from desktop_env/evaluators/metrics/README.md rename to desktop_env/evaluators/README.md index 7ca036c..eff47ac 100644 --- a/desktop_env/evaluators/metrics/README.md +++ b/desktop_env/evaluators/README.md @@ -95,6 +95,12 @@ with sync_playwright() as playwright: ## VLC Media Player + +### Bugs fix +One thing on Ubuntu need to do, enter into the `meida`>`convert/save`>select files>`convert/save` +Then enter the profile of `Audio - MP3`, change the profile for mp3, section audiocodec from "MP3" to "MPEG Audio" +Otherwise the mp3 file will be created but with 0 bytes. It's a bug of VLC. + ### Setting Up VLC's HTTP Interface To enable and use the HTTP interface in VLC Media Player for remote control and status checks, follow these steps: diff --git a/desktop_env/evaluators/getters/__init__.py b/desktop_env/evaluators/getters/__init__.py index a3fe8fe..7f59d4b 100644 --- a/desktop_env/evaluators/getters/__init__.py +++ b/desktop_env/evaluators/getters/__init__.py @@ -1,3 +1,5 @@ from .file import get_cloud_file, get_vm_file, get_cache_file +from .info import get_vm_screen_size, get_vm_window_size, get_vm_wallpaper from .misc import get_rule, get_accessibility_tree from .vlc import get_vlc_playing_info, get_vlc_config +from .chrome import get_default_search_engine, get_bookmarks diff --git a/desktop_env/evaluators/getters/chrome.py b/desktop_env/evaluators/getters/chrome.py new file mode 100644 index 0000000..e87789c --- /dev/null +++ b/desktop_env/evaluators/getters/chrome.py @@ -0,0 +1,126 @@ +import json +import logging +import os +import sqlite3 +from typing import Dict + +logger = logging.getLogger("desktopenv.getters.chrome") + +""" +WARNING: +1. Functions from this script assume that no account is registered on Chrome, otherwise the default file path needs to be changed. +2. The functions are not tested on Windows and Mac, but they should work. +""" + + +def get_default_search_engine(env, config: Dict[str, str]): + os_type = env.vm_platform + if os_type == 'Windows': + preference_file_path = env.controller.execute_python_command("""import os; print(os.path.join(os.getenv('LOCALAPPDATA'), + 'Google\\Chrome\\User Data\\Default\\Preferences'))""")['output'].strip() + elif os_type == 'Darwin': + preference_file_path = env.controller.execute_python_command("import os; print(os.path.join(os.getenv('HOME'), 'Library/Application Support/Google/Chrome/Default/Preferences'))")['output'].strip() + elif os_type == 'Linux': + preference_file_path = env.controller.execute_python_command("import os; print(os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Preferences'))")['output'].strip() + else: + raise Exception('Unsupported operating system') + + try: + content = env.controller.get_file(preference_file_path) + data = json.loads(content) + + # The path within the JSON data to the default search engine might vary + search_engine = data.get('default_search_provider_data', {}).get('template_url_data', {}).get('short_name', + 'Google') + return search_engine + except Exception as e: + logger.error(f"Error: {e}") + return "Google" + + +def get_cookie_data(env, config: Dict[str, str]): + os_type = env.vm_platform + if os_type == 'Windows': + chrome_cookie_file_path = os.path.join(os.getenv('LOCALAPPDATA'), 'Google\\Chrome\\User Data\\Default\\Cookies') + elif os_type == 'Darwin': + chrome_cookie_file_path = os.path.join(os.getenv('HOME'), + 'Library/Application Support/Google/Chrome/Default/Cookies') + elif os_type == 'Linux': + chrome_cookie_file_path = os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Cookies') + else: + raise Exception('Unsupported operating system') + + # todo: add a new controller function to connect the cookie database + ############# + try: + conn = sqlite3.connect(chrome_cookie_file_path) + cursor = conn.cursor() + + # Query to check for OpenAI cookies + cursor.execute("SELECT * FROM cookies") + cookies = cursor.fetchall() + + return cookies + except Exception as e: + logger.error(f"Error: {e}") + return None + ############# + + +def get_bookmarks(env, config: Dict[str, str]): + os_type = env.vm_platform + if os_type == 'Windows': + preference_file_path = os.path.join(os.getenv('LOCALAPPDATA'), + 'Google\\Chrome\\User Data\\Default\\Bookmarks') + elif os_type == 'Darwin': + preference_file_path = os.path.join(os.getenv('HOME'), + 'Library/Application Support/Google/Chrome/Default/Bookmarks') + elif os_type == 'Linux': + preference_file_path = os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Bookmarks') + else: + raise Exception('Unsupported operating system') + + try: + content = env.controller.get_file(preference_file_path) + # make content json variable + data = json.load(content) + + bookmarks = data.get('roots', {}) + return bookmarks + + except Exception as e: + logger.error(f"Error: {e}") + return None + + +# todo: move this to the main.py +def get_extensions_installed_from_shop(env, config: Dict[str, str]): + """Find the Chrome extensions directory based on the operating system.""" + os_type = env.vm_platform + if os_type == 'Windows': + chrome_extension_dir = os.path.expanduser( + '~') + '\\AppData\\Local\\Google\\Chrome\\User Data\\Default\\Extensions\\' + elif os_type == 'Darwin': # macOS + chrome_extension_dir = os.path.expanduser( + '~') + '/Library/Application Support/Google/Chrome/Default/Extensions/' + elif os_type == 'Linux': + chrome_extension_dir = os.path.expanduser('~') + '/.config/google-chrome/Default/Extensions/' + else: + raise Exception('Unsupported operating system') + + manifests = [] + for extension_id in os.listdir(chrome_extension_dir): + extension_path = os.path.join(chrome_extension_dir, extension_id) + if os.path.isdir(extension_path): + # Iterate through version-named subdirectories + for version_dir in os.listdir(extension_path): + version_path = os.path.join(extension_path, version_dir) + manifest_path = os.path.join(version_path, 'manifest.json') + if os.path.isfile(manifest_path): + with open(manifest_path, 'r') as file: + try: + manifest = json.load(file) + manifests.append(manifest) + except json.JSONDecodeError: + logger.error(f"Error reading {manifest_path}") + return manifests diff --git a/desktop_env/evaluators/getters/info.py b/desktop_env/evaluators/getters/info.py new file mode 100644 index 0000000..3d41ee5 --- /dev/null +++ b/desktop_env/evaluators/getters/info.py @@ -0,0 +1,20 @@ +import os +from typing import Union + + +def get_vm_screen_size(env, config: dict) -> dict: + return env.controller.get_vm_screen_size() + + +def get_vm_window_size(env, config: dict) -> dict: + return env.controller.get_vm_window_size(app_class_name=config["app_class_name"]) + + +def get_vm_wallpaper(env, config: dict) -> Union[str, bytes]: + _path = os.path.join(env.cache_dir, config["dest"]) + + content = env.controller.get_vm_wallpaper() + with open(_path, "wb") as f: + f.write(content) + + return _path diff --git a/desktop_env/evaluators/getters/vlc.py b/desktop_env/evaluators/getters/vlc.py index fb4a7d5..64cece9 100644 --- a/desktop_env/evaluators/getters/vlc.py +++ b/desktop_env/evaluators/getters/vlc.py @@ -9,12 +9,12 @@ def get_vlc_playing_info(env, config: Dict[str, str]): """ Gets the current playing information from VLC's HTTP interface. """ - _path = os.path.join(env.cache_dir, config["dest"]) host = env.vm_ip port = 8080 password = 'password' + _path = os.path.join(env.cache_dir, config["dest"]) content = env.controller.get_vlc_status(host, port, password) with open(_path, "wb") as f: f.write(content) @@ -26,22 +26,24 @@ def get_vlc_config(env, config: Dict[str, str]): """ Reads the VLC configuration file to check setting. """ - _path = os.path.join(env.cache_dir, config["dest"]) - os_type = env.controller.execute_python_command("import platform; print(platform.system())")['output'].strip() + os_type = env.vm_platform # fixme: depends on how we config and install the vlc in virtual machine, need to be aligned and double-checked if os_type == "Linux": config_path = \ - env.controller.execute_python_command("import os; print(os.path.expanduser('~/.config/vlc/vlcrc'))")[ - 'output'].strip() + env.controller.execute_python_command("import os; print(os.path.expanduser('~/.config/vlc/vlcrc'))")[ + 'output'].strip() elif os_type == "Darwin": config_path = env.controller.execute_python_command( "import os; print(os.path.expanduser('~/Library/Preferences/org.videolan.vlc/vlcrc'))")['output'].strip() elif os_type == "Windows": config_path = env.controller.execute_python_command( "import os; print(os.path.expanduser('~\\AppData\\Roaming\\vlc\\vlcrc'))")['output'].strip() + else: + raise Exception("Unsupported operating system", os_type) + _path = os.path.join(env.cache_dir, config["dest"]) content = env.controller.get_file(config_path) with open(_path, "wb") as f: f.write(content) diff --git a/desktop_env/evaluators/metrics/__init__.py b/desktop_env/evaluators/metrics/__init__.py index 7a44195..28a53ae 100644 --- a/desktop_env/evaluators/metrics/__init__.py +++ b/desktop_env/evaluators/metrics/__init__.py @@ -1,10 +1,13 @@ -from .table import compare_table -from .table import check_sheet_list, check_xlsx_freeze, check_xlsx_zoom -from .docs import find_default_font, contains_page_break, compare_docx_files, compare_docx_tables, compare_line_spacing, compare_insert_equation from .docs import compare_font_names, compare_subscript_contains, has_page_numbers_in_footers +from .docs import find_default_font, contains_page_break, compare_docx_files, compare_docx_tables, compare_line_spacing, \ + compare_insert_equation from .docs import is_first_line_centered, check_file_exists, compare_contains_image -from .pdf import check_pdf_pages +from .general import exact_match, fuzzy_match, check_csv, check_accessibility_tree, check_list from .libreoffice import check_libre_locale -from .vlc import is_vlc_playing, is_vlc_recordings_folder +from .pdf import check_pdf_pages +from .table import check_sheet_list, check_xlsx_freeze, check_xlsx_zoom +from .table import compare_table +from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, compare_images, compare_audios, \ + compare_videos from .general import check_csv, check_accessibility_tree, check_list from .thunderbird import check_thunderbird_prefs, check_thunderbird_filter diff --git a/desktop_env/evaluators/metrics/chrome.py b/desktop_env/evaluators/metrics/chrome.py index 91cd38d..11aa3ff 100644 --- a/desktop_env/evaluators/metrics/chrome.py +++ b/desktop_env/evaluators/metrics/chrome.py @@ -1,127 +1,15 @@ -import json -import os -import platform -import sqlite3 +import logging from playwright.sync_api import sync_playwright -import logging logger = logging.getLogger("desktopenv.metrics.chrome") -""" -WARNING: -1. Functions from this script assume that no account is registered on Chrome, otherwise the default file path needs to be changed. -2. The functions are not tested on Windows and Mac, but they should work. -""" + # todo: move to getter module # The following ones just need to load info from the files of software, no need to connect to the software -def get_default_search_engine(): - if platform.system() == 'Windows': - preference_file_path = os.path.join(os.getenv('LOCALAPPDATA'), - 'Google\\Chrome\\User Data\\Default\\Preferences') - elif platform.system() == 'Darwin': - preference_file_path = os.path.join(os.getenv('HOME'), - 'Library/Application Support/Google/Chrome/Default/Preferences') - elif platform.system() == 'Linux': - preference_file_path = os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Preferences') - else: - raise Exception('Unsupported operating system') - - try: - with open(preference_file_path, 'r', encoding='utf-8') as file: - data = json.load(file) - - # The path within the JSON data to the default search engine might vary - search_engine = data.get('default_search_provider_data', {}).get('template_url_data', {}).get('short_name', - 'Google') - return search_engine - except Exception as e: - logger.error(f"Error: {e}") - return "Google" - - -def get_cookie_data(): - if platform.system() == 'Windows': - chrome_cookie_file_path = os.path.join(os.getenv('LOCALAPPDATA'), 'Google\\Chrome\\User Data\\Default\\Cookies') - elif platform.system() == 'Darwin': - chrome_cookie_file_path = os.path.join(os.getenv('HOME'), - 'Library/Application Support/Google/Chrome/Default/Cookies') - elif platform.system() == 'Linux': - chrome_cookie_file_path = os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Cookies') - else: - raise Exception('Unsupported operating system') - - try: - conn = sqlite3.connect(chrome_cookie_file_path) - cursor = conn.cursor() - - # Query to check for OpenAI cookies - cursor.execute("SELECT * FROM cookies") - cookies = cursor.fetchall() - - return cookies - except Exception as e: - logger.error(f"Error: {e}") - return None - - -def get_bookmarks(): - if platform.system() == 'Windows': - preference_file_path = os.path.join(os.getenv('LOCALAPPDATA'), - 'Google\\Chrome\\User Data\\Default\\Bookmarks') - elif platform.system() == 'Darwin': - preference_file_path = os.path.join(os.getenv('HOME'), - 'Library/Application Support/Google/Chrome/Default/Bookmarks') - elif platform.system() == 'Linux': - preference_file_path = os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Bookmarks') - else: - raise Exception('Unsupported operating system') - - try: - with open(preference_file_path, 'r', encoding='utf-8') as file: - data = json.load(file) - - bookmarks = data.get('roots', {}) - return bookmarks - - except Exception as e: - logger.error(f"Error: {e}") - return None - - -def get_extensions_installed_from_shop(): - """Find the Chrome extensions directory based on the operating system.""" - os_name = platform.system() - if os_name == 'Windows': - chrome_extension_dir = os.path.expanduser( - '~') + '\\AppData\\Local\\Google\\Chrome\\User Data\\Default\\Extensions\\' - elif os_name == 'Darwin': # macOS - chrome_extension_dir = os.path.expanduser( - '~') + '/Library/Application Support/Google/Chrome/Default/Extensions/' - elif os_name == 'Linux': - chrome_extension_dir = os.path.expanduser('~') + '/.config/google-chrome/Default/Extensions/' - else: - raise Exception('Unsupported operating system') - - manifests = [] - for extension_id in os.listdir(chrome_extension_dir): - extension_path = os.path.join(chrome_extension_dir, extension_id) - if os.path.isdir(extension_path): - # Iterate through version-named subdirectories - for version_dir in os.listdir(extension_path): - version_path = os.path.join(extension_path, version_dir) - manifest_path = os.path.join(version_path, 'manifest.json') - if os.path.isfile(manifest_path): - with open(manifest_path, 'r') as file: - try: - manifest = json.load(file) - manifests.append(manifest) - except json.JSONDecodeError: - logger.error(f"Error reading {manifest_path}") - return manifests # The following ones require Playwright to be installed on the target machine, and the chrome needs to be pre-config on port info to allow remote debugging, see README.md for details diff --git a/desktop_env/evaluators/metrics/general.py b/desktop_env/evaluators/metrics/general.py index 7acf23f..d0795a6 100644 --- a/desktop_env/evaluators/metrics/general.py +++ b/desktop_env/evaluators/metrics/general.py @@ -1,22 +1,32 @@ import csv +import functools +import operator +import re +from numbers import Number +from typing import Callable, Any +from typing import Dict, List, Pattern import lxml.etree -from lxml.etree import _Element from lxml.cssselect import CSSSelector - -from typing import Dict, List, Pattern -from typing import Callable, Any -from numbers import Number - -import operator +from lxml.etree import _Element from rapidfuzz import fuzz -import functools -import re from .utils import _match_record -#def _match_record(pattern: Dict[str, str], item: Dict[str, str]) -> bool: - #return all(k in item and item[k]==val for k, val in pattern.items()) +def exact_match(result, rules) -> float: + expect = rules["expected"] + print(result, expect) + + if result == expect: + return 1. + else: + return 0. + + +def fuzzy_match(result, rules) -> float: + expect = rules["expected"] + + return fuzz.ratio(result, expect) / 100. def check_csv(result: str, rules: Dict[str, List[Dict[str, str]]]) -> float: """ @@ -46,6 +56,7 @@ def check_csv(result: str, rules: Dict[str, List[Dict[str, str]]]) -> float: unexpect_metric = unexpect_metric and not any(_match_record(r, rcd) for r in rules.get("unexpect", [])) return float(all(expect_metrics) and unexpect_metric) + def check_list(result: str, rules: Dict[str, List[str]]) -> float: """ Args: @@ -75,15 +86,18 @@ def check_list(result: str, rules: Dict[str, List[str]]) -> float: unexpect_metric = unexpect_metric and all(r.search(l) is None for r in unexpect_patterns) return float(all(expect_metrics) and unexpect_metric) -_accessibility_ns_map = { "st": "uri:deskat:state.at-spi.gnome.org" - , "attr": "uri:deskat:attributes.at-spi.gnome.org" - , "cp": "uri:deskat:component.at-spi.gnome.org" - , "doc": "uri:deskat:document.at-spi.gnome.org" - , "docattr": "uri:deskat:attributes.document.at-spi.gnome.org" - , "txt": "uri:deskat:text.at-spi.gnome.org" - , "val": "uri:deskat:value.at-spi.gnome.org" - , "act": "uri:deskat:action.at-spi.gnome.org" - } + +_accessibility_ns_map = {"st": "uri:deskat:state.at-spi.gnome.org" + , "attr": "uri:deskat:attributes.at-spi.gnome.org" + , "cp": "uri:deskat:component.at-spi.gnome.org" + , "doc": "uri:deskat:document.at-spi.gnome.org" + , "docattr": "uri:deskat:attributes.document.at-spi.gnome.org" + , "txt": "uri:deskat:text.at-spi.gnome.org" + , "val": "uri:deskat:value.at-spi.gnome.org" + , "act": "uri:deskat:action.at-spi.gnome.org" + } + + def check_accessibility_tree(result: str, rules: Dict[str, Any]) -> float: """ Args: @@ -114,11 +128,12 @@ def check_accessibility_tree(result: str, rules: Dict[str, Any]) -> float: else: raise ValueError("At least one of xpath and selectors is required") - if len(elements)==0: + if len(elements) == 0: return 0. if "text" in rules: - match_func: Callable[[str], Number] = functools.partial( operator.eq if rules["exact"] else fuzz.ratio + match_func: Callable[[str], Number] = functools.partial( operator.eq if rules["exact"]\ + else (lambda a, b: fuzz.ratio(a, b)/100.) , rules["text"] ) match_score: Number = 0 @@ -129,5 +144,5 @@ def check_accessibility_tree(result: str, rules: Dict[str, Any]) -> float: return float(match_score) -#def check_existence(result: str, *args) -> float: - #return 1. - (result is None) +# def check_existence(result: str, *args) -> float: +# return 1. - (result is None) diff --git a/desktop_env/evaluators/metrics/vlc.py b/desktop_env/evaluators/metrics/vlc.py index 21d7b08..dd62968 100644 --- a/desktop_env/evaluators/metrics/vlc.py +++ b/desktop_env/evaluators/metrics/vlc.py @@ -7,7 +7,10 @@ from xml.etree import ElementTree import acoustid import cv2 import imagehash +import librosa +import numpy as np from PIL import Image +from skimage.metrics import structural_similarity as ssim logger = logging.getLogger("desktopenv.metrics.vlc") @@ -37,6 +40,7 @@ def is_vlc_playing(actual_status_path: str, rule: Dict[str, str]) -> float: return 0 +# fixme: part of this function can be moved to getters def is_vlc_recordings_folder(actual_config_path: str, rule: Dict[str, str]) -> float: """ Checks if VLC's recording folder is set to the expected value. @@ -57,24 +61,79 @@ def is_vlc_recordings_folder(actual_config_path: str, rule: Dict[str, str]) -> f current_path = line.split('=')[-1].strip() # Compare with the Desktop path if current_path == expected_recording_file_path: - return True + return 1 else: - return False + return 0 # The configuration key was not found in the file - return False + return 0 except FileNotFoundError: logger.error("VLC configuration file not found.") - return False + return 0 except Exception as e: logger.error(f"An error occurred: {e}") - return False + return 0 def is_vlc_fullscreen(actual_window_size, screen_size): if actual_window_size['width'] == screen_size['width'] and actual_window_size['height'] == screen_size['height']: - return True + return 1 else: - return False + return 0 + + +def compare_images(image1_path, image2_path): + # You would call this function with the paths to the two images you want to compare: + # score = compare_images('path_to_image1', 'path_to_image2') + # print("Similarity score:", score) + + # Open the images and convert to grayscale + image1 = Image.open(image1_path).convert('L') + image2 = Image.open(image2_path).convert('L') + + # Resize images to the smaller one's size for comparison + image1_size = image1.size + image2_size = image2.size + new_size = min(image1_size, image2_size) + + image1 = image1.resize(new_size, Image.Resampling.LANCZOS) + image2 = image2.resize(new_size, Image.Resampling.LANCZOS) + + # Convert images to numpy arrays + image1_array = np.array(image1) + image2_array = np.array(image2) + + # Calculate SSIM between two images + similarity_index = ssim(image1_array, image2_array) + + return similarity_index + + +def compare_audios(audio_path_1, audio_path_2, max_distance=1000): + """ + Compare two audio files and return a similarity score in the range [0, 1]. + audio_path_1, audio_path_2: paths to the audio files to compare + max_distance: an empirically determined maximum expected DTW distance + """ + # Example Usage: + # similarity = compare_audios_simple('path_to_audio1.mp3', 'path_to_audio2.mp3') + # print(f'Similarity Score: {similarity}') + + # Convert to common format if necessary and load audio + y1, sr1 = librosa.load(audio_path_1) + y2, sr2 = librosa.load(audio_path_2) + + # Extract MFCC features + mfcc1 = librosa.feature.mfcc(y=y1, sr=sr1) + mfcc2 = librosa.feature.mfcc(y=y2, sr=sr2) + + # Compute Dynamic Time Warping distance + distance, path = librosa.sequence.dtw(mfcc1.T, mfcc2.T) + + # Normalize distance to get a similarity score + normalized_distance = np.mean(distance) / max_distance + similarity_score = 1 - min(normalized_distance, 1) # Ensure the score is within [0, 1] + + return similarity_score def compare_videos(video_path1, video_path2, max_frames_to_check=100, threshold=5): diff --git a/desktop_env/server/README.md b/desktop_env/server/README.md new file mode 100644 index 0000000..571081a --- /dev/null +++ b/desktop_env/server/README.md @@ -0,0 +1,73 @@ + + + + +### About the Converted Accessibility Tree + +For several applications like Firefox or Thunderbird, you should first enable + +```sh +gsettings set org.gnome.desktop.interface toolkit-accessibility true +``` + +to see their accessibility tree. + +#### Example of AT + +An example of a node: + +```xml +
+ 歡迎使用新的 Outlook.com 帳戶 +
+``` + +An example of a tree: + +```xml + + + ... + + ... + +``` + +#### Useful attributes + +1. `name` - shows the name of application, title of window, or name of some + component +2. `attr:class` - somewhat the same role as `class` in HTML +3. `attr:id` - somewhat the same role as `id` in HTML +4. `cp:screencoord` - absolute coordinator on the screen +5. `cp:windowcoord` - relative coordinator in the window +6. `cp:size` - the size + +Also several states like `st:enabled` and `st:visible` can be indicated. A full +state list is available at +. + +#### How to use it in evaluation + +See example `thunderbird/12086550-11c0-466b-b367-1d9e75b3910e.json` and +function `check_accessibility_tree` in `metrics/general.py`. You can use CSS +selector or XPath to reference a target nodes. You can also check its text +contents. + +An example of a CSS selector: + +```css +application[name=Thunderbird] page-tab-list[attr|id=\"tabmail-tabs\"]>page-tab[name=\"About Profiles\"] +``` + +This selector will select the page tab of profile manager in Thunderbird (if open). + +For usage of CSS selector: . For usage of XPath: . + +#### Manual check + +You can use accerciser to check the accessibility tree on GNOME VM. + +```sh +sudo apt install accerciser +``` diff --git a/desktop_env/server/main.py b/desktop_env/server/main.py index b1f1dc2..11de773 100644 --- a/desktop_env/server/main.py +++ b/desktop_env/server/main.py @@ -360,8 +360,7 @@ def get_wallpaper(): if wallpaper_path: try: # Ensure the filename is secure - filename = secure_filename(os.path.basename(wallpaper_path)) - return send_file(wallpaper_path, attachment_filename=filename) + return send_file(wallpaper_path, mimetype='image/png') except Exception as e: app.logger.error(f"An error occurred while serving the wallpaper file: {e}") abort(500, description="Unable to serve the wallpaper file") diff --git a/evaluation_examples/examples/chrome/06fe7178-4491-4589-810f-2e2bc9502122.json b/evaluation_examples/examples/chrome/06fe7178-4491-4589-810f-2e2bc9502122.json index 78c34c0..96a6041 100644 --- a/evaluation_examples/examples/chrome/06fe7178-4491-4589-810f-2e2bc9502122.json +++ b/evaluation_examples/examples/chrome/06fe7178-4491-4589-810f-2e2bc9502122.json @@ -1,7 +1,7 @@ { "id": "06fe7178-4491-4589-810f-2e2bc9502122", "snapshot": "chrome", - "instruction": "Could you help me open the previously closed tab?", + "instruction": "Can you make my computer bring back the last tab I shut down?", "source": "https://www.wikihow.com/Switch-Tabs-in-Chrome", "config": [], "trajectory": "trajectories/", diff --git a/evaluation_examples/examples/chrome/12086550-11c0-466b-b367-1d9e75b3910e.json b/evaluation_examples/examples/chrome/12086550-11c0-466b-b367-1d9e75b3910e.json index 10c9104..6e2660d 100644 --- a/evaluation_examples/examples/chrome/12086550-11c0-466b-b367-1d9e75b3910e.json +++ b/evaluation_examples/examples/chrome/12086550-11c0-466b-b367-1d9e75b3910e.json @@ -1,7 +1,7 @@ { "id": "12086550-11c0-466b-b367-1d9e75b3910e", "snapshot": "chrome", - "instruction": "Help me find the hidden password of xx website.", + "instruction": "Computer, please navigate to the area in my browser settings where my passwords are stored. I want to check my login information for Etsy without revealing it just yet.", "source": "https://www.quora.com/What-are-the-cool-tricks-to-use-Google-Chrome", "config": [], "trajectory": "trajectories/", diff --git a/evaluation_examples/examples/chrome/7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3.json b/evaluation_examples/examples/chrome/7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3.json index 7e6363e..b914773 100644 --- a/evaluation_examples/examples/chrome/7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3.json +++ b/evaluation_examples/examples/chrome/7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3.json @@ -1,7 +1,7 @@ { "id": "7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3", "snapshot": "chrome", - "instruction": "Could you help me delete cookies from openai website?", + "instruction": "Can you help me clean up my computer by getting rid of all the tracking things that websites like Amazon or eBay might have saved? I want to make sure my browsing is private and those sites don't remember me.", "source": "https://support.google.com/chrome/answer/95647?hl=en&ref_topic=7438325&sjid=16867045591165135686-AP#zippy=%2Cdelete-cookies-from-a-site", "config": [], "trajectory": "trajectories/", diff --git a/evaluation_examples/examples/chrome/bb5e4c0d-f964-439c-97b6-bdb9747de3f4.json b/evaluation_examples/examples/chrome/bb5e4c0d-f964-439c-97b6-bdb9747de3f4.json index 3a8ff96..6f2cbf5 100644 --- a/evaluation_examples/examples/chrome/bb5e4c0d-f964-439c-97b6-bdb9747de3f4.json +++ b/evaluation_examples/examples/chrome/bb5e4c0d-f964-439c-97b6-bdb9747de3f4.json @@ -1,18 +1,33 @@ { "id": "bb5e4c0d-f964-439c-97b6-bdb9747de3f4", "snapshot": "chrome", - "instruction": "Could you help me set my default search engine to be Google?", + "instruction": "Can you make Bing the main search thingy when I look stuff up on the internet?", "source": "https://support.google.com/chrome/answer/95426?sjid=16867045591165135686-AP", - "config": [], + "config": [ + { + "type": "launch", + "parameters": { + "command": [ + "google-chrome", + "--remote-debugging-port=9222" + ] + } + } + ], "trajectory": "trajectories/", "related_apps": [ "chrome" ], "evaluator": { - "func": "", + "func": "exact_match", "result": { + "type": "default_search_engine" }, "expected": { + "type": "rule", + "rules": { + "expected": "Bing" + } } } } diff --git a/evaluation_examples/examples/libreoffice_writer/0810415c-bde4-4443-9047-d5f70165a697.json b/evaluation_examples/examples/libreoffice_writer/0810415c-bde4-4443-9047-d5f70165a697.json index f8f3ef3..09dd8ac 100644 --- a/evaluation_examples/examples/libreoffice_writer/0810415c-bde4-4443-9047-d5f70165a697.json +++ b/evaluation_examples/examples/libreoffice_writer/0810415c-bde4-4443-9047-d5f70165a697.json @@ -10,7 +10,7 @@ "files": [ { "url": "https://drive.usercontent.google.com/download?id=1-svVsH-l2ofufEKuN-cYrIrvXNobtATE&export=download&authuser=0&confirm=t&uuid=be7f891a-f858-48f5-a72d-4e42bbfb8b65&at=APZUnTXzBnaeSJjmxeh4zG03pzA0:1704179807785", - "path": "Desktop/Double_Line_Spacing.docx" + "path": "Desktop/Novels_Intro_Packet.docx" } ] } @@ -18,7 +18,7 @@ { "type": "open", "parameters": { - "path": "Desktop/Double_Line_Spacing.docx" + "path": "Desktop/Novels_Intro_Packet.docx" } } ], @@ -31,12 +31,12 @@ "expected": { "type": "cloud_file", "path": "https://drive.usercontent.google.com/download?id=1-svVsH-l2ofufEKuN-cYrIrvXNobtATE&export=download&authuser=0&confirm=t&uuid=be7f891a-f858-48f5-a72d-4e42bbfb8b65&at=APZUnTXzBnaeSJjmxeh4zG03pzA0:1704179807785", - "dest": "Double_Line_Spacing_Gold.docx" + "dest": "Novels_Intro_Packet_Gold.docx" }, "result": { "type": "vm_file", - "path": "Desktop/Double_Line_Spacing.docx", - "dest": "Double_Line_Spacing.docx" + "path": "Desktop/Novels_Intro_Packet.docx", + "dest": "Novels_Intro_Packet.docx" } } } \ No newline at end of file diff --git a/evaluation_examples/examples/libreoffice_writer/0b17a146-2934-46c7-8727-73ff6b6483e8.json b/evaluation_examples/examples/libreoffice_writer/0b17a146-2934-46c7-8727-73ff6b6483e8.json index 9c7edd3..5b4ad99 100644 --- a/evaluation_examples/examples/libreoffice_writer/0b17a146-2934-46c7-8727-73ff6b6483e8.json +++ b/evaluation_examples/examples/libreoffice_writer/0b17a146-2934-46c7-8727-73ff6b6483e8.json @@ -1,7 +1,7 @@ { "id": "0b17a146-2934-46c7-8727-73ff6b6483e8", "snapshot": "libreoffice_writer", - "instruction": "Change the 2 in H2O to a subscript.", + "instruction": "Help me change the 2 in \"H2O\" to a subscript.", "source": "https://askubuntu.com/questions/245695/how-do-you-insert-subscripts-and-superscripts-into-ordinary-non-formula-text-i", "config": [ { @@ -9,8 +9,8 @@ "parameters": { "files": [ { - "url": "https://drive.usercontent.google.com/download?id=1Nx5AoKNM7tcDRE6y_qjNIDrPOKqhNyfm&export=download&authuser=0&confirm=t&uuid=bb4de348-3bbf-46a2-95b2-e2719c67547a&at=APZUnTUeA-BW7mkQsEw7NGm272zx:1704172916742", - "path": "Desktop/Enter_Subscript.docx" + "url": "https://drive.usercontent.google.com/download?id=1FkorQBeTJ5L2jLuvu4YxHSlBMK4VEEG6&export=download&authuser=0&confirm=t&uuid=cc63dc0b-bae7-4ef6-a40d-e2da721976ef&at=APZUnTWyPZlZPFlqGTWAWXWmS04c:1704976667765", + "path": "Desktop/H2O_Factsheet_WA.docx" } ] } @@ -18,7 +18,7 @@ { "type": "open", "parameters": { - "path": "Desktop/Enter_Subscript.docx" + "path": "Desktop/H2O_Factsheet_WA.docx" } } ], @@ -30,13 +30,13 @@ "func": "compare_docx_files", "result": { "type": "vm_file", - "path": "Desktop/Enter_Subscript.docx", - "dest": "Enter_Subscript.docx" + "path": "Desktop/H2O_Factsheet_WA.docx", + "dest": "H2O_Factsheet_WA.docx" }, "expected": { "type": "cloud_file", - "path": "https://drive.usercontent.google.com/download?id=1AaKXeD9ZgfMykgijZ4G8MEzUjmMJElkq&export=download&authuser=0&confirm=t&uuid=5e347f0d-4efc-4478-878e-d89455d1593b&at=APZUnTWCYWfsD4eCeG52VJiK8-xB:1704172886196", - "dest": "Enter_Subscript_Gold.docx" + "path": "https://drive.usercontent.google.com/download?id=1dM_FSTGDWxSW64VEth_wKMYNkvw0y_tq&export=download&authuser=0&confirm=t&uuid=342f41e2-f48f-41ff-8942-f7dfe5de1dba&at=APZUnTXHfskcX3tvmrSbzCOyQIgb:1704976694506", + "dest": "H2O_Factsheet_WA_Gold.docx" } } } \ No newline at end of file diff --git a/evaluation_examples/examples/libreoffice_writer/0e47de2a-32e0-456c-a366-8c607ef7a9d2.json b/evaluation_examples/examples/libreoffice_writer/0e47de2a-32e0-456c-a366-8c607ef7a9d2.json index ec4182d..f5cc925 100644 --- a/evaluation_examples/examples/libreoffice_writer/0e47de2a-32e0-456c-a366-8c607ef7a9d2.json +++ b/evaluation_examples/examples/libreoffice_writer/0e47de2a-32e0-456c-a366-8c607ef7a9d2.json @@ -10,7 +10,7 @@ "files": [ { "url": "https://drive.usercontent.google.com/download?id=1aDWe-vAmcfQSgtPjFfrncq8ZFnCy4uUK&export=download&authuser=0&confirm=t&uuid=788af72a-ddaf-4ba3-aedb-96f34cc4d815&at=APZUnTVSRSSfMGcjXqLzvMixnkp6:1704179663299", - "path": "Desktop/Add_Page_Number_Bottom_Left.docx" + "path": "Desktop/LibreOffice_Open_Source_Word_Processing.docx" } ] } @@ -18,7 +18,7 @@ { "type": "open", "parameters": { - "path": "Desktop/Add_Page_Number_Bottom_Left.docx" + "path": "Desktop/LibreOffice_Open_Source_Word_Processing.docx" } } ], @@ -30,8 +30,8 @@ "func": "has_page_numbers_in_footers", "result": { "type": "vm_file", - "path": "Desktop/Add_Page_Number_Bottom_Left.docx", - "dest": "Add_Page_Number_Bottom_Left.docx" + "path": "Desktop/LibreOffice_Open_Source_Word_Processing.docx", + "dest": "LibreOffice_Open_Source_Word_Processing.docx" } } } \ No newline at end of file diff --git a/evaluation_examples/examples/libreoffice_writer/0e763496-b6bb-4508-a427-fad0b6c3e195.json b/evaluation_examples/examples/libreoffice_writer/0e763496-b6bb-4508-a427-fad0b6c3e195.json index 265f566..6d886c8 100644 --- a/evaluation_examples/examples/libreoffice_writer/0e763496-b6bb-4508-a427-fad0b6c3e195.json +++ b/evaluation_examples/examples/libreoffice_writer/0e763496-b6bb-4508-a427-fad0b6c3e195.json @@ -10,7 +10,7 @@ "files": [ { "url": "https://drive.usercontent.google.com/download?id=1fIHNzFm8JabWoLKOnxrFM722fQ1d_huK&export=download&authuser=0&confirm=t&uuid=d11a8dda-1e4e-4dc9-b05c-e6b47624dbf0&at=APZUnTVG0ViFnKJa00314wVr3uP9:1704185871014", - "path": "Desktop/Change_Font_Through_File.docx" + "path": "Desktop/Dublin_Zoo_Intro.docx" } ] } @@ -18,7 +18,7 @@ { "type": "open", "parameters": { - "path": "Desktop/Change_Font_Through_File.docx" + "path": "Desktop/Dublin_Zoo_Intro.docx" } } ], @@ -36,8 +36,8 @@ }, "result": { "type": "vm_file", - "path": "Desktop/Change_Font_Through_File.docx", - "dest": "Change_Font_Through_File.docx" + "path": "Desktop/Dublin_Zoo_Intro.docx", + "dest": "Dublin_Zoo_Intro.docx" } } } \ No newline at end of file diff --git a/evaluation_examples/examples/libreoffice_writer/3ef2b351-8a84-4ff2-8724-d86eae9b842e.json b/evaluation_examples/examples/libreoffice_writer/3ef2b351-8a84-4ff2-8724-d86eae9b842e.json index d9d1129..17196f6 100644 --- a/evaluation_examples/examples/libreoffice_writer/3ef2b351-8a84-4ff2-8724-d86eae9b842e.json +++ b/evaluation_examples/examples/libreoffice_writer/3ef2b351-8a84-4ff2-8724-d86eae9b842e.json @@ -1,7 +1,7 @@ { "id": "3ef2b351-8a84-4ff2-8724-d86eae9b842e", "snapshot": "libreoffice_writer", - "instruction": "center-justify the first line", + "instruction": "Help me center align the heading in LibreOffice.", "source": "https://askubuntu.com/questions/1066351/how-do-you-center-align-in-libreoffice#:~:text=Ctrl%20%2B%20e%20will%20Center%20align%20the%20cursor%20for%20you.", "config": [ { @@ -9,8 +9,8 @@ "parameters": { "files": [ { - "url": "https://drive.usercontent.google.com/download?id=1IQ4rKyHMOui71YlyL7huggpLYFYtj923&export=download&authuser=0&confirm=t&uuid=014c2335-c0c6-4712-9d5a-ca8d3217e07f&at=APZUnTVrM698NQgSh4hqYXR8cjDc:1704185072996", - "path": "Desktop/Centering_First_Line.docx" + "url": "https://drive.usercontent.google.com/download?id=1P8QodvDF-3S50rx6UmW4M2D4Kr-p_Q-h&export=download&authuser=0&confirm=t&uuid=eea70a33-4c3f-4e90-885d-dd3df0d605bc&at=APZUnTX7ISvBhOICNrPLoqK0m3G-:1704971931660", + "path": "Desktop/Constitution_Template_With_Guidelines.docx" } ] } @@ -18,7 +18,7 @@ { "type": "open", "parameters": { - "path": "Desktop/Centering_First_Line.docx" + "path": "Desktop/Constitution_Template_With_Guidelines.docx" } } ], @@ -30,8 +30,8 @@ "func": "is_first_line_centered", "result": { "type": "vm_file", - "path": "Desktop/Centering_First_Line.docx", - "dest": "Centering_First_Line.docx" + "path": "Desktop/Constitution_Template_With_Guidelines.docx", + "dest": "Constitution_Template_With_Guidelines.docx" } } } \ No newline at end of file diff --git a/evaluation_examples/examples/libreoffice_writer/45d61a06-6545-4422-97b7-bc76cfa964c1.json b/evaluation_examples/examples/libreoffice_writer/45d61a06-6545-4422-97b7-bc76cfa964c1.json index c892edf..99a960c 100644 --- a/evaluation_examples/examples/libreoffice_writer/45d61a06-6545-4422-97b7-bc76cfa964c1.json +++ b/evaluation_examples/examples/libreoffice_writer/45d61a06-6545-4422-97b7-bc76cfa964c1.json @@ -1,7 +1,7 @@ { "id": "45d61a06-6545-4422-97b7-bc76cfa964c1", "snapshot": "libreoffice_writer", - "instruction": "Replace all newlines with paragraph marks in LibreOffice Write", + "instruction": "Replace all newlines with paragraph marks in LibreOffice Writer", "source": "https://stackoverflow.com/questions/71685737/how-to-replace-all-newlines-with-paragraph-marks-in-libreoffice-write", "config": [ { @@ -9,8 +9,8 @@ "parameters": { "files": [ { - "url": "https://drive.usercontent.google.com/download?id=18XFjPVUnLG_-KOM5sn-Sk74HP_JHivMy&export=download&authuser=0&confirm=t&uuid=d23041bc-2ddd-42c4-84ae-481b953f021c&at=APZUnTVYh0AK0245qsDOCol7SdMB:1704185512767", - "path": "Desktop/Replace_Newlines_with_Paragraph_Marks.docx" + "url": "https://drive.usercontent.google.com/download?id=16lQcSkw-JQ_v8Sg0HkCtnOzyK-4cok8N&export=download&authuser=0&confirm=t&uuid=a3f833ae-2572-4cf3-8a21-6d250e689415&at=APZUnTUfn24NGMtXEzz2Nf7cFLjt:1704975857398", + "path": "Desktop/NOVEL_Submission_Guidelines.docx" } ] } @@ -18,7 +18,7 @@ { "type": "open", "parameters": { - "path": "Desktop/Replace_Newlines_with_Paragraph_Marks.docx" + "path": "Desktop/NOVEL_Submission_Guidelines.docx" } } ], @@ -30,13 +30,13 @@ "func": "compare_line_spacing", "expected": { "type": "cloud_file", - "path": "https://drive.usercontent.google.com/download?id=1bP_noic02MuzrM8CdJIQN7F1gN4N8sel&export=download&authuser=0&confirm=t&uuid=657e0e4f-7b96-4d7e-83f4-99b79c68708f&at=APZUnTX7HsmefsMlzQaCGK2fg5Em:1704185514197", - "dest": "Replace_Newlines_with_Paragraph_Marks_Gold.docx" + "path": "https://drive.usercontent.google.com/download?id=1LaS5ObaOsbyKX1M17vi8ZseZwawgvNmf&export=download&authuser=0&confirm=t&uuid=db523e81-fa22-4002-97ff-e5dff92106a7&at=APZUnTVnT3ZYOGW7ZQdeW4SZP7mX:1704975993684", + "dest": "NOVEL_Submission_Guidelines_Gold.docx" }, "result": { "type": "vm_file", - "path": "Desktop/Double_Line_Spacing.docx", - "dest": "Replace_Newlines_with_Paragraph_Marks.docx" + "path": "Desktop/NOVEL_Submission_Guidelines.docx", + "dest": "NOVEL_Submission_Guidelines.docx" } } -} +} \ No newline at end of file diff --git a/evaluation_examples/examples/libreoffice_writer/4bcb1253-a636-4df4-8cb0-a35c04dfef31.json b/evaluation_examples/examples/libreoffice_writer/4bcb1253-a636-4df4-8cb0-a35c04dfef31.json index c9b72df..ba1cb2c 100644 --- a/evaluation_examples/examples/libreoffice_writer/4bcb1253-a636-4df4-8cb0-a35c04dfef31.json +++ b/evaluation_examples/examples/libreoffice_writer/4bcb1253-a636-4df4-8cb0-a35c04dfef31.json @@ -9,8 +9,8 @@ "parameters": { "files": [ { - "url": "https://drive.usercontent.google.com/download?id=1czb_13MshoiM-yCxvUGYD8OnIIrf-3VX&export=download&authuser=0&confirm=t&uuid=e7c30b67-7fac-4b64-a222-d04bc7c82842&at=APZUnTUA1te5vt7L__zJ7xuMs48e:1704177347643", - "path": "Desktop/Save_Writer_PDF.docx" + "url": "https://drive.usercontent.google.com/download?id=1cn3CaA_4ds1WY8SgBT3TvjRunxocCBiu&export=download&authuser=0&confirm=t&uuid=622f5ae1-6f21-4f31-8a3e-e4ead6ea6bc3&at=APZUnTVKOV40Ww5PovU7at2ELzb9:1704949558060", + "path": "Desktop/View_Person_Organizational_Summary.docx" } ] } @@ -18,7 +18,7 @@ { "type": "open", "parameters": { - "path": "Desktop/Save_Writer_PDF.docx" + "path": "Desktop/View_Person_Organizational_Summary.docx" } } ], @@ -28,7 +28,7 @@ ], "evaluator": { "func": "check_file_exists", - "file_name": "Save_Writer_PDF.pdf", + "file_name": "View_Person_Organizational_Summary.pdf", "directory": "/home/user/Downloads/" } } \ No newline at end of file diff --git a/evaluation_examples/examples/libreoffice_writer/663876c7-3471-43db-ba51-f410b13d9d7d.json b/evaluation_examples/examples/libreoffice_writer/663876c7-3471-43db-ba51-f410b13d9d7d.json index 37af3d2..d04915a 100644 --- a/evaluation_examples/examples/libreoffice_writer/663876c7-3471-43db-ba51-f410b13d9d7d.json +++ b/evaluation_examples/examples/libreoffice_writer/663876c7-3471-43db-ba51-f410b13d9d7d.json @@ -1,7 +1,7 @@ { "id": "663876c7-3471-43db-ba51-f410b13d9d7d", "snapshot": "libreoffice_writer", - "instruction": "Insert the equation \"(a + b)^2 = a^2 + 2 a b + b^2\"", + "instruction": "Insert the equation \"(a + b)^2 = a^2 + 2 a b + b^2\" at the position of the cursor.", "source": "https://askubuntu.com/questions/319593/how-to-type-science-equations-in-libre-office", "config": [ { @@ -9,8 +9,8 @@ "parameters": { "files": [ { - "url": "https://drive.usercontent.google.com/download?id=1FgMp7Ny63eXzeF23qHYhqQux31djlkah&export=download&authuser=0&confirm=t&uuid=d6b5208d-3b3a-4972-a641-ed738a419fdb&at=APZUnTX16Fz8Qg-B0NWpWgC-3Dyu:1704184410221", - "path": "Desktop/Insert_Equation.docx" + "url": "https://drive.usercontent.google.com/download?id=1J3--srr-Fmt2z-wOAyPDmO5GoiJMkUvZ&export=download&authuser=0&confirm=t&uuid=5f275f3d-ab7d-4d27-848a-8012af4bcf7f&at=APZUnTVe5ucwe1Q0oVi9jwiY5PF7:1704971494071", + "path": "Desktop/Factoring_Perfect_Square_Trinomials.docx" } ] } @@ -18,7 +18,7 @@ { "type": "open", "parameters": { - "path": "Desktop/Insert_Equation.docx" + "path": "Desktop/Factoring_Perfect_Square_Trinomials.docx" } } ], @@ -30,13 +30,13 @@ "func": "compare_insert_equation", "expected": { "type": "cloud_file", - "path": "https://drive.usercontent.google.com/download?id=1hMFJnwHs7Iaexz3b9O2LJQUfsJ2wiwZ9&export=download&authuser=0&confirm=t&uuid=2abb49fb-d9c7-46cf-bc21-e69ecb9cefc6&at=APZUnTVzEZjChcUb4MIoxuq4cGea:1704184411805", - "dest": "Insert_Equation_Gold.docx" + "path": "https://drive.usercontent.google.com/download?id=1QGfyQUPEjOX8osycnjR8E5yHeIEGccKI&export=download&authuser=0&confirm=t&uuid=174491d6-7855-4323-9fa5-5146fde48c4a&at=APZUnTXA8bIcIbtw9mUN-K-IU9bT:1704971524974", + "dest": "Factoring_Perfect_Square_Trinomials_Gold.docx" }, "result": { "type": "vm_file", - "path": "Desktop/Insert_Equation.docx", - "dest": "Insert_Equation.docx" + "path": "Desktop/Factoring_Perfect_Square_Trinomials.docx", + "dest": "Factoring_Perfect_Square_Trinomials.docx" } } } \ No newline at end of file diff --git a/evaluation_examples/examples/libreoffice_writer/66399b0d-8fda-4618-95c4-bfc6191617e9.json b/evaluation_examples/examples/libreoffice_writer/66399b0d-8fda-4618-95c4-bfc6191617e9.json index 0004246..a45e7fd 100644 --- a/evaluation_examples/examples/libreoffice_writer/66399b0d-8fda-4618-95c4-bfc6191617e9.json +++ b/evaluation_examples/examples/libreoffice_writer/66399b0d-8fda-4618-95c4-bfc6191617e9.json @@ -1,7 +1,7 @@ { "id": "66399b0d-8fda-4618-95c4-bfc6191617e9", "snapshot": "libreoffice_writer", - "instruction": "Insert a 7*5 empty table", + "instruction": "Could you help me insert a 7*5 empty table at the point of cursor?", "source": "https://www.youtube.com/watch?v=l25Evu4ohKg", "config": [ { @@ -10,7 +10,7 @@ "files": [ { "url": "https://drive.usercontent.google.com/download?id=1kXBP0jMxTeVahzFLYbYHJtjjgmuzrA8R&export=download&authuser=0&confirm=t&uuid=f8b9bad3-415d-4d39-a4fb-05a4cf881cf0&at=APZUnTXaohwzl8_2RDF_tgUsP9cH:1704181463579", - "path": "Desktop/Insert_Empty_Table.docx" + "path": "Desktop/Table_Of_Work_Effort_Instructions.docx" } ] } @@ -18,7 +18,7 @@ { "type": "open", "parameters": { - "path": "Desktop/Insert_Empty_Table.docx" + "path": "Desktop/Table_Of_Work_Effort_Instructions.docx" } } ], @@ -31,12 +31,12 @@ "expected": { "type": "cloud_file", "path": "https://drive.usercontent.google.com/download?id=14JfHsW9GvbhORdtVAtvEbOi00MqEyHfb&export=download&authuser=0&confirm=t&uuid=3dba2459-ac37-4cad-a982-adecd406382a&at=APZUnTVQUqUPq_WacgY2xu4PvAKB:1704181465512", - "dest": "Insert_Empty_Table_Gold.docx" + "dest": "Table_Of_Work_Effort_Instructions_Gold.docx" }, "result": { "type": "vm_file", - "path": "Desktop/Insert_Empty_Table.docx", - "dest": "Insert_Empty_Table.docx" + "path": "Desktop/Table_Of_Work_Effort_Instructions.docx", + "dest": "Table_Of_Work_Effort_Instructions.docx" } } } \ No newline at end of file diff --git a/evaluation_examples/examples/libreoffice_writer/6ada715d-3aae-4a32-a6a7-429b2e43fb93.json b/evaluation_examples/examples/libreoffice_writer/6ada715d-3aae-4a32-a6a7-429b2e43fb93.json index 9528060..ff6fa76 100644 --- a/evaluation_examples/examples/libreoffice_writer/6ada715d-3aae-4a32-a6a7-429b2e43fb93.json +++ b/evaluation_examples/examples/libreoffice_writer/6ada715d-3aae-4a32-a6a7-429b2e43fb93.json @@ -1,7 +1,7 @@ { "id": "6ada715d-3aae-4a32-a6a7-429b2e43fb93", "snapshot": "libreoffice_writer", - "instruction": "Copy the screenshot 1.jpg from the desktop to where my cursor is locatedInsert the image which is in IMAGE_PATH where my cursor is", + "instruction": "Copy the screenshot 1.png from the desktop to where my cursor is located", "source": "https://www.quora.com/How-do-you-insert-images-into-a-LibreOffice-Writer-document", "config": [ { @@ -10,7 +10,7 @@ "files": [ { "url": "https://drive.usercontent.google.com/download?id=1ebLG0gmqYv41ek4UmKWhFsxBnoUSGjKp&export=download&authuser=0&confirm=t&uuid=8f7d7bee-1fe4-4c4c-8b69-8aaf47199c57&at=APZUnTVYUvYTopUXCVs69QWWwPbq:1704173993139", - "path": "Desktop/Insert_Image_At_Cursor.docx" + "path": "Desktop/Viewing_Your_Class_Schedule_and_Textbooks.docx" } ] } @@ -20,8 +20,8 @@ "parameters": { "files": [ { - "url": "https://drive.usercontent.google.com/download?id=1QfjQ4SKtjKDXWpqa2u6mC_KtB3ASEK5O&export=download&authuser=0&confirm=t&uuid=06af00b9-58f3-4691-a6a3-34309c80cbbb&at=APZUnTVZpE1lMxcvGG0cdt5zuxZ_:1704174003198", - "path": "Desktop/1.jpg" + "url": "https://drive.usercontent.google.com/download?id=1-wYHrgPMZl5Y9Losd4AGkMVHWNU7Uemg&export=download&authuser=0&confirm=t&uuid=3bc52ab7-1739-407d-9b14-36c9305c973e&at=APZUnTW95CwED3zm5KRREFmO2mc2:1704952228530", + "path": "Desktop/1.png" } ] } @@ -29,7 +29,7 @@ { "type": "open", "parameters": { - "path": "Desktop/Insert_Image_At_Cursor.docx" + "path": "Desktop/Viewing_Your_Class_Schedule_and_Textbooks.docx" } } ], @@ -41,13 +41,13 @@ "func": "compare_contains_image", "result": { "type": "vm_file", - "path": "Desktop/Insert_Image_At_Cursor.docx", - "dest": "Insert_Image_At_Cursor.docx" + "path": "Desktop/Viewing_Your_Class_Schedule_and_Textbooks.docx", + "dest": "Viewing_Your_Class_Schedule_and_Textbooks.docx" }, "expected": { "type": "cloud_file", "path": "https://drive.usercontent.google.com/download?id=1xbhlfqGrPutHHi2aHg66jwXD-yaZpe9j&export=download&authuser=0&confirm=t&uuid=427765e0-3f97-4a72-92db-a1fe7cdde73b&at=APZUnTUhNLh2PDu4OGkCVQW-LPCd:1704173991269", - "dest": "Insert_Image_At_Cursor_Gold.docx" + "dest": "Viewing_Your_Class_Schedule_and_Textbooks_Gold.docx" } } } \ No newline at end of file diff --git a/evaluation_examples/examples/libreoffice_writer/936321ce-5236-426a-9a20-e0e3c5dc536f.json b/evaluation_examples/examples/libreoffice_writer/936321ce-5236-426a-9a20-e0e3c5dc536f.json index 9b4b172..92591f0 100644 --- a/evaluation_examples/examples/libreoffice_writer/936321ce-5236-426a-9a20-e0e3c5dc536f.json +++ b/evaluation_examples/examples/libreoffice_writer/936321ce-5236-426a-9a20-e0e3c5dc536f.json @@ -1,7 +1,7 @@ { "id": "936321ce-5236-426a-9a20-e0e3c5dc536f", "snapshot": "libreoffice_writer", - "instruction": "Convert the content seperated by commas to a table", + "instruction": "Could you help me convert the text seperated by commas to a table?", "source": "https://www.youtube.com/watch?v=l25Evu4ohKg", "config": [ { @@ -9,8 +9,8 @@ "parameters": { "files": [ { - "url": "https://drive.usercontent.google.com/download?id=139-NmslBR_9qlD7hUO08xj_VFffV95eM&export=download&authuser=0&confirm=t&uuid=64a6c35d-f3ce-4c25-9f83-4a952e24c5ad&at=APZUnTUL1GMR_QbpFQnC9fPwkdqa:1704183959196", - "path": "Desktop/Convert_Text_To_Table.docx" + "url": "https://drive.usercontent.google.com/download?id=1WcHqnDy0OPFiaGtvPSvKXuaLc_rtGzYC&export=download&authuser=0&confirm=t&uuid=0c552800-5dfe-4a3f-a960-8d384481fa7f&at=APZUnTWYjmvuXkU-pb77dy7EgCeD:1704970003044", + "path": "Desktop/Graphemes_Sound_Letter_Patterns.docx" } ] } @@ -18,7 +18,7 @@ { "type": "open", "parameters": { - "path": "Desktop/Convert_Text_To_Table.docx" + "path": "Desktop/Graphemes_Sound_Letter_Patterns.docx" } } ], @@ -30,13 +30,13 @@ "func": "compare_docx_tables", "expected": { "type": "cloud_file", - "path": "https://drive.usercontent.google.com/download?id=1fs2msYaxnEZL9XASENQMZIag2MTxIBJs&export=download&authuser=0&confirm=t&uuid=6c71f008-082c-4f0c-9ffc-0a802f5cbfe6&at=APZUnTVDpucMDfk5P2T-0dZx_KVV:1704183960360", - "dest": "Convert_Text_To_Table_Gold.docx" + "path": "https://drive.usercontent.google.com/download?id=1og3gCBRhAeHk02E1wAsbfau_1ywdoEJL&export=download&authuser=0&confirm=t&uuid=264de3b4-b07c-4382-8a73-a766b1d1fc58&at=APZUnTX8yRMoDXCHi1y1H380GaSX:1704969967989", + "dest": "Graphemes_Sound_Letter_Patterns_Gold.docx" }, "result": { "type": "vm_file", - "path": "Desktop/Convert_Text_To_Table.docx", - "dest": "Convert_Text_To_Table.docx" + "path": "Desktop/Graphemes_Sound_Letter_Patterns.docx", + "dest": "Graphemes_Sound_Letter_Patterns.docx" } } } \ No newline at end of file diff --git a/evaluation_examples/examples/libreoffice_writer/adf5e2c3-64c7-4644-b7b6-d2f0167927e7.json b/evaluation_examples/examples/libreoffice_writer/adf5e2c3-64c7-4644-b7b6-d2f0167927e7.json index 6b1ce4e..a932884 100644 --- a/evaluation_examples/examples/libreoffice_writer/adf5e2c3-64c7-4644-b7b6-d2f0167927e7.json +++ b/evaluation_examples/examples/libreoffice_writer/adf5e2c3-64c7-4644-b7b6-d2f0167927e7.json @@ -1,7 +1,7 @@ { "id": "adf5e2c3-64c7-4644-b7b6-d2f0167927e7", "snapshot": "libreoffice_writer", - "instruction": "Helping me adding \"C. Luo and M. J. Carey, \"LSM-based storage techniques: a survey,\" The VLDB Journal, vol. 29, no. 1, pp. 393–418, 2020.\" to my reference list, and add a cross reference at the end of the first paragraph", + "instruction": "Help me adding \"Steinberg, F. M., Bearden, M. M., & Keen, C. L. (2003). Cocoa and chocolate flavonoids: Implications for cardiovascular health. Journal of the American Dietetic Association, 103(2), 215-223. doi: 10.1053/jada.2003.50028\" to my reference list, and add a cross reference where my cursor is located (in the fourth paragraph).", "source": "https://seekstar.github.io/2022/04/11/libreoffice%E5%BC%95%E7%94%A8%E6%96%87%E7%8C%AE/", "config": [ { @@ -9,8 +9,8 @@ "parameters": { "files": [ { - "url": "https://drive.usercontent.google.com/download?id=1xOfwImgkPzdmjQomj-MCFd8nQS75OjaH&export=download&authuser=0&confirm=t&uuid=7eb91c26-dad5-4480-b1ec-35e506cde1e4&at=APZUnTW01MvBI_gkC8yoiyAVs7yi:1704188254979", - "path": "Desktop/Add_Citation_Cross_Reference.docx" + "url": "https://drive.usercontent.google.com/download?id=1ShGL4gWSV7nzamAb0V2KqjoCOhyodcKU&export=download&authuser=0&confirm=t&uuid=5f67edb8-cbbf-4a83-b46e-f193ad55e1e8&at=APZUnTVRJenYCM--vETagQ5ACTT5:1704979226579", + "path": "Desktop/Essay_Writing_English_for_uni.docx" } ] } @@ -18,7 +18,7 @@ { "type": "open", "parameters": { - "path": "Desktop/Add_Citation_Cross_Reference.docx" + "path": "Desktop/Essay_Writing_English_for_uni.docx" } } ], @@ -30,13 +30,13 @@ "func": "compare_docx_files", "expected": { "type": "cloud_file", - "path": "https://drive.usercontent.google.com/download?id=1wFQU7hkAT2wmSHTgM22F9Ep4WXmymEMW&export=download&authuser=0&confirm=t&uuid=a7ea1eec-678b-4407-b023-df13cc6f8c54&at=APZUnTW3WoqOfS9A1BW79XfV8jKh:1704188260410", - "dest": "Add_Citation_Cross_Reference_Gold.docx" + "path": "https://drive.usercontent.google.com/download?id=1bf7IKjjxGAQfbOlq9rLPzjTtkOZ3dREj&export=download&authuser=0&confirm=t&uuid=0420cfa5-7e51-4688-8c93-74748914ce52&at=APZUnTWIShIleccvgVBIr6ZsSnOw:1704979256798", + "dest": "Essay_Writing_English_for_uni_Gold.docx" }, "result": { "type": "vm_file", - "path": "Desktop/Add_Citation_Cross_Reference.docx", - "dest": "Add_Citation_Cross_Reference.docx" + "path": "Desktop/Essay_Writing_English_for_uni.docx", + "dest": "Essay_Writing_English_for_uni.docx" } } } \ No newline at end of file diff --git a/evaluation_examples/examples/libreoffice_writer/e528b65e-1107-4b8c-8988-490e4fece599.json b/evaluation_examples/examples/libreoffice_writer/e528b65e-1107-4b8c-8988-490e4fece599.json index a3a2340..451a68d 100644 --- a/evaluation_examples/examples/libreoffice_writer/e528b65e-1107-4b8c-8988-490e4fece599.json +++ b/evaluation_examples/examples/libreoffice_writer/e528b65e-1107-4b8c-8988-490e4fece599.json @@ -9,8 +9,8 @@ "parameters": { "files": [ { - "url": "https://drive.usercontent.google.com/download?id=1LZ1_U9CyR8oOkqfbY6HMLWr4wBVYhCLR&export=download&authuser=0&confirm=t&uuid=574d533a-8df9-4f33-bebd-689b623f27a9&at=APZUnTVruCDRxY661_PVT9BA3839:1704180420603", - "path": "Desktop/Capitalize_First_Letter.docx" + "url": "https://drive.usercontent.google.com/download?id=181hyG_NZUJaUp7kUdeFoUpNGpOcjzYFV&export=download&authuser=0&confirm=t&uuid=fdb8aee4-ea1f-43c1-bc66-9d44099512e1&at=APZUnTVUrypBeVnTfkJsriv7S2GO:1704969486759", + "path": "Geography_And_Magical_Realism.docx" } ] } @@ -18,7 +18,7 @@ { "type": "open", "parameters": { - "path": "Desktop/Capitalize_First_Letter.docx" + "path": "Desktop/Geography_And_Magical_Realism.docx" } } ], @@ -30,13 +30,13 @@ "func": "compare_docx_files", "expected": { "type": "cloud_file", - "path": "https://drive.usercontent.google.com/download?id=1ykIGg48GjYUOSw8t44evShJE2HX7R5w3&export=download&authuser=0&confirm=t&uuid=ed81d6cd-6044-49f1-be86-35adeaeeea00&at=APZUnTUxW8WLyPr-_smA2Mnwpuuv:1704180422490https://drive.usercontent.google.com/download?id=1ykIGg48GjYUOSw8t44evShJE2HX7R5w3&export=download&authuser=0&confirm=t&uuid=ed81d6cd-6044-49f1-be86-35adeaeeea00&at=APZUnTUxW8WLyPr-_smA2Mnwpuuv:1704180422490", - "dest": "Capitalize_First_Letter_Gold.docx" + "path": "https://drive.usercontent.google.com/download?id=1qwoJMuMhLxWwONRLmxwMdUuVIFBAhhNT&export=download&authuser=0&confirm=t&uuid=8fa4d503-ec55-419f-ac90-f5915a5f67b5&at=APZUnTVEntT4oMiEfOuWcUKaInBs:1704969347952", + "dest": "Geography_And_Magical_Realism_Gold.docx" }, "result": { "type": "vm_file", - "path": "Desktop/Capitalize_First_Letter.docx", - "dest": "Capitalize_First_Letter.docx" + "path": "Desktop/Geography_And_Magical_Realism.docx", + "dest": "Geography_And_Magical_Realism.docx" } } } \ No newline at end of file diff --git a/evaluation_examples/examples/libreoffice_writer/ecc2413d-8a48-416e-a3a2-d30106ca36cb.json b/evaluation_examples/examples/libreoffice_writer/ecc2413d-8a48-416e-a3a2-d30106ca36cb.json index a99aa0e..25a9d38 100644 --- a/evaluation_examples/examples/libreoffice_writer/ecc2413d-8a48-416e-a3a2-d30106ca36cb.json +++ b/evaluation_examples/examples/libreoffice_writer/ecc2413d-8a48-416e-a3a2-d30106ca36cb.json @@ -1,7 +1,7 @@ { "id": "ecc2413d-8a48-416e-a3a2-d30106ca36cb", "snapshot": "libreoffice_writer", - "instruction": "Insert a blank page", + "instruction": "Help me insert a blank page where my cursor is located.", "source": "https://www.quora.com/How-can-I-insert-a-blank-page-on-libreoffice", "config": [ { @@ -10,7 +10,7 @@ "files": [ { "url": "https://drive.usercontent.google.com/download?id=1sDufDSC4foI379-Jikya9WK7FBUSqgrt&export=download&authuser=0&confirm=t&uuid=0abd82d6-2b2c-49bc-af5e-49bfe1c99278&at=APZUnTURIqTNJcIHBcMP2BxEaGXr:1704174850900", - "path": "Desktop/Insert_Blank_Page.docx" + "path": "Desktop/Sample_Statutory_Declaration.docx" } ] } @@ -18,7 +18,7 @@ { "type": "open", "parameters": { - "path": "Desktop/Insert_Blank_Page.docx" + "path": "Desktop/Sample_Statutory_Declaration.docx" } } ], @@ -30,8 +30,8 @@ "func": "contains_page_break", "result": { "type": "vm_file", - "path": "Desktop/Insert_Blank_Page.docx", - "dest": "Insert_Blank_Page.docx" + "path": "Desktop/Sample_Statutory_Declaration.docx", + "dest": "Sample_Statutory_Declaration.docx" } } } \ No newline at end of file diff --git a/evaluation_examples/examples/libreoffice_writer/f178a4a9-d090-4b56-bc4c-4b72a61a035d.json b/evaluation_examples/examples/libreoffice_writer/f178a4a9-d090-4b56-bc4c-4b72a61a035d.json index 1d710c3..315ea4d 100644 --- a/evaluation_examples/examples/libreoffice_writer/f178a4a9-d090-4b56-bc4c-4b72a61a035d.json +++ b/evaluation_examples/examples/libreoffice_writer/f178a4a9-d090-4b56-bc4c-4b72a61a035d.json @@ -9,8 +9,8 @@ "parameters": { "files": [ { - "url": "https://drive.usercontent.google.com/download?id=1X2XTU2ZFuMXOhm7T400e6AOe6eBYxWzD&export=download&authuser=0&confirm=t&uuid=1318923f-6d54-4148-aa80-a454b9963cec&at=APZUnTU-h1nmcjBO_ytWVxXuh8l9:1704187013730", - "path": "Desktop/Set_Default_Font.docx" + "url": "https://drive.usercontent.google.com/download?id=1xREbNAu_2wLTs8EQT0NnHLpIkAVAfpyk&export=download&authuser=0&confirm=t&uuid=dd5cb525-ff4b-41a2-8123-d488f2f21fad&at=APZUnTXaYBqLT9fRtGYZHOedq-PG:1704977194647", + "path": "Desktop/loa-one-time-submission-sealand.docx" } ] } @@ -18,7 +18,7 @@ { "type": "open", "parameters": { - "path": "Desktop/Set_Default_Font.docx" + "path": "Desktop/loa-one-time-submission-sealand.docx" } } ], diff --git a/evaluation_examples/examples/vlc/59f21cfb-0120-4326-b255-a5b827b38967.json b/evaluation_examples/examples/vlc/59f21cfb-0120-4326-b255-a5b827b38967.json index b65ddfb..32d7570 100644 --- a/evaluation_examples/examples/vlc/59f21cfb-0120-4326-b255-a5b827b38967.json +++ b/evaluation_examples/examples/vlc/59f21cfb-0120-4326-b255-a5b827b38967.json @@ -1,7 +1,7 @@ { "id": "59f21cfb-0120-4326-b255-a5b827b38967", "snapshot": "base_setup", - "instruction": "Play the music video on my desktop", + "instruction": "Could you play the music video that's saved on my desktop for me?", "source": "https://docs.videolan.me/vlc-user/desktop/3.0/en/basic/media.html#playing-a-file", "config": [ { diff --git a/evaluation_examples/examples/vlc/8f080098-ddb1-424c-b438-4e96e5e4786e.json b/evaluation_examples/examples/vlc/8f080098-ddb1-424c-b438-4e96e5e4786e.json index db15765..a39bd31 100644 --- a/evaluation_examples/examples/vlc/8f080098-ddb1-424c-b438-4e96e5e4786e.json +++ b/evaluation_examples/examples/vlc/8f080098-ddb1-424c-b438-4e96e5e4786e.json @@ -1,7 +1,7 @@ { "id": "8f080098-ddb1-424c-b438-4e96e5e4786e", "snapshot": "base_setup", - "instruction": "Could you help me extract MP3 Audio to AUDIO_PATH from Video at VIDEO_PATH using VLC Media Player?", + "instruction": "Could you download the song from this music video and save it as an MP3 file? I'd like to have it on my device to play whenever I want. Please title the file \"Baby Justin Bieber.mp3.\" I really appreciate your help!", "source": "https://medium.com/@jetscribe_ai/how-to-extract-mp3-audio-from-videos-using-vlc-media-player-beeef644ebfb", "config": [ { @@ -10,7 +10,7 @@ "files": [ { "url": "https://drive.usercontent.google.com/download?id=19jBiz8sb0M7KHHATO9qeTPr17aWm4me-&export=download&authuser=0&confirm=t&uuid=7a2261f4-3905-433f-b53f-a52dd0845651&at=APZUnTU1nmXSa1ObrA5NHYt8t1-p:1704710908141", - "path": "Baby Justin Bieber.mp4" + "path": "Desktop/Baby Justin Bieber.mp4" } ] } @@ -18,7 +18,11 @@ { "type": "launch", "parameters": { - "command": "vlc" + "command": [ + "vlc", + "--start-time=73", + "Desktop/Baby Justin Bieber.mp4" + ] } } ], @@ -27,16 +31,16 @@ "vlc" ], "evaluator": { - "func": "is_vlc_recordings_folder", + "func": "compare_audios", "expected": { - "type": "rule", - "rules": { - "recording_file_path": "/home/user/Desktop" - } + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=1eRuuiUAOmckyn2zQebx1nWQ40kpjSJ_N&export=download&authuser=0&confirm=t&uuid=8012703a-9e45-4d91-9bda-90c119e42254&at=APZUnTW0uyCSeg7FClSc50UJVRz-:1704898635455", + "dest": "baby_gold.mp3" }, "result": { - "type": "vlc_config", - "dest": "vlcrc" + "type": "vm_file", + "path": "Desktop/Baby Justin Bieber.mp3", + "dest": "baby.mp3" } } } diff --git a/evaluation_examples/examples/vlc/a1c3ab35-02de-4999-a7ed-2fd12c972c6e.json b/evaluation_examples/examples/vlc/a1c3ab35-02de-4999-a7ed-2fd12c972c6e.json index 676092a..b279cb5 100644 --- a/evaluation_examples/examples/vlc/a1c3ab35-02de-4999-a7ed-2fd12c972c6e.json +++ b/evaluation_examples/examples/vlc/a1c3ab35-02de-4999-a7ed-2fd12c972c6e.json @@ -1,7 +1,7 @@ { "id": "a1c3ab35-02de-4999-a7ed-2fd12c972c6e", "snapshot": "base_setup", - "instruction": "Could you help me compress the video to DIR_PATH?", + "instruction": "Could you help me compress the video to MPEG-4 format and save with name with prefix _?", "source": "https://www.quora.com/How-do-I-compress-a-video-with-VLC", "config": [], "trajectory": "trajectories/", diff --git a/evaluation_examples/examples/vlc/bba3381f-b5eb-4439-bd9e-80c22218d5a7.json b/evaluation_examples/examples/vlc/bba3381f-b5eb-4439-bd9e-80c22218d5a7.json index 670aa5a..a5cd48d 100644 --- a/evaluation_examples/examples/vlc/bba3381f-b5eb-4439-bd9e-80c22218d5a7.json +++ b/evaluation_examples/examples/vlc/bba3381f-b5eb-4439-bd9e-80c22218d5a7.json @@ -1,7 +1,7 @@ { "id": "bba3381f-b5eb-4439-bd9e-80c22218d5a7", "snapshot": "base_setup", - "instruction": "Help me play the online video at https://www.youtube.com/watch?v=pgBsyTKAwLw", + "instruction": "Can you start streaming the video from this link for me? https://www.youtube.com/watch?v=pgBsyTKAwLw", "source": "https://www.quora.com/How-do-I-play-online-videos-using-the-VLC-media-player", "config": [ { diff --git a/evaluation_examples/examples/vlc/efcf0d81-0835-4880-b2fd-d866e8bc2294.json b/evaluation_examples/examples/vlc/efcf0d81-0835-4880-b2fd-d866e8bc2294.json index 3238837..3b07f3c 100644 --- a/evaluation_examples/examples/vlc/efcf0d81-0835-4880-b2fd-d866e8bc2294.json +++ b/evaluation_examples/examples/vlc/efcf0d81-0835-4880-b2fd-d866e8bc2294.json @@ -1,30 +1,47 @@ { "id": "efcf0d81-0835-4880-b2fd-d866e8bc2294", "snapshot": "base_setup", - "instruction": "Set this frame of the current video as my wallpaper", - "source": "https://www.youtube.com/watch?v=XHprwDJ0-fU&t=436s", + "instruction": "Make this part of the video my computer's background picture", + "source": "https://www.youtube.com/watch?v=XHprwDJ0-fU&t=436s, https://help.ubuntu.com/stable/ubuntu-help/look-background.html.en", "config": [ { "type": "download", "parameters": { "files": [ { - "url": "", - "path": "" + "url": "https://drive.usercontent.google.com/download?id=1H9D0jSkzpzEWsJZG0HdNNNHDMi3fnGli&export=download&authuser=0&confirm=t&uuid=a6d03223-db7a-48bd-b5f8-50b51f4d3d68&at=APZUnTVi3cqgvRV49c6sluV-nMo_:1704879367756", + "path": "Desktop/Interstellar Movie - Official Trailer.mp4" } ] } }, { "type": "launch", - "parameters": { - "command": ["vlc", "/path/to/your/video.mp4", "--start-time=0", "--run-time=10", "vlc://quit", "&&", "vlc", "/path/to/your/video.mp4", "--start-time=10"] - } + "parameters": { + "command": [ + "vlc", + "--start-time=120.5", + "--stop-time=121", + "--play-and-pause", + "Desktop/Interstellar Movie - Official Trailer.mp4" + ] + } } ], "trajectory": "trajectories/", "related_apps": [ "vlc" ], - "evaluator": "evaluation_dir" + "evaluator": { + "func": "compare_images", + "expected": { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=10P8kLkWeYhtK3Gl15nB-gH7VLO4S7Xal&export=download&authuser=0&confirm=t&uuid=d5c5db29-435d-404f-8900-683bf56fbc66&at=APZUnTXm1ZHS7CbGiWaB04WySH-t:1704891263184", + "dest": "interstellar_wallpaper_gold.png" + }, + "result": { + "type": "vm_wallpaper", + "dest": "result_wallpaper.png" + } + } } diff --git a/evaluation_examples/examples/vlc/fba2c100-79e8-42df-ae74-b592418d54f4.json b/evaluation_examples/examples/vlc/fba2c100-79e8-42df-ae74-b592418d54f4.json index f564f73..0cd51d9 100644 --- a/evaluation_examples/examples/vlc/fba2c100-79e8-42df-ae74-b592418d54f4.json +++ b/evaluation_examples/examples/vlc/fba2c100-79e8-42df-ae74-b592418d54f4.json @@ -1,12 +1,48 @@ { "id": "fba2c100-79e8-42df-ae74-b592418d54f4", "snapshot": "base_setup", - "instruction": "Screenshot the current frame of the video", + "instruction": "Snap a photo of the current video scene, save it as 'interstellar.png', and put it on the Desktop, please.", "source": "https://www.youtube.com/watch?v=XHprwDJ0-fU&t=436s", - "config": [], + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1u7CFVr5bWti7OnhvyjtSdQolzy0lYicm&export=download&authuser=0&confirm=t&uuid=18498a7c-9b52-4c00-a2cc-2e8b521c4d9c&at=APZUnTX0R_TRccU_UmtJWXZ6On3x:1704890439561", + "path": "Desktop/Interstellar Movie - Official Trailer.mp4" + } + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "vlc", + "--start-time=120.5", + "--stop-time=121", + "--play-and-pause", + "Desktop/Interstellar Movie - Official Trailer.mp4" + ] + } + } + ], "trajectory": "trajectories/", "related_apps": [ "vlc" ], - "evaluator": "evaluation_dir" + "evaluator": { + "func": "compare_images", + "expected": { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=1shU1TQ3ao9QWfhmC63vbD10YFbhv7PYv&export=download&authuser=0&confirm=t&uuid=6a22a47a-6d35-4996-b02e-b24b33a1c449&at=APZUnTVlvDKpxW7QKmBBmdNq2jRw:1704891215314", + "dest": "interstellar_gold.png" + }, + "result": { + "type": "vm_file", + "path": "Desktop/interstellar.png", + "dest": "interstellar.png" + } + } } diff --git a/requirements.txt b/requirements.txt index 49268ac..558098b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,3 +27,4 @@ rapidfuzz pyacoustid opencv-python ImageHash +scikit-image