diff --git a/desktop_env/controllers/python.py b/desktop_env/controllers/python.py
index 6d8f7d1..39afb33 100644
--- a/desktop_env/controllers/python.py
+++ b/desktop_env/controllers/python.py
@@ -228,6 +228,13 @@ class PythonController:
else:
raise Exception(f"Unknown action type: {action_type}")
+ # Additional info
+ def get_vm_platform(self):
+ """
+ Gets the size of the vm screen.
+ """
+ return self.execute_python_command("import platform; print(platform.system())")['output'].strip()
+
def get_vm_screen_size(self):
"""
Gets the size of the vm screen.
@@ -250,6 +257,19 @@ class PythonController:
logger.error("Failed to get window size. Status code: %d", response.status_code)
return None
+ def get_vm_wallpaper(self):
+ """
+ Gets the wallpaper of the vm.
+ """
+ response = requests.post(self.http_server + "/wallpaper")
+ if response.status_code == 200:
+ logger.info("Wallpaper downloaded successfully")
+ return response.content
+ else:
+ logger.error("Failed to get wallpaper. Status code: %d", response.status_code)
+ return None
+
+ # VLC
def get_vlc_status(self, host='localhost', port=8080, password='password'):
url = f'http://{host}:{port}/requests/status.xml'
diff --git a/desktop_env/envs/desktop_env.py b/desktop_env/envs/desktop_env.py
index e256f7c..a309489 100644
--- a/desktop_env/envs/desktop_env.py
+++ b/desktop_env/envs/desktop_env.py
@@ -35,7 +35,10 @@ def _execute_command(command: List[str]) -> None:
class DesktopEnv(gym.Env):
- """DesktopEnv with OpenAI Gym interface."""
+ """
+ DesktopEnv with OpenAI Gym interface.
+ Fixme: refactor the logic when implementing the multi-process version
+ """
def __init__(
self,
@@ -81,6 +84,10 @@ class DesktopEnv(gym.Env):
self.controller = PythonController(http_server=self.host)
self.setup_controller = SetupController(http_server=self.host, cache_dir=self.cache_dir)
+ # Meta info of the VM
+ self.vm_platform = self.controller.get_vm_platform()
+ self.vm_screen_size = self.controller.get_vm_screen_size()
+
# mode: human or machine
assert action_space in ["computer_13", "pyautogui"]
self.action_space = action_space
@@ -225,7 +232,12 @@ class DesktopEnv(gym.Env):
self.setup_controller.setup(self.evaluator.get("postconfig", []))
- result_state = self.result_getter(self, self.evaluator["result"])
+ try:
+ result_state = self.result_getter(self, self.evaluator["result"])
+ except FileNotFoundError:
+ logger.error("File not found!")
+ return 0
+
expected_state = self.expected_getter(self, self.evaluator["expected"]) if "expected" in self.evaluator \
else None
diff --git a/desktop_env/evaluators/metrics/README.md b/desktop_env/evaluators/README.md
similarity index 93%
rename from desktop_env/evaluators/metrics/README.md
rename to desktop_env/evaluators/README.md
index 7ca036c..eff47ac 100644
--- a/desktop_env/evaluators/metrics/README.md
+++ b/desktop_env/evaluators/README.md
@@ -95,6 +95,12 @@ with sync_playwright() as playwright:
## VLC Media Player
+
+### Bugs fix
+One thing on Ubuntu need to do, enter into the `meida`>`convert/save`>select files>`convert/save`
+Then enter the profile of `Audio - MP3`, change the profile for mp3, section audiocodec from "MP3" to "MPEG Audio"
+Otherwise the mp3 file will be created but with 0 bytes. It's a bug of VLC.
+
### Setting Up VLC's HTTP Interface
To enable and use the HTTP interface in VLC Media Player for remote control and status checks, follow these steps:
diff --git a/desktop_env/evaluators/getters/__init__.py b/desktop_env/evaluators/getters/__init__.py
index a3fe8fe..7f59d4b 100644
--- a/desktop_env/evaluators/getters/__init__.py
+++ b/desktop_env/evaluators/getters/__init__.py
@@ -1,3 +1,5 @@
from .file import get_cloud_file, get_vm_file, get_cache_file
+from .info import get_vm_screen_size, get_vm_window_size, get_vm_wallpaper
from .misc import get_rule, get_accessibility_tree
from .vlc import get_vlc_playing_info, get_vlc_config
+from .chrome import get_default_search_engine, get_bookmarks
diff --git a/desktop_env/evaluators/getters/chrome.py b/desktop_env/evaluators/getters/chrome.py
new file mode 100644
index 0000000..e87789c
--- /dev/null
+++ b/desktop_env/evaluators/getters/chrome.py
@@ -0,0 +1,126 @@
+import json
+import logging
+import os
+import sqlite3
+from typing import Dict
+
+logger = logging.getLogger("desktopenv.getters.chrome")
+
+"""
+WARNING:
+1. Functions from this script assume that no account is registered on Chrome, otherwise the default file path needs to be changed.
+2. The functions are not tested on Windows and Mac, but they should work.
+"""
+
+
+def get_default_search_engine(env, config: Dict[str, str]):
+ os_type = env.vm_platform
+ if os_type == 'Windows':
+ preference_file_path = env.controller.execute_python_command("""import os; print(os.path.join(os.getenv('LOCALAPPDATA'),
+ 'Google\\Chrome\\User Data\\Default\\Preferences'))""")['output'].strip()
+ elif os_type == 'Darwin':
+ preference_file_path = env.controller.execute_python_command("import os; print(os.path.join(os.getenv('HOME'), 'Library/Application Support/Google/Chrome/Default/Preferences'))")['output'].strip()
+ elif os_type == 'Linux':
+ preference_file_path = env.controller.execute_python_command("import os; print(os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Preferences'))")['output'].strip()
+ else:
+ raise Exception('Unsupported operating system')
+
+ try:
+ content = env.controller.get_file(preference_file_path)
+ data = json.loads(content)
+
+ # The path within the JSON data to the default search engine might vary
+ search_engine = data.get('default_search_provider_data', {}).get('template_url_data', {}).get('short_name',
+ 'Google')
+ return search_engine
+ except Exception as e:
+ logger.error(f"Error: {e}")
+ return "Google"
+
+
+def get_cookie_data(env, config: Dict[str, str]):
+ os_type = env.vm_platform
+ if os_type == 'Windows':
+ chrome_cookie_file_path = os.path.join(os.getenv('LOCALAPPDATA'), 'Google\\Chrome\\User Data\\Default\\Cookies')
+ elif os_type == 'Darwin':
+ chrome_cookie_file_path = os.path.join(os.getenv('HOME'),
+ 'Library/Application Support/Google/Chrome/Default/Cookies')
+ elif os_type == 'Linux':
+ chrome_cookie_file_path = os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Cookies')
+ else:
+ raise Exception('Unsupported operating system')
+
+ # todo: add a new controller function to connect the cookie database
+ #############
+ try:
+ conn = sqlite3.connect(chrome_cookie_file_path)
+ cursor = conn.cursor()
+
+ # Query to check for OpenAI cookies
+ cursor.execute("SELECT * FROM cookies")
+ cookies = cursor.fetchall()
+
+ return cookies
+ except Exception as e:
+ logger.error(f"Error: {e}")
+ return None
+ #############
+
+
+def get_bookmarks(env, config: Dict[str, str]):
+ os_type = env.vm_platform
+ if os_type == 'Windows':
+ preference_file_path = os.path.join(os.getenv('LOCALAPPDATA'),
+ 'Google\\Chrome\\User Data\\Default\\Bookmarks')
+ elif os_type == 'Darwin':
+ preference_file_path = os.path.join(os.getenv('HOME'),
+ 'Library/Application Support/Google/Chrome/Default/Bookmarks')
+ elif os_type == 'Linux':
+ preference_file_path = os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Bookmarks')
+ else:
+ raise Exception('Unsupported operating system')
+
+ try:
+ content = env.controller.get_file(preference_file_path)
+ # make content json variable
+ data = json.load(content)
+
+ bookmarks = data.get('roots', {})
+ return bookmarks
+
+ except Exception as e:
+ logger.error(f"Error: {e}")
+ return None
+
+
+# todo: move this to the main.py
+def get_extensions_installed_from_shop(env, config: Dict[str, str]):
+ """Find the Chrome extensions directory based on the operating system."""
+ os_type = env.vm_platform
+ if os_type == 'Windows':
+ chrome_extension_dir = os.path.expanduser(
+ '~') + '\\AppData\\Local\\Google\\Chrome\\User Data\\Default\\Extensions\\'
+ elif os_type == 'Darwin': # macOS
+ chrome_extension_dir = os.path.expanduser(
+ '~') + '/Library/Application Support/Google/Chrome/Default/Extensions/'
+ elif os_type == 'Linux':
+ chrome_extension_dir = os.path.expanduser('~') + '/.config/google-chrome/Default/Extensions/'
+ else:
+ raise Exception('Unsupported operating system')
+
+ manifests = []
+ for extension_id in os.listdir(chrome_extension_dir):
+ extension_path = os.path.join(chrome_extension_dir, extension_id)
+ if os.path.isdir(extension_path):
+ # Iterate through version-named subdirectories
+ for version_dir in os.listdir(extension_path):
+ version_path = os.path.join(extension_path, version_dir)
+ manifest_path = os.path.join(version_path, 'manifest.json')
+ if os.path.isfile(manifest_path):
+ with open(manifest_path, 'r') as file:
+ try:
+ manifest = json.load(file)
+ manifests.append(manifest)
+ except json.JSONDecodeError:
+ logger.error(f"Error reading {manifest_path}")
+ return manifests
diff --git a/desktop_env/evaluators/getters/info.py b/desktop_env/evaluators/getters/info.py
new file mode 100644
index 0000000..3d41ee5
--- /dev/null
+++ b/desktop_env/evaluators/getters/info.py
@@ -0,0 +1,20 @@
+import os
+from typing import Union
+
+
+def get_vm_screen_size(env, config: dict) -> dict:
+ return env.controller.get_vm_screen_size()
+
+
+def get_vm_window_size(env, config: dict) -> dict:
+ return env.controller.get_vm_window_size(app_class_name=config["app_class_name"])
+
+
+def get_vm_wallpaper(env, config: dict) -> Union[str, bytes]:
+ _path = os.path.join(env.cache_dir, config["dest"])
+
+ content = env.controller.get_vm_wallpaper()
+ with open(_path, "wb") as f:
+ f.write(content)
+
+ return _path
diff --git a/desktop_env/evaluators/getters/vlc.py b/desktop_env/evaluators/getters/vlc.py
index fb4a7d5..64cece9 100644
--- a/desktop_env/evaluators/getters/vlc.py
+++ b/desktop_env/evaluators/getters/vlc.py
@@ -9,12 +9,12 @@ def get_vlc_playing_info(env, config: Dict[str, str]):
"""
Gets the current playing information from VLC's HTTP interface.
"""
- _path = os.path.join(env.cache_dir, config["dest"])
host = env.vm_ip
port = 8080
password = 'password'
+ _path = os.path.join(env.cache_dir, config["dest"])
content = env.controller.get_vlc_status(host, port, password)
with open(_path, "wb") as f:
f.write(content)
@@ -26,22 +26,24 @@ def get_vlc_config(env, config: Dict[str, str]):
"""
Reads the VLC configuration file to check setting.
"""
- _path = os.path.join(env.cache_dir, config["dest"])
- os_type = env.controller.execute_python_command("import platform; print(platform.system())")['output'].strip()
+ os_type = env.vm_platform
# fixme: depends on how we config and install the vlc in virtual machine, need to be aligned and double-checked
if os_type == "Linux":
config_path = \
- env.controller.execute_python_command("import os; print(os.path.expanduser('~/.config/vlc/vlcrc'))")[
- 'output'].strip()
+ env.controller.execute_python_command("import os; print(os.path.expanduser('~/.config/vlc/vlcrc'))")[
+ 'output'].strip()
elif os_type == "Darwin":
config_path = env.controller.execute_python_command(
"import os; print(os.path.expanduser('~/Library/Preferences/org.videolan.vlc/vlcrc'))")['output'].strip()
elif os_type == "Windows":
config_path = env.controller.execute_python_command(
"import os; print(os.path.expanduser('~\\AppData\\Roaming\\vlc\\vlcrc'))")['output'].strip()
+ else:
+ raise Exception("Unsupported operating system", os_type)
+ _path = os.path.join(env.cache_dir, config["dest"])
content = env.controller.get_file(config_path)
with open(_path, "wb") as f:
f.write(content)
diff --git a/desktop_env/evaluators/metrics/__init__.py b/desktop_env/evaluators/metrics/__init__.py
index 7a44195..28a53ae 100644
--- a/desktop_env/evaluators/metrics/__init__.py
+++ b/desktop_env/evaluators/metrics/__init__.py
@@ -1,10 +1,13 @@
-from .table import compare_table
-from .table import check_sheet_list, check_xlsx_freeze, check_xlsx_zoom
-from .docs import find_default_font, contains_page_break, compare_docx_files, compare_docx_tables, compare_line_spacing, compare_insert_equation
from .docs import compare_font_names, compare_subscript_contains, has_page_numbers_in_footers
+from .docs import find_default_font, contains_page_break, compare_docx_files, compare_docx_tables, compare_line_spacing, \
+ compare_insert_equation
from .docs import is_first_line_centered, check_file_exists, compare_contains_image
-from .pdf import check_pdf_pages
+from .general import exact_match, fuzzy_match, check_csv, check_accessibility_tree, check_list
from .libreoffice import check_libre_locale
-from .vlc import is_vlc_playing, is_vlc_recordings_folder
+from .pdf import check_pdf_pages
+from .table import check_sheet_list, check_xlsx_freeze, check_xlsx_zoom
+from .table import compare_table
+from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, compare_images, compare_audios, \
+ compare_videos
from .general import check_csv, check_accessibility_tree, check_list
from .thunderbird import check_thunderbird_prefs, check_thunderbird_filter
diff --git a/desktop_env/evaluators/metrics/chrome.py b/desktop_env/evaluators/metrics/chrome.py
index 91cd38d..11aa3ff 100644
--- a/desktop_env/evaluators/metrics/chrome.py
+++ b/desktop_env/evaluators/metrics/chrome.py
@@ -1,127 +1,15 @@
-import json
-import os
-import platform
-import sqlite3
+import logging
from playwright.sync_api import sync_playwright
-import logging
logger = logging.getLogger("desktopenv.metrics.chrome")
-"""
-WARNING:
-1. Functions from this script assume that no account is registered on Chrome, otherwise the default file path needs to be changed.
-2. The functions are not tested on Windows and Mac, but they should work.
-"""
+
# todo: move to getter module
# The following ones just need to load info from the files of software, no need to connect to the software
-def get_default_search_engine():
- if platform.system() == 'Windows':
- preference_file_path = os.path.join(os.getenv('LOCALAPPDATA'),
- 'Google\\Chrome\\User Data\\Default\\Preferences')
- elif platform.system() == 'Darwin':
- preference_file_path = os.path.join(os.getenv('HOME'),
- 'Library/Application Support/Google/Chrome/Default/Preferences')
- elif platform.system() == 'Linux':
- preference_file_path = os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Preferences')
- else:
- raise Exception('Unsupported operating system')
-
- try:
- with open(preference_file_path, 'r', encoding='utf-8') as file:
- data = json.load(file)
-
- # The path within the JSON data to the default search engine might vary
- search_engine = data.get('default_search_provider_data', {}).get('template_url_data', {}).get('short_name',
- 'Google')
- return search_engine
- except Exception as e:
- logger.error(f"Error: {e}")
- return "Google"
-
-
-def get_cookie_data():
- if platform.system() == 'Windows':
- chrome_cookie_file_path = os.path.join(os.getenv('LOCALAPPDATA'), 'Google\\Chrome\\User Data\\Default\\Cookies')
- elif platform.system() == 'Darwin':
- chrome_cookie_file_path = os.path.join(os.getenv('HOME'),
- 'Library/Application Support/Google/Chrome/Default/Cookies')
- elif platform.system() == 'Linux':
- chrome_cookie_file_path = os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Cookies')
- else:
- raise Exception('Unsupported operating system')
-
- try:
- conn = sqlite3.connect(chrome_cookie_file_path)
- cursor = conn.cursor()
-
- # Query to check for OpenAI cookies
- cursor.execute("SELECT * FROM cookies")
- cookies = cursor.fetchall()
-
- return cookies
- except Exception as e:
- logger.error(f"Error: {e}")
- return None
-
-
-def get_bookmarks():
- if platform.system() == 'Windows':
- preference_file_path = os.path.join(os.getenv('LOCALAPPDATA'),
- 'Google\\Chrome\\User Data\\Default\\Bookmarks')
- elif platform.system() == 'Darwin':
- preference_file_path = os.path.join(os.getenv('HOME'),
- 'Library/Application Support/Google/Chrome/Default/Bookmarks')
- elif platform.system() == 'Linux':
- preference_file_path = os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Bookmarks')
- else:
- raise Exception('Unsupported operating system')
-
- try:
- with open(preference_file_path, 'r', encoding='utf-8') as file:
- data = json.load(file)
-
- bookmarks = data.get('roots', {})
- return bookmarks
-
- except Exception as e:
- logger.error(f"Error: {e}")
- return None
-
-
-def get_extensions_installed_from_shop():
- """Find the Chrome extensions directory based on the operating system."""
- os_name = platform.system()
- if os_name == 'Windows':
- chrome_extension_dir = os.path.expanduser(
- '~') + '\\AppData\\Local\\Google\\Chrome\\User Data\\Default\\Extensions\\'
- elif os_name == 'Darwin': # macOS
- chrome_extension_dir = os.path.expanduser(
- '~') + '/Library/Application Support/Google/Chrome/Default/Extensions/'
- elif os_name == 'Linux':
- chrome_extension_dir = os.path.expanduser('~') + '/.config/google-chrome/Default/Extensions/'
- else:
- raise Exception('Unsupported operating system')
-
- manifests = []
- for extension_id in os.listdir(chrome_extension_dir):
- extension_path = os.path.join(chrome_extension_dir, extension_id)
- if os.path.isdir(extension_path):
- # Iterate through version-named subdirectories
- for version_dir in os.listdir(extension_path):
- version_path = os.path.join(extension_path, version_dir)
- manifest_path = os.path.join(version_path, 'manifest.json')
- if os.path.isfile(manifest_path):
- with open(manifest_path, 'r') as file:
- try:
- manifest = json.load(file)
- manifests.append(manifest)
- except json.JSONDecodeError:
- logger.error(f"Error reading {manifest_path}")
- return manifests
# The following ones require Playwright to be installed on the target machine, and the chrome needs to be pre-config on port info to allow remote debugging, see README.md for details
diff --git a/desktop_env/evaluators/metrics/general.py b/desktop_env/evaluators/metrics/general.py
index 7acf23f..d0795a6 100644
--- a/desktop_env/evaluators/metrics/general.py
+++ b/desktop_env/evaluators/metrics/general.py
@@ -1,22 +1,32 @@
import csv
+import functools
+import operator
+import re
+from numbers import Number
+from typing import Callable, Any
+from typing import Dict, List, Pattern
import lxml.etree
-from lxml.etree import _Element
from lxml.cssselect import CSSSelector
-
-from typing import Dict, List, Pattern
-from typing import Callable, Any
-from numbers import Number
-
-import operator
+from lxml.etree import _Element
from rapidfuzz import fuzz
-import functools
-import re
from .utils import _match_record
-#def _match_record(pattern: Dict[str, str], item: Dict[str, str]) -> bool:
- #return all(k in item and item[k]==val for k, val in pattern.items())
+def exact_match(result, rules) -> float:
+ expect = rules["expected"]
+ print(result, expect)
+
+ if result == expect:
+ return 1.
+ else:
+ return 0.
+
+
+def fuzzy_match(result, rules) -> float:
+ expect = rules["expected"]
+
+ return fuzz.ratio(result, expect) / 100.
def check_csv(result: str, rules: Dict[str, List[Dict[str, str]]]) -> float:
"""
@@ -46,6 +56,7 @@ def check_csv(result: str, rules: Dict[str, List[Dict[str, str]]]) -> float:
unexpect_metric = unexpect_metric and not any(_match_record(r, rcd) for r in rules.get("unexpect", []))
return float(all(expect_metrics) and unexpect_metric)
+
def check_list(result: str, rules: Dict[str, List[str]]) -> float:
"""
Args:
@@ -75,15 +86,18 @@ def check_list(result: str, rules: Dict[str, List[str]]) -> float:
unexpect_metric = unexpect_metric and all(r.search(l) is None for r in unexpect_patterns)
return float(all(expect_metrics) and unexpect_metric)
-_accessibility_ns_map = { "st": "uri:deskat:state.at-spi.gnome.org"
- , "attr": "uri:deskat:attributes.at-spi.gnome.org"
- , "cp": "uri:deskat:component.at-spi.gnome.org"
- , "doc": "uri:deskat:document.at-spi.gnome.org"
- , "docattr": "uri:deskat:attributes.document.at-spi.gnome.org"
- , "txt": "uri:deskat:text.at-spi.gnome.org"
- , "val": "uri:deskat:value.at-spi.gnome.org"
- , "act": "uri:deskat:action.at-spi.gnome.org"
- }
+
+_accessibility_ns_map = {"st": "uri:deskat:state.at-spi.gnome.org"
+ , "attr": "uri:deskat:attributes.at-spi.gnome.org"
+ , "cp": "uri:deskat:component.at-spi.gnome.org"
+ , "doc": "uri:deskat:document.at-spi.gnome.org"
+ , "docattr": "uri:deskat:attributes.document.at-spi.gnome.org"
+ , "txt": "uri:deskat:text.at-spi.gnome.org"
+ , "val": "uri:deskat:value.at-spi.gnome.org"
+ , "act": "uri:deskat:action.at-spi.gnome.org"
+ }
+
+
def check_accessibility_tree(result: str, rules: Dict[str, Any]) -> float:
"""
Args:
@@ -114,11 +128,12 @@ def check_accessibility_tree(result: str, rules: Dict[str, Any]) -> float:
else:
raise ValueError("At least one of xpath and selectors is required")
- if len(elements)==0:
+ if len(elements) == 0:
return 0.
if "text" in rules:
- match_func: Callable[[str], Number] = functools.partial( operator.eq if rules["exact"] else fuzz.ratio
+ match_func: Callable[[str], Number] = functools.partial( operator.eq if rules["exact"]\
+ else (lambda a, b: fuzz.ratio(a, b)/100.)
, rules["text"]
)
match_score: Number = 0
@@ -129,5 +144,5 @@ def check_accessibility_tree(result: str, rules: Dict[str, Any]) -> float:
return float(match_score)
-#def check_existence(result: str, *args) -> float:
- #return 1. - (result is None)
+# def check_existence(result: str, *args) -> float:
+# return 1. - (result is None)
diff --git a/desktop_env/evaluators/metrics/vlc.py b/desktop_env/evaluators/metrics/vlc.py
index 21d7b08..dd62968 100644
--- a/desktop_env/evaluators/metrics/vlc.py
+++ b/desktop_env/evaluators/metrics/vlc.py
@@ -7,7 +7,10 @@ from xml.etree import ElementTree
import acoustid
import cv2
import imagehash
+import librosa
+import numpy as np
from PIL import Image
+from skimage.metrics import structural_similarity as ssim
logger = logging.getLogger("desktopenv.metrics.vlc")
@@ -37,6 +40,7 @@ def is_vlc_playing(actual_status_path: str, rule: Dict[str, str]) -> float:
return 0
+# fixme: part of this function can be moved to getters
def is_vlc_recordings_folder(actual_config_path: str, rule: Dict[str, str]) -> float:
"""
Checks if VLC's recording folder is set to the expected value.
@@ -57,24 +61,79 @@ def is_vlc_recordings_folder(actual_config_path: str, rule: Dict[str, str]) -> f
current_path = line.split('=')[-1].strip()
# Compare with the Desktop path
if current_path == expected_recording_file_path:
- return True
+ return 1
else:
- return False
+ return 0
# The configuration key was not found in the file
- return False
+ return 0
except FileNotFoundError:
logger.error("VLC configuration file not found.")
- return False
+ return 0
except Exception as e:
logger.error(f"An error occurred: {e}")
- return False
+ return 0
def is_vlc_fullscreen(actual_window_size, screen_size):
if actual_window_size['width'] == screen_size['width'] and actual_window_size['height'] == screen_size['height']:
- return True
+ return 1
else:
- return False
+ return 0
+
+
+def compare_images(image1_path, image2_path):
+ # You would call this function with the paths to the two images you want to compare:
+ # score = compare_images('path_to_image1', 'path_to_image2')
+ # print("Similarity score:", score)
+
+ # Open the images and convert to grayscale
+ image1 = Image.open(image1_path).convert('L')
+ image2 = Image.open(image2_path).convert('L')
+
+ # Resize images to the smaller one's size for comparison
+ image1_size = image1.size
+ image2_size = image2.size
+ new_size = min(image1_size, image2_size)
+
+ image1 = image1.resize(new_size, Image.Resampling.LANCZOS)
+ image2 = image2.resize(new_size, Image.Resampling.LANCZOS)
+
+ # Convert images to numpy arrays
+ image1_array = np.array(image1)
+ image2_array = np.array(image2)
+
+ # Calculate SSIM between two images
+ similarity_index = ssim(image1_array, image2_array)
+
+ return similarity_index
+
+
+def compare_audios(audio_path_1, audio_path_2, max_distance=1000):
+ """
+ Compare two audio files and return a similarity score in the range [0, 1].
+ audio_path_1, audio_path_2: paths to the audio files to compare
+ max_distance: an empirically determined maximum expected DTW distance
+ """
+ # Example Usage:
+ # similarity = compare_audios_simple('path_to_audio1.mp3', 'path_to_audio2.mp3')
+ # print(f'Similarity Score: {similarity}')
+
+ # Convert to common format if necessary and load audio
+ y1, sr1 = librosa.load(audio_path_1)
+ y2, sr2 = librosa.load(audio_path_2)
+
+ # Extract MFCC features
+ mfcc1 = librosa.feature.mfcc(y=y1, sr=sr1)
+ mfcc2 = librosa.feature.mfcc(y=y2, sr=sr2)
+
+ # Compute Dynamic Time Warping distance
+ distance, path = librosa.sequence.dtw(mfcc1.T, mfcc2.T)
+
+ # Normalize distance to get a similarity score
+ normalized_distance = np.mean(distance) / max_distance
+ similarity_score = 1 - min(normalized_distance, 1) # Ensure the score is within [0, 1]
+
+ return similarity_score
def compare_videos(video_path1, video_path2, max_frames_to_check=100, threshold=5):
diff --git a/desktop_env/server/README.md b/desktop_env/server/README.md
new file mode 100644
index 0000000..571081a
--- /dev/null
+++ b/desktop_env/server/README.md
@@ -0,0 +1,73 @@
+
+
+
+
+### About the Converted Accessibility Tree
+
+For several applications like Firefox or Thunderbird, you should first enable
+
+```sh
+gsettings set org.gnome.desktop.interface toolkit-accessibility true
+```
+
+to see their accessibility tree.
+
+#### Example of AT
+
+An example of a node:
+
+```xml
+
+ 歡迎使用新的 Outlook.com 帳戶
+
+```
+
+An example of a tree:
+
+```xml
+
+
+ ...
+
+ ...
+
+```
+
+#### Useful attributes
+
+1. `name` - shows the name of application, title of window, or name of some
+ component
+2. `attr:class` - somewhat the same role as `class` in HTML
+3. `attr:id` - somewhat the same role as `id` in HTML
+4. `cp:screencoord` - absolute coordinator on the screen
+5. `cp:windowcoord` - relative coordinator in the window
+6. `cp:size` - the size
+
+Also several states like `st:enabled` and `st:visible` can be indicated. A full
+state list is available at
+.
+
+#### How to use it in evaluation
+
+See example `thunderbird/12086550-11c0-466b-b367-1d9e75b3910e.json` and
+function `check_accessibility_tree` in `metrics/general.py`. You can use CSS
+selector or XPath to reference a target nodes. You can also check its text
+contents.
+
+An example of a CSS selector:
+
+```css
+application[name=Thunderbird] page-tab-list[attr|id=\"tabmail-tabs\"]>page-tab[name=\"About Profiles\"]
+```
+
+This selector will select the page tab of profile manager in Thunderbird (if open).
+
+For usage of CSS selector: . For usage of XPath: .
+
+#### Manual check
+
+You can use accerciser to check the accessibility tree on GNOME VM.
+
+```sh
+sudo apt install accerciser
+```
diff --git a/desktop_env/server/main.py b/desktop_env/server/main.py
index b1f1dc2..11de773 100644
--- a/desktop_env/server/main.py
+++ b/desktop_env/server/main.py
@@ -360,8 +360,7 @@ def get_wallpaper():
if wallpaper_path:
try:
# Ensure the filename is secure
- filename = secure_filename(os.path.basename(wallpaper_path))
- return send_file(wallpaper_path, attachment_filename=filename)
+ return send_file(wallpaper_path, mimetype='image/png')
except Exception as e:
app.logger.error(f"An error occurred while serving the wallpaper file: {e}")
abort(500, description="Unable to serve the wallpaper file")
diff --git a/evaluation_examples/examples/chrome/06fe7178-4491-4589-810f-2e2bc9502122.json b/evaluation_examples/examples/chrome/06fe7178-4491-4589-810f-2e2bc9502122.json
index 78c34c0..96a6041 100644
--- a/evaluation_examples/examples/chrome/06fe7178-4491-4589-810f-2e2bc9502122.json
+++ b/evaluation_examples/examples/chrome/06fe7178-4491-4589-810f-2e2bc9502122.json
@@ -1,7 +1,7 @@
{
"id": "06fe7178-4491-4589-810f-2e2bc9502122",
"snapshot": "chrome",
- "instruction": "Could you help me open the previously closed tab?",
+ "instruction": "Can you make my computer bring back the last tab I shut down?",
"source": "https://www.wikihow.com/Switch-Tabs-in-Chrome",
"config": [],
"trajectory": "trajectories/",
diff --git a/evaluation_examples/examples/chrome/12086550-11c0-466b-b367-1d9e75b3910e.json b/evaluation_examples/examples/chrome/12086550-11c0-466b-b367-1d9e75b3910e.json
index 10c9104..6e2660d 100644
--- a/evaluation_examples/examples/chrome/12086550-11c0-466b-b367-1d9e75b3910e.json
+++ b/evaluation_examples/examples/chrome/12086550-11c0-466b-b367-1d9e75b3910e.json
@@ -1,7 +1,7 @@
{
"id": "12086550-11c0-466b-b367-1d9e75b3910e",
"snapshot": "chrome",
- "instruction": "Help me find the hidden password of xx website.",
+ "instruction": "Computer, please navigate to the area in my browser settings where my passwords are stored. I want to check my login information for Etsy without revealing it just yet.",
"source": "https://www.quora.com/What-are-the-cool-tricks-to-use-Google-Chrome",
"config": [],
"trajectory": "trajectories/",
diff --git a/evaluation_examples/examples/chrome/7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3.json b/evaluation_examples/examples/chrome/7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3.json
index 7e6363e..b914773 100644
--- a/evaluation_examples/examples/chrome/7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3.json
+++ b/evaluation_examples/examples/chrome/7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3.json
@@ -1,7 +1,7 @@
{
"id": "7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3",
"snapshot": "chrome",
- "instruction": "Could you help me delete cookies from openai website?",
+ "instruction": "Can you help me clean up my computer by getting rid of all the tracking things that websites like Amazon or eBay might have saved? I want to make sure my browsing is private and those sites don't remember me.",
"source": "https://support.google.com/chrome/answer/95647?hl=en&ref_topic=7438325&sjid=16867045591165135686-AP#zippy=%2Cdelete-cookies-from-a-site",
"config": [],
"trajectory": "trajectories/",
diff --git a/evaluation_examples/examples/chrome/bb5e4c0d-f964-439c-97b6-bdb9747de3f4.json b/evaluation_examples/examples/chrome/bb5e4c0d-f964-439c-97b6-bdb9747de3f4.json
index 3a8ff96..6f2cbf5 100644
--- a/evaluation_examples/examples/chrome/bb5e4c0d-f964-439c-97b6-bdb9747de3f4.json
+++ b/evaluation_examples/examples/chrome/bb5e4c0d-f964-439c-97b6-bdb9747de3f4.json
@@ -1,18 +1,33 @@
{
"id": "bb5e4c0d-f964-439c-97b6-bdb9747de3f4",
"snapshot": "chrome",
- "instruction": "Could you help me set my default search engine to be Google?",
+ "instruction": "Can you make Bing the main search thingy when I look stuff up on the internet?",
"source": "https://support.google.com/chrome/answer/95426?sjid=16867045591165135686-AP",
- "config": [],
+ "config": [
+ {
+ "type": "launch",
+ "parameters": {
+ "command": [
+ "google-chrome",
+ "--remote-debugging-port=9222"
+ ]
+ }
+ }
+ ],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
- "func": "",
+ "func": "exact_match",
"result": {
+ "type": "default_search_engine"
},
"expected": {
+ "type": "rule",
+ "rules": {
+ "expected": "Bing"
+ }
}
}
}
diff --git a/evaluation_examples/examples/libreoffice_writer/0810415c-bde4-4443-9047-d5f70165a697.json b/evaluation_examples/examples/libreoffice_writer/0810415c-bde4-4443-9047-d5f70165a697.json
index f8f3ef3..09dd8ac 100644
--- a/evaluation_examples/examples/libreoffice_writer/0810415c-bde4-4443-9047-d5f70165a697.json
+++ b/evaluation_examples/examples/libreoffice_writer/0810415c-bde4-4443-9047-d5f70165a697.json
@@ -10,7 +10,7 @@
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1-svVsH-l2ofufEKuN-cYrIrvXNobtATE&export=download&authuser=0&confirm=t&uuid=be7f891a-f858-48f5-a72d-4e42bbfb8b65&at=APZUnTXzBnaeSJjmxeh4zG03pzA0:1704179807785",
- "path": "Desktop/Double_Line_Spacing.docx"
+ "path": "Desktop/Novels_Intro_Packet.docx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
- "path": "Desktop/Double_Line_Spacing.docx"
+ "path": "Desktop/Novels_Intro_Packet.docx"
}
}
],
@@ -31,12 +31,12 @@
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1-svVsH-l2ofufEKuN-cYrIrvXNobtATE&export=download&authuser=0&confirm=t&uuid=be7f891a-f858-48f5-a72d-4e42bbfb8b65&at=APZUnTXzBnaeSJjmxeh4zG03pzA0:1704179807785",
- "dest": "Double_Line_Spacing_Gold.docx"
+ "dest": "Novels_Intro_Packet_Gold.docx"
},
"result": {
"type": "vm_file",
- "path": "Desktop/Double_Line_Spacing.docx",
- "dest": "Double_Line_Spacing.docx"
+ "path": "Desktop/Novels_Intro_Packet.docx",
+ "dest": "Novels_Intro_Packet.docx"
}
}
}
\ No newline at end of file
diff --git a/evaluation_examples/examples/libreoffice_writer/0b17a146-2934-46c7-8727-73ff6b6483e8.json b/evaluation_examples/examples/libreoffice_writer/0b17a146-2934-46c7-8727-73ff6b6483e8.json
index 9c7edd3..5b4ad99 100644
--- a/evaluation_examples/examples/libreoffice_writer/0b17a146-2934-46c7-8727-73ff6b6483e8.json
+++ b/evaluation_examples/examples/libreoffice_writer/0b17a146-2934-46c7-8727-73ff6b6483e8.json
@@ -1,7 +1,7 @@
{
"id": "0b17a146-2934-46c7-8727-73ff6b6483e8",
"snapshot": "libreoffice_writer",
- "instruction": "Change the 2 in H2O to a subscript.",
+ "instruction": "Help me change the 2 in \"H2O\" to a subscript.",
"source": "https://askubuntu.com/questions/245695/how-do-you-insert-subscripts-and-superscripts-into-ordinary-non-formula-text-i",
"config": [
{
@@ -9,8 +9,8 @@
"parameters": {
"files": [
{
- "url": "https://drive.usercontent.google.com/download?id=1Nx5AoKNM7tcDRE6y_qjNIDrPOKqhNyfm&export=download&authuser=0&confirm=t&uuid=bb4de348-3bbf-46a2-95b2-e2719c67547a&at=APZUnTUeA-BW7mkQsEw7NGm272zx:1704172916742",
- "path": "Desktop/Enter_Subscript.docx"
+ "url": "https://drive.usercontent.google.com/download?id=1FkorQBeTJ5L2jLuvu4YxHSlBMK4VEEG6&export=download&authuser=0&confirm=t&uuid=cc63dc0b-bae7-4ef6-a40d-e2da721976ef&at=APZUnTWyPZlZPFlqGTWAWXWmS04c:1704976667765",
+ "path": "Desktop/H2O_Factsheet_WA.docx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
- "path": "Desktop/Enter_Subscript.docx"
+ "path": "Desktop/H2O_Factsheet_WA.docx"
}
}
],
@@ -30,13 +30,13 @@
"func": "compare_docx_files",
"result": {
"type": "vm_file",
- "path": "Desktop/Enter_Subscript.docx",
- "dest": "Enter_Subscript.docx"
+ "path": "Desktop/H2O_Factsheet_WA.docx",
+ "dest": "H2O_Factsheet_WA.docx"
},
"expected": {
"type": "cloud_file",
- "path": "https://drive.usercontent.google.com/download?id=1AaKXeD9ZgfMykgijZ4G8MEzUjmMJElkq&export=download&authuser=0&confirm=t&uuid=5e347f0d-4efc-4478-878e-d89455d1593b&at=APZUnTWCYWfsD4eCeG52VJiK8-xB:1704172886196",
- "dest": "Enter_Subscript_Gold.docx"
+ "path": "https://drive.usercontent.google.com/download?id=1dM_FSTGDWxSW64VEth_wKMYNkvw0y_tq&export=download&authuser=0&confirm=t&uuid=342f41e2-f48f-41ff-8942-f7dfe5de1dba&at=APZUnTXHfskcX3tvmrSbzCOyQIgb:1704976694506",
+ "dest": "H2O_Factsheet_WA_Gold.docx"
}
}
}
\ No newline at end of file
diff --git a/evaluation_examples/examples/libreoffice_writer/0e47de2a-32e0-456c-a366-8c607ef7a9d2.json b/evaluation_examples/examples/libreoffice_writer/0e47de2a-32e0-456c-a366-8c607ef7a9d2.json
index ec4182d..f5cc925 100644
--- a/evaluation_examples/examples/libreoffice_writer/0e47de2a-32e0-456c-a366-8c607ef7a9d2.json
+++ b/evaluation_examples/examples/libreoffice_writer/0e47de2a-32e0-456c-a366-8c607ef7a9d2.json
@@ -10,7 +10,7 @@
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1aDWe-vAmcfQSgtPjFfrncq8ZFnCy4uUK&export=download&authuser=0&confirm=t&uuid=788af72a-ddaf-4ba3-aedb-96f34cc4d815&at=APZUnTVSRSSfMGcjXqLzvMixnkp6:1704179663299",
- "path": "Desktop/Add_Page_Number_Bottom_Left.docx"
+ "path": "Desktop/LibreOffice_Open_Source_Word_Processing.docx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
- "path": "Desktop/Add_Page_Number_Bottom_Left.docx"
+ "path": "Desktop/LibreOffice_Open_Source_Word_Processing.docx"
}
}
],
@@ -30,8 +30,8 @@
"func": "has_page_numbers_in_footers",
"result": {
"type": "vm_file",
- "path": "Desktop/Add_Page_Number_Bottom_Left.docx",
- "dest": "Add_Page_Number_Bottom_Left.docx"
+ "path": "Desktop/LibreOffice_Open_Source_Word_Processing.docx",
+ "dest": "LibreOffice_Open_Source_Word_Processing.docx"
}
}
}
\ No newline at end of file
diff --git a/evaluation_examples/examples/libreoffice_writer/0e763496-b6bb-4508-a427-fad0b6c3e195.json b/evaluation_examples/examples/libreoffice_writer/0e763496-b6bb-4508-a427-fad0b6c3e195.json
index 265f566..6d886c8 100644
--- a/evaluation_examples/examples/libreoffice_writer/0e763496-b6bb-4508-a427-fad0b6c3e195.json
+++ b/evaluation_examples/examples/libreoffice_writer/0e763496-b6bb-4508-a427-fad0b6c3e195.json
@@ -10,7 +10,7 @@
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1fIHNzFm8JabWoLKOnxrFM722fQ1d_huK&export=download&authuser=0&confirm=t&uuid=d11a8dda-1e4e-4dc9-b05c-e6b47624dbf0&at=APZUnTVG0ViFnKJa00314wVr3uP9:1704185871014",
- "path": "Desktop/Change_Font_Through_File.docx"
+ "path": "Desktop/Dublin_Zoo_Intro.docx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
- "path": "Desktop/Change_Font_Through_File.docx"
+ "path": "Desktop/Dublin_Zoo_Intro.docx"
}
}
],
@@ -36,8 +36,8 @@
},
"result": {
"type": "vm_file",
- "path": "Desktop/Change_Font_Through_File.docx",
- "dest": "Change_Font_Through_File.docx"
+ "path": "Desktop/Dublin_Zoo_Intro.docx",
+ "dest": "Dublin_Zoo_Intro.docx"
}
}
}
\ No newline at end of file
diff --git a/evaluation_examples/examples/libreoffice_writer/3ef2b351-8a84-4ff2-8724-d86eae9b842e.json b/evaluation_examples/examples/libreoffice_writer/3ef2b351-8a84-4ff2-8724-d86eae9b842e.json
index d9d1129..17196f6 100644
--- a/evaluation_examples/examples/libreoffice_writer/3ef2b351-8a84-4ff2-8724-d86eae9b842e.json
+++ b/evaluation_examples/examples/libreoffice_writer/3ef2b351-8a84-4ff2-8724-d86eae9b842e.json
@@ -1,7 +1,7 @@
{
"id": "3ef2b351-8a84-4ff2-8724-d86eae9b842e",
"snapshot": "libreoffice_writer",
- "instruction": "center-justify the first line",
+ "instruction": "Help me center align the heading in LibreOffice.",
"source": "https://askubuntu.com/questions/1066351/how-do-you-center-align-in-libreoffice#:~:text=Ctrl%20%2B%20e%20will%20Center%20align%20the%20cursor%20for%20you.",
"config": [
{
@@ -9,8 +9,8 @@
"parameters": {
"files": [
{
- "url": "https://drive.usercontent.google.com/download?id=1IQ4rKyHMOui71YlyL7huggpLYFYtj923&export=download&authuser=0&confirm=t&uuid=014c2335-c0c6-4712-9d5a-ca8d3217e07f&at=APZUnTVrM698NQgSh4hqYXR8cjDc:1704185072996",
- "path": "Desktop/Centering_First_Line.docx"
+ "url": "https://drive.usercontent.google.com/download?id=1P8QodvDF-3S50rx6UmW4M2D4Kr-p_Q-h&export=download&authuser=0&confirm=t&uuid=eea70a33-4c3f-4e90-885d-dd3df0d605bc&at=APZUnTX7ISvBhOICNrPLoqK0m3G-:1704971931660",
+ "path": "Desktop/Constitution_Template_With_Guidelines.docx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
- "path": "Desktop/Centering_First_Line.docx"
+ "path": "Desktop/Constitution_Template_With_Guidelines.docx"
}
}
],
@@ -30,8 +30,8 @@
"func": "is_first_line_centered",
"result": {
"type": "vm_file",
- "path": "Desktop/Centering_First_Line.docx",
- "dest": "Centering_First_Line.docx"
+ "path": "Desktop/Constitution_Template_With_Guidelines.docx",
+ "dest": "Constitution_Template_With_Guidelines.docx"
}
}
}
\ No newline at end of file
diff --git a/evaluation_examples/examples/libreoffice_writer/45d61a06-6545-4422-97b7-bc76cfa964c1.json b/evaluation_examples/examples/libreoffice_writer/45d61a06-6545-4422-97b7-bc76cfa964c1.json
index c892edf..99a960c 100644
--- a/evaluation_examples/examples/libreoffice_writer/45d61a06-6545-4422-97b7-bc76cfa964c1.json
+++ b/evaluation_examples/examples/libreoffice_writer/45d61a06-6545-4422-97b7-bc76cfa964c1.json
@@ -1,7 +1,7 @@
{
"id": "45d61a06-6545-4422-97b7-bc76cfa964c1",
"snapshot": "libreoffice_writer",
- "instruction": "Replace all newlines with paragraph marks in LibreOffice Write",
+ "instruction": "Replace all newlines with paragraph marks in LibreOffice Writer",
"source": "https://stackoverflow.com/questions/71685737/how-to-replace-all-newlines-with-paragraph-marks-in-libreoffice-write",
"config": [
{
@@ -9,8 +9,8 @@
"parameters": {
"files": [
{
- "url": "https://drive.usercontent.google.com/download?id=18XFjPVUnLG_-KOM5sn-Sk74HP_JHivMy&export=download&authuser=0&confirm=t&uuid=d23041bc-2ddd-42c4-84ae-481b953f021c&at=APZUnTVYh0AK0245qsDOCol7SdMB:1704185512767",
- "path": "Desktop/Replace_Newlines_with_Paragraph_Marks.docx"
+ "url": "https://drive.usercontent.google.com/download?id=16lQcSkw-JQ_v8Sg0HkCtnOzyK-4cok8N&export=download&authuser=0&confirm=t&uuid=a3f833ae-2572-4cf3-8a21-6d250e689415&at=APZUnTUfn24NGMtXEzz2Nf7cFLjt:1704975857398",
+ "path": "Desktop/NOVEL_Submission_Guidelines.docx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
- "path": "Desktop/Replace_Newlines_with_Paragraph_Marks.docx"
+ "path": "Desktop/NOVEL_Submission_Guidelines.docx"
}
}
],
@@ -30,13 +30,13 @@
"func": "compare_line_spacing",
"expected": {
"type": "cloud_file",
- "path": "https://drive.usercontent.google.com/download?id=1bP_noic02MuzrM8CdJIQN7F1gN4N8sel&export=download&authuser=0&confirm=t&uuid=657e0e4f-7b96-4d7e-83f4-99b79c68708f&at=APZUnTX7HsmefsMlzQaCGK2fg5Em:1704185514197",
- "dest": "Replace_Newlines_with_Paragraph_Marks_Gold.docx"
+ "path": "https://drive.usercontent.google.com/download?id=1LaS5ObaOsbyKX1M17vi8ZseZwawgvNmf&export=download&authuser=0&confirm=t&uuid=db523e81-fa22-4002-97ff-e5dff92106a7&at=APZUnTVnT3ZYOGW7ZQdeW4SZP7mX:1704975993684",
+ "dest": "NOVEL_Submission_Guidelines_Gold.docx"
},
"result": {
"type": "vm_file",
- "path": "Desktop/Double_Line_Spacing.docx",
- "dest": "Replace_Newlines_with_Paragraph_Marks.docx"
+ "path": "Desktop/NOVEL_Submission_Guidelines.docx",
+ "dest": "NOVEL_Submission_Guidelines.docx"
}
}
-}
+}
\ No newline at end of file
diff --git a/evaluation_examples/examples/libreoffice_writer/4bcb1253-a636-4df4-8cb0-a35c04dfef31.json b/evaluation_examples/examples/libreoffice_writer/4bcb1253-a636-4df4-8cb0-a35c04dfef31.json
index c9b72df..ba1cb2c 100644
--- a/evaluation_examples/examples/libreoffice_writer/4bcb1253-a636-4df4-8cb0-a35c04dfef31.json
+++ b/evaluation_examples/examples/libreoffice_writer/4bcb1253-a636-4df4-8cb0-a35c04dfef31.json
@@ -9,8 +9,8 @@
"parameters": {
"files": [
{
- "url": "https://drive.usercontent.google.com/download?id=1czb_13MshoiM-yCxvUGYD8OnIIrf-3VX&export=download&authuser=0&confirm=t&uuid=e7c30b67-7fac-4b64-a222-d04bc7c82842&at=APZUnTUA1te5vt7L__zJ7xuMs48e:1704177347643",
- "path": "Desktop/Save_Writer_PDF.docx"
+ "url": "https://drive.usercontent.google.com/download?id=1cn3CaA_4ds1WY8SgBT3TvjRunxocCBiu&export=download&authuser=0&confirm=t&uuid=622f5ae1-6f21-4f31-8a3e-e4ead6ea6bc3&at=APZUnTVKOV40Ww5PovU7at2ELzb9:1704949558060",
+ "path": "Desktop/View_Person_Organizational_Summary.docx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
- "path": "Desktop/Save_Writer_PDF.docx"
+ "path": "Desktop/View_Person_Organizational_Summary.docx"
}
}
],
@@ -28,7 +28,7 @@
],
"evaluator": {
"func": "check_file_exists",
- "file_name": "Save_Writer_PDF.pdf",
+ "file_name": "View_Person_Organizational_Summary.pdf",
"directory": "/home/user/Downloads/"
}
}
\ No newline at end of file
diff --git a/evaluation_examples/examples/libreoffice_writer/663876c7-3471-43db-ba51-f410b13d9d7d.json b/evaluation_examples/examples/libreoffice_writer/663876c7-3471-43db-ba51-f410b13d9d7d.json
index 37af3d2..d04915a 100644
--- a/evaluation_examples/examples/libreoffice_writer/663876c7-3471-43db-ba51-f410b13d9d7d.json
+++ b/evaluation_examples/examples/libreoffice_writer/663876c7-3471-43db-ba51-f410b13d9d7d.json
@@ -1,7 +1,7 @@
{
"id": "663876c7-3471-43db-ba51-f410b13d9d7d",
"snapshot": "libreoffice_writer",
- "instruction": "Insert the equation \"(a + b)^2 = a^2 + 2 a b + b^2\"",
+ "instruction": "Insert the equation \"(a + b)^2 = a^2 + 2 a b + b^2\" at the position of the cursor.",
"source": "https://askubuntu.com/questions/319593/how-to-type-science-equations-in-libre-office",
"config": [
{
@@ -9,8 +9,8 @@
"parameters": {
"files": [
{
- "url": "https://drive.usercontent.google.com/download?id=1FgMp7Ny63eXzeF23qHYhqQux31djlkah&export=download&authuser=0&confirm=t&uuid=d6b5208d-3b3a-4972-a641-ed738a419fdb&at=APZUnTX16Fz8Qg-B0NWpWgC-3Dyu:1704184410221",
- "path": "Desktop/Insert_Equation.docx"
+ "url": "https://drive.usercontent.google.com/download?id=1J3--srr-Fmt2z-wOAyPDmO5GoiJMkUvZ&export=download&authuser=0&confirm=t&uuid=5f275f3d-ab7d-4d27-848a-8012af4bcf7f&at=APZUnTVe5ucwe1Q0oVi9jwiY5PF7:1704971494071",
+ "path": "Desktop/Factoring_Perfect_Square_Trinomials.docx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
- "path": "Desktop/Insert_Equation.docx"
+ "path": "Desktop/Factoring_Perfect_Square_Trinomials.docx"
}
}
],
@@ -30,13 +30,13 @@
"func": "compare_insert_equation",
"expected": {
"type": "cloud_file",
- "path": "https://drive.usercontent.google.com/download?id=1hMFJnwHs7Iaexz3b9O2LJQUfsJ2wiwZ9&export=download&authuser=0&confirm=t&uuid=2abb49fb-d9c7-46cf-bc21-e69ecb9cefc6&at=APZUnTVzEZjChcUb4MIoxuq4cGea:1704184411805",
- "dest": "Insert_Equation_Gold.docx"
+ "path": "https://drive.usercontent.google.com/download?id=1QGfyQUPEjOX8osycnjR8E5yHeIEGccKI&export=download&authuser=0&confirm=t&uuid=174491d6-7855-4323-9fa5-5146fde48c4a&at=APZUnTXA8bIcIbtw9mUN-K-IU9bT:1704971524974",
+ "dest": "Factoring_Perfect_Square_Trinomials_Gold.docx"
},
"result": {
"type": "vm_file",
- "path": "Desktop/Insert_Equation.docx",
- "dest": "Insert_Equation.docx"
+ "path": "Desktop/Factoring_Perfect_Square_Trinomials.docx",
+ "dest": "Factoring_Perfect_Square_Trinomials.docx"
}
}
}
\ No newline at end of file
diff --git a/evaluation_examples/examples/libreoffice_writer/66399b0d-8fda-4618-95c4-bfc6191617e9.json b/evaluation_examples/examples/libreoffice_writer/66399b0d-8fda-4618-95c4-bfc6191617e9.json
index 0004246..a45e7fd 100644
--- a/evaluation_examples/examples/libreoffice_writer/66399b0d-8fda-4618-95c4-bfc6191617e9.json
+++ b/evaluation_examples/examples/libreoffice_writer/66399b0d-8fda-4618-95c4-bfc6191617e9.json
@@ -1,7 +1,7 @@
{
"id": "66399b0d-8fda-4618-95c4-bfc6191617e9",
"snapshot": "libreoffice_writer",
- "instruction": "Insert a 7*5 empty table",
+ "instruction": "Could you help me insert a 7*5 empty table at the point of cursor?",
"source": "https://www.youtube.com/watch?v=l25Evu4ohKg",
"config": [
{
@@ -10,7 +10,7 @@
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1kXBP0jMxTeVahzFLYbYHJtjjgmuzrA8R&export=download&authuser=0&confirm=t&uuid=f8b9bad3-415d-4d39-a4fb-05a4cf881cf0&at=APZUnTXaohwzl8_2RDF_tgUsP9cH:1704181463579",
- "path": "Desktop/Insert_Empty_Table.docx"
+ "path": "Desktop/Table_Of_Work_Effort_Instructions.docx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
- "path": "Desktop/Insert_Empty_Table.docx"
+ "path": "Desktop/Table_Of_Work_Effort_Instructions.docx"
}
}
],
@@ -31,12 +31,12 @@
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=14JfHsW9GvbhORdtVAtvEbOi00MqEyHfb&export=download&authuser=0&confirm=t&uuid=3dba2459-ac37-4cad-a982-adecd406382a&at=APZUnTVQUqUPq_WacgY2xu4PvAKB:1704181465512",
- "dest": "Insert_Empty_Table_Gold.docx"
+ "dest": "Table_Of_Work_Effort_Instructions_Gold.docx"
},
"result": {
"type": "vm_file",
- "path": "Desktop/Insert_Empty_Table.docx",
- "dest": "Insert_Empty_Table.docx"
+ "path": "Desktop/Table_Of_Work_Effort_Instructions.docx",
+ "dest": "Table_Of_Work_Effort_Instructions.docx"
}
}
}
\ No newline at end of file
diff --git a/evaluation_examples/examples/libreoffice_writer/6ada715d-3aae-4a32-a6a7-429b2e43fb93.json b/evaluation_examples/examples/libreoffice_writer/6ada715d-3aae-4a32-a6a7-429b2e43fb93.json
index 9528060..ff6fa76 100644
--- a/evaluation_examples/examples/libreoffice_writer/6ada715d-3aae-4a32-a6a7-429b2e43fb93.json
+++ b/evaluation_examples/examples/libreoffice_writer/6ada715d-3aae-4a32-a6a7-429b2e43fb93.json
@@ -1,7 +1,7 @@
{
"id": "6ada715d-3aae-4a32-a6a7-429b2e43fb93",
"snapshot": "libreoffice_writer",
- "instruction": "Copy the screenshot 1.jpg from the desktop to where my cursor is locatedInsert the image which is in IMAGE_PATH where my cursor is",
+ "instruction": "Copy the screenshot 1.png from the desktop to where my cursor is located",
"source": "https://www.quora.com/How-do-you-insert-images-into-a-LibreOffice-Writer-document",
"config": [
{
@@ -10,7 +10,7 @@
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1ebLG0gmqYv41ek4UmKWhFsxBnoUSGjKp&export=download&authuser=0&confirm=t&uuid=8f7d7bee-1fe4-4c4c-8b69-8aaf47199c57&at=APZUnTVYUvYTopUXCVs69QWWwPbq:1704173993139",
- "path": "Desktop/Insert_Image_At_Cursor.docx"
+ "path": "Desktop/Viewing_Your_Class_Schedule_and_Textbooks.docx"
}
]
}
@@ -20,8 +20,8 @@
"parameters": {
"files": [
{
- "url": "https://drive.usercontent.google.com/download?id=1QfjQ4SKtjKDXWpqa2u6mC_KtB3ASEK5O&export=download&authuser=0&confirm=t&uuid=06af00b9-58f3-4691-a6a3-34309c80cbbb&at=APZUnTVZpE1lMxcvGG0cdt5zuxZ_:1704174003198",
- "path": "Desktop/1.jpg"
+ "url": "https://drive.usercontent.google.com/download?id=1-wYHrgPMZl5Y9Losd4AGkMVHWNU7Uemg&export=download&authuser=0&confirm=t&uuid=3bc52ab7-1739-407d-9b14-36c9305c973e&at=APZUnTW95CwED3zm5KRREFmO2mc2:1704952228530",
+ "path": "Desktop/1.png"
}
]
}
@@ -29,7 +29,7 @@
{
"type": "open",
"parameters": {
- "path": "Desktop/Insert_Image_At_Cursor.docx"
+ "path": "Desktop/Viewing_Your_Class_Schedule_and_Textbooks.docx"
}
}
],
@@ -41,13 +41,13 @@
"func": "compare_contains_image",
"result": {
"type": "vm_file",
- "path": "Desktop/Insert_Image_At_Cursor.docx",
- "dest": "Insert_Image_At_Cursor.docx"
+ "path": "Desktop/Viewing_Your_Class_Schedule_and_Textbooks.docx",
+ "dest": "Viewing_Your_Class_Schedule_and_Textbooks.docx"
},
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1xbhlfqGrPutHHi2aHg66jwXD-yaZpe9j&export=download&authuser=0&confirm=t&uuid=427765e0-3f97-4a72-92db-a1fe7cdde73b&at=APZUnTUhNLh2PDu4OGkCVQW-LPCd:1704173991269",
- "dest": "Insert_Image_At_Cursor_Gold.docx"
+ "dest": "Viewing_Your_Class_Schedule_and_Textbooks_Gold.docx"
}
}
}
\ No newline at end of file
diff --git a/evaluation_examples/examples/libreoffice_writer/936321ce-5236-426a-9a20-e0e3c5dc536f.json b/evaluation_examples/examples/libreoffice_writer/936321ce-5236-426a-9a20-e0e3c5dc536f.json
index 9b4b172..92591f0 100644
--- a/evaluation_examples/examples/libreoffice_writer/936321ce-5236-426a-9a20-e0e3c5dc536f.json
+++ b/evaluation_examples/examples/libreoffice_writer/936321ce-5236-426a-9a20-e0e3c5dc536f.json
@@ -1,7 +1,7 @@
{
"id": "936321ce-5236-426a-9a20-e0e3c5dc536f",
"snapshot": "libreoffice_writer",
- "instruction": "Convert the content seperated by commas to a table",
+ "instruction": "Could you help me convert the text seperated by commas to a table?",
"source": "https://www.youtube.com/watch?v=l25Evu4ohKg",
"config": [
{
@@ -9,8 +9,8 @@
"parameters": {
"files": [
{
- "url": "https://drive.usercontent.google.com/download?id=139-NmslBR_9qlD7hUO08xj_VFffV95eM&export=download&authuser=0&confirm=t&uuid=64a6c35d-f3ce-4c25-9f83-4a952e24c5ad&at=APZUnTUL1GMR_QbpFQnC9fPwkdqa:1704183959196",
- "path": "Desktop/Convert_Text_To_Table.docx"
+ "url": "https://drive.usercontent.google.com/download?id=1WcHqnDy0OPFiaGtvPSvKXuaLc_rtGzYC&export=download&authuser=0&confirm=t&uuid=0c552800-5dfe-4a3f-a960-8d384481fa7f&at=APZUnTWYjmvuXkU-pb77dy7EgCeD:1704970003044",
+ "path": "Desktop/Graphemes_Sound_Letter_Patterns.docx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
- "path": "Desktop/Convert_Text_To_Table.docx"
+ "path": "Desktop/Graphemes_Sound_Letter_Patterns.docx"
}
}
],
@@ -30,13 +30,13 @@
"func": "compare_docx_tables",
"expected": {
"type": "cloud_file",
- "path": "https://drive.usercontent.google.com/download?id=1fs2msYaxnEZL9XASENQMZIag2MTxIBJs&export=download&authuser=0&confirm=t&uuid=6c71f008-082c-4f0c-9ffc-0a802f5cbfe6&at=APZUnTVDpucMDfk5P2T-0dZx_KVV:1704183960360",
- "dest": "Convert_Text_To_Table_Gold.docx"
+ "path": "https://drive.usercontent.google.com/download?id=1og3gCBRhAeHk02E1wAsbfau_1ywdoEJL&export=download&authuser=0&confirm=t&uuid=264de3b4-b07c-4382-8a73-a766b1d1fc58&at=APZUnTX8yRMoDXCHi1y1H380GaSX:1704969967989",
+ "dest": "Graphemes_Sound_Letter_Patterns_Gold.docx"
},
"result": {
"type": "vm_file",
- "path": "Desktop/Convert_Text_To_Table.docx",
- "dest": "Convert_Text_To_Table.docx"
+ "path": "Desktop/Graphemes_Sound_Letter_Patterns.docx",
+ "dest": "Graphemes_Sound_Letter_Patterns.docx"
}
}
}
\ No newline at end of file
diff --git a/evaluation_examples/examples/libreoffice_writer/adf5e2c3-64c7-4644-b7b6-d2f0167927e7.json b/evaluation_examples/examples/libreoffice_writer/adf5e2c3-64c7-4644-b7b6-d2f0167927e7.json
index 6b1ce4e..a932884 100644
--- a/evaluation_examples/examples/libreoffice_writer/adf5e2c3-64c7-4644-b7b6-d2f0167927e7.json
+++ b/evaluation_examples/examples/libreoffice_writer/adf5e2c3-64c7-4644-b7b6-d2f0167927e7.json
@@ -1,7 +1,7 @@
{
"id": "adf5e2c3-64c7-4644-b7b6-d2f0167927e7",
"snapshot": "libreoffice_writer",
- "instruction": "Helping me adding \"C. Luo and M. J. Carey, \"LSM-based storage techniques: a survey,\" The VLDB Journal, vol. 29, no. 1, pp. 393–418, 2020.\" to my reference list, and add a cross reference at the end of the first paragraph",
+ "instruction": "Help me adding \"Steinberg, F. M., Bearden, M. M., & Keen, C. L. (2003). Cocoa and chocolate flavonoids: Implications for cardiovascular health. Journal of the American Dietetic Association, 103(2), 215-223. doi: 10.1053/jada.2003.50028\" to my reference list, and add a cross reference where my cursor is located (in the fourth paragraph).",
"source": "https://seekstar.github.io/2022/04/11/libreoffice%E5%BC%95%E7%94%A8%E6%96%87%E7%8C%AE/",
"config": [
{
@@ -9,8 +9,8 @@
"parameters": {
"files": [
{
- "url": "https://drive.usercontent.google.com/download?id=1xOfwImgkPzdmjQomj-MCFd8nQS75OjaH&export=download&authuser=0&confirm=t&uuid=7eb91c26-dad5-4480-b1ec-35e506cde1e4&at=APZUnTW01MvBI_gkC8yoiyAVs7yi:1704188254979",
- "path": "Desktop/Add_Citation_Cross_Reference.docx"
+ "url": "https://drive.usercontent.google.com/download?id=1ShGL4gWSV7nzamAb0V2KqjoCOhyodcKU&export=download&authuser=0&confirm=t&uuid=5f67edb8-cbbf-4a83-b46e-f193ad55e1e8&at=APZUnTVRJenYCM--vETagQ5ACTT5:1704979226579",
+ "path": "Desktop/Essay_Writing_English_for_uni.docx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
- "path": "Desktop/Add_Citation_Cross_Reference.docx"
+ "path": "Desktop/Essay_Writing_English_for_uni.docx"
}
}
],
@@ -30,13 +30,13 @@
"func": "compare_docx_files",
"expected": {
"type": "cloud_file",
- "path": "https://drive.usercontent.google.com/download?id=1wFQU7hkAT2wmSHTgM22F9Ep4WXmymEMW&export=download&authuser=0&confirm=t&uuid=a7ea1eec-678b-4407-b023-df13cc6f8c54&at=APZUnTW3WoqOfS9A1BW79XfV8jKh:1704188260410",
- "dest": "Add_Citation_Cross_Reference_Gold.docx"
+ "path": "https://drive.usercontent.google.com/download?id=1bf7IKjjxGAQfbOlq9rLPzjTtkOZ3dREj&export=download&authuser=0&confirm=t&uuid=0420cfa5-7e51-4688-8c93-74748914ce52&at=APZUnTWIShIleccvgVBIr6ZsSnOw:1704979256798",
+ "dest": "Essay_Writing_English_for_uni_Gold.docx"
},
"result": {
"type": "vm_file",
- "path": "Desktop/Add_Citation_Cross_Reference.docx",
- "dest": "Add_Citation_Cross_Reference.docx"
+ "path": "Desktop/Essay_Writing_English_for_uni.docx",
+ "dest": "Essay_Writing_English_for_uni.docx"
}
}
}
\ No newline at end of file
diff --git a/evaluation_examples/examples/libreoffice_writer/e528b65e-1107-4b8c-8988-490e4fece599.json b/evaluation_examples/examples/libreoffice_writer/e528b65e-1107-4b8c-8988-490e4fece599.json
index a3a2340..451a68d 100644
--- a/evaluation_examples/examples/libreoffice_writer/e528b65e-1107-4b8c-8988-490e4fece599.json
+++ b/evaluation_examples/examples/libreoffice_writer/e528b65e-1107-4b8c-8988-490e4fece599.json
@@ -9,8 +9,8 @@
"parameters": {
"files": [
{
- "url": "https://drive.usercontent.google.com/download?id=1LZ1_U9CyR8oOkqfbY6HMLWr4wBVYhCLR&export=download&authuser=0&confirm=t&uuid=574d533a-8df9-4f33-bebd-689b623f27a9&at=APZUnTVruCDRxY661_PVT9BA3839:1704180420603",
- "path": "Desktop/Capitalize_First_Letter.docx"
+ "url": "https://drive.usercontent.google.com/download?id=181hyG_NZUJaUp7kUdeFoUpNGpOcjzYFV&export=download&authuser=0&confirm=t&uuid=fdb8aee4-ea1f-43c1-bc66-9d44099512e1&at=APZUnTVUrypBeVnTfkJsriv7S2GO:1704969486759",
+ "path": "Geography_And_Magical_Realism.docx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
- "path": "Desktop/Capitalize_First_Letter.docx"
+ "path": "Desktop/Geography_And_Magical_Realism.docx"
}
}
],
@@ -30,13 +30,13 @@
"func": "compare_docx_files",
"expected": {
"type": "cloud_file",
- "path": "https://drive.usercontent.google.com/download?id=1ykIGg48GjYUOSw8t44evShJE2HX7R5w3&export=download&authuser=0&confirm=t&uuid=ed81d6cd-6044-49f1-be86-35adeaeeea00&at=APZUnTUxW8WLyPr-_smA2Mnwpuuv:1704180422490https://drive.usercontent.google.com/download?id=1ykIGg48GjYUOSw8t44evShJE2HX7R5w3&export=download&authuser=0&confirm=t&uuid=ed81d6cd-6044-49f1-be86-35adeaeeea00&at=APZUnTUxW8WLyPr-_smA2Mnwpuuv:1704180422490",
- "dest": "Capitalize_First_Letter_Gold.docx"
+ "path": "https://drive.usercontent.google.com/download?id=1qwoJMuMhLxWwONRLmxwMdUuVIFBAhhNT&export=download&authuser=0&confirm=t&uuid=8fa4d503-ec55-419f-ac90-f5915a5f67b5&at=APZUnTVEntT4oMiEfOuWcUKaInBs:1704969347952",
+ "dest": "Geography_And_Magical_Realism_Gold.docx"
},
"result": {
"type": "vm_file",
- "path": "Desktop/Capitalize_First_Letter.docx",
- "dest": "Capitalize_First_Letter.docx"
+ "path": "Desktop/Geography_And_Magical_Realism.docx",
+ "dest": "Geography_And_Magical_Realism.docx"
}
}
}
\ No newline at end of file
diff --git a/evaluation_examples/examples/libreoffice_writer/ecc2413d-8a48-416e-a3a2-d30106ca36cb.json b/evaluation_examples/examples/libreoffice_writer/ecc2413d-8a48-416e-a3a2-d30106ca36cb.json
index a99aa0e..25a9d38 100644
--- a/evaluation_examples/examples/libreoffice_writer/ecc2413d-8a48-416e-a3a2-d30106ca36cb.json
+++ b/evaluation_examples/examples/libreoffice_writer/ecc2413d-8a48-416e-a3a2-d30106ca36cb.json
@@ -1,7 +1,7 @@
{
"id": "ecc2413d-8a48-416e-a3a2-d30106ca36cb",
"snapshot": "libreoffice_writer",
- "instruction": "Insert a blank page",
+ "instruction": "Help me insert a blank page where my cursor is located.",
"source": "https://www.quora.com/How-can-I-insert-a-blank-page-on-libreoffice",
"config": [
{
@@ -10,7 +10,7 @@
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1sDufDSC4foI379-Jikya9WK7FBUSqgrt&export=download&authuser=0&confirm=t&uuid=0abd82d6-2b2c-49bc-af5e-49bfe1c99278&at=APZUnTURIqTNJcIHBcMP2BxEaGXr:1704174850900",
- "path": "Desktop/Insert_Blank_Page.docx"
+ "path": "Desktop/Sample_Statutory_Declaration.docx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
- "path": "Desktop/Insert_Blank_Page.docx"
+ "path": "Desktop/Sample_Statutory_Declaration.docx"
}
}
],
@@ -30,8 +30,8 @@
"func": "contains_page_break",
"result": {
"type": "vm_file",
- "path": "Desktop/Insert_Blank_Page.docx",
- "dest": "Insert_Blank_Page.docx"
+ "path": "Desktop/Sample_Statutory_Declaration.docx",
+ "dest": "Sample_Statutory_Declaration.docx"
}
}
}
\ No newline at end of file
diff --git a/evaluation_examples/examples/libreoffice_writer/f178a4a9-d090-4b56-bc4c-4b72a61a035d.json b/evaluation_examples/examples/libreoffice_writer/f178a4a9-d090-4b56-bc4c-4b72a61a035d.json
index 1d710c3..315ea4d 100644
--- a/evaluation_examples/examples/libreoffice_writer/f178a4a9-d090-4b56-bc4c-4b72a61a035d.json
+++ b/evaluation_examples/examples/libreoffice_writer/f178a4a9-d090-4b56-bc4c-4b72a61a035d.json
@@ -9,8 +9,8 @@
"parameters": {
"files": [
{
- "url": "https://drive.usercontent.google.com/download?id=1X2XTU2ZFuMXOhm7T400e6AOe6eBYxWzD&export=download&authuser=0&confirm=t&uuid=1318923f-6d54-4148-aa80-a454b9963cec&at=APZUnTU-h1nmcjBO_ytWVxXuh8l9:1704187013730",
- "path": "Desktop/Set_Default_Font.docx"
+ "url": "https://drive.usercontent.google.com/download?id=1xREbNAu_2wLTs8EQT0NnHLpIkAVAfpyk&export=download&authuser=0&confirm=t&uuid=dd5cb525-ff4b-41a2-8123-d488f2f21fad&at=APZUnTXaYBqLT9fRtGYZHOedq-PG:1704977194647",
+ "path": "Desktop/loa-one-time-submission-sealand.docx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
- "path": "Desktop/Set_Default_Font.docx"
+ "path": "Desktop/loa-one-time-submission-sealand.docx"
}
}
],
diff --git a/evaluation_examples/examples/vlc/59f21cfb-0120-4326-b255-a5b827b38967.json b/evaluation_examples/examples/vlc/59f21cfb-0120-4326-b255-a5b827b38967.json
index b65ddfb..32d7570 100644
--- a/evaluation_examples/examples/vlc/59f21cfb-0120-4326-b255-a5b827b38967.json
+++ b/evaluation_examples/examples/vlc/59f21cfb-0120-4326-b255-a5b827b38967.json
@@ -1,7 +1,7 @@
{
"id": "59f21cfb-0120-4326-b255-a5b827b38967",
"snapshot": "base_setup",
- "instruction": "Play the music video on my desktop",
+ "instruction": "Could you play the music video that's saved on my desktop for me?",
"source": "https://docs.videolan.me/vlc-user/desktop/3.0/en/basic/media.html#playing-a-file",
"config": [
{
diff --git a/evaluation_examples/examples/vlc/8f080098-ddb1-424c-b438-4e96e5e4786e.json b/evaluation_examples/examples/vlc/8f080098-ddb1-424c-b438-4e96e5e4786e.json
index db15765..a39bd31 100644
--- a/evaluation_examples/examples/vlc/8f080098-ddb1-424c-b438-4e96e5e4786e.json
+++ b/evaluation_examples/examples/vlc/8f080098-ddb1-424c-b438-4e96e5e4786e.json
@@ -1,7 +1,7 @@
{
"id": "8f080098-ddb1-424c-b438-4e96e5e4786e",
"snapshot": "base_setup",
- "instruction": "Could you help me extract MP3 Audio to AUDIO_PATH from Video at VIDEO_PATH using VLC Media Player?",
+ "instruction": "Could you download the song from this music video and save it as an MP3 file? I'd like to have it on my device to play whenever I want. Please title the file \"Baby Justin Bieber.mp3.\" I really appreciate your help!",
"source": "https://medium.com/@jetscribe_ai/how-to-extract-mp3-audio-from-videos-using-vlc-media-player-beeef644ebfb",
"config": [
{
@@ -10,7 +10,7 @@
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=19jBiz8sb0M7KHHATO9qeTPr17aWm4me-&export=download&authuser=0&confirm=t&uuid=7a2261f4-3905-433f-b53f-a52dd0845651&at=APZUnTU1nmXSa1ObrA5NHYt8t1-p:1704710908141",
- "path": "Baby Justin Bieber.mp4"
+ "path": "Desktop/Baby Justin Bieber.mp4"
}
]
}
@@ -18,7 +18,11 @@
{
"type": "launch",
"parameters": {
- "command": "vlc"
+ "command": [
+ "vlc",
+ "--start-time=73",
+ "Desktop/Baby Justin Bieber.mp4"
+ ]
}
}
],
@@ -27,16 +31,16 @@
"vlc"
],
"evaluator": {
- "func": "is_vlc_recordings_folder",
+ "func": "compare_audios",
"expected": {
- "type": "rule",
- "rules": {
- "recording_file_path": "/home/user/Desktop"
- }
+ "type": "cloud_file",
+ "path": "https://drive.usercontent.google.com/download?id=1eRuuiUAOmckyn2zQebx1nWQ40kpjSJ_N&export=download&authuser=0&confirm=t&uuid=8012703a-9e45-4d91-9bda-90c119e42254&at=APZUnTW0uyCSeg7FClSc50UJVRz-:1704898635455",
+ "dest": "baby_gold.mp3"
},
"result": {
- "type": "vlc_config",
- "dest": "vlcrc"
+ "type": "vm_file",
+ "path": "Desktop/Baby Justin Bieber.mp3",
+ "dest": "baby.mp3"
}
}
}
diff --git a/evaluation_examples/examples/vlc/a1c3ab35-02de-4999-a7ed-2fd12c972c6e.json b/evaluation_examples/examples/vlc/a1c3ab35-02de-4999-a7ed-2fd12c972c6e.json
index 676092a..b279cb5 100644
--- a/evaluation_examples/examples/vlc/a1c3ab35-02de-4999-a7ed-2fd12c972c6e.json
+++ b/evaluation_examples/examples/vlc/a1c3ab35-02de-4999-a7ed-2fd12c972c6e.json
@@ -1,7 +1,7 @@
{
"id": "a1c3ab35-02de-4999-a7ed-2fd12c972c6e",
"snapshot": "base_setup",
- "instruction": "Could you help me compress the video to DIR_PATH?",
+ "instruction": "Could you help me compress the video to MPEG-4 format and save with name with prefix _?",
"source": "https://www.quora.com/How-do-I-compress-a-video-with-VLC",
"config": [],
"trajectory": "trajectories/",
diff --git a/evaluation_examples/examples/vlc/bba3381f-b5eb-4439-bd9e-80c22218d5a7.json b/evaluation_examples/examples/vlc/bba3381f-b5eb-4439-bd9e-80c22218d5a7.json
index 670aa5a..a5cd48d 100644
--- a/evaluation_examples/examples/vlc/bba3381f-b5eb-4439-bd9e-80c22218d5a7.json
+++ b/evaluation_examples/examples/vlc/bba3381f-b5eb-4439-bd9e-80c22218d5a7.json
@@ -1,7 +1,7 @@
{
"id": "bba3381f-b5eb-4439-bd9e-80c22218d5a7",
"snapshot": "base_setup",
- "instruction": "Help me play the online video at https://www.youtube.com/watch?v=pgBsyTKAwLw",
+ "instruction": "Can you start streaming the video from this link for me? https://www.youtube.com/watch?v=pgBsyTKAwLw",
"source": "https://www.quora.com/How-do-I-play-online-videos-using-the-VLC-media-player",
"config": [
{
diff --git a/evaluation_examples/examples/vlc/efcf0d81-0835-4880-b2fd-d866e8bc2294.json b/evaluation_examples/examples/vlc/efcf0d81-0835-4880-b2fd-d866e8bc2294.json
index 3238837..3b07f3c 100644
--- a/evaluation_examples/examples/vlc/efcf0d81-0835-4880-b2fd-d866e8bc2294.json
+++ b/evaluation_examples/examples/vlc/efcf0d81-0835-4880-b2fd-d866e8bc2294.json
@@ -1,30 +1,47 @@
{
"id": "efcf0d81-0835-4880-b2fd-d866e8bc2294",
"snapshot": "base_setup",
- "instruction": "Set this frame of the current video as my wallpaper",
- "source": "https://www.youtube.com/watch?v=XHprwDJ0-fU&t=436s",
+ "instruction": "Make this part of the video my computer's background picture",
+ "source": "https://www.youtube.com/watch?v=XHprwDJ0-fU&t=436s, https://help.ubuntu.com/stable/ubuntu-help/look-background.html.en",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
- "url": "",
- "path": ""
+ "url": "https://drive.usercontent.google.com/download?id=1H9D0jSkzpzEWsJZG0HdNNNHDMi3fnGli&export=download&authuser=0&confirm=t&uuid=a6d03223-db7a-48bd-b5f8-50b51f4d3d68&at=APZUnTVi3cqgvRV49c6sluV-nMo_:1704879367756",
+ "path": "Desktop/Interstellar Movie - Official Trailer.mp4"
}
]
}
},
{
"type": "launch",
- "parameters": {
- "command": ["vlc", "/path/to/your/video.mp4", "--start-time=0", "--run-time=10", "vlc://quit", "&&", "vlc", "/path/to/your/video.mp4", "--start-time=10"]
- }
+ "parameters": {
+ "command": [
+ "vlc",
+ "--start-time=120.5",
+ "--stop-time=121",
+ "--play-and-pause",
+ "Desktop/Interstellar Movie - Official Trailer.mp4"
+ ]
+ }
}
],
"trajectory": "trajectories/",
"related_apps": [
"vlc"
],
- "evaluator": "evaluation_dir"
+ "evaluator": {
+ "func": "compare_images",
+ "expected": {
+ "type": "cloud_file",
+ "path": "https://drive.usercontent.google.com/download?id=10P8kLkWeYhtK3Gl15nB-gH7VLO4S7Xal&export=download&authuser=0&confirm=t&uuid=d5c5db29-435d-404f-8900-683bf56fbc66&at=APZUnTXm1ZHS7CbGiWaB04WySH-t:1704891263184",
+ "dest": "interstellar_wallpaper_gold.png"
+ },
+ "result": {
+ "type": "vm_wallpaper",
+ "dest": "result_wallpaper.png"
+ }
+ }
}
diff --git a/evaluation_examples/examples/vlc/fba2c100-79e8-42df-ae74-b592418d54f4.json b/evaluation_examples/examples/vlc/fba2c100-79e8-42df-ae74-b592418d54f4.json
index f564f73..0cd51d9 100644
--- a/evaluation_examples/examples/vlc/fba2c100-79e8-42df-ae74-b592418d54f4.json
+++ b/evaluation_examples/examples/vlc/fba2c100-79e8-42df-ae74-b592418d54f4.json
@@ -1,12 +1,48 @@
{
"id": "fba2c100-79e8-42df-ae74-b592418d54f4",
"snapshot": "base_setup",
- "instruction": "Screenshot the current frame of the video",
+ "instruction": "Snap a photo of the current video scene, save it as 'interstellar.png', and put it on the Desktop, please.",
"source": "https://www.youtube.com/watch?v=XHprwDJ0-fU&t=436s",
- "config": [],
+ "config": [
+ {
+ "type": "download",
+ "parameters": {
+ "files": [
+ {
+ "url": "https://drive.usercontent.google.com/download?id=1u7CFVr5bWti7OnhvyjtSdQolzy0lYicm&export=download&authuser=0&confirm=t&uuid=18498a7c-9b52-4c00-a2cc-2e8b521c4d9c&at=APZUnTX0R_TRccU_UmtJWXZ6On3x:1704890439561",
+ "path": "Desktop/Interstellar Movie - Official Trailer.mp4"
+ }
+ ]
+ }
+ },
+ {
+ "type": "launch",
+ "parameters": {
+ "command": [
+ "vlc",
+ "--start-time=120.5",
+ "--stop-time=121",
+ "--play-and-pause",
+ "Desktop/Interstellar Movie - Official Trailer.mp4"
+ ]
+ }
+ }
+ ],
"trajectory": "trajectories/",
"related_apps": [
"vlc"
],
- "evaluator": "evaluation_dir"
+ "evaluator": {
+ "func": "compare_images",
+ "expected": {
+ "type": "cloud_file",
+ "path": "https://drive.usercontent.google.com/download?id=1shU1TQ3ao9QWfhmC63vbD10YFbhv7PYv&export=download&authuser=0&confirm=t&uuid=6a22a47a-6d35-4996-b02e-b24b33a1c449&at=APZUnTVlvDKpxW7QKmBBmdNq2jRw:1704891215314",
+ "dest": "interstellar_gold.png"
+ },
+ "result": {
+ "type": "vm_file",
+ "path": "Desktop/interstellar.png",
+ "dest": "interstellar.png"
+ }
+ }
}
diff --git a/requirements.txt b/requirements.txt
index 49268ac..558098b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -27,3 +27,4 @@ rapidfuzz
pyacoustid
opencv-python
ImageHash
+scikit-image