lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into zdy

Browse Source
This commit is contained in:
David Chang
2024-01-11 23:02:00 +08:00
parent 3c04872dcf b30d87bba8
commit 127a101994
39 changed files with 580 additions and 282 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
desktop_env/controllers/python.py
View File

@@ -228,6 +228,13 @@ class PythonController:
else:
raise Exception(f"Unknown action type: {action_type}")
# Additional info
def get_vm_platform(self):
"""
Gets the size of the vm screen.
"""
return self.execute_python_command("import platform; print(platform.system())")['output'].strip()
def get_vm_screen_size(self):
"""
Gets the size of the vm screen.
@@ -250,6 +257,19 @@ class PythonController:
logger.error("Failed to get window size. Status code: %d", response.status_code)
return None
def get_vm_wallpaper(self):
"""
Gets the wallpaper of the vm.
"""
response = requests.post(self.http_server + "/wallpaper")
if response.status_code == 200:
logger.info("Wallpaper downloaded successfully")
return response.content
else:
logger.error("Failed to get wallpaper. Status code: %d", response.status_code)
return None
# VLC
def get_vlc_status(self, host='localhost', port=8080, password='password'):
url = f'http://{host}:{port}/requests/status.xml'

16
desktop_env/envs/desktop_env.py
View File

@@ -35,7 +35,10 @@ def _execute_command(command: List[str]) -> None:
class DesktopEnv(gym.Env):
"""DesktopEnv with OpenAI Gym interface."""
"""
DesktopEnv with OpenAI Gym interface.
Fixme: refactor the logic when implementing the multi-process version
"""
def __init__(
self,
@@ -81,6 +84,10 @@ class DesktopEnv(gym.Env):
self.controller = PythonController(http_server=self.host)
self.setup_controller = SetupController(http_server=self.host, cache_dir=self.cache_dir)
# Meta info of the VM
self.vm_platform = self.controller.get_vm_platform()
self.vm_screen_size = self.controller.get_vm_screen_size()
# mode: human or machine
assert action_space in ["computer_13", "pyautogui"]
self.action_space = action_space
@@ -225,7 +232,12 @@ class DesktopEnv(gym.Env):
self.setup_controller.setup(self.evaluator.get("postconfig", []))
result_state = self.result_getter(self, self.evaluator["result"])
try:
result_state = self.result_getter(self, self.evaluator["result"])
except FileNotFoundError:
logger.error("File not found!")
return 0
expected_state = self.expected_getter(self, self.evaluator["expected"]) if "expected" in self.evaluator \
else None

6
desktop_env/evaluators/metrics/README.md → desktop_env/evaluators/README.md
View File

@@ -95,6 +95,12 @@ with sync_playwright() as playwright:
## VLC Media Player
### Bugs fix
One thing on Ubuntu need to do, enter into the `meida`>`convert/save`>select files>`convert/save`
Then enter the profile of `Audio - MP3`, change the profile for mp3, section audiocodec from "MP3" to "MPEG Audio"
Otherwise the mp3 file will be created but with 0 bytes. It's a bug of VLC.
### Setting Up VLC's HTTP Interface
To enable and use the HTTP interface in VLC Media Player for remote control and status checks, follow these steps:

2
desktop_env/evaluators/getters/__init__.py
View File

@@ -1,3 +1,5 @@
from .file import get_cloud_file, get_vm_file, get_cache_file
from .info import get_vm_screen_size, get_vm_window_size, get_vm_wallpaper
from .misc import get_rule, get_accessibility_tree
from .vlc import get_vlc_playing_info, get_vlc_config
from .chrome import get_default_search_engine, get_bookmarks

126
desktop_env/evaluators/getters/chrome.py Normal file
View File

@@ -0,0 +1,126 @@
import json
import logging
import os
import sqlite3
from typing import Dict
logger = logging.getLogger("desktopenv.getters.chrome")
"""
WARNING:
1. Functions from this script assume that no account is registered on Chrome, otherwise the default file path needs to be changed.
2. The functions are not tested on Windows and Mac, but they should work.
"""
def get_default_search_engine(env, config: Dict[str, str]):
os_type = env.vm_platform
if os_type == 'Windows':
preference_file_path = env.controller.execute_python_command("""import os; print(os.path.join(os.getenv('LOCALAPPDATA'),
'Google\\Chrome\\User Data\\Default\\Preferences'))""")['output'].strip()
elif os_type == 'Darwin':
preference_file_path = env.controller.execute_python_command("import os; print(os.path.join(os.getenv('HOME'), 'Library/Application Support/Google/Chrome/Default/Preferences'))")['output'].strip()
elif os_type == 'Linux':
preference_file_path = env.controller.execute_python_command("import os; print(os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Preferences'))")['output'].strip()
else:
raise Exception('Unsupported operating system')
try:
content = env.controller.get_file(preference_file_path)
data = json.loads(content)
# The path within the JSON data to the default search engine might vary
search_engine = data.get('default_search_provider_data', {}).get('template_url_data', {}).get('short_name',
'Google')
return search_engine
except Exception as e:
logger.error(f"Error: {e}")
return "Google"
def get_cookie_data(env, config: Dict[str, str]):
os_type = env.vm_platform
if os_type == 'Windows':
chrome_cookie_file_path = os.path.join(os.getenv('LOCALAPPDATA'), 'Google\\Chrome\\User Data\\Default\\Cookies')
elif os_type == 'Darwin':
chrome_cookie_file_path = os.path.join(os.getenv('HOME'),
'Library/Application Support/Google/Chrome/Default/Cookies')
elif os_type == 'Linux':
chrome_cookie_file_path = os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Cookies')
else:
raise Exception('Unsupported operating system')
# todo: add a new controller function to connect the cookie database
#############
try:
conn = sqlite3.connect(chrome_cookie_file_path)
cursor = conn.cursor()
# Query to check for OpenAI cookies
cursor.execute("SELECT * FROM cookies")
cookies = cursor.fetchall()
return cookies
except Exception as e:
logger.error(f"Error: {e}")
return None
#############
def get_bookmarks(env, config: Dict[str, str]):
os_type = env.vm_platform
if os_type == 'Windows':
preference_file_path = os.path.join(os.getenv('LOCALAPPDATA'),
'Google\\Chrome\\User Data\\Default\\Bookmarks')
elif os_type == 'Darwin':
preference_file_path = os.path.join(os.getenv('HOME'),
'Library/Application Support/Google/Chrome/Default/Bookmarks')
elif os_type == 'Linux':
preference_file_path = os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Bookmarks')
else:
raise Exception('Unsupported operating system')
try:
content = env.controller.get_file(preference_file_path)
# make content json variable
data = json.load(content)
bookmarks = data.get('roots', {})
return bookmarks
except Exception as e:
logger.error(f"Error: {e}")
return None
# todo: move this to the main.py
def get_extensions_installed_from_shop(env, config: Dict[str, str]):
"""Find the Chrome extensions directory based on the operating system."""
os_type = env.vm_platform
if os_type == 'Windows':
chrome_extension_dir = os.path.expanduser(
'~') + '\\AppData\\Local\\Google\\Chrome\\User Data\\Default\\Extensions\\'
elif os_type == 'Darwin': # macOS
chrome_extension_dir = os.path.expanduser(
'~') + '/Library/Application Support/Google/Chrome/Default/Extensions/'
elif os_type == 'Linux':
chrome_extension_dir = os.path.expanduser('~') + '/.config/google-chrome/Default/Extensions/'
else:
raise Exception('Unsupported operating system')
manifests = []
for extension_id in os.listdir(chrome_extension_dir):
extension_path = os.path.join(chrome_extension_dir, extension_id)
if os.path.isdir(extension_path):
# Iterate through version-named subdirectories
for version_dir in os.listdir(extension_path):
version_path = os.path.join(extension_path, version_dir)
manifest_path = os.path.join(version_path, 'manifest.json')
if os.path.isfile(manifest_path):
with open(manifest_path, 'r') as file:
try:
manifest = json.load(file)
manifests.append(manifest)
except json.JSONDecodeError:
logger.error(f"Error reading {manifest_path}")
return manifests

20
desktop_env/evaluators/getters/info.py Normal file
View File

@@ -0,0 +1,20 @@
import os
from typing import Union
def get_vm_screen_size(env, config: dict) -> dict:
return env.controller.get_vm_screen_size()
def get_vm_window_size(env, config: dict) -> dict:
return env.controller.get_vm_window_size(app_class_name=config["app_class_name"])
def get_vm_wallpaper(env, config: dict) -> Union[str, bytes]:
_path = os.path.join(env.cache_dir, config["dest"])
content = env.controller.get_vm_wallpaper()
with open(_path, "wb") as f:
f.write(content)
return _path

12
desktop_env/evaluators/getters/vlc.py
View File

@@ -9,12 +9,12 @@ def get_vlc_playing_info(env, config: Dict[str, str]):
"""
Gets the current playing information from VLC's HTTP interface.
"""
_path = os.path.join(env.cache_dir, config["dest"])
host = env.vm_ip
port = 8080
password = 'password'
_path = os.path.join(env.cache_dir, config["dest"])
content = env.controller.get_vlc_status(host, port, password)
with open(_path, "wb") as f:
f.write(content)
@@ -26,22 +26,24 @@ def get_vlc_config(env, config: Dict[str, str]):
"""
Reads the VLC configuration file to check setting.
"""
_path = os.path.join(env.cache_dir, config["dest"])
os_type = env.controller.execute_python_command("import platform; print(platform.system())")['output'].strip()
os_type = env.vm_platform
# fixme: depends on how we config and install the vlc in virtual machine, need to be aligned and double-checked
if os_type == "Linux":
config_path = \
env.controller.execute_python_command("import os; print(os.path.expanduser('~/.config/vlc/vlcrc'))")[
'output'].strip()
env.controller.execute_python_command("import os; print(os.path.expanduser('~/.config/vlc/vlcrc'))")[
'output'].strip()
elif os_type == "Darwin":
config_path = env.controller.execute_python_command(
"import os; print(os.path.expanduser('~/Library/Preferences/org.videolan.vlc/vlcrc'))")['output'].strip()
elif os_type == "Windows":
config_path = env.controller.execute_python_command(
"import os; print(os.path.expanduser('~\\AppData\\Roaming\\vlc\\vlcrc'))")['output'].strip()
else:
raise Exception("Unsupported operating system", os_type)
_path = os.path.join(env.cache_dir, config["dest"])
content = env.controller.get_file(config_path)
with open(_path, "wb") as f:
f.write(content)

13
desktop_env/evaluators/metrics/__init__.py
View File

@@ -1,10 +1,13 @@
from .table import compare_table
from .table import check_sheet_list, check_xlsx_freeze, check_xlsx_zoom
from .docs import find_default_font, contains_page_break, compare_docx_files, compare_docx_tables, compare_line_spacing, compare_insert_equation
from .docs import compare_font_names, compare_subscript_contains, has_page_numbers_in_footers
from .docs import find_default_font, contains_page_break, compare_docx_files, compare_docx_tables, compare_line_spacing, \
compare_insert_equation
from .docs import is_first_line_centered, check_file_exists, compare_contains_image
from .pdf import check_pdf_pages
from .general import exact_match, fuzzy_match, check_csv, check_accessibility_tree, check_list
from .libreoffice import check_libre_locale
from .vlc import is_vlc_playing, is_vlc_recordings_folder
from .pdf import check_pdf_pages
from .table import check_sheet_list, check_xlsx_freeze, check_xlsx_zoom
from .table import compare_table
from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, compare_images, compare_audios, \
compare_videos
from .general import check_csv, check_accessibility_tree, check_list
from .thunderbird import check_thunderbird_prefs, check_thunderbird_filter

116
desktop_env/evaluators/metrics/chrome.py
View File

@@ -1,127 +1,15 @@
import json
import os
import platform
import sqlite3
import logging
from playwright.sync_api import sync_playwright
import logging
logger = logging.getLogger("desktopenv.metrics.chrome")
"""
WARNING:
1. Functions from this script assume that no account is registered on Chrome, otherwise the default file path needs to be changed.
2. The functions are not tested on Windows and Mac, but they should work.
"""
# todo: move to getter module
# The following ones just need to load info from the files of software, no need to connect to the software
def get_default_search_engine():
if platform.system() == 'Windows':
preference_file_path = os.path.join(os.getenv('LOCALAPPDATA'),
'Google\\Chrome\\User Data\\Default\\Preferences')
elif platform.system() == 'Darwin':
preference_file_path = os.path.join(os.getenv('HOME'),
'Library/Application Support/Google/Chrome/Default/Preferences')
elif platform.system() == 'Linux':
preference_file_path = os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Preferences')
else:
raise Exception('Unsupported operating system')
try:
with open(preference_file_path, 'r', encoding='utf-8') as file:
data = json.load(file)
# The path within the JSON data to the default search engine might vary
search_engine = data.get('default_search_provider_data', {}).get('template_url_data', {}).get('short_name',
'Google')
return search_engine
except Exception as e:
logger.error(f"Error: {e}")
return "Google"
def get_cookie_data():
if platform.system() == 'Windows':
chrome_cookie_file_path = os.path.join(os.getenv('LOCALAPPDATA'), 'Google\\Chrome\\User Data\\Default\\Cookies')
elif platform.system() == 'Darwin':
chrome_cookie_file_path = os.path.join(os.getenv('HOME'),
'Library/Application Support/Google/Chrome/Default/Cookies')
elif platform.system() == 'Linux':
chrome_cookie_file_path = os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Cookies')
else:
raise Exception('Unsupported operating system')
try:
conn = sqlite3.connect(chrome_cookie_file_path)
cursor = conn.cursor()
# Query to check for OpenAI cookies
cursor.execute("SELECT * FROM cookies")
cookies = cursor.fetchall()
return cookies
except Exception as e:
logger.error(f"Error: {e}")
return None
def get_bookmarks():
if platform.system() == 'Windows':
preference_file_path = os.path.join(os.getenv('LOCALAPPDATA'),
'Google\\Chrome\\User Data\\Default\\Bookmarks')
elif platform.system() == 'Darwin':
preference_file_path = os.path.join(os.getenv('HOME'),
'Library/Application Support/Google/Chrome/Default/Bookmarks')
elif platform.system() == 'Linux':
preference_file_path = os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Bookmarks')
else:
raise Exception('Unsupported operating system')
try:
with open(preference_file_path, 'r', encoding='utf-8') as file:
data = json.load(file)
bookmarks = data.get('roots', {})
return bookmarks
except Exception as e:
logger.error(f"Error: {e}")
return None
def get_extensions_installed_from_shop():
"""Find the Chrome extensions directory based on the operating system."""
os_name = platform.system()
if os_name == 'Windows':
chrome_extension_dir = os.path.expanduser(
'~') + '\\AppData\\Local\\Google\\Chrome\\User Data\\Default\\Extensions\\'
elif os_name == 'Darwin': # macOS
chrome_extension_dir = os.path.expanduser(
'~') + '/Library/Application Support/Google/Chrome/Default/Extensions/'
elif os_name == 'Linux':
chrome_extension_dir = os.path.expanduser('~') + '/.config/google-chrome/Default/Extensions/'
else:
raise Exception('Unsupported operating system')
manifests = []
for extension_id in os.listdir(chrome_extension_dir):
extension_path = os.path.join(chrome_extension_dir, extension_id)
if os.path.isdir(extension_path):
# Iterate through version-named subdirectories
for version_dir in os.listdir(extension_path):
version_path = os.path.join(extension_path, version_dir)
manifest_path = os.path.join(version_path, 'manifest.json')
if os.path.isfile(manifest_path):
with open(manifest_path, 'r') as file:
try:
manifest = json.load(file)
manifests.append(manifest)
except json.JSONDecodeError:
logger.error(f"Error reading {manifest_path}")
return manifests
# The following ones require Playwright to be installed on the target machine, and the chrome needs to be pre-config on port info to allow remote debugging, see README.md for details

63
desktop_env/evaluators/metrics/general.py
View File

@@ -1,22 +1,32 @@
import csv
import functools
import operator
import re
from numbers import Number
from typing import Callable, Any
from typing import Dict, List, Pattern
import lxml.etree
from lxml.etree import _Element
from lxml.cssselect import CSSSelector
from typing import Dict, List, Pattern
from typing import Callable, Any
from numbers import Number
import operator
from lxml.etree import _Element
from rapidfuzz import fuzz
import functools
import re
from .utils import _match_record
#def _match_record(pattern: Dict[str, str], item: Dict[str, str]) -> bool:
#return all(k in item and item[k]==val for k, val in pattern.items())
def exact_match(result, rules) -> float:
expect = rules["expected"]
print(result, expect)
if result == expect:
return 1.
else:
return 0.
def fuzzy_match(result, rules) -> float:
expect = rules["expected"]
return fuzz.ratio(result, expect) / 100.
def check_csv(result: str, rules: Dict[str, List[Dict[str, str]]]) -> float:
"""
@@ -46,6 +56,7 @@ def check_csv(result: str, rules: Dict[str, List[Dict[str, str]]]) -> float:
unexpect_metric = unexpect_metric and not any(_match_record(r, rcd) for r in rules.get("unexpect", []))
return float(all(expect_metrics) and unexpect_metric)
def check_list(result: str, rules: Dict[str, List[str]]) -> float:
"""
Args:
@@ -75,15 +86,18 @@ def check_list(result: str, rules: Dict[str, List[str]]) -> float:
unexpect_metric = unexpect_metric and all(r.search(l) is None for r in unexpect_patterns)
return float(all(expect_metrics) and unexpect_metric)
_accessibility_ns_map = { "st": "uri:deskat:state.at-spi.gnome.org"
, "attr": "uri:deskat:attributes.at-spi.gnome.org"
, "cp": "uri:deskat:component.at-spi.gnome.org"
, "doc": "uri:deskat:document.at-spi.gnome.org"
, "docattr": "uri:deskat:attributes.document.at-spi.gnome.org"
, "txt": "uri:deskat:text.at-spi.gnome.org"
, "val": "uri:deskat:value.at-spi.gnome.org"
, "act": "uri:deskat:action.at-spi.gnome.org"
}
_accessibility_ns_map = {"st": "uri:deskat:state.at-spi.gnome.org"
, "attr": "uri:deskat:attributes.at-spi.gnome.org"
, "cp": "uri:deskat:component.at-spi.gnome.org"
, "doc": "uri:deskat:document.at-spi.gnome.org"
, "docattr": "uri:deskat:attributes.document.at-spi.gnome.org"
, "txt": "uri:deskat:text.at-spi.gnome.org"
, "val": "uri:deskat:value.at-spi.gnome.org"
, "act": "uri:deskat:action.at-spi.gnome.org"
}
def check_accessibility_tree(result: str, rules: Dict[str, Any]) -> float:
"""
Args:
@@ -114,11 +128,12 @@ def check_accessibility_tree(result: str, rules: Dict[str, Any]) -> float:
else:
raise ValueError("At least one of xpath and selectors is required")
if len(elements)==0:
if len(elements) == 0:
return 0.
if "text" in rules:
match_func: Callable[[str], Number] = functools.partial( operator.eq if rules["exact"] else fuzz.ratio
match_func: Callable[[str], Number] = functools.partial( operator.eq if rules["exact"]\
else (lambda a, b: fuzz.ratio(a, b)/100.)
, rules["text"]
)
match_score: Number = 0
@@ -129,5 +144,5 @@ def check_accessibility_tree(result: str, rules: Dict[str, Any]) -> float:
return float(match_score)
#def check_existence(result: str, *args) -> float:
#return 1. - (result is None)
# def check_existence(result: str, *args) -> float:
# return 1. - (result is None)

73
desktop_env/evaluators/metrics/vlc.py
View File

@@ -7,7 +7,10 @@ from xml.etree import ElementTree
import acoustid
import cv2
import imagehash
import librosa
import numpy as np
from PIL import Image
from skimage.metrics import structural_similarity as ssim
logger = logging.getLogger("desktopenv.metrics.vlc")
@@ -37,6 +40,7 @@ def is_vlc_playing(actual_status_path: str, rule: Dict[str, str]) -> float:
return 0
# fixme: part of this function can be moved to getters
def is_vlc_recordings_folder(actual_config_path: str, rule: Dict[str, str]) -> float:
"""
Checks if VLC's recording folder is set to the expected value.
@@ -57,24 +61,79 @@ def is_vlc_recordings_folder(actual_config_path: str, rule: Dict[str, str]) -> f
current_path = line.split('=')[-1].strip()
# Compare with the Desktop path
if current_path == expected_recording_file_path:
return True
return 1
else:
return False
return 0
# The configuration key was not found in the file
return False
return 0
except FileNotFoundError:
logger.error("VLC configuration file not found.")
return False
return 0
except Exception as e:
logger.error(f"An error occurred: {e}")
return False
return 0
def is_vlc_fullscreen(actual_window_size, screen_size):
if actual_window_size['width'] == screen_size['width'] and actual_window_size['height'] == screen_size['height']:
return True
return 1
else:
return False
return 0
def compare_images(image1_path, image2_path):
# You would call this function with the paths to the two images you want to compare:
# score = compare_images('path_to_image1', 'path_to_image2')
# print("Similarity score:", score)
# Open the images and convert to grayscale
image1 = Image.open(image1_path).convert('L')
image2 = Image.open(image2_path).convert('L')
# Resize images to the smaller one's size for comparison
image1_size = image1.size
image2_size = image2.size
new_size = min(image1_size, image2_size)
image1 = image1.resize(new_size, Image.Resampling.LANCZOS)
image2 = image2.resize(new_size, Image.Resampling.LANCZOS)
# Convert images to numpy arrays
image1_array = np.array(image1)
image2_array = np.array(image2)
# Calculate SSIM between two images
similarity_index = ssim(image1_array, image2_array)
return similarity_index
def compare_audios(audio_path_1, audio_path_2, max_distance=1000):
"""
Compare two audio files and return a similarity score in the range [0, 1].
audio_path_1, audio_path_2: paths to the audio files to compare
max_distance: an empirically determined maximum expected DTW distance
"""
# Example Usage:
# similarity = compare_audios_simple('path_to_audio1.mp3', 'path_to_audio2.mp3')
# print(f'Similarity Score: {similarity}')
# Convert to common format if necessary and load audio
y1, sr1 = librosa.load(audio_path_1)
y2, sr2 = librosa.load(audio_path_2)
# Extract MFCC features
mfcc1 = librosa.feature.mfcc(y=y1, sr=sr1)
mfcc2 = librosa.feature.mfcc(y=y2, sr=sr2)
# Compute Dynamic Time Warping distance
distance, path = librosa.sequence.dtw(mfcc1.T, mfcc2.T)
# Normalize distance to get a similarity score
normalized_distance = np.mean(distance) / max_distance
similarity_score = 1 - min(normalized_distance, 1) # Ensure the score is within [0, 1]
return similarity_score
def compare_videos(video_path1, video_path2, max_frames_to_check=100, threshold=5):

73
desktop_env/server/README.md Normal file
View File

@@ -0,0 +1,73 @@
<!-- vimc: call SyntaxRange#Include('```xml', '```', 'xml', 'NonText'): -->
<!-- vimc: call SyntaxRange#Include('```css', '```', 'css', 'NonText'): -->
<!-- vimc: call SyntaxRange#Include('```sh', '```', 'sh', 'NonText'): -->
### About the Converted Accessibility Tree
For several applications like Firefox or Thunderbird, you should first enable
```sh
gsettings set org.gnome.desktop.interface toolkit-accessibility true
```
to see their accessibility tree.
#### Example of AT
An example of a node:
```xml
<section xmlns:attr="uri:deskat:attributes.at-spi.gnome.org" attr:class="subject" st:enabled="true" cp:screencoord="(1525, 169)", cp:windowcoord="(342, 162)", cp:size="(327, 21)">
歡迎使用新的 Outlook.com 帳戶
</section>
```
An example of a tree:
```xml
<desktop-frame ...>
<application name="Thunderbird" ...>
... <!-- nodes of windows -->
</application>
...
</desktop-frame>
```
#### Useful attributes
1. `name` - shows the name of application, title of window, or name of some
component
2. `attr:class` - somewhat the same role as `class` in HTML
3. `attr:id` - somewhat the same role as `id` in HTML
4. `cp:screencoord` - absolute coordinator on the screen
5. `cp:windowcoord` - relative coordinator in the window
6. `cp:size` - the size
Also several states like `st:enabled` and `st:visible` can be indicated. A full
state list is available at
<https://gitlab.gnome.org/GNOME/pyatspi2/-/blob/master/pyatspi/state.py?ref_type=heads>.
#### How to use it in evaluation
See example `thunderbird/12086550-11c0-466b-b367-1d9e75b3910e.json` and
function `check_accessibility_tree` in `metrics/general.py`. You can use CSS
selector or XPath to reference a target nodes. You can also check its text
contents.
An example of a CSS selector:
```css
application[name=Thunderbird] page-tab-list[attr|id=\"tabmail-tabs\"]>page-tab[name=\"About Profiles\"]
```
This selector will select the page tab of profile manager in Thunderbird (if open).
For usage of CSS selector: <https://www.w3.org/TR/selectors-3/>. For usage of XPath: <https://www.w3.org/TR/xpath-31/>.
#### Manual check
You can use accerciser to check the accessibility tree on GNOME VM.
```sh
sudo apt install accerciser
```

3
desktop_env/server/main.py
View File

@@ -360,8 +360,7 @@ def get_wallpaper():
if wallpaper_path:
try:
# Ensure the filename is secure
filename = secure_filename(os.path.basename(wallpaper_path))
return send_file(wallpaper_path, attachment_filename=filename)
return send_file(wallpaper_path, mimetype='image/png')
except Exception as e:
app.logger.error(f"An error occurred while serving the wallpaper file: {e}")
abort(500, description="Unable to serve the wallpaper file")

2
evaluation_examples/examples/chrome/06fe7178-4491-4589-810f-2e2bc9502122.json
View File

@@ -1,7 +1,7 @@
{
"id": "06fe7178-4491-4589-810f-2e2bc9502122",
"snapshot": "chrome",
"instruction": "Could you help me open the previously closed tab?",
"instruction": "Can you make my computer bring back the last tab I shut down?",
"source": "https://www.wikihow.com/Switch-Tabs-in-Chrome",
"config": [],
"trajectory": "trajectories/",

2
evaluation_examples/examples/chrome/12086550-11c0-466b-b367-1d9e75b3910e.json
View File

@@ -1,7 +1,7 @@
{
"id": "12086550-11c0-466b-b367-1d9e75b3910e",
"snapshot": "chrome",
"instruction": "Help me find the hidden password of xx website.",
"instruction": "Computer, please navigate to the area in my browser settings where my passwords are stored. I want to check my login information for Etsy without revealing it just yet.",
"source": "https://www.quora.com/What-are-the-cool-tricks-to-use-Google-Chrome",
"config": [],
"trajectory": "trajectories/",

2
evaluation_examples/examples/chrome/7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3.json
View File

@@ -1,7 +1,7 @@
{
"id": "7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3",
"snapshot": "chrome",
"instruction": "Could you help me delete cookies from openai website?",
"instruction": "Can you help me clean up my computer by getting rid of all the tracking things that websites like Amazon or eBay might have saved? I want to make sure my browsing is private and those sites don't remember me.",
"source": "https://support.google.com/chrome/answer/95647?hl=en&ref_topic=7438325&sjid=16867045591165135686-AP#zippy=%2Cdelete-cookies-from-a-site",
"config": [],
"trajectory": "trajectories/",

21
evaluation_examples/examples/chrome/bb5e4c0d-f964-439c-97b6-bdb9747de3f4.json
View File

@@ -1,18 +1,33 @@
{
"id": "bb5e4c0d-f964-439c-97b6-bdb9747de3f4",
"snapshot": "chrome",
"instruction": "Could you help me set my default search engine to be Google?",
"instruction": "Can you make Bing the main search thingy when I look stuff up on the internet?",
"source": "https://support.google.com/chrome/answer/95426?sjid=16867045591165135686-AP",
"config": [],
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=9222"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "",
"func": "exact_match",
"result": {
"type": "default_search_engine"
},
"expected": {
"type": "rule",
"rules": {
"expected": "Bing"
}
}
}
}

10
evaluation_examples/examples/libreoffice_writer/0810415c-bde4-4443-9047-d5f70165a697.json
View File

@@ -10,7 +10,7 @@
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1-svVsH-l2ofufEKuN-cYrIrvXNobtATE&export=download&authuser=0&confirm=t&uuid=be7f891a-f858-48f5-a72d-4e42bbfb8b65&at=APZUnTXzBnaeSJjmxeh4zG03pzA0:1704179807785",
"path": "Desktop/Double_Line_Spacing.docx"
"path": "Desktop/Novels_Intro_Packet.docx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
"path": "Desktop/Double_Line_Spacing.docx"
"path": "Desktop/Novels_Intro_Packet.docx"
}
}
],
@@ -31,12 +31,12 @@
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1-svVsH-l2ofufEKuN-cYrIrvXNobtATE&export=download&authuser=0&confirm=t&uuid=be7f891a-f858-48f5-a72d-4e42bbfb8b65&at=APZUnTXzBnaeSJjmxeh4zG03pzA0:1704179807785",
"dest": "Double_Line_Spacing_Gold.docx"
"dest": "Novels_Intro_Packet_Gold.docx"
},
"result": {
"type": "vm_file",
"path": "Desktop/Double_Line_Spacing.docx",
"dest": "Double_Line_Spacing.docx"
"path": "Desktop/Novels_Intro_Packet.docx",
"dest": "Novels_Intro_Packet.docx"
}
}
}

16
evaluation_examples/examples/libreoffice_writer/0b17a146-2934-46c7-8727-73ff6b6483e8.json
View File

@@ -1,7 +1,7 @@
{
"id": "0b17a146-2934-46c7-8727-73ff6b6483e8",
"snapshot": "libreoffice_writer",
"instruction": "Change the 2 in H2O to a subscript.",
"instruction": "Help me change the 2 in \"H2O\" to a subscript.",
"source": "https://askubuntu.com/questions/245695/how-do-you-insert-subscripts-and-superscripts-into-ordinary-non-formula-text-i",
"config": [
{
@@ -9,8 +9,8 @@
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1Nx5AoKNM7tcDRE6y_qjNIDrPOKqhNyfm&export=download&authuser=0&confirm=t&uuid=bb4de348-3bbf-46a2-95b2-e2719c67547a&at=APZUnTUeA-BW7mkQsEw7NGm272zx:1704172916742",
"path": "Desktop/Enter_Subscript.docx"
"url": "https://drive.usercontent.google.com/download?id=1FkorQBeTJ5L2jLuvu4YxHSlBMK4VEEG6&export=download&authuser=0&confirm=t&uuid=cc63dc0b-bae7-4ef6-a40d-e2da721976ef&at=APZUnTWyPZlZPFlqGTWAWXWmS04c:1704976667765",
"path": "Desktop/H2O_Factsheet_WA.docx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
"path": "Desktop/Enter_Subscript.docx"
"path": "Desktop/H2O_Factsheet_WA.docx"
}
}
],
@@ -30,13 +30,13 @@
"func": "compare_docx_files",
"result": {
"type": "vm_file",
"path": "Desktop/Enter_Subscript.docx",
"dest": "Enter_Subscript.docx"
"path": "Desktop/H2O_Factsheet_WA.docx",
"dest": "H2O_Factsheet_WA.docx"
},
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1AaKXeD9ZgfMykgijZ4G8MEzUjmMJElkq&export=download&authuser=0&confirm=t&uuid=5e347f0d-4efc-4478-878e-d89455d1593b&at=APZUnTWCYWfsD4eCeG52VJiK8-xB:1704172886196",
"dest": "Enter_Subscript_Gold.docx"
"path": "https://drive.usercontent.google.com/download?id=1dM_FSTGDWxSW64VEth_wKMYNkvw0y_tq&export=download&authuser=0&confirm=t&uuid=342f41e2-f48f-41ff-8942-f7dfe5de1dba&at=APZUnTXHfskcX3tvmrSbzCOyQIgb:1704976694506",
"dest": "H2O_Factsheet_WA_Gold.docx"
}
}
}

8
evaluation_examples/examples/libreoffice_writer/0e47de2a-32e0-456c-a366-8c607ef7a9d2.json
View File

@@ -10,7 +10,7 @@
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1aDWe-vAmcfQSgtPjFfrncq8ZFnCy4uUK&export=download&authuser=0&confirm=t&uuid=788af72a-ddaf-4ba3-aedb-96f34cc4d815&at=APZUnTVSRSSfMGcjXqLzvMixnkp6:1704179663299",
"path": "Desktop/Add_Page_Number_Bottom_Left.docx"
"path": "Desktop/LibreOffice_Open_Source_Word_Processing.docx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
"path": "Desktop/Add_Page_Number_Bottom_Left.docx"
"path": "Desktop/LibreOffice_Open_Source_Word_Processing.docx"
}
}
],
@@ -30,8 +30,8 @@
"func": "has_page_numbers_in_footers",
"result": {
"type": "vm_file",
"path": "Desktop/Add_Page_Number_Bottom_Left.docx",
"dest": "Add_Page_Number_Bottom_Left.docx"
"path": "Desktop/LibreOffice_Open_Source_Word_Processing.docx",
"dest": "LibreOffice_Open_Source_Word_Processing.docx"
}
}
}

8
evaluation_examples/examples/libreoffice_writer/0e763496-b6bb-4508-a427-fad0b6c3e195.json
View File

@@ -10,7 +10,7 @@
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1fIHNzFm8JabWoLKOnxrFM722fQ1d_huK&export=download&authuser=0&confirm=t&uuid=d11a8dda-1e4e-4dc9-b05c-e6b47624dbf0&at=APZUnTVG0ViFnKJa00314wVr3uP9:1704185871014",
"path": "Desktop/Change_Font_Through_File.docx"
"path": "Desktop/Dublin_Zoo_Intro.docx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
"path": "Desktop/Change_Font_Through_File.docx"
"path": "Desktop/Dublin_Zoo_Intro.docx"
}
}
],
@@ -36,8 +36,8 @@
},
"result": {
"type": "vm_file",
"path": "Desktop/Change_Font_Through_File.docx",
"dest": "Change_Font_Through_File.docx"
"path": "Desktop/Dublin_Zoo_Intro.docx",
"dest": "Dublin_Zoo_Intro.docx"
}
}
}

12
evaluation_examples/examples/libreoffice_writer/3ef2b351-8a84-4ff2-8724-d86eae9b842e.json
View File

@@ -1,7 +1,7 @@
{
"id": "3ef2b351-8a84-4ff2-8724-d86eae9b842e",
"snapshot": "libreoffice_writer",
"instruction": "center-justify the first line",
"instruction": "Help me center align the heading in LibreOffice.",
"source": "https://askubuntu.com/questions/1066351/how-do-you-center-align-in-libreoffice#:~:text=Ctrl%20%2B%20e%20will%20Center%20align%20the%20cursor%20for%20you.",
"config": [
{
@@ -9,8 +9,8 @@
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1IQ4rKyHMOui71YlyL7huggpLYFYtj923&export=download&authuser=0&confirm=t&uuid=014c2335-c0c6-4712-9d5a-ca8d3217e07f&at=APZUnTVrM698NQgSh4hqYXR8cjDc:1704185072996",
"path": "Desktop/Centering_First_Line.docx"
"url": "https://drive.usercontent.google.com/download?id=1P8QodvDF-3S50rx6UmW4M2D4Kr-p_Q-h&export=download&authuser=0&confirm=t&uuid=eea70a33-4c3f-4e90-885d-dd3df0d605bc&at=APZUnTX7ISvBhOICNrPLoqK0m3G-:1704971931660",
"path": "Desktop/Constitution_Template_With_Guidelines.docx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
"path": "Desktop/Centering_First_Line.docx"
"path": "Desktop/Constitution_Template_With_Guidelines.docx"
}
}
],
@@ -30,8 +30,8 @@
"func": "is_first_line_centered",
"result": {
"type": "vm_file",
"path": "Desktop/Centering_First_Line.docx",
"dest": "Centering_First_Line.docx"
"path": "Desktop/Constitution_Template_With_Guidelines.docx",
"dest": "Constitution_Template_With_Guidelines.docx"
}
}
}

18
evaluation_examples/examples/libreoffice_writer/45d61a06-6545-4422-97b7-bc76cfa964c1.json
View File

@@ -1,7 +1,7 @@
{
"id": "45d61a06-6545-4422-97b7-bc76cfa964c1",
"snapshot": "libreoffice_writer",
"instruction": "Replace all newlines with paragraph marks in LibreOffice Write",
"instruction": "Replace all newlines with paragraph marks in LibreOffice Writer",
"source": "https://stackoverflow.com/questions/71685737/how-to-replace-all-newlines-with-paragraph-marks-in-libreoffice-write",
"config": [
{
@@ -9,8 +9,8 @@
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=18XFjPVUnLG_-KOM5sn-Sk74HP_JHivMy&export=download&authuser=0&confirm=t&uuid=d23041bc-2ddd-42c4-84ae-481b953f021c&at=APZUnTVYh0AK0245qsDOCol7SdMB:1704185512767",
"path": "Desktop/Replace_Newlines_with_Paragraph_Marks.docx"
"url": "https://drive.usercontent.google.com/download?id=16lQcSkw-JQ_v8Sg0HkCtnOzyK-4cok8N&export=download&authuser=0&confirm=t&uuid=a3f833ae-2572-4cf3-8a21-6d250e689415&at=APZUnTUfn24NGMtXEzz2Nf7cFLjt:1704975857398",
"path": "Desktop/NOVEL_Submission_Guidelines.docx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
"path": "Desktop/Replace_Newlines_with_Paragraph_Marks.docx"
"path": "Desktop/NOVEL_Submission_Guidelines.docx"
}
}
],
@@ -30,13 +30,13 @@
"func": "compare_line_spacing",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1bP_noic02MuzrM8CdJIQN7F1gN4N8sel&export=download&authuser=0&confirm=t&uuid=657e0e4f-7b96-4d7e-83f4-99b79c68708f&at=APZUnTX7HsmefsMlzQaCGK2fg5Em:1704185514197",
"dest": "Replace_Newlines_with_Paragraph_Marks_Gold.docx"
"path": "https://drive.usercontent.google.com/download?id=1LaS5ObaOsbyKX1M17vi8ZseZwawgvNmf&export=download&authuser=0&confirm=t&uuid=db523e81-fa22-4002-97ff-e5dff92106a7&at=APZUnTVnT3ZYOGW7ZQdeW4SZP7mX:1704975993684",
"dest": "NOVEL_Submission_Guidelines_Gold.docx"
},
"result": {
"type": "vm_file",
"path": "Desktop/Double_Line_Spacing.docx",
"dest": "Replace_Newlines_with_Paragraph_Marks.docx"
"path": "Desktop/NOVEL_Submission_Guidelines.docx",
"dest": "NOVEL_Submission_Guidelines.docx"
}
}
}
}

8
evaluation_examples/examples/libreoffice_writer/4bcb1253-a636-4df4-8cb0-a35c04dfef31.json
View File

@@ -9,8 +9,8 @@
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1czb_13MshoiM-yCxvUGYD8OnIIrf-3VX&export=download&authuser=0&confirm=t&uuid=e7c30b67-7fac-4b64-a222-d04bc7c82842&at=APZUnTUA1te5vt7L__zJ7xuMs48e:1704177347643",
"path": "Desktop/Save_Writer_PDF.docx"
"url": "https://drive.usercontent.google.com/download?id=1cn3CaA_4ds1WY8SgBT3TvjRunxocCBiu&export=download&authuser=0&confirm=t&uuid=622f5ae1-6f21-4f31-8a3e-e4ead6ea6bc3&at=APZUnTVKOV40Ww5PovU7at2ELzb9:1704949558060",
"path": "Desktop/View_Person_Organizational_Summary.docx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
"path": "Desktop/Save_Writer_PDF.docx"
"path": "Desktop/View_Person_Organizational_Summary.docx"
}
}
],
@@ -28,7 +28,7 @@
],
"evaluator": {
"func": "check_file_exists",
"file_name": "Save_Writer_PDF.pdf",
"file_name": "View_Person_Organizational_Summary.pdf",
"directory": "/home/user/Downloads/"
}
}

16
evaluation_examples/examples/libreoffice_writer/663876c7-3471-43db-ba51-f410b13d9d7d.json
View File

@@ -1,7 +1,7 @@
{
"id": "663876c7-3471-43db-ba51-f410b13d9d7d",
"snapshot": "libreoffice_writer",
"instruction": "Insert the equation \"(a + b)^2 = a^2 + 2 a b + b^2\"",
"instruction": "Insert the equation \"(a + b)^2 = a^2 + 2 a b + b^2\" at the position of the cursor.",
"source": "https://askubuntu.com/questions/319593/how-to-type-science-equations-in-libre-office",
"config": [
{
@@ -9,8 +9,8 @@
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1FgMp7Ny63eXzeF23qHYhqQux31djlkah&export=download&authuser=0&confirm=t&uuid=d6b5208d-3b3a-4972-a641-ed738a419fdb&at=APZUnTX16Fz8Qg-B0NWpWgC-3Dyu:1704184410221",
"path": "Desktop/Insert_Equation.docx"
"url": "https://drive.usercontent.google.com/download?id=1J3--srr-Fmt2z-wOAyPDmO5GoiJMkUvZ&export=download&authuser=0&confirm=t&uuid=5f275f3d-ab7d-4d27-848a-8012af4bcf7f&at=APZUnTVe5ucwe1Q0oVi9jwiY5PF7:1704971494071",
"path": "Desktop/Factoring_Perfect_Square_Trinomials.docx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
"path": "Desktop/Insert_Equation.docx"
"path": "Desktop/Factoring_Perfect_Square_Trinomials.docx"
}
}
],
@@ -30,13 +30,13 @@
"func": "compare_insert_equation",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1hMFJnwHs7Iaexz3b9O2LJQUfsJ2wiwZ9&export=download&authuser=0&confirm=t&uuid=2abb49fb-d9c7-46cf-bc21-e69ecb9cefc6&at=APZUnTVzEZjChcUb4MIoxuq4cGea:1704184411805",
"dest": "Insert_Equation_Gold.docx"
"path": "https://drive.usercontent.google.com/download?id=1QGfyQUPEjOX8osycnjR8E5yHeIEGccKI&export=download&authuser=0&confirm=t&uuid=174491d6-7855-4323-9fa5-5146fde48c4a&at=APZUnTXA8bIcIbtw9mUN-K-IU9bT:1704971524974",
"dest": "Factoring_Perfect_Square_Trinomials_Gold.docx"
},
"result": {
"type": "vm_file",
"path": "Desktop/Insert_Equation.docx",
"dest": "Insert_Equation.docx"
"path": "Desktop/Factoring_Perfect_Square_Trinomials.docx",
"dest": "Factoring_Perfect_Square_Trinomials.docx"
}
}
}

12
evaluation_examples/examples/libreoffice_writer/66399b0d-8fda-4618-95c4-bfc6191617e9.json
View File

@@ -1,7 +1,7 @@
{
"id": "66399b0d-8fda-4618-95c4-bfc6191617e9",
"snapshot": "libreoffice_writer",
"instruction": "Insert a 7*5 empty table",
"instruction": "Could you help me insert a 7*5 empty table at the point of cursor?",
"source": "https://www.youtube.com/watch?v=l25Evu4ohKg",
"config": [
{
@@ -10,7 +10,7 @@
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1kXBP0jMxTeVahzFLYbYHJtjjgmuzrA8R&export=download&authuser=0&confirm=t&uuid=f8b9bad3-415d-4d39-a4fb-05a4cf881cf0&at=APZUnTXaohwzl8_2RDF_tgUsP9cH:1704181463579",
"path": "Desktop/Insert_Empty_Table.docx"
"path": "Desktop/Table_Of_Work_Effort_Instructions.docx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
"path": "Desktop/Insert_Empty_Table.docx"
"path": "Desktop/Table_Of_Work_Effort_Instructions.docx"
}
}
],
@@ -31,12 +31,12 @@
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=14JfHsW9GvbhORdtVAtvEbOi00MqEyHfb&export=download&authuser=0&confirm=t&uuid=3dba2459-ac37-4cad-a982-adecd406382a&at=APZUnTVQUqUPq_WacgY2xu4PvAKB:1704181465512",
"dest": "Insert_Empty_Table_Gold.docx"
"dest": "Table_Of_Work_Effort_Instructions_Gold.docx"
},
"result": {
"type": "vm_file",
"path": "Desktop/Insert_Empty_Table.docx",
"dest": "Insert_Empty_Table.docx"
"path": "Desktop/Table_Of_Work_Effort_Instructions.docx",
"dest": "Table_Of_Work_Effort_Instructions.docx"
}
}
}

16
evaluation_examples/examples/libreoffice_writer/6ada715d-3aae-4a32-a6a7-429b2e43fb93.json
View File

@@ -1,7 +1,7 @@
{
"id": "6ada715d-3aae-4a32-a6a7-429b2e43fb93",
"snapshot": "libreoffice_writer",
"instruction": "Copy the screenshot 1.jpg from the desktop to where my cursor is locatedInsert the image which is in IMAGE_PATH where my cursor is",
"instruction": "Copy the screenshot 1.png from the desktop to where my cursor is located",
"source": "https://www.quora.com/How-do-you-insert-images-into-a-LibreOffice-Writer-document",
"config": [
{
@@ -10,7 +10,7 @@
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1ebLG0gmqYv41ek4UmKWhFsxBnoUSGjKp&export=download&authuser=0&confirm=t&uuid=8f7d7bee-1fe4-4c4c-8b69-8aaf47199c57&at=APZUnTVYUvYTopUXCVs69QWWwPbq:1704173993139",
"path": "Desktop/Insert_Image_At_Cursor.docx"
"path": "Desktop/Viewing_Your_Class_Schedule_and_Textbooks.docx"
}
]
}
@@ -20,8 +20,8 @@
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1QfjQ4SKtjKDXWpqa2u6mC_KtB3ASEK5O&export=download&authuser=0&confirm=t&uuid=06af00b9-58f3-4691-a6a3-34309c80cbbb&at=APZUnTVZpE1lMxcvGG0cdt5zuxZ_:1704174003198",
"path": "Desktop/1.jpg"
"url": "https://drive.usercontent.google.com/download?id=1-wYHrgPMZl5Y9Losd4AGkMVHWNU7Uemg&export=download&authuser=0&confirm=t&uuid=3bc52ab7-1739-407d-9b14-36c9305c973e&at=APZUnTW95CwED3zm5KRREFmO2mc2:1704952228530",
"path": "Desktop/1.png"
}
]
}
@@ -29,7 +29,7 @@
{
"type": "open",
"parameters": {
"path": "Desktop/Insert_Image_At_Cursor.docx"
"path": "Desktop/Viewing_Your_Class_Schedule_and_Textbooks.docx"
}
}
],
@@ -41,13 +41,13 @@
"func": "compare_contains_image",
"result": {
"type": "vm_file",
"path": "Desktop/Insert_Image_At_Cursor.docx",
"dest": "Insert_Image_At_Cursor.docx"
"path": "Desktop/Viewing_Your_Class_Schedule_and_Textbooks.docx",
"dest": "Viewing_Your_Class_Schedule_and_Textbooks.docx"
},
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1xbhlfqGrPutHHi2aHg66jwXD-yaZpe9j&export=download&authuser=0&confirm=t&uuid=427765e0-3f97-4a72-92db-a1fe7cdde73b&at=APZUnTUhNLh2PDu4OGkCVQW-LPCd:1704173991269",
"dest": "Insert_Image_At_Cursor_Gold.docx"
"dest": "Viewing_Your_Class_Schedule_and_Textbooks_Gold.docx"
}
}
}

16
evaluation_examples/examples/libreoffice_writer/936321ce-5236-426a-9a20-e0e3c5dc536f.json
View File

@@ -1,7 +1,7 @@
{
"id": "936321ce-5236-426a-9a20-e0e3c5dc536f",
"snapshot": "libreoffice_writer",
"instruction": "Convert the content seperated by commas to a table",
"instruction": "Could you help me convert the text seperated by commas to a table?",
"source": "https://www.youtube.com/watch?v=l25Evu4ohKg",
"config": [
{
@@ -9,8 +9,8 @@
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=139-NmslBR_9qlD7hUO08xj_VFffV95eM&export=download&authuser=0&confirm=t&uuid=64a6c35d-f3ce-4c25-9f83-4a952e24c5ad&at=APZUnTUL1GMR_QbpFQnC9fPwkdqa:1704183959196",
"path": "Desktop/Convert_Text_To_Table.docx"
"url": "https://drive.usercontent.google.com/download?id=1WcHqnDy0OPFiaGtvPSvKXuaLc_rtGzYC&export=download&authuser=0&confirm=t&uuid=0c552800-5dfe-4a3f-a960-8d384481fa7f&at=APZUnTWYjmvuXkU-pb77dy7EgCeD:1704970003044",
"path": "Desktop/Graphemes_Sound_Letter_Patterns.docx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
"path": "Desktop/Convert_Text_To_Table.docx"
"path": "Desktop/Graphemes_Sound_Letter_Patterns.docx"
}
}
],
@@ -30,13 +30,13 @@
"func": "compare_docx_tables",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1fs2msYaxnEZL9XASENQMZIag2MTxIBJs&export=download&authuser=0&confirm=t&uuid=6c71f008-082c-4f0c-9ffc-0a802f5cbfe6&at=APZUnTVDpucMDfk5P2T-0dZx_KVV:1704183960360",
"dest": "Convert_Text_To_Table_Gold.docx"
"path": "https://drive.usercontent.google.com/download?id=1og3gCBRhAeHk02E1wAsbfau_1ywdoEJL&export=download&authuser=0&confirm=t&uuid=264de3b4-b07c-4382-8a73-a766b1d1fc58&at=APZUnTX8yRMoDXCHi1y1H380GaSX:1704969967989",
"dest": "Graphemes_Sound_Letter_Patterns_Gold.docx"
},
"result": {
"type": "vm_file",
"path": "Desktop/Convert_Text_To_Table.docx",
"dest": "Convert_Text_To_Table.docx"
"path": "Desktop/Graphemes_Sound_Letter_Patterns.docx",
"dest": "Graphemes_Sound_Letter_Patterns.docx"
}
}
}

16
evaluation_examples/examples/libreoffice_writer/adf5e2c3-64c7-4644-b7b6-d2f0167927e7.json
View File

@@ -1,7 +1,7 @@
{
"id": "adf5e2c3-64c7-4644-b7b6-d2f0167927e7",
"snapshot": "libreoffice_writer",
"instruction": "Helping me adding \"C. Luo and M. J. Carey, \"LSM-based storage techniques: a survey,\" The VLDB Journal, vol. 29, no. 1, pp. 393–418, 2020.\" to my reference list, and add a cross reference at the end of the first paragraph",
"instruction": "Help me adding \"Steinberg, F. M., Bearden, M. M., & Keen, C. L. (2003). Cocoa and chocolate flavonoids: Implications for cardiovascular health. Journal of the American Dietetic Association, 103(2), 215-223. doi: 10.1053/jada.2003.50028\" to my reference list, and add a cross reference where my cursor is located (in the fourth paragraph).",
"source": "https://seekstar.github.io/2022/04/11/libreoffice%E5%BC%95%E7%94%A8%E6%96%87%E7%8C%AE/",
"config": [
{
@@ -9,8 +9,8 @@
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1xOfwImgkPzdmjQomj-MCFd8nQS75OjaH&export=download&authuser=0&confirm=t&uuid=7eb91c26-dad5-4480-b1ec-35e506cde1e4&at=APZUnTW01MvBI_gkC8yoiyAVs7yi:1704188254979",
"path": "Desktop/Add_Citation_Cross_Reference.docx"
"url": "https://drive.usercontent.google.com/download?id=1ShGL4gWSV7nzamAb0V2KqjoCOhyodcKU&export=download&authuser=0&confirm=t&uuid=5f67edb8-cbbf-4a83-b46e-f193ad55e1e8&at=APZUnTVRJenYCM--vETagQ5ACTT5:1704979226579",
"path": "Desktop/Essay_Writing_English_for_uni.docx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
"path": "Desktop/Add_Citation_Cross_Reference.docx"
"path": "Desktop/Essay_Writing_English_for_uni.docx"
}
}
],
@@ -30,13 +30,13 @@
"func": "compare_docx_files",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1wFQU7hkAT2wmSHTgM22F9Ep4WXmymEMW&export=download&authuser=0&confirm=t&uuid=a7ea1eec-678b-4407-b023-df13cc6f8c54&at=APZUnTW3WoqOfS9A1BW79XfV8jKh:1704188260410",
"dest": "Add_Citation_Cross_Reference_Gold.docx"
"path": "https://drive.usercontent.google.com/download?id=1bf7IKjjxGAQfbOlq9rLPzjTtkOZ3dREj&export=download&authuser=0&confirm=t&uuid=0420cfa5-7e51-4688-8c93-74748914ce52&at=APZUnTWIShIleccvgVBIr6ZsSnOw:1704979256798",
"dest": "Essay_Writing_English_for_uni_Gold.docx"
},
"result": {
"type": "vm_file",
"path": "Desktop/Add_Citation_Cross_Reference.docx",
"dest": "Add_Citation_Cross_Reference.docx"
"path": "Desktop/Essay_Writing_English_for_uni.docx",
"dest": "Essay_Writing_English_for_uni.docx"
}
}
}

14
evaluation_examples/examples/libreoffice_writer/e528b65e-1107-4b8c-8988-490e4fece599.json
View File

@@ -9,8 +9,8 @@
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1LZ1_U9CyR8oOkqfbY6HMLWr4wBVYhCLR&export=download&authuser=0&confirm=t&uuid=574d533a-8df9-4f33-bebd-689b623f27a9&at=APZUnTVruCDRxY661_PVT9BA3839:1704180420603",
"path": "Desktop/Capitalize_First_Letter.docx"
"url": "https://drive.usercontent.google.com/download?id=181hyG_NZUJaUp7kUdeFoUpNGpOcjzYFV&export=download&authuser=0&confirm=t&uuid=fdb8aee4-ea1f-43c1-bc66-9d44099512e1&at=APZUnTVUrypBeVnTfkJsriv7S2GO:1704969486759",
"path": "Geography_And_Magical_Realism.docx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
"path": "Desktop/Capitalize_First_Letter.docx"
"path": "Desktop/Geography_And_Magical_Realism.docx"
}
}
],
@@ -30,13 +30,13 @@
"func": "compare_docx_files",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1ykIGg48GjYUOSw8t44evShJE2HX7R5w3&export=download&authuser=0&confirm=t&uuid=ed81d6cd-6044-49f1-be86-35adeaeeea00&at=APZUnTUxW8WLyPr-_smA2Mnwpuuv:1704180422490https://drive.usercontent.google.com/download?id=1ykIGg48GjYUOSw8t44evShJE2HX7R5w3&export=download&authuser=0&confirm=t&uuid=ed81d6cd-6044-49f1-be86-35adeaeeea00&at=APZUnTUxW8WLyPr-_smA2Mnwpuuv:1704180422490",
"dest": "Capitalize_First_Letter_Gold.docx"
"path": "https://drive.usercontent.google.com/download?id=1qwoJMuMhLxWwONRLmxwMdUuVIFBAhhNT&export=download&authuser=0&confirm=t&uuid=8fa4d503-ec55-419f-ac90-f5915a5f67b5&at=APZUnTVEntT4oMiEfOuWcUKaInBs:1704969347952",
"dest": "Geography_And_Magical_Realism_Gold.docx"
},
"result": {
"type": "vm_file",
"path": "Desktop/Capitalize_First_Letter.docx",
"dest": "Capitalize_First_Letter.docx"
"path": "Desktop/Geography_And_Magical_Realism.docx",
"dest": "Geography_And_Magical_Realism.docx"
}
}
}

10
evaluation_examples/examples/libreoffice_writer/ecc2413d-8a48-416e-a3a2-d30106ca36cb.json
View File

@@ -1,7 +1,7 @@
{
"id": "ecc2413d-8a48-416e-a3a2-d30106ca36cb",
"snapshot": "libreoffice_writer",
"instruction": "Insert a blank page",
"instruction": "Help me insert a blank page where my cursor is located.",
"source": "https://www.quora.com/How-can-I-insert-a-blank-page-on-libreoffice",
"config": [
{
@@ -10,7 +10,7 @@
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1sDufDSC4foI379-Jikya9WK7FBUSqgrt&export=download&authuser=0&confirm=t&uuid=0abd82d6-2b2c-49bc-af5e-49bfe1c99278&at=APZUnTURIqTNJcIHBcMP2BxEaGXr:1704174850900",
"path": "Desktop/Insert_Blank_Page.docx"
"path": "Desktop/Sample_Statutory_Declaration.docx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
"path": "Desktop/Insert_Blank_Page.docx"
"path": "Desktop/Sample_Statutory_Declaration.docx"
}
}
],
@@ -30,8 +30,8 @@
"func": "contains_page_break",
"result": {
"type": "vm_file",
"path": "Desktop/Insert_Blank_Page.docx",
"dest": "Insert_Blank_Page.docx"
"path": "Desktop/Sample_Statutory_Declaration.docx",
"dest": "Sample_Statutory_Declaration.docx"
}
}
}

6
evaluation_examples/examples/libreoffice_writer/f178a4a9-d090-4b56-bc4c-4b72a61a035d.json
View File

@@ -9,8 +9,8 @@
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1X2XTU2ZFuMXOhm7T400e6AOe6eBYxWzD&export=download&authuser=0&confirm=t&uuid=1318923f-6d54-4148-aa80-a454b9963cec&at=APZUnTU-h1nmcjBO_ytWVxXuh8l9:1704187013730",
"path": "Desktop/Set_Default_Font.docx"
"url": "https://drive.usercontent.google.com/download?id=1xREbNAu_2wLTs8EQT0NnHLpIkAVAfpyk&export=download&authuser=0&confirm=t&uuid=dd5cb525-ff4b-41a2-8123-d488f2f21fad&at=APZUnTXaYBqLT9fRtGYZHOedq-PG:1704977194647",
"path": "Desktop/loa-one-time-submission-sealand.docx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
"path": "Desktop/Set_Default_Font.docx"
"path": "Desktop/loa-one-time-submission-sealand.docx"
}
}
],

2
evaluation_examples/examples/vlc/59f21cfb-0120-4326-b255-a5b827b38967.json
View File

@@ -1,7 +1,7 @@
{
"id": "59f21cfb-0120-4326-b255-a5b827b38967",
"snapshot": "base_setup",
"instruction": "Play the music video on my desktop",
"instruction": "Could you play the music video that's saved on my desktop for me?",
"source": "https://docs.videolan.me/vlc-user/desktop/3.0/en/basic/media.html#playing-a-file",
"config": [
{

24
evaluation_examples/examples/vlc/8f080098-ddb1-424c-b438-4e96e5e4786e.json
View File

@@ -1,7 +1,7 @@
{
"id": "8f080098-ddb1-424c-b438-4e96e5e4786e",
"snapshot": "base_setup",
"instruction": "Could you help me extract MP3 Audio to AUDIO_PATH from Video at VIDEO_PATH using VLC Media Player?",
"instruction": "Could you download the song from this music video and save it as an MP3 file? I'd like to have it on my device to play whenever I want. Please title the file \"Baby Justin Bieber.mp3.\" I really appreciate your help!",
"source": "https://medium.com/@jetscribe_ai/how-to-extract-mp3-audio-from-videos-using-vlc-media-player-beeef644ebfb",
"config": [
{
@@ -10,7 +10,7 @@
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=19jBiz8sb0M7KHHATO9qeTPr17aWm4me-&export=download&authuser=0&confirm=t&uuid=7a2261f4-3905-433f-b53f-a52dd0845651&at=APZUnTU1nmXSa1ObrA5NHYt8t1-p:1704710908141",
"path": "Baby Justin Bieber.mp4"
"path": "Desktop/Baby Justin Bieber.mp4"
}
]
}
@@ -18,7 +18,11 @@
{
"type": "launch",
"parameters": {
"command": "vlc"
"command": [
"vlc",
"--start-time=73",
"Desktop/Baby Justin Bieber.mp4"
]
}
}
],
@@ -27,16 +31,16 @@
"vlc"
],
"evaluator": {
"func": "is_vlc_recordings_folder",
"func": "compare_audios",
"expected": {
"type": "rule",
"rules": {
"recording_file_path": "/home/user/Desktop"
}
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1eRuuiUAOmckyn2zQebx1nWQ40kpjSJ_N&export=download&authuser=0&confirm=t&uuid=8012703a-9e45-4d91-9bda-90c119e42254&at=APZUnTW0uyCSeg7FClSc50UJVRz-:1704898635455",
"dest": "baby_gold.mp3"
},
"result": {
"type": "vlc_config",
"dest": "vlcrc"
"type": "vm_file",
"path": "Desktop/Baby Justin Bieber.mp3",
"dest": "baby.mp3"
}
}
}

2
evaluation_examples/examples/vlc/a1c3ab35-02de-4999-a7ed-2fd12c972c6e.json
View File

@@ -1,7 +1,7 @@
{
"id": "a1c3ab35-02de-4999-a7ed-2fd12c972c6e",
"snapshot": "base_setup",
"instruction": "Could you help me compress the video to DIR_PATH?",
"instruction": "Could you help me compress the video to MPEG-4 format and save with name with prefix _?",
"source": "https://www.quora.com/How-do-I-compress-a-video-with-VLC",
"config": [],
"trajectory": "trajectories/",

2
evaluation_examples/examples/vlc/bba3381f-b5eb-4439-bd9e-80c22218d5a7.json
View File

@@ -1,7 +1,7 @@
{
"id": "bba3381f-b5eb-4439-bd9e-80c22218d5a7",
"snapshot": "base_setup",
"instruction": "Help me play the online video at https://www.youtube.com/watch?v=pgBsyTKAwLw",
"instruction": "Can you start streaming the video from this link for me? https://www.youtube.com/watch?v=pgBsyTKAwLw",
"source": "https://www.quora.com/How-do-I-play-online-videos-using-the-VLC-media-player",
"config": [
{

33
evaluation_examples/examples/vlc/efcf0d81-0835-4880-b2fd-d866e8bc2294.json
View File

@@ -1,30 +1,47 @@
{
"id": "efcf0d81-0835-4880-b2fd-d866e8bc2294",
"snapshot": "base_setup",
"instruction": "Set this frame of the current video as my wallpaper",
"source": "https://www.youtube.com/watch?v=XHprwDJ0-fU&t=436s",
"instruction": "Make this part of the video my computer's background picture",
"source": "https://www.youtube.com/watch?v=XHprwDJ0-fU&t=436s, https://help.ubuntu.com/stable/ubuntu-help/look-background.html.en",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "",
"path": ""
"url": "https://drive.usercontent.google.com/download?id=1H9D0jSkzpzEWsJZG0HdNNNHDMi3fnGli&export=download&authuser=0&confirm=t&uuid=a6d03223-db7a-48bd-b5f8-50b51f4d3d68&at=APZUnTVi3cqgvRV49c6sluV-nMo_:1704879367756",
"path": "Desktop/Interstellar Movie - Official Trailer.mp4"
}
]
}
},
{
"type": "launch",
"parameters": {
"command": ["vlc", "/path/to/your/video.mp4", "--start-time=0", "--run-time=10", "vlc://quit", "&&", "vlc", "/path/to/your/video.mp4", "--start-time=10"]
}
"parameters": {
"command": [
"vlc",
"--start-time=120.5",
"--stop-time=121",
"--play-and-pause",
"Desktop/Interstellar Movie - Official Trailer.mp4"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"vlc"
],
"evaluator": "evaluation_dir"
"evaluator": {
"func": "compare_images",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=10P8kLkWeYhtK3Gl15nB-gH7VLO4S7Xal&export=download&authuser=0&confirm=t&uuid=d5c5db29-435d-404f-8900-683bf56fbc66&at=APZUnTXm1ZHS7CbGiWaB04WySH-t:1704891263184",
"dest": "interstellar_wallpaper_gold.png"
},
"result": {
"type": "vm_wallpaper",
"dest": "result_wallpaper.png"
}
}
}

42
evaluation_examples/examples/vlc/fba2c100-79e8-42df-ae74-b592418d54f4.json
View File

@@ -1,12 +1,48 @@
{
"id": "fba2c100-79e8-42df-ae74-b592418d54f4",
"snapshot": "base_setup",
"instruction": "Screenshot the current frame of the video",
"instruction": "Snap a photo of the current video scene, save it as 'interstellar.png', and put it on the Desktop, please.",
"source": "https://www.youtube.com/watch?v=XHprwDJ0-fU&t=436s",
"config": [],
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1u7CFVr5bWti7OnhvyjtSdQolzy0lYicm&export=download&authuser=0&confirm=t&uuid=18498a7c-9b52-4c00-a2cc-2e8b521c4d9c&at=APZUnTX0R_TRccU_UmtJWXZ6On3x:1704890439561",
"path": "Desktop/Interstellar Movie - Official Trailer.mp4"
}
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"vlc",
"--start-time=120.5",
"--stop-time=121",
"--play-and-pause",
"Desktop/Interstellar Movie - Official Trailer.mp4"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"vlc"
],
"evaluator": "evaluation_dir"
"evaluator": {
"func": "compare_images",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1shU1TQ3ao9QWfhmC63vbD10YFbhv7PYv&export=download&authuser=0&confirm=t&uuid=6a22a47a-6d35-4996-b02e-b24b33a1c449&at=APZUnTVlvDKpxW7QKmBBmdNq2jRw:1704891215314",
"dest": "interstellar_gold.png"
},
"result": {
"type": "vm_file",
"path": "Desktop/interstellar.png",
"dest": "interstellar.png"
}
}
}

1
requirements.txt
View File

@@ -27,3 +27,4 @@ rapidfuzz
pyacoustid
opencv-python
ImageHash
scikit-image