lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix OS examples annotated by Yitao

Browse Source
This commit is contained in:
Timothyxxx
2024-01-25 19:57:32 +08:00
parent 0c34fccc15
commit b9ae4174b1
24 changed files with 327 additions and 358 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
desktop_env/controllers/setup.py
View File

@@ -257,7 +257,7 @@ class SetupController:
with open(os.path.join(self.cache_dir, stderr), "w") as f:
f.write(results["error"])
logger.info("Command executed successfully: %s -> %s"
, " ".join(command)
, " ".join(command) if isinstance(command, list) else command
, response.text
)
else:

2
desktop_env/envs/desktop_env.py
View File

@@ -213,7 +213,7 @@ class DesktopEnv(gym.Env):
self.cache_dir: str = os.path.join(self.cache_dir_base, self.task_id)
os.makedirs(self.cache_dir, exist_ok=True)
self.instruction = task_config["instruction"]
self.config = task_config["config"]
self.config = task_config["config"] if "config" in task_config else []
# evaluator dict
# func -> metric function string, or list of metric function strings

3
desktop_env/evaluators/getters/general.py
View File

@@ -9,8 +9,9 @@ def get_vm_command_line(env, config: Dict[str, str]):
vm_ip = env.vm_ip
port = 5000
command = config["command"]
shell = config.get("shell", False)
response = requests.post(f"http://{vm_ip}:{port}/execute", json={"command": command})
response = requests.post(f"http://{vm_ip}:{port}/execute", json={"command": command, "shell": shell})
if response.status_code == 200:
return response.json()["output"]

27
desktop_env/evaluators/metrics/__init__.py
View File

@@ -1,23 +1,28 @@
from .chrome import is_expected_tabs, is_expected_bookmarks, compare_pdfs, is_cookie_deleted, is_shortcut_on_desktop, check_font_size, \
check_enabled_experiments, check_history_deleted, is_expected_search_query
from .chrome import is_expected_tabs, is_expected_bookmarks, compare_pdfs, is_cookie_deleted, is_shortcut_on_desktop, \
check_font_size, \
check_enabled_experiments, check_history_deleted, is_expected_search_query
from .docs import compare_font_names, compare_subscript_contains, has_page_numbers_in_footers, compare_docx_lines
from .docs import find_default_font, contains_page_break, compare_docx_files, compare_docx_tables, compare_line_spacing, \
compare_insert_equation, compare_highlighted_text
from .docs import is_first_line_centered, check_file_exists, compare_contains_image
from .docs import evaluate_colored_words_in_tables, check_highlighted_words, evaluate_strike_through_last_paragraph, \
evaluate_conversion, evaluate_spacing, check_italic_font_size_14, evaluate_alignment, get_unique_train_ids, \
check_no_duplicates
from .general import exact_match, fuzzy_match
from .docs import find_default_font, contains_page_break, compare_docx_files, compare_docx_tables, compare_line_spacing, \
compare_insert_equation, compare_highlighted_text
from .docs import is_first_line_centered, check_file_exists, compare_contains_image
from .general import check_csv, check_accessibility_tree, run_sqlite3, check_json
from .general import exact_match, fuzzy_match, check_include_exclude
from .gimp import increase_saturation, decrease_brightness, check_file_exists, compare_triangle_positions
from .slides import check_presenter_console_disable, check_image_stretch_and_center, check_slide_numbers_color, compare_pptx_files, check_strikethrough, \
check_slide_orientation_Portrait, evaluate_presentation_fill_to_rgb_distance, check_left_panel
from .libreoffice import check_libre_locale
from .pdf import check_pdf_pages
#from .table import check_sheet_list, check_xlsx_freeze, check_xlsx_zoom, check_data_validations
from .slides import check_presenter_console_disable, check_image_stretch_and_center, check_slide_numbers_color, \
compare_pptx_files, check_strikethrough, \
check_slide_orientation_Portrait, evaluate_presentation_fill_to_rgb_distance, check_left_panel
# from .table import check_sheet_list, check_xlsx_freeze, check_xlsx_zoom, check_data_validations
from .table import compare_table
from .thunderbird import check_thunderbird_prefs, check_thunderbird_filter
from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, compare_images, compare_audios, \
compare_videos, check_qt_bgcone, check_one_instance_when_started_from_file,check_qt_minimal_view, check_qt_max_volume, \
compare_videos, check_qt_bgcone, check_one_instance_when_started_from_file, check_qt_minimal_view, \
check_qt_max_volume, \
check_qt_slider_colours, check_global_key_play_pause
from .vscode import compare_text_file, compare_config, compare_answer, is_extension_installed, check_json_settings, check_json_keybindings
from .vscode import compare_text_file, compare_config, compare_answer, is_extension_installed, check_json_settings, \
check_json_keybindings
from .os import check_gnome_favorite_apps

18
desktop_env/evaluators/metrics/general.py
View File

@@ -1,8 +1,9 @@
import csv
import json
import functools
import json
import operator
import re
import sqlite3
from numbers import Number
from typing import Callable, Any, Union
from typing import Dict, List, Pattern
@@ -14,13 +15,14 @@ from rapidfuzz import fuzz
from .utils import _match_record, _match_value_to_rule
import sqlite3
def check_include_exclude(result: str, rules: Dict[str, List[str]]) -> float:
print(result, rules)
include = rules.get("include", [])
exclude = rules.get("exclude", [])
return all(r in result for r in include) and all(r not in result for r in exclude)
def exact_match(result, rules) -> float:
expect = rules["expected"]
print(result, expect)
@@ -36,6 +38,7 @@ def fuzzy_match(result, rules) -> float:
return fuzz.ratio(result, expect) / 100.
def check_csv(result: str, rules: Dict[str, List[Dict[str, str]]]) -> float:
"""
Args:
@@ -140,10 +143,10 @@ def check_accessibility_tree(result: str, rules: Dict[str, Any]) -> float:
return 0.
if "text" in rules:
match_func: Callable[[str], Number] = functools.partial( operator.eq if rules["exact"]\
else (lambda a, b: fuzz.ratio(a, b)/100.)
, rules["text"]
)
match_func: Callable[[str], Number] = functools.partial(operator.eq if rules["exact"] \
else (lambda a, b: fuzz.ratio(a, b) / 100.)
, rules["text"]
)
match_score: Number = 0
for elm in elements:
match_score = max(match_score, match_func(elm.text or None))
@@ -152,6 +155,7 @@ def check_accessibility_tree(result: str, rules: Dict[str, Any]) -> float:
return float(match_score)
# def check_existence(result: str, *args) -> float:
# return 1. - (result is None)
@@ -160,6 +164,7 @@ def run_sqlite3(result: str, rules: Dict[str, Any]) -> float:
cursor: sqlite3.Cursor = connection.execute(rules["sql"])
return float(cursor.fetchone()[0] or 0)
def check_json(result: str, rules: Dict[str, List[Dict[str, Union[List[str], str]]]]) -> float:
"""
Args:
@@ -200,4 +205,3 @@ def check_json(result: str, rules: Dict[str, List[Dict[str, Union[List[str], str
value = value[k]
metric = metric and not _match_value_to_rule(value, r)
return metric

102
desktop_env/evaluators/metrics/os.py
View File

@@ -1,58 +1,20 @@
import os
import subprocess
def is_spotify_installed() -> bool:
# Use the 'which' command to check if the 'spotify' executable exists
try:
subprocess.check_output(['which', 'spotify'])
return True
except subprocess.CalledProcessError:
return False
def check_gnome_favorite_apps(apps_str: str, rule):
# parse the string like "['thunderbird.desktop', 'vim.desktop', 'google-chrome.desktop']"
# to a list of strings
apps = eval(apps_str)
expected_apps = rule["expected"]
def is_dim_screen_inactive_off():
# Get the current value of "Dim screen when inactive" setting
result = subprocess.run(["gsettings", "get", "org.gnome.desktop.session", "idle-delay"], capture_output=True,
text=True)
if len(apps) != len(expected_apps):
return 0
# Check if the setting is set to "uint32 0"
if result.stdout.strip() == "uint32 0":
return 1 # Task successful
if set(apps) == set(expected_apps):
return 1
else:
return 0 # Task not successful
def evaluate_create_test_directory():
try:
# Specify the path to the directory
path = '/test'
# Check if the directory already exists
if os.path.exists(path):
return 0 # Directory already exists, task not successful
# Check if the user has sufficient permissions to create the directory
if not os.access("/", os.W_OK | os.X_OK):
return 0 # Insufficient permissions to create the directory, task not successful
return 1 # Task can be considered successful
except Exception as e:
print(f"Error: {str(e)}")
return 0 # Any other errors, task not successful
# TODO: create a file named test.txt and a directory dir1 at home dir before running this
def is_file_in_directory():
# Specify the paths of the file and directory
file_path = "/test.txt"
directory_path = "/dir1"
# Check if the file exists in the directory
if os.path.isfile(os.path.join(directory_path, os.path.basename(file_path))):
return 1 # Task successful
else:
return 0 # Task not successful
return 0
# TODO: log in to the system before running this
@@ -64,47 +26,6 @@ def is_logout_successful():
return 1 # Task successful
def is_do_not_disturb_mode_enabled():
try:
subprocess.run(["gsettings", "set", "org.gnome.desktop.notifications", "show-banners", "false"], check=True)
return 1 # Task successful
except subprocess.CalledProcessError:
return 0 # Task not successful
def get_default_browser(answer: str):
try:
result = subprocess.run(["xdg-settings", "get", "default-web-browser"], capture_output=True, text=True)
default_browser = result.stdout.strip()
return default_browser == answer
except subprocess.CalledProcessError:
return 0
# TODO: should confirm initial state, i.e. initial list of favorite apps & the first app
first_app = "thunderbird.desktop" # to be changed
def is_first_favorite_app_removed():
try:
result = subprocess.run(["gsettings", "get", "org.gnome.shell", "favorite-apps"], capture_output=True,
text=True)
output = result.stdout.strip()
# Remove brackets and spaces from the output
favorites = output[1:-1].replace(" ", "")
# Split output by comma to get individual favorite apps
favorite_apps = favorites.split(",")
if favorite_apps[0] != first_app:
return 1 # First favorite app removed
else:
return 0 # First favorite app not removed
except subprocess.CalledProcessError:
return 0 # Task not successful
def is_battery_percentage_displayed():
# GNOME schema and key for the setting
schema = "org.gnome.desktop.interface"
@@ -147,6 +68,3 @@ def check_auto_lock_settings():
except Exception as e:
return 0.

38
evaluation_examples/examples/os/28cc3b7e-b194-4bc9-8353-d04c0f4d56d2.json
View File

@@ -1,17 +1,25 @@
{
"id": "28cc3b7e-b194-4bc9-8353-d04c0f4d56d2",
"snapshot": "os",
"instruction": "The volume of my system is too small. Can you help me turn up to the max volumn?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/sound-volume.html.en",
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
"id": "28cc3b7e-b194-4bc9-8353-d04c0f4d56d2",
"snapshot": "os",
"instruction": "The volume of my system is too small. Can you help me turn up to the max volume?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/sound-volume.html.en",
"trajectory": "trajectories/",
"config": [],
"related_apps": [
"os"
],
"evaluator": {
"func": "exact_match",
"result": {
"type": "vm_command_line",
"command": "pactl list sinks | grep '^[[:space:]]Volume:' | head -n 1 | awk '{print $5}' | sed 's/%//g'",
"shell": true
},
"expected": {
"type": "rule",
"rules":{
"expected": "100\n"
}
}
}
}
}

31
evaluation_examples/examples/os/3ce045a0-877b-42aa-8d2c-b4a863336ab8.json
View File

@@ -1,17 +1,18 @@
{
"id": "3ce045a0-877b-42aa-8d2c-b4a863336ab8",
"snapshot": "os",
"instruction": "I want to make the text on the screen larger",
"source": "https://help.ubuntu.com/lts/ubuntu-help/a11y-font-size.html.en",
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
"id": "3ce045a0-877b-42aa-8d2c-b4a863336ab8",
"snapshot": "os",
"instruction": "I want to make the text on the screen larger",
"source": "https://help.ubuntu.com/lts/ubuntu-help/a11y-font-size.html.en",
"trajectory": "trajectories/",
"config": [],
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
}
}
}

7
evaluation_examples/examples/os/43c2d64c-bab5-4dcb-a30c-b888321c319a.json
View File

@@ -12,14 +12,13 @@
"func": "exact_match",
"result": {
"type": "vm_command_line",
"config": {
"command": "[ -d '/path/to/directory' ] && echo 'Directory exists.' || echo 'Directory does not exist.'"
}
"command": "[ -d '/home/user/Desktop/test' ] && echo 'Directory exists.' || echo 'Directory does not exist.'",
"shell": true
},
"expected": {
"type": "rule",
"rules":{
"expected": "Directory exists."
"expected": "Directory exists.\n"
}
}
}

31
evaluation_examples/examples/os/4d2b519e-e872-4100-8ea3-fe71ab0f9133.json
View File

@@ -1,17 +1,18 @@
{
"id": "4d2b519e-e872-4100-8ea3-fe71ab0f9133",
"snapshot": "os",
"instruction": "Could you please help me to add a new entry \"/home/david/pear/bin\" to the PATH variable in the ZSH terminal?",
"source": "https://stackoverflow.com/questions/11530090/adding-a-new-entry-to-the-path-variable-in-zsh",
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
"id": "4d2b519e-e872-4100-8ea3-fe71ab0f9133",
"snapshot": "os",
"instruction": "Could you please help me to add a new entry \"/home/david/pear/bin\" to the PATH variable in the ZSH terminal?",
"source": "https://stackoverflow.com/questions/11530090/adding-a-new-entry-to-the-path-variable-in-zsh",
"trajectory": "trajectories/",
"config": [],
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
}
}
}

31
evaluation_examples/examples/os/5812b315-e7bd-4265-b51f-863c02174c28.json
View File

@@ -1,17 +1,18 @@
{
"id": "5812b315-e7bd-4265-b51f-863c02174c28",
"snapshot": "os",
"instruction": "Please create an SSH user on Ubuntu who is only allowed to access the folder \"test1\".",
"source": "https://superuser.com/questions/149404/create-an-ssh-user-who-only-has-permission-to-access-specific-folders",
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
"id": "5812b315-e7bd-4265-b51f-863c02174c28",
"snapshot": "os",
"instruction": "Please create an SSH user on Ubuntu who is only allowed to access the folder \"test1\".",
"source": "https://superuser.com/questions/149404/create-an-ssh-user-who-only-has-permission-to-access-specific-folders",
"config": [],
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
}
}
}

31
evaluation_examples/examples/os/5c433d22-ed9a-4e31-91f5-54cf3e8acd63.json
View File

@@ -1,17 +1,18 @@
{
"id": "5c433d22-ed9a-4e31-91f5-54cf3e8acd63",
"snapshot": "os",
"instruction": "I want to change my system language to Chinese(simplified). Can you help me?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/session-language.html.zh-CN",
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
"id": "5c433d22-ed9a-4e31-91f5-54cf3e8acd63",
"snapshot": "os",
"instruction": "I want to change my system language to Chinese(simplified). Can you help me?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/session-language.html.zh-CN",
"trajectory": "trajectories/",
"config": [],
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
}
}
}

31
evaluation_examples/examples/os/5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57.json
View File

@@ -1,17 +1,18 @@
{
"id": "5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57",
"snapshot": "os",
"instruction": "I am currently using an Ubuntu system, and I have wrongly deleted a file named \"test\". Could you help me recover it from the Trash?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/files-recover.html.en",
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
"id": "5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57",
"snapshot": "os",
"instruction": "I am currently using an Ubuntu system, and I have wrongly deleted a file named \"test\". Could you help me recover it from the Trash?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/files-recover.html.en",
"trajectory": "trajectories/",
"config": [],
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
}
}
}

31
evaluation_examples/examples/os/6ebbfb01-ea72-4226-a2a6-dc428e111ed2.json
View File

@@ -1,17 +1,18 @@
{
"id": "6ebbfb01-ea72-4226-a2a6-dc428e111ed2",
"snapshot": "os",
"instruction": "Could you please help me to set Bash as my default shell on the current Ubuntu system?",
"source": "https://superuser.com/questions/46748/how-do-i-make-bash-my-default-shell-on-ubuntu",
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
"id": "6ebbfb01-ea72-4226-a2a6-dc428e111ed2",
"snapshot": "os",
"instruction": "Could you please help me to set Bash as my default shell on the current Ubuntu system?",
"source": "https://superuser.com/questions/46748/how-do-i-make-bash-my-default-shell-on-ubuntu",
"trajectory": "trajectories/",
"config": [],
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
}
}
}

31
evaluation_examples/examples/os/765d2b74-88a7-4d50-bf51-34e4106fd24a.json
View File

@@ -1,17 +1,18 @@
{
"id": "765d2b74-88a7-4d50-bf51-34e4106fd24a",
"snapshot": "os",
"instruction": "Can you help me delete the \"test\" file on my desktop?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/files-delete.html.en",
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
"id": "765d2b74-88a7-4d50-bf51-34e4106fd24a",
"snapshot": "os",
"instruction": "Can you help me delete the \"test\" file on my desktop?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/files-delete.html.en",
"config": [],
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
}
}
}

19
evaluation_examples/examples/os/7688b85f-87a4-4e4a-b2f8-f3d6c3f29b82.json
View File

@@ -1,13 +1,21 @@
{
"id": "7688b85f-87a4-4e4a-b2f8-f3d6c3f29b82",
"snapshot": "os",
"instruction": "Can you move the file with the path '/test.txt' to the directory with the path '/dir1'?",
"instruction": "Can you move the file with the path 'todo.txt' to the directory with the path 'done'?",
"source": "https://ubuntu.com/tutorials/command-line-for-beginners#5-moving-and-manipulating-files",
"config": [
{
"type": "execute",
"parameters": {
"command": "touch /test.txt && mkdir /dir1"
"command": "echo 'password' | sudo -S touch ~/Desktop/todo.txt && sudo mkdir ~/Desktop/done",
"shell": true
}
},
{
"type": "execute",
"parameters": {
"command": "echo 'password' | sudo -S chmod 777 ~/Desktop/todo.txt && sudo chmod 777 ~/Desktop/done",
"shell": true
}
}
],
@@ -19,14 +27,13 @@
"func": "exact_match",
"result": {
"type": "vm_command_line",
"config": {
"command": "if [ -f '/dir/test.txt' ]; then echo 'File exists.'; else echo 'File does not exist.'; fi"
}
"command": "if [ -f ~/Desktop/done/todo.txt ]; then echo 'File exists.'; else echo 'File does not exist.'; fi",
"shell": true
},
"expected": {
"type": "rule",
"rules":{
"expected": "File exists."
"expected": "File exists.\n"
}
}
}

49
evaluation_examples/examples/os/94d95f96-9699-4208-98ba-3c3119edf9c2.json
View File

@@ -1,26 +1,29 @@
{
"id": "94d95f96-9699-4208-98ba-3c3119edf9c2",
"snapshot": "os",
"instruction": "I want to install Spotify on my current system. Could you please help me?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/addremove-install.html.en",
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "check_include_exclude",
"result": {
"type": "vm_command_line",
"config": {
"command": "which spotify"
}
},
"expected": {
"type": "rule",
"rules": {
"include": [],
"exclude": ["not found"]
}
"id": "94d95f96-9699-4208-98ba-3c3119edf9c2",
"snapshot": "os",
"instruction": "I want to install Spotify on my current system. Could you please help me?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/addremove-install.html.en",
"trajectory": "trajectories/",
"config": [],
"related_apps": [
"os"
],
"evaluator": {
"func": "check_include_exclude",
"result": {
"type": "vm_command_line",
"command": "which spotify"
},
"expected": {
"type": "rule",
"rules": {
"include": [
"spotify"
],
"exclude": [
"not found"
]
}
}
}
}
}

8
evaluation_examples/examples/os/a462a795-fdc7-4b23-b689-e8b6df786b78.json
View File

@@ -7,7 +7,8 @@
{
"type": "execute",
"parameters": {
"command": "echo password | sudo -S su - charles"
"command": "echo password | sudo -S su - charles",
"shell": true
}
}
],
@@ -19,9 +20,8 @@
"func": "exact_match",
"result": {
"type": "vm_command_line",
"config": {
"command": "if [ '$(whoami)' = 'charles' ]; then echo 'Current user is charles.'; else echo 'Current user is not charles'; fi"
}
"command": "if [ '$(whoami)' = 'charles' ]; then echo 'Current user is charles.'; else echo 'Current user is not charles'; fi",
"shell": true
},
"expected": {
"type": "rule",

4
evaluation_examples/examples/os/ae039631-2b12-4637-84f6-c67d51511be3.json
View File

@@ -19,9 +19,7 @@
"func": "exact_match",
"result": {
"type": "vm_command_line",
"config": {
"command": "xdg-settings get default-web-browser"
}
"command": "xdg-settings get default-web-browser"
},
"expected": {
"type": "rule",

31
evaluation_examples/examples/os/b3d4a89c-53f2-4d6b-8b6a-541fb5d205fa.json
View File

@@ -1,17 +1,18 @@
{
"id": "b3d4a89c-53f2-4d6b-8b6a-541fb5d205fa",
"snapshot": "os",
"instruction": "I want to switch off the Bluetooth. Can you help me?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/bluetooth-turn-on-off.html.en",
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
"id": "b3d4a89c-53f2-4d6b-8b6a-541fb5d205fa",
"snapshot": "os",
"instruction": "I want to switch off the Bluetooth. Can you help me?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/bluetooth-turn-on-off.html.en",
"trajectory": "trajectories/",
"config": [],
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
}
}
}

31
evaluation_examples/examples/os/b6781586-6346-41cd-935a-a6b1487918fc.json
View File

@@ -1,17 +1,18 @@
{
"id": "b6781586-6346-41cd-935a-a6b1487918fc",
"snapshot": "os",
"instruction": "I want to set my current time zone to UTC+0. Can you help me?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/clock-timezone.html.en",
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
"id": "b6781586-6346-41cd-935a-a6b1487918fc",
"snapshot": "os",
"instruction": "I want to set my current time zone to UTC+0. Can you help me?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/clock-timezone.html.en",
"trajectory": "trajectories/",
"config": [],
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
}
}
}

48
evaluation_examples/examples/os/bedcedc4-4d72-425e-ad62-21960b11fe0d.json
View File

@@ -1,25 +1,29 @@
{
"id": "bedcedc4-4d72-425e-ad62-21960b11fe0d",
"snapshot": "os",
"instruction": "Could you set the 'Dim screen when inactive' to on in setting?",
"source": "https://www.youtube.com/watch?v=D4WyNjt_hbQ&t=2s",
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "exact_match",
"result": {
"type": "vm_command_line",
"config": {
"command": "gsettings get org.gnome.desktop.session idle-delay"
}
},
"expected": {
"type": "rule",
"rules":{
"expected": "0"
}
"id": "bedcedc4-4d72-425e-ad62-21960b11fe0d",
"snapshot": "os",
"instruction": "Could you set the 'Dim screen when inactive' to on in setting?",
"source": "https://www.youtube.com/watch?v=D4WyNjt_hbQ&t=2s",
"trajectory": "trajectories/",
"config": [],
"related_apps": [
"os"
],
"evaluator": {
"func": "exact_match",
"result": {
"type": "vm_command_line",
"command": [
"gsettings",
"get",
"org.gnome.desktop.session",
"idle-delay"
]
},
"expected": {
"type": "rule",
"rules": {
"expected": "uint32 0\n"
}
}
}
}
}

69
evaluation_examples/examples/os/ec4e3f68-9ea4-4c18-a5c9-69f89d1178b3.json
View File

@@ -1,33 +1,46 @@
{
"id": "ec4e3f68-9ea4-4c18-a5c9-69f89d1178b3",
"snapshot": "os",
"instruction": "Can you remove the first favorite app from 'favorites'?",
"source": "https://www.youtube.com/watch?v=D4WyNjt_hbQ&t=2s",
"config": [
{
"type": "execute",
"parameters": {
"command": "gsettings set org.gnome.shell favorite-apps \"['thunderbird.desktop', 'firefox.desktop', 'libreoffice-writer.desktop']\""
}
"id": "ec4e3f68-9ea4-4c18-a5c9-69f89d1178b3",
"snapshot": "os",
"instruction": "Can you remove the first favorite app from 'favorites'?",
"source": "https://www.youtube.com/watch?v=D4WyNjt_hbQ&t=2s",
"config": [
{
"type": "execute",
"parameters": {
"command": "gsettings set org.gnome.shell favorite-apps \"['thunderbird.desktop']\""
}
],
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "exact_match",
"result": {
"type": "vm_command_line",
"config": {
"command": "gsettings get org.gnome.shell favorite-apps"
}
},
"expected": {
"type": "rule",
"rules":{
"expected": "['firefox.desktop', 'libreoffice-writer.desktop']"
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": "gsettings set org.gnome.shell favorite-apps \"['thunderbird.desktop', 'vim.desktop', 'google-chrome.desktop']\""
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "check_gnome_favorite_apps",
"result": {
"type": "vm_command_line",
"command": "gsettings get org.gnome.shell favorite-apps"
},
"expected": {
"type": "rule",
"rules": {
"expected": [
"google-chrome.desktop",
"thunderbird.desktop"
]
}
}
}
}

10
evaluation_examples/examples/os/f9be0997-4b7c-45c5-b05c-4612b44a6118.json
View File

@@ -7,7 +7,8 @@
{
"type": "execute",
"parameters": {
"command": "gsettings set org.gnome.desktop.notifications show-banners true"
"command": "gsettings set org.gnome.desktop.notifications show-banners true",
"shell": true
}
}
],
@@ -19,14 +20,13 @@
"func": "exact_match",
"result": {
"type": "vm_command_line",
"config": {
"command": "gsettings get org.gnome.desktop.notifications show-banners"
}
"command": "gsettings get org.gnome.desktop.notifications show-banners",
"shell": true
},
"expected": {
"type": "rule",
"rules":{
"expected": "false"
"expected": "false\n"
}
}
}