lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update OS evals

Browse Source
This commit is contained in:
Liu Yitao
2024-01-25 10:45:51 +08:00
parent 09f3e776ae
commit 93b4ff7d95
10 changed files with 311 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
desktop_env/evaluators/metrics/general.py
View File

@@ -16,6 +16,11 @@ from .utils import _match_record, _match_value_to_rule
import sqlite3
def check_include_exclude(result: str, rules: Dict[str, List[str]]) -> float:
include = rules.get("include", [])
exclude = rules.get("exclude", [])
return all(r in result for r in include) and all(r not in result for r in exclude)
def exact_match(result, rules) -> float:
expect = rules["expected"]
print(result, expect)
@@ -195,3 +200,4 @@ def check_json(result: str, rules: Dict[str, List[Dict[str, Union[List[str], str
value = value[k]
metric = metric and not _match_value_to_rule(value, r)
return metric

93
desktop_env/evaluators/metrics/os.py Normal file
View File

@@ -0,0 +1,93 @@
import subprocess
import os
def is_spotify_installed() -> bool:
# Use the 'which' command to check if the 'spotify' executable exists
try:
subprocess.check_output(['which', 'spotify'])
return True
except subprocess.CalledProcessError:
return False
def is_dim_screen_inactive_off():
# Get the current value of "Dim screen when inactive" setting
result = subprocess.run(["gsettings", "get", "org.gnome.desktop.session", "idle-delay"], capture_output=True, text=True)
# Check if the setting is set to "uint32 0"
if result.stdout.strip() == "uint32 0":
return 1 # Task successful
else:
return 0 # Task not successful
def evaluate_create_test_directory():
try:
# Specify the path to the directory
path = '/test'
# Check if the directory already exists
if os.path.exists(path):
return 0 # Directory already exists, task not successful
# Check if the user has sufficient permissions to create the directory
if not os.access("/", os.W_OK | os.X_OK):
return 0 # Insufficient permissions to create the directory, task not successful
return 1 # Task can be considered successful
except Exception as e:
print(f"Error: {str(e)}")
return 0 # Any other errors, task not successful
# TODO: create a file named test.txt and a directory dir1 at home dir before running this
def is_file_in_directory():
# Specify the paths of the file and directory
file_path = "/test.txt"
directory_path = "/dir1"
# Check if the file exists in the directory
if os.path.isfile(os.path.join(directory_path, os.path.basename(file_path))):
return 1 # Task successful
else:
return 0 # Task not successful
# TODO: log in to the system before running this
def is_logout_successful():
try:
subprocess.run(["whoami"])
return 0 # Task not successful
except subprocess.CalledProcessError:
return 1 # Task successful
def is_do_not_disturb_mode_enabled():
try:
subprocess.run(["gsettings", "set", "org.gnome.desktop.notifications", "show-banners", "false"], check=True)
return 1 # Task successful
except subprocess.CalledProcessError:
return 0 # Task not successful
def get_default_browser(answer: str):
try:
result = subprocess.run(["xdg-settings", "get", "default-web-browser"], capture_output=True, text=True)
default_browser = result.stdout.strip()
return default_browser == answer
except subprocess.CalledProcessError:
return 0
# TODO: should confirm initial state, i.e. initial list of favorite apps & the first app
first_app = "thunderbird.desktop" # to be changed
def is_first_favorite_app_removed():
try:
result = subprocess.run(["gsettings", "get", "org.gnome.shell", "favorite-apps"], capture_output=True, text=True)
output = result.stdout.strip()
# Remove brackets and spaces from the output
favorites = output[1:-1].replace(" ", "")
# Split output by comma to get individual favorite apps
favorite_apps = favorites.split(",")
if favorite_apps[0] != first_app:
return 1 # First favorite app removed
else:
return 0 # First favorite app not removed
except subprocess.CalledProcessError:
return 0 # Task not successful

26
evaluation_examples/examples/os/43c2d64c-bab5-4dcb-a30c-b888321c319a.json Normal file
View File

@@ -0,0 +1,26 @@
{
"id": "43c2d64c-bab5-4dcb-a30c-b888321c319a",
"snapshot": "os",
"instruction": "Could you please help me create a dir named 'test' in the root directory of current computer?",
"source": "https://ubuntu.com/tutorials/command-line-for-beginners#4-creating-folders-and-files",
"config": [],
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "exact_match",
"result": {
"type": "vm_command_line",
"config": {
"command": "[ -d '/path/to/directory' ] && echo 'Directory exists.' || echo 'Directory does not exist.'"
}
},
"expected": {
"type": "rule",
"rules":{
"expected": "Directory exists."
}
}
}
}

33
evaluation_examples/examples/os/7688b85f-87a4-4e4a-b2f8-f3d6c3f29b82.json Normal file
View File

@@ -0,0 +1,33 @@
{
"id": "7688b85f-87a4-4e4a-b2f8-f3d6c3f29b82",
"snapshot": "os",
"instruction": "Can you move the file with the path '/test.txt' to the directory with the path '/dir1'?",
"source": "https://ubuntu.com/tutorials/command-line-for-beginners#5-moving-and-manipulating-files",
"config": [
{
"type": "execute",
"parameters": {
"command": "touch /test.txt && mkdir /dir1"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "exact_match",
"result": {
"type": "vm_command_line",
"config": {
"command": "if [ -f '/dir/test.txt' ]; then echo 'File exists.'; else echo 'File does not exist.'; fi"
}
},
"expected": {
"type": "rule",
"rules":{
"expected": "File exists."
}
}
}
}

26
evaluation_examples/examples/os/94d95f96-9699-4208-98ba-3c3119edf9c2.json Normal file
View File

@@ -0,0 +1,26 @@
{
"id": "94d95f96-9699-4208-98ba-3c3119edf9c2",
"snapshot": "os",
"instruction": "I want to install Spotify on my current system. Could you please help me?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/addremove-install.html.en",
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "check_include_exclude",
"result": {
"type": "vm_command_line",
"config": {
"command": "which spotify"
}
},
"expected": {
"type": "rule",
"rules": {
"include": [],
"exclude": ["not found"]
}
}
}
}

33
evaluation_examples/examples/os/a462a795-fdc7-4b23-b689-e8b6df786b78.json Normal file
View File

@@ -0,0 +1,33 @@
{
"id": "a462a795-fdc7-4b23-b689-e8b6df786b78",
"snapshot": "os",
"instruction": "Could you please help me to switch to the user 'Charles' with password of 'Ex@mpleP@55w0rd!'?",
"source": "https://ubuntu.com/tutorials/command-line-for-beginners#5-moving-and-manipulating-files",
"config": [
{
"type": "execute",
"parameters": {
"command": "echo password | sudo -S su - charles"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "exact_match",
"result": {
"type": "vm_command_line",
"config": {
"command": "if [ '$(whoami)' = 'charles' ]; then echo 'Current user is charles.'; else echo 'Current user is not charles'; fi"
}
},
"expected": {
"type": "rule",
"rules":{
"expected": "Current user is charles"
}
}
}
}

33
evaluation_examples/examples/os/ae039631-2b12-4637-84f6-c67d51511be3.json Normal file
View File

@@ -0,0 +1,33 @@
{
"id": "ae039631-2b12-4637-84f6-c67d51511be3",
"snapshot": "os",
"instruction": "Could you please help me check out my current default browser?",
"source": "https://ubuntu.com/tutorials/command-line-for-beginners#5-moving-and-manipulating-files",
"config": [
{
"type": "execute",
"parameters": {
"command": "xdg-settings set default-web-browser firefox.desktop"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "exact_match",
"result": {
"type": "vm_command_line",
"config": {
"command": "xdg-settings get default-web-browser"
}
},
"expected": {
"type": "rule",
"rules":{
"expected": "firefox.desktop"
}
}
}
}

25
evaluation_examples/examples/os/bedcedc4-4d72-425e-ad62-21960b11fe0d.json Normal file
View File

@@ -0,0 +1,25 @@
{
"id": "bedcedc4-4d72-425e-ad62-21960b11fe0d",
"snapshot": "os",
"instruction": "Could you set the 'Dim screen when inactive' to on in setting?",
"source": "https://www.youtube.com/watch?v=D4WyNjt_hbQ&t=2s",
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "exact_match",
"result": {
"type": "vm_command_line",
"config": {
"command": "gsettings get org.gnome.desktop.session idle-delay"
}
},
"expected": {
"type": "rule",
"rules":{
"expected": "0"
}
}
}
}

18
evaluation_examples/examples/os/ec4e3f68-9ea4-4c18-a5c9-69f89d1178b3.json Normal file
View File

@@ -0,0 +1,18 @@
{
"id": "ec4e3f68-9ea4-4c18-a5c9-69f89d1178b3",
"snapshot": "os",
"instruction": "Can you remove the first favorite app from 'favorites'?",
"source": "https://www.youtube.com/watch?v=D4WyNjt_hbQ&t=2s",
"config": [],
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "is_first_favorite_app_removed",
"result": {
},
"expected": {
}
}
}

18
evaluation_examples/examples/os/f9be0997-4b7c-45c5-b05c-4612b44a6118.json Normal file
View File

@@ -0,0 +1,18 @@
{
"id": "f9be0997-4b7c-45c5-b05c-4612b44a6118",
"snapshot": "os",
"instruction": "I am currently working on a ubuntu system but I do not want the notifications to bother me. Can you help me to switch to 'Do not disturb mode'?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/shell-notifications.html.en",
"config": [],
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "is_do_not_disturb_mode_enabled",
"result": {
},
"expected": {
}
}
}