lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Load OS Ubuntu examples batch 2

Browse Source
This commit is contained in:
Timothyxxx
2024-01-26 00:15:04 +08:00
parent b9ae4174b1
commit 64594701ae
14 changed files with 324 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
desktop_env/evaluators/getters/__init__.py
View File

@@ -4,7 +4,7 @@ from .chrome import get_default_search_engine, get_cookie_data, get_bookmarks, g
from .file import get_cloud_file, get_vm_file, get_cache_file
from .general import get_vm_command_line
from .impress import get_audio_in_slide
from .info import get_vm_screen_size, get_vm_window_size, get_vm_wallpaper
from .info import get_vm_screen_size, get_vm_window_size, get_vm_wallpaper, get_list_directory
from .misc import get_rule, get_accessibility_tree
from .replay import get_replay
from .vlc import get_vlc_playing_info, get_vlc_config

4
desktop_env/evaluators/getters/info.py
View File

@@ -18,3 +18,7 @@ def get_vm_wallpaper(env, config: dict) -> Union[str, bytes]:
f.write(content)
return _path
def get_list_directory(env, config: dict) -> dict:
return env.controller.get_vm_directory_tree(config["path"])

2
desktop_env/evaluators/metrics/__init__.py
View File

@@ -25,4 +25,4 @@ from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, co
check_qt_slider_colours, check_global_key_play_pause
from .vscode import compare_text_file, compare_config, compare_answer, is_extension_installed, check_json_settings, \
check_json_keybindings
from .os import check_gnome_favorite_apps
from .os import check_gnome_favorite_apps, is_utc_0, check_text_enlarged, check_moved_jpgs

77
desktop_env/evaluators/metrics/os.py
View File

@@ -17,54 +17,43 @@ def check_gnome_favorite_apps(apps_str: str, rule):
return 0
# TODO: log in to the system before running this
def is_logout_successful():
try:
subprocess.run(["whoami"])
return 0 # Task not successful
except subprocess.CalledProcessError:
return 1 # Task successful
def is_utc_0(timedatectl_output):
"""
Format as:
Local time: Thu 2024-01-25 12:56:06 WET
Universal time: Thu 2024-01-25 12:56:06 UTC
RTC time: Thu 2024-01-25 12:56:05
Time zone: Atlantic/Faroe (WET, +0000)
System clock synchronized: yes
NTP service: inactive
RTC in local TZ: no
"""
utc_line = timedatectl_output.split("\n")[3]
if utc_line.endswith("+0000)"):
return 1
else:
return 0
def is_battery_percentage_displayed():
# GNOME schema and key for the setting
schema = "org.gnome.desktop.interface"
key = "show-battery-percentage"
try:
# use gsettings to get the current setting
result = subprocess.run(['gsettings', 'get', schema, key], capture_output=True, text=True)
# examine the output is 'true'
if result.stdout.strip() == 'true':
return 1.
else:
return 0.
except Exception as e:
print(f"An error occurred: {e}")
return 0.
def check_text_enlarged(scaling_factor_str):
scaling_factor = float(scaling_factor_str)
if scaling_factor > 1.0:
return 1
else:
return 0
def check_auto_lock_settings():
# The schema and keys for the GNOME desktop lock settings
schema = "org.gnome.desktop.screensaver"
lock_enabled_key = "lock-enabled"
lock_delay_key = "lock-delay"
def check_moved_jpgs(directory_list, rule):
try:
# Check if the screen lock is enabled
lock_enabled = subprocess.run(['gsettings', 'get', schema, lock_enabled_key], capture_output=True, text=True)
is_lock_enabled = lock_enabled.stdout.strip() == 'true'
expected_jpgs = rule["expected"]
moved_jpgs = [node['name'] for node in directory_list['children']]
# Check the delay before the screen is locked
lock_delay = subprocess.run(['gsettings', 'get', schema, lock_delay_key], capture_output=True, text=True)
# Extract the numerical value from the output
delay_seconds = lock_delay
if len(moved_jpgs) != len(expected_jpgs):
return 0
# Print the results
if is_lock_enabled:
return 1.
else:
return 0.
except Exception as e:
return 0.
if set(moved_jpgs) == set(expected_jpgs):
return 1
else:
return 0

112
evaluation_examples/examples/os/23393935-50c7-4a86-aeea-2b78fd089c5c.json
View File

@@ -1,17 +1,101 @@
{
"id": "23393935-50c7-4a86-aeea-2b78fd089c5c",
"snapshot": "os",
"instruction": "Recursively go through the folders of the 'photos' directory and copy any .jpg files found into another directory named 'cpjpg'.",
"source": "https://superuser.com/questions/91307/copying-only-jpg-from-a-directory-structure-to-another-location-linux",
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
"id": "23393935-50c7-4a86-aeea-2b78fd089c5c",
"snapshot": "os",
"instruction": "Recursively go through the folders of the 'photos' directory and copy any .jpg files found into another directory named 'cpjpg'.",
"source": "https://superuser.com/questions/91307/copying-only-jpg-from-a-directory-structure-to-another-location-linux",
"trajectory": "trajectories/",
"config": [
{
"type": "execute",
"parameters": {
"command": "mkdir -p ~/Desktop/photos && mkdir -p ~/Desktop/cpjpg",
"shell": true
}
},
{
"type": "execute",
"parameters": {
"command": "mkdir -p ~/Desktop/photos/vacation && mkdir -p ~/Desktop/photos/family && mkdir -p ~/Desktop/photos/events",
"shell": true
}
},
{
"type": "execute",
"parameters": {
"command": "mkdir -p ~/Desktop/photos/vacation/thailand && mkdir -p ~/Desktop/photos/vacation/hk",
"shell": true
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1H7R6flpv6vgyBOHurzR0CyvPRIobuIpI&export=download&authuser=0&confirm=t&uuid=1b41e87d-7a84-4aa6-af40-cf940e80d956&at=APZUnTWus2n1pP6l2wmIyXkCUHGs:1706196492307",
"path": "/home/user/Desktop/photos/vacation/thailand/monk252520thailand252520wat252520arun252520scaled25255B225255D.jpg"
}
]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1Mh3kIViHaayxNC88TpgpjbmIEl17LA3d&export=download&authuser=0&confirm=t&uuid=a8355b84-aa2e-4502-8d0f-3723ffe7d442&at=APZUnTW4nzDFgDgNeERMRKEQ4bTU:1706196493503",
"path": "/home/user/Desktop/photos/vacation/hk/hong-kong-china.jpg"
}
]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EhmbZ3QP_mdvvppC5zKA3o--Ce2F-ocO&export=download&authuser=0&confirm=t&uuid=70e05133-dcac-4168-b555-82da15a10d58&at=APZUnTXJ371jG7qiPLHBLt2zy7ij:1706196494612",
"path": "/home/user/Desktop/photos/vacation/hk/hk_group_photo.jpg"
}
]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1p_r_BMCLFG872uH9Qa-sLnqXppHvDx95&export=download&authuser=0&confirm=t&uuid=af59d107-8e46-4d2c-b72a-e4b42655959a&at=APZUnTU2hBidSnefJuUQTyJCEVFx:1706196491030",
"path": "/home/user/Desktop/photos/family/us_3.png"
}
]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1YxryNHfLPcFODnUh31CYGHGj3ZFRhvbA&export=download&authuser=0&confirm=t&uuid=0925ae33-ab33-42b9-901c-d90f11e0afcd&at=APZUnTVey7VeC9zYOmTwl0x02Iwf:1706196495770",
"path": "/home/user/Desktop/photos/events/emnlp2023.jpg"
}
]
}
}
}
],
"related_apps": [
"os"
],
"evaluator": {
"func": "check_moved_jpgs",
"result": {
"type": "list_directory",
"path": "/home/user/Desktop/cpjpg"
},
"expected": {
"type": "rule",
"rules": {
"expected": ["emnlp2023.jpg", "hk_group_photo.jpg", "hong-kong-china.jpg", "monk252520thailand252520wat252520arun252520scaled25255B225255D.jpg"]
}
}
}
}

9
evaluation_examples/examples/os/3ce045a0-877b-42aa-8d2c-b4a863336ab8.json
View File

@@ -1,7 +1,7 @@
{
"id": "3ce045a0-877b-42aa-8d2c-b4a863336ab8",
"snapshot": "os",
"instruction": "I want to make the text on the screen larger",
"instruction": "My glasses are broken, and I'm having trouble seeing small things clearly. Could you help me enlarge the text on my screen so it's easier to read?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/a11y-font-size.html.en",
"trajectory": "trajectories/",
"config": [],
@@ -9,10 +9,11 @@
"os"
],
"evaluator": {
"func": "",
"func": "check_text_enlarged",
"result": {
},
"expected": {
"type": "vm_command_line",
"command": "gsettings get org.gnome.desktop.interface text-scaling-factor",
"shell": true
}
}
}

2
evaluation_examples/examples/os/5c433d22-ed9a-4e31-91f5-54cf3e8acd63.json
View File

@@ -1,7 +1,7 @@
{
"id": "5c433d22-ed9a-4e31-91f5-54cf3e8acd63",
"snapshot": "os",
"instruction": "I want to change my system language to Chinese(simplified). Can you help me?",
"instruction": "I want to change my system language to Chinese (simplified). Can you help me?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/session-language.html.zh-CN",
"trajectory": "trajectories/",
"config": [],

40
evaluation_examples/examples/os/5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57.json
View File

@@ -1,18 +1,44 @@
{
"id": "5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57",
"snapshot": "os",
"instruction": "I am currently using an Ubuntu system, and I have wrongly deleted a file named \"test\". Could you help me recover it from the Trash?",
"instruction": "I am currently using an Ubuntu system, and I have wrongly deleted a poster of party night. Could you help me recover it from the Trash?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/files-recover.html.en",
"trajectory": "trajectories/",
"config": [],
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1XaTnC_lLbR_tGTz8tcN2Tp6cNrMlNW3R&export=download&authuser=0&confirm=t&uuid=89e69a23-43cf-4316-833a-fb9d3e281460&at=APZUnTWn5zZTH4GlClO6lV1i4WwP:1706184669922",
"path": "poster_party_night.webp"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": "mv ~/poster_party_night.webp ~/.local/share/Trash/files/",
"shell": true
}
}
],
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
"func": "exact_match",
"result": {
"type": "vm_command_line",
"command": "[ -f /home/user/Desktop/poster_party_night.webp ] && echo 'File exists.' || echo 'File does not exist.'",
"shell": true
},
"expected": {
"type": "rule",
"rules":{
"expected": "File exists.\n"
}
}
}
}

57
evaluation_examples/examples/os/6ebbfb01-ea72-4226-a2a6-dc428e111ed2.json
View File

@@ -4,15 +4,60 @@
"instruction": "Could you please help me to set Bash as my default shell on the current Ubuntu system?",
"source": "https://superuser.com/questions/46748/how-do-i-make-bash-my-default-shell-on-ubuntu",
"trajectory": "trajectories/",
"config": [],
"config": [
{
"type": "execute",
"parameters": {
"command": "echo password | sudo -S apt-get install zsh -y",
"shell": true
}
},
{
"type": "execute",
"parameters": {
"command": "echo password | sudo -S chsh -s $(which zsh)",
"shell": true
}
},
{
"type": "execute",
"parameters": {
"command": "echo password | sudo -S loginctl terminate-user user",
"shell": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 10
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; time.sleep(2); pyautogui.press(\"enter\"); pyautogui.type(\"password\"); time.sleep(2); pyautogui.press(\"enter\");"
]
}
}
],
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
"func": "exact_match",
"result": {
"type": "vm_command_line",
"command": "echo $SHELL",
"shell": true
},
"expected": {
"type": "rule",
"rules":{
"expected": "/bin/bash\n"
}
}
}
}

35
evaluation_examples/examples/os/a4d98375-215b-4a4d-aee9-3d4370fccc41.json
View File

@@ -1,17 +1,24 @@
{
"id": "a4d98375-215b-4a4d-aee9-3d4370fccc41",
"snapshot": "os",
"instruction": "I want to have my computer automatically locked after I leaved. Can you help me?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/privacy-screen-lock.html.en",
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
"id": "a4d98375-215b-4a4d-aee9-3d4370fccc41",
"snapshot": "os",
"instruction": "I want to have my computer automatically locked after I leaved. Can you help me?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/privacy-screen-lock.html.en",
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "exact_match",
"result": {
"type": "vm_command_line",
"command": "gsettings get org.gnome.desktop.screensaver lock-enabled",
"shell": true
},
"expected": {
"type": "rule",
"rules": {
"expected": "true\n"
}
}
}
}
}

10
evaluation_examples/examples/os/b6781586-6346-41cd-935a-a6b1487918fc.json
View File

@@ -4,15 +4,17 @@
"instruction": "I want to set my current time zone to UTC+0. Can you help me?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/clock-timezone.html.en",
"trajectory": "trajectories/",
"config": [],
"config": [
],
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"func": "is_utc_0",
"result": {
},
"expected": {
"type": "vm_command_line",
"command": "timedatectl status",
"shell": true
}
}
}

45
evaluation_examples/examples/os/ddc75b62-7311-4af8-bfb3-859558542b36.json
View File

@@ -1,17 +1,36 @@
{
"id": "ddc75b62-7311-4af8-bfb3-859558542b36",
"snapshot": "os",
"instruction": "I want to uninstall the Firefox on my system. Can you help me",
"source": "https://help.ubuntu.com/lts/ubuntu-help/addremove-remove.html.en",
"trajectory": "trajectories/",
"related_apps": [
"os"
"id": "ddc75b62-7311-4af8-bfb3-859558542b36",
"snapshot": "os",
"instruction": "I want to uninstall the Mahjongg on my system. Can you help me?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/addremove-remove.html.en",
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"postconfig": [
{
"type": "sleep",
"parameters": {
"seconds": 1
}
}
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
"func": "check_include_exclude",
"result": {
"type": "vm_command_line",
"command": "dpkg -l | grep mahjongg",
"shell": true
},
"expected": {
"type": "rule",
"rules": {
"include": [
],
"exclude": [
"mahjongg"
]
}
}
}
}
}

44
evaluation_examples/examples/os/e0df059f-28a6-4169-924f-b9623e7184cc.json
View File

@@ -1,17 +1,33 @@
{
"id": "e0df059f-28a6-4169-924f-b9623e7184cc",
"snapshot": "os",
"instruction": "I have a directory named \"test1\". Can you help me change its name into \"test2\"?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/files-rename.html.en",
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
"id": "e0df059f-28a6-4169-924f-b9623e7184cc",
"snapshot": "os",
"instruction": "I have a directory named \"todo_list_Jan_1\". Can you help me change its name into \"todo_list_Jan_2\"?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/files-rename.html.en",
"trajectory": "trajectories/",
"config": [
{
"type": "execute",
"parameters": {
"command": "echo 'password' | sudo -S mkdir ~/Desktop/todo_list_Jan_1",
"shell": true
}
}
}
],
"related_apps": [
"os"
],
"evaluator": {
"func": "exact_match",
"result": {
"type": "vm_command_line",
"command": "[ -d ~/Desktop/todo_list_Jan_2 ] && echo 'Directory exists.' || echo 'Directory does not exist.'",
"shell": true
},
"expected": {
"type": "rule",
"rules":{
"expected": "Directory exists.\n"
}
}
}
}

9
evaluation_examples/examples/os/fe41f596-a71b-4c2f-9b2f-9dcd40b568c3.json
View File

@@ -8,10 +8,17 @@
"os"
],
"evaluator": {
"func": "",
"func": "exact_match",
"result": {
"type": "vm_command_line",
"command": "gsettings get org.gnome.desktop.interface show-battery-percentage",
"shell": true
},
"expected": {
"type": "rule",
"rules":{
"expected": "true\n"
}
}
}
}