lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update OS evals

Browse Source
This commit is contained in:
Liu Yitao
2024-01-25 10:55:49 +08:00
parent 93b4ff7d95
commit 344e7db55c
2 changed files with 35 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
evaluation_examples/examples/os/ec4e3f68-9ea4-4c18-a5c9-69f89d1178b3.json
View File

@@ -3,16 +3,31 @@
"snapshot": "os", "snapshot": "os",
"instruction": "Can you remove the first favorite app from 'favorites'?", "instruction": "Can you remove the first favorite app from 'favorites'?",
"source": "https://www.youtube.com/watch?v=D4WyNjt_hbQ&t=2s", "source": "https://www.youtube.com/watch?v=D4WyNjt_hbQ&t=2s",
"config": [], "config": [
{
"type": "execute",
"parameters": {
"command": "gsettings set org.gnome.shell favorite-apps \"['thunderbird.desktop', 'firefox.desktop', 'libreoffice-writer.desktop']\""
}
}
],
"trajectory": "trajectories/", "trajectory": "trajectories/",
"related_apps": [ "related_apps": [
"os" "os"
], ],
"evaluator": { "evaluator": {
"func": "is_first_favorite_app_removed", "func": "exact_match",
"result": { "result": {
"type": "vm_command_line",
"config": {
"command": "gsettings get org.gnome.shell favorite-apps"
}
}, },
"expected": { "expected": {
"type": "rule",
"rules":{
"expected": "['firefox.desktop', 'libreoffice-writer.desktop']"
}
} }
} }
} }

19
evaluation_examples/examples/os/f9be0997-4b7c-45c5-b05c-4612b44a6118.json
View File

@@ -3,16 +3,31 @@
"snapshot": "os", "snapshot": "os",
"instruction": "I am currently working on a ubuntu system but I do not want the notifications to bother me. Can you help me to switch to 'Do not disturb mode'?", "instruction": "I am currently working on a ubuntu system but I do not want the notifications to bother me. Can you help me to switch to 'Do not disturb mode'?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/shell-notifications.html.en", "source": "https://help.ubuntu.com/lts/ubuntu-help/shell-notifications.html.en",
"config": [], "config": [
{
"type": "execute",
"parameters": {
"command": "gsettings set org.gnome.desktop.notifications show-banners true"
}
}
],
"trajectory": "trajectories/", "trajectory": "trajectories/",
"related_apps": [ "related_apps": [
"os" "os"
], ],
"evaluator": { "evaluator": {
"func": "is_do_not_disturb_mode_enabled", "func": "exact_match",
"result": { "result": {
"type": "vm_command_line",
"config": {
"command": "gsettings get org.gnome.desktop.notifications show-banners"
}
}, },
"expected": { "expected": {
"type": "rule",
"rules":{
"expected": "false"
}
} }
} }
} }