lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix OS examples annotated by Yitao

Browse Source
This commit is contained in:
Timothyxxx
2024-01-25 19:57:32 +08:00
parent 0c34fccc15
commit b9ae4174b1
24 changed files with 327 additions and 358 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
evaluation_examples/examples/os/28cc3b7e-b194-4bc9-8353-d04c0f4d56d2.json
View File

@@ -1,17 +1,25 @@
{
"id": "28cc3b7e-b194-4bc9-8353-d04c0f4d56d2",
"snapshot": "os",
"instruction": "The volume of my system is too small. Can you help me turn up to the max volumn?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/sound-volume.html.en",
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
"id": "28cc3b7e-b194-4bc9-8353-d04c0f4d56d2",
"snapshot": "os",
"instruction": "The volume of my system is too small. Can you help me turn up to the max volume?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/sound-volume.html.en",
"trajectory": "trajectories/",
"config": [],
"related_apps": [
"os"
],
"evaluator": {
"func": "exact_match",
"result": {
"type": "vm_command_line",
"command": "pactl list sinks | grep '^[[:space:]]Volume:' | head -n 1 | awk '{print $5}' | sed 's/%//g'",
"shell": true
},
"expected": {
"type": "rule",
"rules":{
"expected": "100\n"
}
}
}
}
}

31
evaluation_examples/examples/os/3ce045a0-877b-42aa-8d2c-b4a863336ab8.json
View File

@@ -1,17 +1,18 @@
{
"id": "3ce045a0-877b-42aa-8d2c-b4a863336ab8",
"snapshot": "os",
"instruction": "I want to make the text on the screen larger",
"source": "https://help.ubuntu.com/lts/ubuntu-help/a11y-font-size.html.en",
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
"id": "3ce045a0-877b-42aa-8d2c-b4a863336ab8",
"snapshot": "os",
"instruction": "I want to make the text on the screen larger",
"source": "https://help.ubuntu.com/lts/ubuntu-help/a11y-font-size.html.en",
"trajectory": "trajectories/",
"config": [],
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
}
}
}

7
evaluation_examples/examples/os/43c2d64c-bab5-4dcb-a30c-b888321c319a.json
View File

@@ -12,14 +12,13 @@
"func": "exact_match",
"result": {
"type": "vm_command_line",
"config": {
"command": "[ -d '/path/to/directory' ] && echo 'Directory exists.' || echo 'Directory does not exist.'"
}
"command": "[ -d '/home/user/Desktop/test' ] && echo 'Directory exists.' || echo 'Directory does not exist.'",
"shell": true
},
"expected": {
"type": "rule",
"rules":{
"expected": "Directory exists."
"expected": "Directory exists.\n"
}
}
}

31
evaluation_examples/examples/os/4d2b519e-e872-4100-8ea3-fe71ab0f9133.json
View File

@@ -1,17 +1,18 @@
{
"id": "4d2b519e-e872-4100-8ea3-fe71ab0f9133",
"snapshot": "os",
"instruction": "Could you please help me to add a new entry \"/home/david/pear/bin\" to the PATH variable in the ZSH terminal?",
"source": "https://stackoverflow.com/questions/11530090/adding-a-new-entry-to-the-path-variable-in-zsh",
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
"id": "4d2b519e-e872-4100-8ea3-fe71ab0f9133",
"snapshot": "os",
"instruction": "Could you please help me to add a new entry \"/home/david/pear/bin\" to the PATH variable in the ZSH terminal?",
"source": "https://stackoverflow.com/questions/11530090/adding-a-new-entry-to-the-path-variable-in-zsh",
"trajectory": "trajectories/",
"config": [],
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
}
}
}

31
evaluation_examples/examples/os/5812b315-e7bd-4265-b51f-863c02174c28.json
View File

@@ -1,17 +1,18 @@
{
"id": "5812b315-e7bd-4265-b51f-863c02174c28",
"snapshot": "os",
"instruction": "Please create an SSH user on Ubuntu who is only allowed to access the folder \"test1\".",
"source": "https://superuser.com/questions/149404/create-an-ssh-user-who-only-has-permission-to-access-specific-folders",
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
"id": "5812b315-e7bd-4265-b51f-863c02174c28",
"snapshot": "os",
"instruction": "Please create an SSH user on Ubuntu who is only allowed to access the folder \"test1\".",
"source": "https://superuser.com/questions/149404/create-an-ssh-user-who-only-has-permission-to-access-specific-folders",
"config": [],
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
}
}
}

31
evaluation_examples/examples/os/5c433d22-ed9a-4e31-91f5-54cf3e8acd63.json
View File

@@ -1,17 +1,18 @@
{
"id": "5c433d22-ed9a-4e31-91f5-54cf3e8acd63",
"snapshot": "os",
"instruction": "I want to change my system language to Chinese(simplified). Can you help me?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/session-language.html.zh-CN",
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
"id": "5c433d22-ed9a-4e31-91f5-54cf3e8acd63",
"snapshot": "os",
"instruction": "I want to change my system language to Chinese(simplified). Can you help me?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/session-language.html.zh-CN",
"trajectory": "trajectories/",
"config": [],
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
}
}
}

31
evaluation_examples/examples/os/5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57.json
View File

@@ -1,17 +1,18 @@
{
"id": "5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57",
"snapshot": "os",
"instruction": "I am currently using an Ubuntu system, and I have wrongly deleted a file named \"test\". Could you help me recover it from the Trash?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/files-recover.html.en",
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
"id": "5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57",
"snapshot": "os",
"instruction": "I am currently using an Ubuntu system, and I have wrongly deleted a file named \"test\". Could you help me recover it from the Trash?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/files-recover.html.en",
"trajectory": "trajectories/",
"config": [],
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
}
}
}

31
evaluation_examples/examples/os/6ebbfb01-ea72-4226-a2a6-dc428e111ed2.json
View File

@@ -1,17 +1,18 @@
{
"id": "6ebbfb01-ea72-4226-a2a6-dc428e111ed2",
"snapshot": "os",
"instruction": "Could you please help me to set Bash as my default shell on the current Ubuntu system?",
"source": "https://superuser.com/questions/46748/how-do-i-make-bash-my-default-shell-on-ubuntu",
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
"id": "6ebbfb01-ea72-4226-a2a6-dc428e111ed2",
"snapshot": "os",
"instruction": "Could you please help me to set Bash as my default shell on the current Ubuntu system?",
"source": "https://superuser.com/questions/46748/how-do-i-make-bash-my-default-shell-on-ubuntu",
"trajectory": "trajectories/",
"config": [],
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
}
}
}

31
evaluation_examples/examples/os/765d2b74-88a7-4d50-bf51-34e4106fd24a.json
View File

@@ -1,17 +1,18 @@
{
"id": "765d2b74-88a7-4d50-bf51-34e4106fd24a",
"snapshot": "os",
"instruction": "Can you help me delete the \"test\" file on my desktop?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/files-delete.html.en",
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
"id": "765d2b74-88a7-4d50-bf51-34e4106fd24a",
"snapshot": "os",
"instruction": "Can you help me delete the \"test\" file on my desktop?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/files-delete.html.en",
"config": [],
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
}
}
}

19
evaluation_examples/examples/os/7688b85f-87a4-4e4a-b2f8-f3d6c3f29b82.json
View File

@@ -1,13 +1,21 @@
{
"id": "7688b85f-87a4-4e4a-b2f8-f3d6c3f29b82",
"snapshot": "os",
"instruction": "Can you move the file with the path '/test.txt' to the directory with the path '/dir1'?",
"instruction": "Can you move the file with the path 'todo.txt' to the directory with the path 'done'?",
"source": "https://ubuntu.com/tutorials/command-line-for-beginners#5-moving-and-manipulating-files",
"config": [
{
"type": "execute",
"parameters": {
"command": "touch /test.txt && mkdir /dir1"
"command": "echo 'password' | sudo -S touch ~/Desktop/todo.txt && sudo mkdir ~/Desktop/done",
"shell": true
}
},
{
"type": "execute",
"parameters": {
"command": "echo 'password' | sudo -S chmod 777 ~/Desktop/todo.txt && sudo chmod 777 ~/Desktop/done",
"shell": true
}
}
],
@@ -19,14 +27,13 @@
"func": "exact_match",
"result": {
"type": "vm_command_line",
"config": {
"command": "if [ -f '/dir/test.txt' ]; then echo 'File exists.'; else echo 'File does not exist.'; fi"
}
"command": "if [ -f ~/Desktop/done/todo.txt ]; then echo 'File exists.'; else echo 'File does not exist.'; fi",
"shell": true
},
"expected": {
"type": "rule",
"rules":{
"expected": "File exists."
"expected": "File exists.\n"
}
}
}

49
evaluation_examples/examples/os/94d95f96-9699-4208-98ba-3c3119edf9c2.json
View File

@@ -1,26 +1,29 @@
{
"id": "94d95f96-9699-4208-98ba-3c3119edf9c2",
"snapshot": "os",
"instruction": "I want to install Spotify on my current system. Could you please help me?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/addremove-install.html.en",
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "check_include_exclude",
"result": {
"type": "vm_command_line",
"config": {
"command": "which spotify"
}
},
"expected": {
"type": "rule",
"rules": {
"include": [],
"exclude": ["not found"]
}
"id": "94d95f96-9699-4208-98ba-3c3119edf9c2",
"snapshot": "os",
"instruction": "I want to install Spotify on my current system. Could you please help me?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/addremove-install.html.en",
"trajectory": "trajectories/",
"config": [],
"related_apps": [
"os"
],
"evaluator": {
"func": "check_include_exclude",
"result": {
"type": "vm_command_line",
"command": "which spotify"
},
"expected": {
"type": "rule",
"rules": {
"include": [
"spotify"
],
"exclude": [
"not found"
]
}
}
}
}
}

8
evaluation_examples/examples/os/a462a795-fdc7-4b23-b689-e8b6df786b78.json
View File

@@ -7,7 +7,8 @@
{
"type": "execute",
"parameters": {
"command": "echo password | sudo -S su - charles"
"command": "echo password | sudo -S su - charles",
"shell": true
}
}
],
@@ -19,9 +20,8 @@
"func": "exact_match",
"result": {
"type": "vm_command_line",
"config": {
"command": "if [ '$(whoami)' = 'charles' ]; then echo 'Current user is charles.'; else echo 'Current user is not charles'; fi"
}
"command": "if [ '$(whoami)' = 'charles' ]; then echo 'Current user is charles.'; else echo 'Current user is not charles'; fi",
"shell": true
},
"expected": {
"type": "rule",

4
evaluation_examples/examples/os/ae039631-2b12-4637-84f6-c67d51511be3.json
View File

@@ -19,9 +19,7 @@
"func": "exact_match",
"result": {
"type": "vm_command_line",
"config": {
"command": "xdg-settings get default-web-browser"
}
"command": "xdg-settings get default-web-browser"
},
"expected": {
"type": "rule",

31
evaluation_examples/examples/os/b3d4a89c-53f2-4d6b-8b6a-541fb5d205fa.json
View File

@@ -1,17 +1,18 @@
{
"id": "b3d4a89c-53f2-4d6b-8b6a-541fb5d205fa",
"snapshot": "os",
"instruction": "I want to switch off the Bluetooth. Can you help me?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/bluetooth-turn-on-off.html.en",
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
"id": "b3d4a89c-53f2-4d6b-8b6a-541fb5d205fa",
"snapshot": "os",
"instruction": "I want to switch off the Bluetooth. Can you help me?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/bluetooth-turn-on-off.html.en",
"trajectory": "trajectories/",
"config": [],
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
}
}
}

31
evaluation_examples/examples/os/b6781586-6346-41cd-935a-a6b1487918fc.json
View File

@@ -1,17 +1,18 @@
{
"id": "b6781586-6346-41cd-935a-a6b1487918fc",
"snapshot": "os",
"instruction": "I want to set my current time zone to UTC+0. Can you help me?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/clock-timezone.html.en",
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
"id": "b6781586-6346-41cd-935a-a6b1487918fc",
"snapshot": "os",
"instruction": "I want to set my current time zone to UTC+0. Can you help me?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/clock-timezone.html.en",
"trajectory": "trajectories/",
"config": [],
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
}
}
}

48
evaluation_examples/examples/os/bedcedc4-4d72-425e-ad62-21960b11fe0d.json
View File

@@ -1,25 +1,29 @@
{
"id": "bedcedc4-4d72-425e-ad62-21960b11fe0d",
"snapshot": "os",
"instruction": "Could you set the 'Dim screen when inactive' to on in setting?",
"source": "https://www.youtube.com/watch?v=D4WyNjt_hbQ&t=2s",
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "exact_match",
"result": {
"type": "vm_command_line",
"config": {
"command": "gsettings get org.gnome.desktop.session idle-delay"
}
},
"expected": {
"type": "rule",
"rules":{
"expected": "0"
}
"id": "bedcedc4-4d72-425e-ad62-21960b11fe0d",
"snapshot": "os",
"instruction": "Could you set the 'Dim screen when inactive' to on in setting?",
"source": "https://www.youtube.com/watch?v=D4WyNjt_hbQ&t=2s",
"trajectory": "trajectories/",
"config": [],
"related_apps": [
"os"
],
"evaluator": {
"func": "exact_match",
"result": {
"type": "vm_command_line",
"command": [
"gsettings",
"get",
"org.gnome.desktop.session",
"idle-delay"
]
},
"expected": {
"type": "rule",
"rules": {
"expected": "uint32 0\n"
}
}
}
}
}

69
evaluation_examples/examples/os/ec4e3f68-9ea4-4c18-a5c9-69f89d1178b3.json
View File

@@ -1,33 +1,46 @@
{
"id": "ec4e3f68-9ea4-4c18-a5c9-69f89d1178b3",
"snapshot": "os",
"instruction": "Can you remove the first favorite app from 'favorites'?",
"source": "https://www.youtube.com/watch?v=D4WyNjt_hbQ&t=2s",
"config": [
{
"type": "execute",
"parameters": {
"command": "gsettings set org.gnome.shell favorite-apps \"['thunderbird.desktop', 'firefox.desktop', 'libreoffice-writer.desktop']\""
}
"id": "ec4e3f68-9ea4-4c18-a5c9-69f89d1178b3",
"snapshot": "os",
"instruction": "Can you remove the first favorite app from 'favorites'?",
"source": "https://www.youtube.com/watch?v=D4WyNjt_hbQ&t=2s",
"config": [
{
"type": "execute",
"parameters": {
"command": "gsettings set org.gnome.shell favorite-apps \"['thunderbird.desktop']\""
}
],
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "exact_match",
"result": {
"type": "vm_command_line",
"config": {
"command": "gsettings get org.gnome.shell favorite-apps"
}
},
"expected": {
"type": "rule",
"rules":{
"expected": "['firefox.desktop', 'libreoffice-writer.desktop']"
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": "gsettings set org.gnome.shell favorite-apps \"['thunderbird.desktop', 'vim.desktop', 'google-chrome.desktop']\""
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "check_gnome_favorite_apps",
"result": {
"type": "vm_command_line",
"command": "gsettings get org.gnome.shell favorite-apps"
},
"expected": {
"type": "rule",
"rules": {
"expected": [
"google-chrome.desktop",
"thunderbird.desktop"
]
}
}
}
}

10
evaluation_examples/examples/os/f9be0997-4b7c-45c5-b05c-4612b44a6118.json
View File

@@ -7,7 +7,8 @@
{
"type": "execute",
"parameters": {
"command": "gsettings set org.gnome.desktop.notifications show-banners true"
"command": "gsettings set org.gnome.desktop.notifications show-banners true",
"shell": true
}
}
],
@@ -19,14 +20,13 @@
"func": "exact_match",
"result": {
"type": "vm_command_line",
"config": {
"command": "gsettings get org.gnome.desktop.notifications show-banners"
}
"command": "gsettings get org.gnome.desktop.notifications show-banners",
"shell": true
},
"expected": {
"type": "rule",
"rules":{
"expected": "false"
"expected": "false\n"
}
}
}