lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1 from xlang-ai/lyt

Browse Source 
OS evals update
This commit is contained in:
Tianbao Xie
2024-01-25 10:59:22 +08:00
committed by GitHub
parent 8837d8051a 344e7db55c
commit 8a9f8e116b
10 changed files with 341 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
evaluation_examples/examples/os/43c2d64c-bab5-4dcb-a30c-b888321c319a.json Normal file
View File

@@ -0,0 +1,26 @@
{
"id": "43c2d64c-bab5-4dcb-a30c-b888321c319a",
"snapshot": "os",
"instruction": "Could you please help me create a dir named 'test' in the root directory of current computer?",
"source": "https://ubuntu.com/tutorials/command-line-for-beginners#4-creating-folders-and-files",
"config": [],
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "exact_match",
"result": {
"type": "vm_command_line",
"config": {
"command": "[ -d '/path/to/directory' ] && echo 'Directory exists.' || echo 'Directory does not exist.'"
}
},
"expected": {
"type": "rule",
"rules":{
"expected": "Directory exists."
}
}
}
}

33
evaluation_examples/examples/os/7688b85f-87a4-4e4a-b2f8-f3d6c3f29b82.json Normal file
View File

@@ -0,0 +1,33 @@
{
"id": "7688b85f-87a4-4e4a-b2f8-f3d6c3f29b82",
"snapshot": "os",
"instruction": "Can you move the file with the path '/test.txt' to the directory with the path '/dir1'?",
"source": "https://ubuntu.com/tutorials/command-line-for-beginners#5-moving-and-manipulating-files",
"config": [
{
"type": "execute",
"parameters": {
"command": "touch /test.txt && mkdir /dir1"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "exact_match",
"result": {
"type": "vm_command_line",
"config": {
"command": "if [ -f '/dir/test.txt' ]; then echo 'File exists.'; else echo 'File does not exist.'; fi"
}
},
"expected": {
"type": "rule",
"rules":{
"expected": "File exists."
}
}
}
}

26
evaluation_examples/examples/os/94d95f96-9699-4208-98ba-3c3119edf9c2.json Normal file
View File

@@ -0,0 +1,26 @@
{
"id": "94d95f96-9699-4208-98ba-3c3119edf9c2",
"snapshot": "os",
"instruction": "I want to install Spotify on my current system. Could you please help me?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/addremove-install.html.en",
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "check_include_exclude",
"result": {
"type": "vm_command_line",
"config": {
"command": "which spotify"
}
},
"expected": {
"type": "rule",
"rules": {
"include": [],
"exclude": ["not found"]
}
}
}
}

33
evaluation_examples/examples/os/a462a795-fdc7-4b23-b689-e8b6df786b78.json Normal file
View File

@@ -0,0 +1,33 @@
{
"id": "a462a795-fdc7-4b23-b689-e8b6df786b78",
"snapshot": "os",
"instruction": "Could you please help me to switch to the user 'Charles' with password of 'Ex@mpleP@55w0rd!'?",
"source": "https://ubuntu.com/tutorials/command-line-for-beginners#5-moving-and-manipulating-files",
"config": [
{
"type": "execute",
"parameters": {
"command": "echo password | sudo -S su - charles"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "exact_match",
"result": {
"type": "vm_command_line",
"config": {
"command": "if [ '$(whoami)' = 'charles' ]; then echo 'Current user is charles.'; else echo 'Current user is not charles'; fi"
}
},
"expected": {
"type": "rule",
"rules":{
"expected": "Current user is charles"
}
}
}
}

33
evaluation_examples/examples/os/ae039631-2b12-4637-84f6-c67d51511be3.json Normal file
View File

@@ -0,0 +1,33 @@
{
"id": "ae039631-2b12-4637-84f6-c67d51511be3",
"snapshot": "os",
"instruction": "Could you please help me check out my current default browser?",
"source": "https://ubuntu.com/tutorials/command-line-for-beginners#5-moving-and-manipulating-files",
"config": [
{
"type": "execute",
"parameters": {
"command": "xdg-settings set default-web-browser firefox.desktop"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "exact_match",
"result": {
"type": "vm_command_line",
"config": {
"command": "xdg-settings get default-web-browser"
}
},
"expected": {
"type": "rule",
"rules":{
"expected": "firefox.desktop"
}
}
}
}

25
evaluation_examples/examples/os/bedcedc4-4d72-425e-ad62-21960b11fe0d.json Normal file
View File

@@ -0,0 +1,25 @@
{
"id": "bedcedc4-4d72-425e-ad62-21960b11fe0d",
"snapshot": "os",
"instruction": "Could you set the 'Dim screen when inactive' to on in setting?",
"source": "https://www.youtube.com/watch?v=D4WyNjt_hbQ&t=2s",
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "exact_match",
"result": {
"type": "vm_command_line",
"config": {
"command": "gsettings get org.gnome.desktop.session idle-delay"
}
},
"expected": {
"type": "rule",
"rules":{
"expected": "0"
}
}
}
}

33
evaluation_examples/examples/os/ec4e3f68-9ea4-4c18-a5c9-69f89d1178b3.json Normal file
View File

@@ -0,0 +1,33 @@
{
"id": "ec4e3f68-9ea4-4c18-a5c9-69f89d1178b3",
"snapshot": "os",
"instruction": "Can you remove the first favorite app from 'favorites'?",
"source": "https://www.youtube.com/watch?v=D4WyNjt_hbQ&t=2s",
"config": [
{
"type": "execute",
"parameters": {
"command": "gsettings set org.gnome.shell favorite-apps \"['thunderbird.desktop', 'firefox.desktop', 'libreoffice-writer.desktop']\""
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "exact_match",
"result": {
"type": "vm_command_line",
"config": {
"command": "gsettings get org.gnome.shell favorite-apps"
}
},
"expected": {
"type": "rule",
"rules":{
"expected": "['firefox.desktop', 'libreoffice-writer.desktop']"
}
}
}
}

33
evaluation_examples/examples/os/f9be0997-4b7c-45c5-b05c-4612b44a6118.json Normal file
View File

@@ -0,0 +1,33 @@
{
"id": "f9be0997-4b7c-45c5-b05c-4612b44a6118",
"snapshot": "os",
"instruction": "I am currently working on a ubuntu system but I do not want the notifications to bother me. Can you help me to switch to 'Do not disturb mode'?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/shell-notifications.html.en",
"config": [
{
"type": "execute",
"parameters": {
"command": "gsettings set org.gnome.desktop.notifications show-banners true"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "exact_match",
"result": {
"type": "vm_command_line",
"config": {
"command": "gsettings get org.gnome.desktop.notifications show-banners"
}
},
"expected": {
"type": "rule",
"rules":{
"expected": "false"
}
}
}
}