lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix Duplicate ids; Remove unused JSON files across multiple applications

Browse Source
This commit is contained in:
Timothyxxx
2025-02-10 15:49:54 +08:00
parent f4750701d4
commit 2f0f3f31aa
139 changed files with 17 additions and 4711 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
evaluation_examples/examples/os/0b35aee5-6f2a-4c71-b2da-aed0105fdbde.json
View File

@@ -1,59 +0,0 @@
{
"id": "0b35aee5-6f2a-4c71-b2da-aed0105fdbde",
"snapshot": "os",
"instruction": "Calculate the md5 sum of \"submission.cpp\" with less sensitivity to superficial changes like comments or whitespace",
"source": "NL2Bash",
"config": [
{
"type": "execute",
"parameters": {
"command": "echo password | sudo -S apt-get install astyle cpp md5sum -y",
"shell": true
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1UiDgRdPqGGPmklZyXhwO1-UyOlhJsxt0&export=download&authuser=0&confirm=t&uuid=7795d6f5-87df-4ebb-8540-e9756362736a&at=APZUnTVk-7AnKL4ofVZGKQeHxjHY:1707987053722",
"path": "/home/user/Desktop/submission.cpp"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"alt\", \"t\"); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "check_include_exclude",
"result": {
"type": "vm_terminal_output"
},
"expected": {
"type": "calc_insensitive_md5"
}
}
}

36
evaluation_examples/examples/os/43c2d64c-bab5-4dcb-a30c-b888321c319a.json
View File

@@ -1,36 +0,0 @@
{
"id": "43c2d64c-bab5-4dcb-a30c-b888321c319a",
"snapshot": "os",
"instruction": "Could you please help me create a dir named 'test' in the root directory of current computer?",
"source": "https://ubuntu.com/tutorials/command-line-for-beginners#4-creating-folders-and-files",
"config": [
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "exact_match",
"result": {
"type": "vm_command_line",
"command": "[ -d '/test' ] && echo 'Directory exists.' || echo 'Directory does not exist.'",
"shell": true
},
"expected": {
"type": "rule",
"rules":{
"expected": "Directory exists.\n"
}
}
}
}

18
evaluation_examples/examples/os/4d2b519e-e872-4100-8ea3-fe71ab0f9133.json
View File

@@ -1,18 +0,0 @@
{
"id": "4d2b519e-e872-4100-8ea3-fe71ab0f9133",
"snapshot": "os",
"instruction": "Could you please help me to add a new entry \"/home/david/pear/bin\" to the PATH variable in the ZSH terminal?",
"source": "https://stackoverflow.com/questions/11530090/adding-a-new-entry-to-the-path-variable-in-zsh",
"trajectory": "trajectories/",
"config": [],
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
}
}

18
evaluation_examples/examples/os/5c433d22-ed9a-4e31-91f5-54cf3e8acd63.json
View File

@@ -1,18 +0,0 @@
{
"id": "5c433d22-ed9a-4e31-91f5-54cf3e8acd63",
"snapshot": "os",
"instruction": "I want to change my system language to Chinese (simplified). Can you help me?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/session-language.html.zh-CN",
"trajectory": "trajectories/",
"config": [],
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
}
}

63
evaluation_examples/examples/os/6ebbfb01-ea72-4226-a2a6-dc428e111ed2.json
View File

@@ -1,63 +0,0 @@
{
"id": "6ebbfb01-ea72-4226-a2a6-dc428e111ed2",
"snapshot": "os",
"instruction": "Could you please help me to set Bash as my default shell on the current Ubuntu system?",
"source": "https://superuser.com/questions/46748/how-do-i-make-bash-my-default-shell-on-ubuntu",
"trajectory": "trajectories/",
"config": [
{
"type": "execute",
"parameters": {
"command": "echo password | sudo -S apt-get install zsh -y",
"shell": true
}
},
{
"type": "execute",
"parameters": {
"command": "echo password | sudo -S chsh -s $(which zsh)",
"shell": true
}
},
{
"type": "execute",
"parameters": {
"command": "echo password | sudo -S loginctl terminate-user user",
"shell": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 10
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; time.sleep(2); pyautogui.press(\"enter\"); pyautogui.type(\"password\"); time.sleep(2); pyautogui.press(\"enter\");"
]
}
}
],
"related_apps": [
"os"
],
"evaluator": {
"func": "exact_match",
"result": {
"type": "vm_command_line",
"command": "echo $SHELL",
"shell": true
},
"expected": {
"type": "rule",
"rules":{
"expected": "/bin/bash\n"
}
}
}
}

29
evaluation_examples/examples/os/765d2b74-88a7-4d50-bf51-34e4106fd24a.json
View File

@@ -1,29 +0,0 @@
{
"id": "765d2b74-88a7-4d50-bf51-34e4106fd24a",
"snapshot": "os",
"instruction": "Can you help me delete the \"test\" file on my desktop?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/files-delete.html.en",
"config": [
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
}
}

50
evaluation_examples/examples/os/7688b85f-87a4-4e4a-b2f8-f3d6c3f29b82.json
View File

@@ -1,50 +0,0 @@
{
"id": "7688b85f-87a4-4e4a-b2f8-f3d6c3f29b82",
"snapshot": "os",
"instruction": "Can you move the file 'todo.txt' on the Desktop to the directory 'done/' on the Desktop?",
"source": "https://ubuntu.com/tutorials/command-line-for-beginners#5-moving-and-manipulating-files",
"config": [
{
"type": "execute",
"parameters": {
"command": "echo 'password' | sudo -S touch ~/Desktop/todo.txt && sudo mkdir ~/Desktop/done",
"shell": true
}
},
{
"type": "execute",
"parameters": {
"command": "echo 'password' | sudo -S chmod 777 ~/Desktop/todo.txt && sudo chmod 777 ~/Desktop/done",
"shell": true
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "exact_match",
"result": {
"type": "vm_command_line",
"command": "if [ -f ~/Desktop/done/todo.txt ]; then echo 'File exists.'; else echo 'File does not exist.'; fi",
"shell": true
},
"expected": {
"type": "rule",
"rules":{
"expected": "File exists.\n"
}
}
}
}

41
evaluation_examples/examples/os/ae039631-2b12-4637-84f6-c67d51511be3.json
View File

@@ -1,41 +0,0 @@
{
"id": "ae039631-2b12-4637-84f6-c67d51511be3",
"snapshot": "os",
"instruction": "Could you please help me check out my current default browser?",
"source": "https://ubuntu.com/tutorials/command-line-for-beginners#5-moving-and-manipulating-files",
"config": [
{
"type": "execute",
"parameters": {
"command": "xdg-settings set default-web-browser firefox.desktop"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "exact_match",
"result": {
"type": "vm_command_line",
"command": "xdg-settings get default-web-browser"
},
"expected": {
"type": "rule",
"rules":{
"expected": "firefox.desktop"
}
}
}
}

34
evaluation_examples/examples/os/c56de254-a3ec-414e-81a6-83d2ce8c41fa.json
View File

@@ -1,34 +0,0 @@
{
"id": "c56de254-a3ec-414e-81a6-83d2ce8c41fa",
"snapshot": "os",
"instruction": "I want to install the lxml package on my ubuntu system. Can you help me?",
"source": "https://superuser.com/questions/28426/how-to-extract-text-with-ocr-from-a-pdf-on-linux",
"config": [
{
"type": "execute",
"parameters": {
"command": "sudo aptget update && sudo apt-get install -y python3",
"shell": true
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "exact_match",
"result": {
"type": "vm_command_line",
"command": "python3 -c \"import lxml\" >/dev/null 2>&1 && echo \"lxml is installed\" || echo \"lxml is not installed\"",
"shell": true
},
"expected": {
"type": "rule",
"rules":{
"expected": "lxml is installed\n"
}
}
}
}

45
evaluation_examples/examples/os/cc9d4f34-1ca0-4a1b-8ff2-09302696acb9.json
View File

@@ -1,45 +0,0 @@
{
"id": "cc9d4f34-1ca0-4a1b-8ff2-09302696acb9",
"snapshot": "os",
"instruction": "Currently, the program my_process.sh is running in the system and I want to end the process. Can you help me?",
"source": "https://stackoverflow.com/questions/9346211/how-to-kill-a-process-on-a-port-on-ubuntu",
"config": [
{
"type": "execute",
"parameters": {
"command": "echo -e '#!/bin/bash\n\nwhile true; do\necho \"Running my_process...\"\nsleep 1\ndone' > my_process.sh && chmod +x my_process.sh && nohup ./my_process.sh > /dev/null 2>&1 &",
"shell": true
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"postconfig": [
{
"type": "sleep",
"parameters": {
"seconds": 5
}
}
],
"func": "check_include_exclude",
"result": {
"type": "vm_command_line",
"command": "ps -ef | grep my_process.sh",
"shell": true
},
"expected": {
"type": "rule",
"rules": {
"include": [
],
"exclude": [
"/bin/sh ./my_process.sh"
]
}
}
}
}

48
evaluation_examples/examples/os/ddc75b62-7311-4af8-bfb3-859558542b36.json
View File

@@ -1,48 +0,0 @@
{
"id": "ddc75b62-7311-4af8-bfb3-859558542b36",
"snapshot": "os",
"instruction": "I want to uninstall the Mahjongg on my system. Can you help me?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/addremove-remove.html.en",
"trajectory": "trajectories/",
"config": [
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"related_apps": [
"os"
],
"evaluator": {
"postconfig": [
{
"type": "sleep",
"parameters": {
"seconds": 1
}
}
],
"func": "check_include_exclude",
"result": {
"type": "vm_command_line",
"command": "dpkg -l | grep mahjongg",
"shell": true
},
"expected": {
"type": "rule",
"rules": {
"include": [
],
"exclude": [
"mahjongg"
]
}
}
}
}

17
evaluation_examples/examples/os/f10b16e1-c160-4cb3-989f-7b2ec89bc073.json
View File

@@ -1,17 +0,0 @@
{
"id": "f10b16e1-c160-4cb3-989f-7b2ec89bc073",
"snapshot": "os",
"instruction": "Please install the latest version of the GNOME Desktop on the current Ubuntu system.",
"source": "https://www.wikihow.com/Install-Gnome-on-Ubuntu",
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
}
}