lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' of github.com:ztjhz/DesktopEnv

Browse Source
This commit is contained in:
Siheng Zhao
2024-01-31 10:37:31 +08:00
parent cffed446fc 0e5eefbb86
commit 43a752c3e6
29 changed files with 1002 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
evaluation_examples/examples/libreoffice_calc/01b269ae-2111-4a07-81fd-3fcd711993b0-win.json Normal file
View File

@@ -0,0 +1,76 @@
{
"id": "01b269ae-2111-4a07-81fd-3fcd711993b0",
"snapshot": "libreoffice_calc",
"instruction": "Fill all the blank cells with the value in the cell above it",
"source": "https://www.youtube.com/shorts/VrUzPTIwQ04",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1FuOZ-5YoKgLLwl_oZd4R3D8pZACf_ukS&export=download&authuser=0&confirm=t&uuid=2051e7a6-5930-4cef-8d77-20ebf66ec6e6&at=APZUnTX1fXqlxy6rluq-Kw-LUhS5:1705919461032",
"path": "C:\\Users\\user\\Student_Level_Fill_Blank.xlsx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "C:\\Users\\user\\Student_Level_Fill_Blank.xlsx"
}
}
],
"trajectory": "trajectories/01b269ae-2111-4a07-81fd-3fcd711993b0",
"related_apps": [
"msoffice_excel"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "Student_Level_Fill_Blank.xlsx - Excel",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
}
],
"func": "compare_table",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1HTle3vgdZSjJIK_wjXyjtWwbiYJeguwv&export=download&authuser=0&confirm=t&uuid=c5d0868b-bed2-48fb-949b-8a9f3f61e8cf&at=APZUnTVqS9CTZFJ1rPqCGQPDCv3p:1705919542916",
"dest": "Student_Level_Fill_Blank_gold.xlsx"
},
"result": {
"type": "vm_file",
"path": "C:\\Users\\user\\Student_Level_Fill_Blank.xlsx",
"dest": "Student_Level_Fill_Blank.xlsx"
},
"options": {
"rules": [
{
"type": "sheet_data",
"sheet_idx0": 0,
"sheet_idx1": "EI0"
}
]
}
}
}

2
evaluation_examples/examples/libreoffice_impress/455d3c66-7dc6-4537-a39a-36d3e9119df7.json
View File

@@ -1,7 +1,7 @@
{
"id": "455d3c66-7dc6-4537-a39a-36d3e9119df7",
"snapshot": "libreoffice_impress",
"instruction": "Could you help me export an Impress file to a .jpg image file and save it as res.jpg on the Desktop? ",
"instruction": "Could you help me export an Impress file to a .png image file and save it as res.png on the Desktop? ",
"source": "https://stackoverflow.com/questions/75626383/how-export-libreoffice-impress-to-image",
"config": [
{

2
evaluation_examples/examples/libreoffice_impress/ef9d12bd-bcee-4ba0-a40e-918400f43ddf.json
View File

@@ -19,7 +19,7 @@
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5); pyautogui.press('esc'); time.sleep(0.3); pyautogui.press('f10'); time.sleep(0.3); pyautogui.press('right', presses=2, interval=0.1); time.sleep(0.3); pyautogui.press('down', presses=11, interval=0.1); pyautogui.press('enter')"
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(5); pyautogui.press('esc'); time.sleep(0.3); pyautogui.press('f10'); time.sleep(0.3); pyautogui.press('right', presses=2, interval=0.1); time.sleep(0.3); pyautogui.press('down', presses=11, interval=0.1); pyautogui.press('enter')"
]
}
}

4
evaluation_examples/examples/libreoffice_writer/0810415c-bde4-4443-9047-d5f70165a697.json
View File

@@ -9,7 +9,7 @@
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1-svVsH-l2ofufEKuN-cYrIrvXNobtATE&export=download&authuser=0&confirm=t&uuid=be7f891a-f858-48f5-a72d-4e42bbfb8b65&at=APZUnTXzBnaeSJjmxeh4zG03pzA0:1704179807785",
"url": "https://drive.usercontent.google.com/download?id=1YhHNlRsL7lJBsqRYctz4CmEoD1g8oAm0&export=download&authuser=0&confirm=t&uuid=16776039-9eae-4ee7-ae0b-8b2d71cb25e1&at=APZUnTWVT6sfD3MQEADssAEc4Pwn:1706622286569",
"path": "Desktop/Novels_Intro_Packet.docx"
}
]
@@ -64,4 +64,4 @@
"dest": "Novels_Intro_Packet.docx"
}
}
}
}

59
evaluation_examples/examples/multi_apps/2b9493d7-49b8-493a-a71b-56cd1f4d6908.json
View File

@@ -36,7 +36,8 @@
"parameters": {
"command": [
"gnome-terminal",
"--maximize"
"--maximize",
"--working-directory=/home/user/"
]
}
}
@@ -59,43 +60,27 @@
}
}
],
"func": ["check_include_exclude", "check_include_exclude"],
"conj": "and",
"result": [
{
"type": "vm_command_line",
"command": [
"/bin/bash",
"-c",
"output=$(ps aux | grep \"[s]office\"]); if [ -z \"$output\" ]; then echo \"true\"; else echo \"false\"; fi"
]
},
{
"type": "vm_command_line",
"command": [
"/bin/bash",
"-c",
"output=$(cat ~/.bash_history | grep \"[k]ill\"); if [ -z \"$output\" ]; then echo \"false\"; else echo \"true\"; fi"
"func": "check_include_exclude",
"result": {
"type": "vm_command_line",
"command": [
"/bin/bash",
"-c",
"output=$(ps aux | grep \"[s]office\"]); if [ -z \"$output\" ]; then echo \"no libreoffice is running\"; else echo \"libreoffice is still running\"; fi; output=$(cat ~/.bash_history | grep \"[k]ill\"); if [ -z \"$output\" ]; then echo \"not killed from terminal\"; else echo \"killed from terminal\"; fi"
]
},
"expected": {
"type": "rule",
"rules": {
"include": [
"no libreoffice is running",
"killed from terminal"
],
"exclude": [
"libreoffice is still running",
"not killed from terminal"
]
}
],
"expected": [
{
"type": "rule",
"rules": {
"include": [
"true\n"
]
}
},
{
"type": "rule",
"rules": {
"include": [
"true\n"
]
}
}
]
}
}
}

51
evaluation_examples/examples/multi_apps/2c9fc0de-3ee7-45e1-a5df-c86206ad78b5.json
View File

@@ -6,15 +6,12 @@
"config": [
{
"type": "download",
"path": "",
},
{
"type": "execute",
"parameters": {
"command": [
"/bin/bash",
"-c",
"git config --global user.name \"xlang\" && git config --global user.email \"xlang2024anonym@gmail.com\" && mkdir -p /home/user/projects/remote_project && cd /home/user/projects/remote_project && git init --initial-branch=main --bare .git"
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1aANBCEHT6K8GmHDMEYtL1LQXdApqVoGv&export=download",
"path": "/home/user/Downloads/binder.zip"
}
]
}
},
@@ -24,7 +21,17 @@
"command": [
"/bin/bash",
"-c",
"mkdir -p /home/user/projects/hello_world && cd /home/user/projects/hello_world && git init --initial-branch main && git remote add origin /home/user/projects/remote_project && echo \"Hello World!\" > README.md"
"git config --global user.name \"xlang\" && git config --global user.email \"xlang2024anonym@gmail.com\" && mkdir -p /home/user/projects/remote_project && cd /home/user/projects/remote_project && git init --initial-branch=main && git config receive.denyCurrentBranch ignore "
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"/bin/bash",
"-c",
"unzip -q /home/user/Downloads/binder.zip -d /home/user/projects/ && cd /home/user/projects/binder && git init --initial-branch main && git remote add origin /home/user/projects/remote_project"
]
}
},
@@ -34,7 +41,7 @@
"command": [
"gnome-terminal",
"--maximize",
"--working-directory=/home/user/projects/hello_world"
"--working-directory=/home/user/projects/binder"
]
}
}
@@ -45,19 +52,23 @@
"terminal"
],
"evaluator": {
"func": "compare_docx_files",
"func": "check_include_exclude",
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/notes.docx",
"dest": "notes.docx"
"type": "vm_command_line",
"command": [
"/bin/bash",
"-c",
"cd /home/user/projects/remote_project; git log --oneline | head -n 1 | awk '{for (i=2; i<=NF; i++) {printf \"%s%s\", $i, (i==NF ? \"\" : \" \")}; print \"\"}'; remote_id=$(git log --oneline | head -n 1 | awk '{print $1}'); cd /home/user/projects/binder; local_id=$(git log --oneline | head -n 1 | awk '{print $1}'); if [ \"${local_id}\" = \"${remote_id}\" ]; then echo \"repo is synchronous\"; else echo \"repo is not synchronous\"; fi"
]
},
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1Xl6tgQ0K5qA1BDA2fKTK2xFLzXwbtkZ6&export=download",
"dest": "notes_gold.docx"
},
"options": {
"ignore_blanks": true
"type": "rule",
"rules": {
"include": [
"daily update",
"repo is synchronous"
]
}
}
}
}

53
evaluation_examples/examples/multi_apps/2fe4b718-3bd7-46ec-bdce-b184f5653624.json Normal file
View File

@@ -0,0 +1,53 @@
{
"id": "2fe4b718-3bd7-46ec-bdce-b184f5653624",
"snapshot": "vlc",
"instruction": "Could you help me create an Animated GIF src_clip.gif from a video file using VLC and GIMP from the source of video \"src.mp4\" in the desktop, 5-second clip beginning at 00:03?",
"source": "https://www.maketecheasier.com/create-gif-from-video-gimp/",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"gimp"
]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1krQ_vN3QiboMttWrjDY-YROCWZq6cBtb&export=download",
"path": "/home/user/Desktop/src.mp4"
}
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"vlc"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"vlc",
"gimp"
],
"evaluator": {
"func": "compare_images",
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/src_clip.gif",
"dest": "src_clip.gif"
},
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1kDq8pakgdZvKh8CFpR5368stf14gwFCp&export=download",
"dest": "src_clip_gold.gif"
}
}
}

71
evaluation_examples/examples/multi_apps/3680a5ee-6870-426a-a997-eba929a0d25c.json Normal file
View File

@@ -0,0 +1,71 @@
{
"id": "3680a5ee-6870-426a-a997-eba929a0d25c",
"snapshot": "libreoffice_calc",
"instruction": "I have file1.xlsx and file2.ods on the Desktop and each has one column. Help me use only the command line to merge these two columns into one LibreOffice Calc file called output.csv and open it from terminal.",
"source": "https://unix.stackexchange.com/questions/510850/how-to-open-calc-from-terminal-and-insert-files",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1ofMPSBPXtt0h6t0arjYo0GMtKAAhfEPz&export=download",
"path": "/home/user/Desktop/file1.xlsx"
},
{
"url": "https://drive.usercontent.google.com/download?id=15QOxr0_MiHR80ppMX02aHnnsWs25aXgZ&export=download",
"path": "/home/user/Desktop/file2.ods"
}
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"gnome-terminal",
"--maximize",
"--working-directory=/home/user/Desktop"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"libreoffice_calc",
"terminal"
],
"evaluator": {
"func": ["check_include_exclude", "compare_csv"],
"result": [
{
"type": "vm_command_line",
"command": [
"/bin/bash",
"-c",
"output=$(ps aux | grep \"[s]office\" | awk '{print $7}' | grep -E \"pts/|tty\"); if [ -z \"$output\" ]; then echo \"use no terminal\"; else echo \"use terminal\"; fi;"
]
},
{
"type": "vm_file",
"path": "/home/user/Desktop/output.csv",
"dest": "output.csv"
}
],
"expected": [
{
"type": "rule",
"rules": {
"include": [
"use terminal"
]
}
},
{
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1OBQXIJ06HQeodaCwU7z0Kq_-d3YuGNqE&export=download",
"dest": "output_gold.csv"
}
]
}
}

6
evaluation_examples/examples/multi_apps/46407397-a7d5-4c6b-92c6-dbe038b1457b.json
View File

@@ -1,7 +1,7 @@
{
"id": "46407397-a7d5-4c6b-92c6-dbe038b1457b",
"snapshot": "chrome",
"instruction": "Help me export charts, graph or other images from docx files received in email xxx in Thunderbird and upload these png files to the figures/ folder in Google Drive for later use (use numbers to name them).",
"instruction": "Help me export charts, graph or other images from docx files received in email \"Lecture Document\" in Notes folder and upload these png files to the figures/ folder in Google Drive for later use (use numbers to name them).",
"source": "https://marketplace.uipath.com/listings/merge-pdfs-from-gmail-email-attachments-and-upload-to-gogle-drive",
"config": [
{
@@ -58,7 +58,7 @@
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=11b93787-7076-47ba-b04b-b63a4e9aab02&at=APZUnTV_3ASC-R55FmBXgTLcC46e:1706187828620",
"url": "https://drive.usercontent.google.com/download?id=18jdi0OanMtAQenm4ODTivsxTSzdj4HUV&export=download&authuser=0&confirm=t&uuid=e858d3cc-4535-4419-a651-8856ac517d19&at=APZUnTW7g4ygfrkKTPBWCO13twRj:1706611460571",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
@@ -93,7 +93,7 @@
"chrome"
],
"evaluator": {
"func": "compare_images",
"func": "compare_image_list",
"result": {
"type": "googledrive_file",
"settings_file": "evaluation_examples/settings/googledrive/settings.yml",

116
evaluation_examples/examples/multi_apps/510f64c8-9bcc-4be1-8d30-638705850618.json Normal file
View File

@@ -0,0 +1,116 @@
{
"id": "510f64c8-9bcc-4be1-8d30-638705850618",
"snapshot": "vscode",
"instruction": "Could you start VS Code in folder ~/Desktop/project from the terminal?",
"source": "https://www.geeksforgeeks.org/how-to-start-vs-code-from-the-terminal-command-line/",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1PnC-zxTtntYvuGlnIV2_05jiHfkX_1E-&export=download",
"path": "/home/user/Downloads/vscodeEvalExtension.zip"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"/bin/bash",
"-c",
"cd /home/user/Downloads && unzip -q vscodeEvalExtension.zip && code --install-extension vscodeEvalExtension/eval-0.0.1.vsix && rm -rf vscodeEvalExtension vscodeEvalExtension.zip && mkdir -p ~/Desktop/project/.vscode && history -c && echo > ~/.bash_history"
]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1akdsiRVdq6CUtT-FX8Dpf8ruPTq6DcFn&export=download&authuser=0&confirm=t&uuid=ce2fa96a-454e-43d9-bbe3-98553b7eed0d&at=APZUnTVw_YQ1URTvP34vrmKcw0b4:1705222451052",
"path": "/home/user/Desktop/project/main.py"
},
{
"url": "https://drive.usercontent.google.com/download?id=1BkwtqtAzv_K2CrTbJZ0HbMHBffzdD9vc&export=download&authuser=0&confirm=t&uuid=28f77090-deef-49a1-b156-91317881e75e&at=APZUnTXuaR6i_3t3Prslk535GaO5:1705222457290",
"path": "/home/user/Desktop/project/README.md"
},
{
"url": "https://drive.usercontent.google.com/download?id=1ea_zF2tbcXOB8w9neBV-U5xI2nnPzIw_&export=download&authuser=0&confirm=t&uuid=9cf8c5bb-a880-475c-b80b-967a0c4fbea4&at=APZUnTUdjIj80F3Mbgi72eZDTZLO:1705222462443",
"path": "/home/user/Desktop/project/.vscode/settings.json"
}
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"gnome-terminal",
"--maximize",
"--working-directory=/home/user/"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"vscode",
"terminal"
],
"evaluator": {
"postconfig": [
{
"type": "execute",
"parameters": {
"command": [
"/bin/bash",
"-c",
"killall gnome-terminal-server"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Visual Studio Code"
}
}
],
"func": ["check_include_exclude", "compare_config"],
"result": [
{
"type": "vm_command_line",
"command": [
"/bin/bash",
"-c",
"use_terminal=$(cat ~/.bash_history | grep \"[c]ode \"); if [ -z \"$use_terminal\" ]; then echo \"false\"; else echo \"true\"; fi"
]
},
{
"type": "vscode_config",
"vscode_extension_command": "OpenProject",
"path": "/home/user/OpenProject.txt",
"dest": "OpenProject.txt"
}
],
"expected": [
{
"type": "rule",
"rules": {
"include": [
"true"
]
}
},
{
"type": "rule",
"rules": {
"expect": "project"
}
}
]
}
}

93
evaluation_examples/examples/multi_apps/58565672-7bfe-48ab-b828-db349231de6b.json Normal file
View File

@@ -0,0 +1,93 @@
{
"id": "58565672-7bfe-48ab-b828-db349231de6b",
"snapshot": "chrome",
"instruction": "Can you assist me by opening the first link in the latest email in Bills folder from Thunderbird and displaying it in a new Chrome tab?",
"source": "https://superuser.com/questions/1792660/open-link-from-other-application-does-not-open-the-url-in-firefox",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://www.apple.com/",
"https://en.sjtu.edu.cn/",
"https://scholar.google.com/"
]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=88b71c06-b5b9-4108-a516-24c46fe9273d&at=APZUnTXK2q16yG7u43034ou5uCoA:1706600462917",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xz",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"thunderbird",
"chrome"
],
"evaluator": {
"func": "is_expected_tabs",
"result": {
"type": "open_tabs_info"
},
"expected": {
"type": "rule",
"rules": {
"type": "url",
"urls": [
"https://www.apple.com/",
"https://en.sjtu.edu.cn/",
"https://scholar.google.com/",
"https://www.amazon.com/"
]
}
}
}
}

26
evaluation_examples/examples/multi_apps/937087b6-f668-4ba6-9110-60682ee33441.json Normal file
View File

@@ -0,0 +1,26 @@
{
"id": "937087b6-f668-4ba6-9110-60682ee33441",
"snapshot": "vlc",
"instruction": "I am currently using a ubuntu system. Could you help me set the default video player as VLC?",
"source": "https://superuser.com/questions/187440/set-default-ubuntu-video-player-as-vlc",
"config": [],
"trajectory": "trajectories/",
"related_apps": [
"vlc",
"os"
],
"evaluator": {
"func": "check_include_exclude",
"result": {
"type": "default_video_player"
},
"expected": {
"type": "rule",
"rules": {
"include": [
"vlc.desktop"
]
}
}
}
}

6
evaluation_examples/examples/multi_apps/b52b40a5-ad70-4c53-b5b0-5650a8387052.json
View File

@@ -1,7 +1,7 @@
{
"id": "b52b40a5-ad70-4c53-b5b0-5650a8387052",
"snapshot": "chrome",
"instruction": "Could you help me merge all PDF files in the email attachment in Thunderbird into one file and upload it to attachment_full.pdf in Google Drive?",
"instruction": "Could you help me merge all PDF files in the \"Paper Recommendation\" email attachment in Thunderbird into one file and upload it to attachment_full.pdf in Google Drive?",
"source": "https://marketplace.uipath.com/listings/merge-pdfs-from-gmail-email-attachments-and-upload-to-gogle-drive",
"config": [
{
@@ -58,7 +58,7 @@
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=11b93787-7076-47ba-b04b-b63a4e9aab02&at=APZUnTV_3ASC-R55FmBXgTLcC46e:1706187828620",
"url": "https://drive.usercontent.google.com/download?id=1_SujjgvE6SkrIINB7bd-4quti7ICGaB_&export=download&authuser=0&confirm=t&uuid=aa95822d-55ca-46e0-8b52-14d495b41995&at=APZUnTX7f8cbJ00STW_kV8kMY1KW:1706609819112",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
@@ -104,7 +104,7 @@
},
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1-FaONI6f5g9XyJAlx8vPFgTP_SEZ-1vV&export=download&authuser=0&confirm=t&uuid=b0ad08b7-5b4e-4372-99fa-0952cd096144&at=APZUnTVrZK9_alT_gchTKE6ZYeod:1706194464805",
"path": "https://drive.usercontent.google.com/download?id=1-FaONI6f5g9XyJAlx8vPFgTP_SEZ-1vV&export=download&authuser=0&confirm=t&uuid=de7b274a-25b8-4361-a614-4910a88ba4bd&at=APZUnTVRw21zrIY4ydp9sagr9ZhZ:1706609799766",
"dest": "attachment_full_gold.pdf"
}
}

95
evaluation_examples/examples/multi_apps/ee9a3c83-f437-4879-8918-be5efbb9fac7.json Normal file
View File

@@ -0,0 +1,95 @@
{
"id": "ee9a3c83-f437-4879-8918-be5efbb9fac7",
"snapshot": "libreoffice_calc",
"instruction": "Could you help me convert the opened ods file in the desktop to csv file with the same file name using command line when Libreoffice instance is running?",
"source": "https://stackoverflow.com/questions/64589140/convert-ods-to-csv-using-command-line-when-libreoffice-instance-is-running",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1V6e4KB6Sabp6nAHkq5Seun0qbjwEfdaF&export=download",
"path": "/home/user/Desktop/file_example_ODS_5000.ods"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/Desktop/file_example_ODS_5000.ods"
}
},
{
"type": "execute",
"parameters": {
"command": [
"/bin/bash",
"-c",
"history -c && echo > ~/.bash_history && sleep 3"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"gnome-terminal",
"--maximize",
"--working-directory=/home/user/Desktop"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"libreoffice_calc",
"terminal"
],
"evaluator": {
"postconfig": [
{
"type": "execute",
"parameters": {
"command": [
"/bin/bash",
"-c",
"killall gnome-terminal-server"
]
}
}
],
"func": ["check_include_exclude", "compare_csv"],
"result": [
{
"type": "vm_command_line",
"command": [
"/bin/bash",
"-c",
"use_terminal=$(cat ~/.bash_history | grep \"\\(soffice\\|libreoffice\\).\\+--convert-to\\s\\+csv\"); if [ -z \"$use_terminal\" ]; then echo \"use no terminal\"; else echo \"use terminal\"; fi"
]
},
{
"type": "vm_file",
"path": "/home/user/Desktop/file_example_ODS_5000.csv",
"dest": "file_example_ODS_5000.csv"
}
],
"expected": [
{
"type": "rule",
"rules": {
"include": [
"use terminal"
]
}
},
{
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1L8cJjHYDtZymQp4-QdNZ9i6jzZzsXJm1&export=download",
"dest": "file_example_ODS_5000_gold.csv"
}
]
}
}

21
evaluation_examples/examples/os/5812b315-e7bd-4265-b51f-863c02174c28.json
View File

@@ -1,18 +1,33 @@
{
"id": "5812b315-e7bd-4265-b51f-863c02174c28",
"snapshot": "os",
"instruction": "Please create an SSH user on Ubuntu who is only allowed to access the folder \"test1\".",
"instruction": "Please create an SSH user named \"charles\" with password \"Ex@mpleP@55w0rd!\" on Ubuntu who is only allowed to access the folder \"test1\".",
"source": "https://superuser.com/questions/149404/create-an-ssh-user-who-only-has-permission-to-access-specific-folders",
"config": [],
"config": [
{
"type": "execute",
"parameters": {
"command": "mkdir /test1",
"shell": true
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"func": "exact_match",
"result": {
"type": "vm_command_line",
"command": "[[ $(getent passwd charles) && $(getent passwd charles | cut -d: -f6) = \"/test1\" ]] && echo \"User charles exists and has /test1 as their home directory\" || echo \"User charles does not exist or doesn't have /test1 as their home directory\"",
"shell": true
},
"expected": {
"type": "rule",
"rules":{
"expected": "User charles exists and has /test1 as their home directory"
}
}
}
}

23
evaluation_examples/examples/os/c56de254-a3ec-414e-81a6-83d2ce8c41fa.json
View File

@@ -1,17 +1,34 @@
{
"id": "c56de254-a3ec-414e-81a6-83d2ce8c41fa",
"snapshot": "os",
"instruction": "Could you please help me to extract text from a non-indexed PDF document on the current system?",
"instruction": "I want to install the lxml package on my ubuntu system. Can you help me?",
"source": "https://superuser.com/questions/28426/how-to-extract-text-with-ocr-from-a-pdf-on-linux",
"config": [
{
"type": "execute",
"parameters": {
"command": "sudo apt-get update && sudo apt-get install -y python3",
"shell": true
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"func": "exact_match",
"result": {
"type": "vm_command_line",
"command": "python3 -c \"import lxml\" >/dev/null 2>&1 && echo \"lxml is installed\" || echo \"lxml is not installed\"",
"shell": true
},
"expected": {
"type": "rule",
"rules":{
"expected": "lxml is installed"
}
}
}
}
}

22
evaluation_examples/examples/os/cc9d4f34-1ca0-4a1b-8ff2-09302696acb9.json
View File

@@ -1,17 +1,33 @@
{
"id": "cc9d4f34-1ca0-4a1b-8ff2-09302696acb9",
"snapshot": "os",
"instruction": "I need you to execute a specific process or script from the terminal and ensure that it continues to run independently, even after the terminal session is terminated. This involves redirecting the process's output to a file and disassociating the process from the terminal session's control.",
"source": "https://superuser.com/questions/178587/how-do-i-detach-a-process-from-terminal-entirely",
"instruction": "Currently, the program my_process.sh is running in the system and I want to end the process. Can you help me?",
"source": "https://stackoverflow.com/questions/9346211/how-to-kill-a-process-on-a-port-on-ubuntu",
"config": [
{
"type": "execute",
"parameters": {
"command": "echo -e '#!/bin/bash\n\nwhile true; do\necho \"Running my_process...\"\nsleep 1\ndone' > my_process.sh && chmod +x my_process.sh && nohup ./my_process.sh > /dev/null 2>&1 &",
"shell": true
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"func": "exact_match",
"result": {
"type": "vm_command_line",
"command": "[[ -z $(ps -ef | grep -v grep | grep my_process) ]] && echo \"Process killed\" || echo \"Process still running\"",
"shell": true
},
"expected": {
"type": "rule",
"rules":{
"expected": "Process killed"
}
}
}
}

21
evaluation_examples/examples/vs_code/53ad5833-3455-407b-bbc6-45b4c79ab8fb.json
View File

@@ -4,6 +4,27 @@
"instruction": "Please help me use VS Code to open the \"project\" in the \"user\" folder under \"home\".",
"source": "https://www.youtube.com/watch?v=VqCgcpAypFQ",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1PnC-zxTtntYvuGlnIV2_05jiHfkX_1E-&export=download",
"path": "/home/user/Downloads/vscodeEvalExtension.zip"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"/bin/bash",
"-c",
"cd /home/user/Downloads && unzip -q vscodeEvalExtension.zip && code --install-extension vscodeEvalExtension/eval-0.0.1.vsix && rm -rf vscodeEvalExtension vscodeEvalExtension.zip"
]
}
},
{
"type": "launch",
"parameters": {