lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #12 from xlang-ai/zhoujun/multi-app

Browse Source 
Update multi-app examples
This commit is contained in:
Tianbao Xie
2024-03-08 20:41:14 +08:00
committed by GitHub
parent 2b119d59b4 7e41955eb7
commit 4b841c199a
12 changed files with 605 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
evaluation_examples/examples/gimp/d16c99dc-2a1e-46f2-b350-d97c86c85c15.json
View File

@@ -84,16 +84,34 @@
}
}
],
"func": "check_image_size_and_structure_sim",
"expected":{
"func": [
"check_image_size",
"check_structure_sim"
],
"expected": [
{
"type": "vm_file",
"path": "/home/user/Desktop/dog_with_background.png",
"dest": "dog_with_background.png"
},
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/resized.png",
"dest": "resized.png"
}
{
"type": "vm_file",
"path": "/home/user/Desktop/dog_with_background.png",
"dest": "dog_with_background.png"
}
],
"result": [
{
"type": "rule",
"rules": {
"height": 512
}
},
{
"type": "vm_file",
"path": "/home/user/Desktop/resized.png",
"dest": "resized.png"
}
]
}
}

37
evaluation_examples/examples/multi_apps/3c8f201a-009d-4bbe-8b65-a6f8b35bb57f.json Normal file
View File

@@ -0,0 +1,37 @@
{
"id": "3c8f201a-009d-4bbe-8b65-a6f8b35bb57f",
"snapshot": "gimp",
"instruction": "Use `gdown` to download the image from \"https://drive.google.com/uc?export=download&id=1i8j5dGS57sA07jEuPNAlQW-sn5uqUnuK\", and then use GIMP to compress it to under 600KB. Resize if needed.",
"source": "",
"config": [
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"alt\", \"t\"); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"gimp",
"os"
],
"evaluator": {
"func": "check_image_file_size",
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/compressed.jpeg",
"dest": "compressed.jpeg"
},
"expected": {
"type": "rule",
"rules": {
"max_size": 600000
}
}
}
}

71
evaluation_examples/examples/multi_apps/42f4d1c7-4521-4161-b646-0a8934e36081.json Normal file
View File

@@ -0,0 +1,71 @@
{
"id": "42f4d1c7-4521-4161-b646-0a8934e36081",
"snapshot": "gimp",
"instruction": "Configure VS Code to edit GIMP script-fu scripts effectively by installing lisp extension. Test by writing code to resizing the image as 128 * 128 as \"resized.png\"",
"source": "",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1yrWU5HimYPNUjdtvw1a218kh50fPVtZ3",
"path": "/home/user/Desktop/character.png"
}
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"code"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"gimp",
"vs_code"
],
"evaluator": {
"func": [
"is_extension_installed",
"check_image_size"
],
"result": [
{
"type": "vm_command_line",
"command": [
"code",
"--list-extensions",
"|",
"grep",
"mattn.lisp"
]
},
{
"type": "vm_file",
"path": "/home/user/Desktop/resized.png",
"dest": "resized.png"
}
],
"expected": [
{
"type": "rule",
"rules": {
"type": "contain",
"expected": "mattn.lisp"
}
},
{
"type": "rule",
"rules": {
"height": 128,
"width": 128
}
}
]
}
}

43
evaluation_examples/examples/multi_apps/7f35355e-02a6-45b5-b140-f0be698bcf85.json Normal file
View File

@@ -0,0 +1,43 @@
{
"id": "7f35355e-02a6-45b5-b140-f0be698bcf85",
"snapshot": "libreoffice_calc",
"instruction": "Export the table to a CSV file and then help me write code to find the medium price (fill empty value with average). Save the result in \"result.txt\".",
"source": "",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://docs.google.com/spreadsheets/d/13YL-KC__pav2qp3sFDs1BT2wZnpWGp7s/export?format=xlsx",
"path": "/home/user/Desktop/stock.xlsx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/Desktop/stock.xlsx"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"libreoffice_calc",
"vs_code"
],
"evaluator": {
"func": "compare_result_files",
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/result.txt",
"dest": "result.txt"
},
"expected": {
"type": "cloud_file",
"path": "https://drive.google.com/uc?export=download&id=1oPPW_dozWGII5MRmdXdKKoEK5iBkd_8Q",
"dest": "result_gold.txt"
}
}
}

51
evaluation_examples/examples/multi_apps/91190194-f406-4cd6-b3f9-c43fac942b22.json Normal file
View File

@@ -0,0 +1,51 @@
{
"id": "91190194-f406-4cd6-b3f9-c43fac942b22",
"snapshot": "gimp",
"instruction": "Launch GIMP from the command line to edit \"cola.png\" and crop the top 20% off the image for my avatar as \"cropped.png\".",
"source": "",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1bmSRNNh4JkF6izrKrmynUHarf0pFES50",
"path": "/home/user/Desktop/cola.png"
},
{
"url": "https://drive.google.com/uc?export=download&id=1MayrIPJWRK7cMEVe3TxYmgkAbVMrYcQA",
"path": "/home/user/Desktop/cropped_gold.png"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"alt\", \"t\"); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"gimp",
"os"
],
"evaluator": {
"func": "check_structure_sim",
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/cropped.png",
"dest": "cropped.png"
},
"expected": {
"type": "vm_file",
"path": "/home/user/Desktop/cropped_gold.png",
"dest": "cropped_gold.png"
}
}
}

54
evaluation_examples/examples/multi_apps/98e8e339-5f91-4ed2-b2b2-12647cb134f4.json Normal file
View File

@@ -0,0 +1,54 @@
{
"id": "98e8e339-5f91-4ed2-b2b2-12647cb134f4",
"snapshot": "vs_code",
"instruction": "Merge the contents of all .txt files from your vscode project into a single document in Writer. No merging separator is needed. Ensure to set the overall font size of the document to 10.",
"source": "",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1dqbi6j3zPOqFTMFHVVaEOXSdHz5zIey1",
"path": "/home/user/Desktop/doc_proc.zip"
}
]
}
},
{
"type": "command",
"parameters": {
"command": "mkdir -p /home/user/Desktop/doc_proc/; unzip /home/user/Desktop/doc_proc.zip -d /home/user/Desktop/doc_proc/",
"shell": true
}
},
{
"type": "launch",
"parameters": {
"command": [
"code",
"-g",
"/home/user/Desktop/doc_proc/"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"vs_code",
"libreoffice_writer"
],
"evaluator": {
"func": "compare_docx_files",
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/concat.docx",
"dest": "concat.docx"
},
"expected": {
"type": "cloud_file",
"path": "https://drive.google.com/uc?export=download&id=10k_8T7Hk9KQ2I-AbNK56hvXR6yv0Pemc",
"dest": "concat_gold.docx"
}
}
}

69
evaluation_examples/examples/multi_apps/bb7db4c2-30b5-4be7-8dd7-b8c4ec7d3108.json Normal file
View File

@@ -0,0 +1,69 @@
{
"id": "bb7db4c2-30b5-4be7-8dd7-b8c4ec7d3108",
"snapshot": "gimp",
"instruction": "I have a background image named 'anmi.png'. I need the image to be enhanced for better sharpness, as 'anmi_sharper.png'. Next, please use it as the background image for my 'index.html' page.",
"source": "",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1BAZvR0BflPh70ClL2bHrbJ-xyV-nNQ1o",
"path": "/home/user/Desktop/anmi.png"
},
{
"url": "https://drive.google.com/uc?export=download&id=1zcyY7y_deKStc-AtU8pFFrJItOuyfrNK",
"path": "/home/user/Desktop/index.html"
}
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"gimp",
"/home/user/Desktop/anmi.png"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"gimp",
"vs_code"
],
"evaluator": {
"func": [
"check_sharper",
"check_html_background_image"
],
"result": [
{
"type": "vm_file",
"path": "/home/user/Desktop/anmi_sharper.png",
"dest": "anmi_sharper.png"
},
{
"type": "vm_file",
"path": "/home/user/Desktop/index.html",
"dest": "index.html"
}
],
"expected": [
{
"type": "vm_file",
"path": "/home/user/Desktop/anmi.png",
"dest": "anmi.png"
},
{
"type": "rule",
"rules": {
"type:": "value",
"value": "anmi_sharper.png"
}
}
]
}
}

51
evaluation_examples/examples/multi_apps/d68204bf-11c1-4b13-b48b-d303c73d4bf6.json Normal file
View File

@@ -0,0 +1,51 @@
{
"id": "d68204bf-11c1-4b13-b48b-d303c73d4bf6",
"snapshot": "gimp",
"instruction": "Divide my image vertically into three equal sections with command line. Then rearrange them in order with a gradient of warm tones, progressively becoming warmer from left to right as a new image \"rearranged.png\".",
"source": "",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1CPGW_OZsfSWDdTU7CFrTjpzSAASyLy4w",
"path": "/home/user/Desktop/tilearray.png"
},
{
"url": "https://drive.google.com/uc?export=download&id=1aHwmnxL2CKEh_FhVpevY452-BQH2t5rG",
"path": "/home/user/Desktop/rearranged_gold.png"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"alt\", \"t\"); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"gimp",
"os"
],
"evaluator": {
"func": "check_structure_sim",
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/rearranged.png",
"dest": "rearranged.png"
},
"expected": {
"type": "vm_file",
"path": "/home/user/Desktop/rearranged_gold.png",
"dest": "rearranged_gold.png"
}
}
}

67
evaluation_examples/examples/multi_apps/e8172110-ec08-421b-a6f5-842e6451911f.json Normal file
View File

@@ -0,0 +1,67 @@
{
"id": "e8172110-ec08-421b-a6f5-842e6451911f",
"snapshot": "gimp",
"instruction": "Open 'character.png' in GIMP and extract the pixel art character. Save the selected character as 'character_gimp.png'. Additionally, write a Python script to automate this selection process, ensuring it precisely mimics the manual extraction done in GIMP. Output the result from the script as 'character_code.png'.",
"source": "",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1Tf7MvqBMHuIhvKmk-Ryr_qmzmkACFjxB",
"path": "/home/user/Desktop/character.png"
},
{
"url": "https://drive.google.com/uc?export=download&id=1wKG5X6LaN0tShsAK4lFg3OyFg8OGYYZg",
"path": "/home/user/Desktop/character_no_background_gold.png"
}
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"gimp",
"/home/user/Desktop/character.png"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"gimp",
"vs_code"
],
"evaluator": {
"func": [
"check_structure_sim",
"check_structure_sim"
],
"result": [
{
"type": "vm_file",
"path": "/home/user/Desktop/character_gimp.png",
"dest": "character_gimp.png"
},
{
"type": "vm_file",
"path": "/home/user/Desktop/character_code.png",
"dest": "character_code.png"
}
],
"expected": [
{
"type": "vm_file",
"path": "/home/user/Desktop/character_no_background_gold.png",
"dest": "character_no_background_gold.png"
},
{
"type": "vm_file",
"path": "/home/user/Desktop/character_no_background_gold.png",
"dest": "character_no_background_gold.png"
}
]
}
}