lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

merge

Browse Source
This commit is contained in:
tsuky_chen
2024-03-09 18:53:27 +08:00
parent 5b07ec17bf 2291af394f
commit aae848196b
48 changed files with 2106 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
evaluation_examples/examples/multi_apps/2373b66a-092d-44cb-bfd7-82e86e7a3b4d.json Normal file
View File

@@ -0,0 +1,49 @@
{
"id": "2373b66a-092d-44cb-bfd7-82e86e7a3b4d",
"snapshot": "multiapps",
"instruction": "I want to understand the resource usage of my Ubuntu system under normal workloads. Please use the `sar` command in the `sysstat` toolkit to monitor the system CPU usage, evaluate the status once every second for 30 seconds, output the results to \"System_Resources_Report.txt\" under Desktop, and convert the txt to csv file with the same name.",
"source": "author",
"config": [
{
"type": "command",
"parameters":{
"command": "echo password | sudo -S apt-get update && echo password | sudo -S apt-get install sysstat",
"shell": "true"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"os", "calc"
],
"evaluator": {
"func": ["file_contains", "check_csv_line_number"],
"result":
[
{
"type": "vm_file",
"path": "/home/user/Desktop/System_Resources_Report.txt",
"dest": "System_Resources_Report.txt"
},
{
"type": "vm_file",
"path": "/home/user/Desktop/System_Resources_Report.csv",
"dest": "System_Resources_Report.csv"
}
],
"expected":
[
{
"type": "rule",
"rules" :{
"expected": ["tps", "rtps", "wtps", "dtps", "bread/s", "bwrtn/s", "bdscd/s", "Average", "Linux"]
}
},
{
"type": "rule",
"rules": {
"expected": "33"
}
}]
}
}

86
evaluation_examples/examples/multi_apps/26660ad1-6ebb-4f59-8cba-a8432dfe8d38.json
View File

@@ -1,25 +1,69 @@
{
"id": "26660ad1-6ebb-4f59-8cba-a8432dfe8d38",
"snapshot": "libreoffice_calc",
"instruction": "I want to test the quality of the network environment my laptop is currently in. Please measure my network situation through speedtest.net, export the measurement results, and save them to ~/Test/Speed.",
"source": "authors",
"config": [
],
"trajectory": "trajectories/26660ad1-6ebb-4f59-8cba-a8432dfe8d38",
"related_apps": [
],
"evaluator": {
"postconfig": [],
"func": "",
"result": {
"id": "26660ad1-6ebb-4f59-8cba-a8432dfe8d38",
"snapshot": "multiapps",
"instruction": "I want to test the quality of the network environment my laptop is currently in. Please measure my network situation through speedtest.net, export the measurement results, and save them to ~/Test/Speed (if the dir does not exist, create it).",
"source": "https://www.speedtest.net/",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
"expected": {
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
"options": {
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://www.speedtest.net/"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Google Chrome"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",
"related_apps":[
"os",
"browser"
],
"evaluator":{
"func": "compare_time_in_speedtest_results",
"result":{
"type": "vm_file",
"path": "/home/user/Test/Speed/Speedtest Results Export-.csv",
"dest": "Speedtest Results Export-.csv",
"time_suffix": true
},
"expected":{
"type": "time_diff_range",
"diff_range_in_minutes": "60"
}
}
}
}
}

37
evaluation_examples/examples/multi_apps/3c8f201a-009d-4bbe-8b65-a6f8b35bb57f.json Normal file
View File

@@ -0,0 +1,37 @@
{
"id": "3c8f201a-009d-4bbe-8b65-a6f8b35bb57f",
"snapshot": "gimp",
"instruction": "Use `gdown` to download the image from \"https://drive.google.com/uc?export=download&id=1i8j5dGS57sA07jEuPNAlQW-sn5uqUnuK\", and then use GIMP to compress it to under 600KB. Resize if needed.",
"source": "",
"config": [
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"alt\", \"t\"); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"gimp",
"os"
],
"evaluator": {
"func": "check_image_file_size",
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/compressed.jpeg",
"dest": "compressed.jpeg"
},
"expected": {
"type": "rule",
"rules": {
"max_size": 600000
}
}
}
}

52
evaluation_examples/examples/multi_apps/3eb2a122-a5e3-4f89-9820-f7fa1a582969.json Normal file
View File

@@ -0,0 +1,52 @@
{
"id": "3eb2a122-a5e3-4f89-9820-f7fa1a582969",
"snapshot": "multiapps",
"instruction": "Please search online for the submission deadline and venue of the ICLR main conference in 2035, and copy it to my clipboard. If not yet publicized, copy None.",
"source": "author",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "command",
"parameters":{
"command": "echo password | sudo -S apt install xsel && xsel -bc",
"shell": "true"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"os", "chrome"
],
"evaluator": {
"func": "is_in_vm_clickboard",
"expected": {
"type": "vm_command_line",
"command": "xsel --clipboard --output",
"shell": "true"
},
"result": {
"type": "rule",
"rules": {
"expected": ["None"]
}
}
}
}

71
evaluation_examples/examples/multi_apps/42f4d1c7-4521-4161-b646-0a8934e36081.json Normal file
View File

@@ -0,0 +1,71 @@
{
"id": "42f4d1c7-4521-4161-b646-0a8934e36081",
"snapshot": "gimp",
"instruction": "Configure VS Code to edit GIMP script-fu scripts effectively by installing lisp extension. Test by writing code to resizing the image as 128 * 128 as \"resized.png\"",
"source": "",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1yrWU5HimYPNUjdtvw1a218kh50fPVtZ3",
"path": "/home/user/Desktop/character.png"
}
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"code"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"gimp",
"vs_code"
],
"evaluator": {
"func": [
"is_extension_installed",
"check_image_size"
],
"result": [
{
"type": "vm_command_line",
"command": [
"code",
"--list-extensions",
"|",
"grep",
"mattn.lisp"
]
},
{
"type": "vm_file",
"path": "/home/user/Desktop/resized.png",
"dest": "resized.png"
}
],
"expected": [
{
"type": "rule",
"rules": {
"type": "contain",
"expected": "mattn.lisp"
}
},
{
"type": "rule",
"rules": {
"height": 128,
"width": 128
}
}
]
}
}

70
evaluation_examples/examples/multi_apps/5bc63fb9-276a-4439-a7c1-9dc76401737f.json Normal file
View File

@@ -0,0 +1,70 @@
{
"id": "5bc63fb9-276a-4439-a7c1-9dc76401737f",
"snapshot": "multiapps",
"instruction": "I have a JSON-formatted data file opened now that stores the responses of several large language models on a specific dataset. Now, I want to filter out all the responses from Gemini and specifically look at the sentences in the responses that contain \"Iliad\". Please copy all of Gemini's responses(values) from the JSON file, paste them into a Word document named \"gemini_results.docx\" under Desktop. Each response should be a new paragraph and sepreated by a new line. Highlight the all the \"Iliad\" word.",
"source": "",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1OtN8poSktw5s9w7y28wr5JmXszMOpn2q",
"path": "/home/user/Desktop/llm_answers.json"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/Desktop/llm_answers.json"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"libreoffice_writer"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "gemini_results.docx - LibreOffice Writer",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
]
}
}
],
"func": "compare_docx_files_and_ignore_new_lines",
"expected": {
"type": "cloud_file",
"path": "https://drive.google.com/uc?export=download&id=1gmg5mC7qfmPvspC-ICOIJJ3aRjuSvP8u",
"dest": "gemini_results_Gold.docx"
},
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/gemini_results.docx",
"dest": "gemini_results.docx"
},
"options": {
"ignore_blanks": false
}
}
}

70
evaluation_examples/examples/multi_apps/65e62ec6-4603-4c67-b4d2-07830deb285b.json Normal file
View File

@@ -0,0 +1,70 @@
{
"id": "65e62ec6-4603-4c67-b4d2-07830deb285b",
"snapshot": "multiapps",
"instruction": "I've recently become very interested in large language models. I tried entering some questions into several large language models and stored the answers in a file named \"answers.json\" under Desktop. Now, I want to focus only on the answers from GPT-4. Please help me copy all the GPT-4 answers into a Word document named \"results.docx\" under Desktop, one answer per paragraph.",
"source": "",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1XIJHJ6jtYITRG6vA1-rQlnKegIk0u6j6",
"path": "/home/user/Desktop/answers.json"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/Desktop/answers.json"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"libreoffice_writer"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "results.docx - LibreOffice Writer",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
]
}
}
],
"func": "compare_docx_files_and_ignore_new_lines",
"expected": {
"type": "cloud_file",
"path": "https://drive.google.com/uc?export=download&id=1edtU4kHecg7oxerf1jvd1-ak-2WmEla3",
"dest": "results_Gold.docx"
},
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/results.docx",
"dest": "results.docx"
},
"options": {
"ignore_blanks": false
}
}
}

72
evaluation_examples/examples/multi_apps/6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a.json Normal file
View File

@@ -0,0 +1,72 @@
{
"id": "6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a",
"snapshot": "multiapps",
"instruction": "I now want to count the meeting cities of the three machine learning conferences in the past ten years from 2013 to 2019(including 2013 and 2019). I have listed the names and years of the conferences in excel. Please fill in the vacant locations.",
"source": "author",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=19wUxTQeoKr6ihJWJ_9cu2tzKQH0cnxWH",
"path": "/home/user/Desktop/ConferenceCity.xlsx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/Desktop/ConferenceCity.xlsx"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"calc", "chrome", "os"
],
"evaluator": {
"postconfig":[
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1ZcITkIOs2Z86S5L6MShSohFs3_xVfeCP",
"path": "/home/user/Desktop/ConferenceCity_Gold.xlsx"
}
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "ConferenceCity.xlsx - LibreOffice Calc"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5); pyautogui.press(\"enter\");"
]
}
}
],
"func": "compare_conference_city_in_order",
"expected": {
"type": "rule",
"rules":{
"expected": ["Scottsdale","Atlanta","Lake Tahoe","Banff","Beijing","Montreal","San Diego","Lille","Montreal","San Juan","New York","Barcelona","Toulon","Sydney","Long Beach","Vancouver","Stockholm","Montréal","New Orleans","Long Beach","Vancouver"]
}
},
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/ConferenceCity.xlsx",
"dest": "ConferenceCity.xlsx"
}
}
}

51
evaluation_examples/examples/multi_apps/716a6079-22da-47f1-ba73-c9d58f986a38.json Normal file
View File

@@ -0,0 +1,51 @@
{
"id": "716a6079-22da-47f1-ba73-c9d58f986a38",
"snapshot": "multiapps",
"instruction": "I remember there is a file named \"secret.docx\" on this computer, but I can't remember where it is. Please find the path where this file is stored and copy it to the clipboard.",
"source": "",
"config": [
{
"type": "command",
"parameters": {
"command": "mkdir -p /home/user/Data1/List1 && mkdir -p /home/user/Data2/List2 && mkdir -p /home/user/Data3/List3 && mkdir -p /home/user/Data4/List4 && mkdir -p /home/user/Data5/List5",
"shell" :"true"
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1iaqtXHZs2sA11nmoLHPK9VXGO0_6xq2l",
"path": "/home/user/Data3/List3/secret.docx"
}
]
}
},
{
"type": "command",
"parameters":{
"command": "echo password | sudo -S apt install xsel && xsel -bc",
"shell": "true"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"os", "terminal"
],
"evaluator": {
"func": "is_in_vm_clickboard",
"expected": {
"type": "vm_command_line",
"command": "xsel --clipboard --output",
"shell": "true"
},
"result": {
"type": "rule",
"rules": {
"expected": "/home/user/Data3/List3/secret.docx"
}
}
}
}

43
evaluation_examples/examples/multi_apps/7f35355e-02a6-45b5-b140-f0be698bcf85.json Normal file
View File

@@ -0,0 +1,43 @@
{
"id": "7f35355e-02a6-45b5-b140-f0be698bcf85",
"snapshot": "libreoffice_calc",
"instruction": "Export the table to a CSV file and then help me write code to find the medium price (fill empty value with average). Save the result in \"result.txt\".",
"source": "",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://docs.google.com/spreadsheets/d/13YL-KC__pav2qp3sFDs1BT2wZnpWGp7s/export?format=xlsx",
"path": "/home/user/Desktop/stock.xlsx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/Desktop/stock.xlsx"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"libreoffice_calc",
"vs_code"
],
"evaluator": {
"func": "compare_result_files",
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/result.txt",
"dest": "result.txt"
},
"expected": {
"type": "cloud_file",
"path": "https://drive.google.com/uc?export=download&id=1oPPW_dozWGII5MRmdXdKKoEK5iBkd_8Q",
"dest": "result_gold.txt"
}
}
}

83
evaluation_examples/examples/multi_apps/7ff48d5b-2df2-49da-b500-a5150ffc7f18.json Normal file
View File

@@ -0,0 +1,83 @@
{
"id": "7ff48d5b-2df2-49da-b500-a5150ffc7f18",
"snapshot": "multiapps",
"instruction": "I am a Chinese citizen and I want to go to Macau to watch a concert recently, but I have not yet applied for a visa for Macau. I live in Futian District, Shenzhen City. I heard that Shenzhen currently has 24-hour self-service check-in machines. Please help me find the addresses of 5 24-hour self-service check-in machines in Futian District and save them in Chinese in this open word document.",
"source": "authors",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1H-GVXzyn30zxBKw02w5p5ZQzcwEZSlE4&export=download&authuser=0&confirm=t&uuid=da987f7a-7aa5-43d9-8426-741a97393726&at=APZUnTWyce0qNBfoOjOZ5pNRwIvX:1709971794981",
"path": "/home/user/Desktop/AllLocations.docx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/Desktop/AllLocations.docx"
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "AllLocations.docx - LibreOffice Writer"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome", "os", "writer"
],
"evaluator": {
"func": "fuzzy_place_math",
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/AllLocations.docx",
"dest": "AllLocations.docx"
},
"expected": {
"type": "rule",
"rules":{
"expected": [
"深圳市福田区益田路5055号信息枢纽大厦西门一楼",
"深圳市福田区福华三路111号北三门会展中心警务室",
"深圳市福田区正义街1号",
"福田区莲科路18号莲花一村警务室",
"深圳市福田区彩云路2-8长城盛世家园一期C座一楼一期管理处旁边",
"深圳市福田区香梅路2002-4号",
"福田区水围村龙景楼一楼",
"深圳市福田区梅林路与梅康路交汇处卓悦汇4号、5号门对面",
"深圳市福田区福强路3028号金沙嘴大厦",
"深圳市福田区天安数码城昌泰公寓一楼",
"福田区泰然五路5号天安数码城9栋",
"深圳市福田区振兴路108号",
"深圳市福田区滨河大道2033号",
"深圳市福田区上沙四十八栋一巷11",
"深圳市福田区北环大道与香蜜湖路交汇处香蜜原著警务室",
"深圳市福田区八卦路38号八卦岭派出所",
"深圳市福田区宝能城市公馆B栋一楼竹园警务室",
"深圳市福田区竹子林五路12号",
"福田区福强路3028号金沙嘴大厦",
"福田区彩云路2-8长城盛世家园一期C座一楼一期管理处旁边",
"福田区益田路5055号信息枢纽大厦西门一楼",
"福田区正义街1号",
"福田区香梅路2002-4号",
"福田区梅林路与梅康路交汇处卓悦汇4号、5号门对面",
"福田区天安数码城昌泰公寓一楼",
"福田区振兴路108号",
"福田区滨河大道2033号",
"福田区上沙四十八栋一巷11",
"福田区北环大道与香蜜湖路交汇处香蜜原著警务室",
"福田区八卦路38号八卦岭派出所",
"福田区宝能城市公馆B栋一楼竹园警务室",
"福田区竹子林五路12号"
]
}
}
}
}

66
evaluation_examples/examples/multi_apps/873cafdd-a581-47f6-8b33-b9696ddb7b05.json Normal file
View File

@@ -0,0 +1,66 @@
{
"id": "873cafdd-a581-47f6-8b33-b9696ddb7b05",
"snapshot": "multiapps",
"instruction": "My friend is a \"plugin guru\" and he recommended some good plug-ins to me. Please go to the Chrome plug-in store and install all the listed plug-ins.",
"source": "author",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1EMpoFNW3reLgZE0lj6scRFmL-OPjWuzf",
"path": "/home/user/Desktop/Recommended_plugin_list.docx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/Desktop/Recommended_plugin_list.docx"
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Recommended_plugin_list.docx - LibreOffice Writer"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"os", "chrome", "Writer"
],
"evaluator": {
"func": "is_expected_installed_extensions",
"expected": {
"type": "rule",
"rules":{
"expected": ["Zoom Chrome Extension", "Speechify Text to Speech Voice Reader", "React Developer Tools", "Momentum", "Google Translate", "Web Store", "Chromium PDF Viewer", "Google Hangouts"]
}
},
"result": {
"type": "find_installed_extension_name"
}
}
}

51
evaluation_examples/examples/multi_apps/91190194-f406-4cd6-b3f9-c43fac942b22.json Normal file
View File

@@ -0,0 +1,51 @@
{
"id": "91190194-f406-4cd6-b3f9-c43fac942b22",
"snapshot": "gimp",
"instruction": "Launch GIMP from the command line to edit \"cola.png\" and crop the top 20% off the image for my avatar as \"cropped.png\".",
"source": "",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1bmSRNNh4JkF6izrKrmynUHarf0pFES50",
"path": "/home/user/Desktop/cola.png"
},
{
"url": "https://drive.google.com/uc?export=download&id=1MayrIPJWRK7cMEVe3TxYmgkAbVMrYcQA",
"path": "/home/user/Desktop/cropped_gold.png"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"alt\", \"t\"); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"gimp",
"os"
],
"evaluator": {
"func": "check_structure_sim",
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/cropped.png",
"dest": "cropped.png"
},
"expected": {
"type": "vm_file",
"path": "/home/user/Desktop/cropped_gold.png",
"dest": "cropped_gold.png"
}
}
}

54
evaluation_examples/examples/multi_apps/98e8e339-5f91-4ed2-b2b2-12647cb134f4.json Normal file
View File

@@ -0,0 +1,54 @@
{
"id": "98e8e339-5f91-4ed2-b2b2-12647cb134f4",
"snapshot": "vs_code",
"instruction": "Merge the contents of all .txt files from your vscode project into a single document in Writer. No merging separator is needed. Ensure to set the overall font size of the document to 10.",
"source": "",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1dqbi6j3zPOqFTMFHVVaEOXSdHz5zIey1",
"path": "/home/user/Desktop/doc_proc.zip"
}
]
}
},
{
"type": "command",
"parameters": {
"command": "mkdir -p /home/user/Desktop/doc_proc/; unzip /home/user/Desktop/doc_proc.zip -d /home/user/Desktop/doc_proc/",
"shell": true
}
},
{
"type": "launch",
"parameters": {
"command": [
"code",
"-g",
"/home/user/Desktop/doc_proc/"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"vs_code",
"libreoffice_writer"
],
"evaluator": {
"func": "compare_docx_files",
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/concat.docx",
"dest": "concat.docx"
},
"expected": {
"type": "cloud_file",
"path": "https://drive.google.com/uc?export=download&id=10k_8T7Hk9KQ2I-AbNK56hvXR6yv0Pemc",
"dest": "concat_gold.docx"
}
}
}

81
evaluation_examples/examples/multi_apps/9f3bb592-209d-43bc-bb47-d77d9df56504.json Normal file
View File

@@ -0,0 +1,81 @@
{
"id": "9f3bb592-209d-43bc-bb47-d77d9df56504",
"snapshot": "multiapps",
"instruction": "I downloaded a video to practice listening, but I don't know how to remove the subtitles. Please help me remove the subtitles from the video and export it as \"subtitles.srt\" and store it in the same directory as the video.",
"source": "authors",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=17vIS65RVA7e9n5u8lLNqwjBl56MQpkk8&export=download&authuser=0&confirm=t&uuid=1c8882db-dd55-450d-a998-52c03bfaf801&at=APZUnTWPObUX-3Pqtusro4T5Ga5B:1709972109572",
"path": "/home/user/video.mp4"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/video.mp4"
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "video.mp4 - VLC media player"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"os", "vlc"
],
"evaluator": {
"postconfig":[
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1oMCDx2hqj0xzkQP4le0Mn09y-vvCYXlt&export=download&authuser=0&confirm=t&uuid=2a2cac5c-f031-41f7-8504-4a013c02ee15&at=APZUnTUa188AtQ93qZG8HInS_o90:1709972317568",
"path": "/home/user/subtitles_Gold.srt"
}
]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=19ooP4gPY2HYQVI9n31CtMlJ3q-1dr6cU&export=download&authuser=0&confirm=t&uuid=ba968219-3a9b-44d1-bee6-64e289ce4ffe&at=APZUnTXlL0jQBoia7KuzELsBCVxE:1709972303196",
"path": "/home/user/Desktop/subtitles_script.py"
}
]
}
},
{
"type": "command",
"parameters": {
"command": "echo password | sudo -S pip install pysrt",
"shell" :"true"
}
}
],
"func": "exact_match",
"result": {
"type": "vm_command_line",
"command": "python /home/user/Desktop/subtitles_script.py",
"shell" :"true"
},
"expected": {
"type": "rule",
"rules":{
"expected": "true\n"
}
}
}
}

62
evaluation_examples/examples/multi_apps/a74b607e-6bb5-4ea8-8a7c-5d97c7bbcd2a.json Normal file
View File

@@ -0,0 +1,62 @@
{
"id": "a74b607e-6bb5-4ea8-8a7c-5d97c7bbcd2a",
"snapshot": "multiapps",
"instruction": "I have developed a new Chrome extension myself, so it needs to be installed manually. Please help me install the extension located in the Desktop directory into the Chrome browser.",
"source": "https://support.google.com/chrome/thread/205881926/it-s-possible-to-load-unpacked-extension-automatically-in-chrome?hl=en",
"config": [
{
"type": "download",
"parameters": {
"files":[
{
"url":"https://drive.google.com/uc?export=download&id=1fe-MuG57hf12cEi4qI4dEI3OYJOHYoY2",
"path":"/home/user/Desktop/helloExtension.zip"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": "echo password | sudo -S apt-get update -y && echo password | sudo -S apt-get install unzip -y && unzip /home/user/Desktop/helloExtension.zip -d /home/user/Desktop/ && rm /home/user/Desktop/helloExtension.zip",
"shell": true
}
},
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome", "os"
],
"evaluator": {
"func": "is_in_list",
"result": {
"type": "find_unpacked_extension_path"
},
"expected": {
"type": "rule",
"rules": {
"expected": "/home/user/Desktop/helloExtension"
}
}
}
}

14
evaluation_examples/examples/multi_apps/a82b78bb-7fde-4cb3-94a4-035baf10bcf0.json
View File

@@ -23,6 +23,12 @@
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 2
}
},
{
"type": "download",
"parameters": {
@@ -75,10 +81,10 @@
"Liked Authors"
],
"urls": [
"https://jimfan.me/",
"https://ai.stanford.edu/~dahuang/",
"https://yukezhu.me/",
"https://www.eas.caltech.edu/people/anima"
["https://jimfan.me/", "https://research.nvidia.com/person/linxi-jim-fan"],
["https://research.nvidia.com/person/de-an-huang", "https://ai.stanford.edu/~dahuang/"],
["https://yukezhu.me/", "https://www.cs.utexas.edu/people/faculty-researchers/yuke-zhu", "https://experts.utexas.edu/yuke_zhu", "https://research.nvidia.com/person/yuke-zhu"],
["http://tensorlab.cms.caltech.edu/users/anima/", "https://www.eas.caltech.edu/people/anima"]
]
}
}

69
evaluation_examples/examples/multi_apps/bb7db4c2-30b5-4be7-8dd7-b8c4ec7d3108.json Normal file
View File

@@ -0,0 +1,69 @@
{
"id": "bb7db4c2-30b5-4be7-8dd7-b8c4ec7d3108",
"snapshot": "gimp",
"instruction": "I have a background image named 'anmi.png'. I need the image to be enhanced for better sharpness, as 'anmi_sharper.png'. Next, please use it as the background image for my 'index.html' page.",
"source": "",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1BAZvR0BflPh70ClL2bHrbJ-xyV-nNQ1o",
"path": "/home/user/Desktop/anmi.png"
},
{
"url": "https://drive.google.com/uc?export=download&id=1zcyY7y_deKStc-AtU8pFFrJItOuyfrNK",
"path": "/home/user/Desktop/index.html"
}
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"gimp",
"/home/user/Desktop/anmi.png"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"gimp",
"vs_code"
],
"evaluator": {
"func": [
"check_sharper",
"check_html_background_image"
],
"result": [
{
"type": "vm_file",
"path": "/home/user/Desktop/anmi_sharper.png",
"dest": "anmi_sharper.png"
},
{
"type": "vm_file",
"path": "/home/user/Desktop/index.html",
"dest": "index.html"
}
],
"expected": [
{
"type": "vm_file",
"path": "/home/user/Desktop/anmi.png",
"dest": "anmi.png"
},
{
"type": "rule",
"rules": {
"type:": "value",
"value": "anmi_sharper.png"
}
}
]
}
}

80
evaluation_examples/examples/multi_apps/ce2b64a2-ddc1-4f91-8c7d-a88be7121aac.json Normal file
View File

@@ -0,0 +1,80 @@
{
"id": "ce2b64a2-ddc1-4f91-8c7d-a88be7121aac",
"snapshot": "multiapps",
"instruction": "There are several pictures of mountains in my Pictures directory, but I don’t know the names of these mountains. Please help me identify these pictures and change the names of these pictures to the names of the mountains in the pictures.",
"source": "authors",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1sKdxRx4Uxs-wxohYkmxPfxOUpw72BINV&export=download&authuser=0&confirm=t&uuid=2cd18b66-93a6-4771-a92b-a1b1f0adf12d&at=APZUnTUvT9iZ0IdRnnNf1ys4mZaE:1709972639977",
"path": "/home/user/Pictures/picture1.jpg"
}
]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1iquZNI4ktx2915srQ1MMlhcwCuXnmq5T&export=download&authuser=0&confirm=t&uuid=e0bfd86f-a0f1-4a9e-8344-0f645eb379dd&at=APZUnTWQnCd7bNtf8ZN4KfkQUaJ9:1709972645590",
"path": "/home/user/Pictures/picture2.jpg"
}
]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1ie3rhHzsW293xhnmgeZm6wQj_uaFmPS7&export=download&authuser=0&confirm=t&uuid=633c8b0c-0d3f-4f07-a772-945148956244&at=APZUnTX6OIN5YfudwPVT3Mxntd9E:1709972650733",
"path": "/home/user/Pictures/picture3.jpg"
}
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome", "os", "image"
],
"evaluator": {
"postconfig":[
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1eWGQ3z-Ps5p784NZjj9KyhHfX7NbHgWd&export=download&authuser=0&confirm=t&uuid=8ba167f4-8669-477b-89ee-a8f6996f43d7&at=APZUnTWShYYueBb8hRnrjjabU1ZD:1709972787510",
"path": "/home/user/Desktop/image_script.py"
}
]
}
}
],
"func": "check_direct_json_object",
"result":{
"type": "vm_command_line",
"command":"python /home/user/Desktop/image_script.py",
"shell": "true"
},
"expected": {
"type": "rule",
"rules":{
"expected":{
"ec076282f61ba74642e94b5a6a1250c6988204d59d9b02936606b6b8ef1e4433": "Kilimanjaro",
"999957c8de835bfa420d198270e7a6b079ee20ff53a3f214491e8134768a7c0b": "Himalayas",
"79f45d40d8413d4e81f1b9734ea39e58622cafd79e12bab32959643fc245147c": "Hua"
},
"expect_in_result": true
}
}
}
}

51
evaluation_examples/examples/multi_apps/d68204bf-11c1-4b13-b48b-d303c73d4bf6.json Normal file
View File

@@ -0,0 +1,51 @@
{
"id": "d68204bf-11c1-4b13-b48b-d303c73d4bf6",
"snapshot": "gimp",
"instruction": "Divide my image vertically into three equal sections with command line. Then rearrange them in order with a gradient of warm tones, progressively becoming warmer from left to right as a new image \"rearranged.png\".",
"source": "",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1CPGW_OZsfSWDdTU7CFrTjpzSAASyLy4w",
"path": "/home/user/Desktop/tilearray.png"
},
{
"url": "https://drive.google.com/uc?export=download&id=1aHwmnxL2CKEh_FhVpevY452-BQH2t5rG",
"path": "/home/user/Desktop/rearranged_gold.png"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"alt\", \"t\"); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"gimp",
"os"
],
"evaluator": {
"func": "check_structure_sim",
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/rearranged.png",
"dest": "rearranged.png"
},
"expected": {
"type": "vm_file",
"path": "/home/user/Desktop/rearranged_gold.png",
"dest": "rearranged_gold.png"
}
}
}

75
evaluation_examples/examples/multi_apps/da922383-bfa4-4cd3-bbad-6bebab3d7742.json Normal file
View File

@@ -0,0 +1,75 @@
{
"id": "da922383-bfa4-4cd3-bbad-6bebab3d7742",
"snapshot": "multiapps",
"instruction": "I browsed a lot of interesting blog articles today. I hope to store these articles in my local designated folder just like zotero stores papers. Please download the blogs you are opening now in pdf format to /home/user/Documents/Blogs.",
"source": "authors",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://lilianweng.github.io/posts/2023-06-23-agent/",
"https://lilianweng.github.io/posts/2024-02-05-human-data-quality/"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome", "os"
],
"evaluator": {
"postconfig":[
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1fm4wqp91_3yzVSlenc9k2Tlh3-7QuRvu&export=download&authuser=0&confirm=t&uuid=5bb46f9e-ef04-4b52-bceb-a88f0058f696&at=APZUnTU6aWNzsiqLwAlyr_8pxKUd:1709971656456",
"path": "/home/user/Desktop/script.py"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": "pip install PyMuPDF",
"shell": "true"
}
}
],
"func": "exact_match",
"result":{
"type": "vm_command_line",
"command":"python /home/user/Desktop/script.py",
"shell": "true"
},
"expected": {
"type": "rule",
"rules":{
"expected": "[1, 1]\n"
}
}
}
}

53
evaluation_examples/examples/multi_apps/dd60633f-2c72-42ba-8547-6f2c8cb0fdb0.json Normal file
View File

@@ -0,0 +1,53 @@
{
"id": "dd60633f-2c72-42ba-8547-6f2c8cb0fdb0",
"snapshot": "multiapps",
"instruction": "I ran some simple code demos on the currently open google colab, and I think the effect is pretty good. Please help me extract the code in all code boxes, merge it into a \"task.py\" file and store it in the local Home directory.",
"source": "authors",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://colab.research.google.com/github/stanfordnlp/dspy/blob/main/intro.ipynb#scrollTo=L1BHAoL_LRd7"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome", "os", "writer"
],
"evaluator": {
"func": "compare_python_pure_text",
"result": {
"type": "vm_file",
"path": "/home/user/colab.py",
"dest": "colab.py"
},
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1r0VyQZf5Hi8n8R9w9iGKMsqQaxAjYN7O&export=download&authuser=0&confirm=t&uuid=56efd2ff-4494-4026-9e96-82783b5ed600&at=APZUnTUL7BOMtV3FIoc0kXe18LWz:1709972550788",
"dest": "colab_Gold.py"
}
}
}

67
evaluation_examples/examples/multi_apps/e8172110-ec08-421b-a6f5-842e6451911f.json Normal file
View File

@@ -0,0 +1,67 @@
{
"id": "e8172110-ec08-421b-a6f5-842e6451911f",
"snapshot": "gimp",
"instruction": "Open 'character.png' in GIMP and extract the pixel art character. Save the selected character as 'character_gimp.png'. Additionally, write a Python script to automate this selection process, ensuring it precisely mimics the manual extraction done in GIMP. Output the result from the script as 'character_code.png'.",
"source": "",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1Tf7MvqBMHuIhvKmk-Ryr_qmzmkACFjxB",
"path": "/home/user/Desktop/character.png"
},
{
"url": "https://drive.google.com/uc?export=download&id=1wKG5X6LaN0tShsAK4lFg3OyFg8OGYYZg",
"path": "/home/user/Desktop/character_no_background_gold.png"
}
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"gimp",
"/home/user/Desktop/character.png"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"gimp",
"vs_code"
],
"evaluator": {
"func": [
"check_structure_sim",
"check_structure_sim"
],
"result": [
{
"type": "vm_file",
"path": "/home/user/Desktop/character_gimp.png",
"dest": "character_gimp.png"
},
{
"type": "vm_file",
"path": "/home/user/Desktop/character_code.png",
"dest": "character_code.png"
}
],
"expected": [
{
"type": "vm_file",
"path": "/home/user/Desktop/character_no_background_gold.png",
"dest": "character_no_background_gold.png"
},
{
"type": "vm_file",
"path": "/home/user/Desktop/character_no_background_gold.png",
"dest": "character_no_background_gold.png"
}
]
}
}