lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ver Mar6thv3

Browse Source 
new multi_app tasks and metrics
This commit is contained in:
David Chang
2024-03-06 23:29:01 +08:00
parent e045eee901
commit 054e016aff
8 changed files with 548 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

153
evaluation_examples/examples/multi_apps/3f05f3b9-29ba-4b6b-95aa-2204697ffc06.json Normal file
View File

@@ -0,0 +1,153 @@
{
"id": "3f05f3b9-29ba-4b6b-95aa-2204697ffc06",
"snapshot": "os",
"instruction": "I have a collection of MP3s named with their artist and title. However, their meta data are messy. I don't know how to quickly fix them. Some people tells that Picard or Kid3 may help, but I have never used them. Please help me to do this.",
"source": "authors",
"config": [
{
"type": "execute",
"parameters": {
"command": ["mkdir", "-p", "/home/user/Documents/Finance/receipts", "/home/user/Documents/Projects/OSWorld", "/home/user/Documents/Novels/4th Year in Tsinghua", "/home/user/Documents/Novels/Pass Through"]
}
},
{
"type": "download",
"parameters": {
"files": [
{"path": "/home/user/Documents/Finance/receipts/aws-invoice-2308.pdf", "url": "https://drive.google.com/uc?id=1azRFXf4A7fvW0S7r9upHvleMEi-92hHM&export=download"},
{"path": "/home/user/Documents/Finance/receipts/aws-invoice-2309.pdf", "url": "https://drive.google.com/uc?id=1x-lpHm8U4U7uRPZ74-9wq9KzW2R55ln1&export=download"},
{"path": "/home/user/Documents/Finance/receipts/aws-invoice-2310.pdf", "url": "https://drive.google.com/uc?id=1pcrgV9G6NO4ekMEQBiupwXtq6mmke7b_&export=download"},
{"path": "/home/user/Documents/Finance/receipts/aws-invoice-2311.pdf", "url": "https://drive.google.com/uc?id=1JzbCK_nIY8X_3QZjnkzTtb-cRoq9zNT-&export=download"},
{"path": "/home/user/Documents/Finance/receipts/aws-invoice-2312.pdf", "url": "https://drive.google.com/uc?id=1RqbulzKG_HeYb1GZmLABOzlohlFg02UU&export=download"},
{"path": "/home/user/Documents/Finance/receipts/X-receipt-2312.pdf", "url": "https://drive.google.com/uc?id=1QzWjNzvNosG_yQr7VVonvYb3cUYF5f3u&export=download"},
{"path": "/home/user/Documents/Finance/tally_book.xlsx", "url": "https://drive.google.com/uc?id=13yuLhBPmouoWR-DybfgaIbWUOxbY_jhL&export=download"},
{"path": "/home/user/.projects.tar.xz", "url": "https://drive.google.com/uc?id=1oJcxpjqF474Wm16i1aZc8DlCEfAvc4t_&export=download"},
{"path": "/home/user/.old-chapters.tar.xz", "url": "https://drive.google.com/uc?id=11y-kLI83sQz6ncmP1-tkIR6mQJ7SfX5-&export=download"},
{"path": "/home/user/.pass-through.tar.xz", "url": "https://drive.google.com/uc?id=1z7QGTWMHhoZCAd7O1O6CWWKy49oKUowf&export=download"},
{"path": "/home/user/Music/Cheng Xiang - Missing You.mp3", "url": "https://drive.google.com/uc?id=1dqAiQoHhXn0UIatHs0iUzL266w8OhuzI&export=download"},
{"path": "/home/user/Music/Han Baoyi - Tears of Dancing Girl.mp3", "url": "https://drive.google.com/uc?id=1ePZMj14XfdzSERB6iveUMTF7nvfx45K7&export=download"},
{"path": "/home/user/Music/Huang An - I Know Missing is Painful.mp3", "url": "https://drive.google.com/uc?id=1h790X11CHFXZyB6SiftNtiq-3kd2Q12j&export=download"},
{"path": "/home/user/Music/Chen Shaohua - Red Daughter.mp3", "url": "https://drive.google.com/uc?id=1GHhlAw4KZglAwV9C5bhHDCHIFb_adJUO&export=download"},
{"path": "/home/user/Music/Zhou Xuan - Nights in Shanghai.mp3", "url": "https://drive.google.com/uc?id=1MGTK-Sr00CihcitI9ebei-HFzJNhtgIo&export=download"}
]
}
},
{
"type": "execute",
"parameters": {
"command": ["tar", "-xJvf", "/home/user/.projects.tar.xz", "-C", "/home/user/Documents/Projects/OSWorld"]
}
},
{
"type": "execute",
"parameters": {
"command": ["tar", "-xJvf", "/home/user/.old-chapters.tar.xz", "-C", "/home/user/Documents/Novels/4th Year in Tsinghua"]
}
},
{
"type": "execute",
"parameters": {
"command": ["tar", "-xJvf", "/home/user/.pass-through.tar.xz", "-C", "/home/user/Documents/Novels/Pass Through"]
}
}
],
"trajectory": "trajectories/3f05f3b9-29ba-4b6b-95aa-2204697ffc06",
"related_apps": ["os", "picard"],
"evaluator": {
"func": ["check_mp3_meta", "check_mp3_meta", "check_mp3_meta", "check_mp3_meta", "check_mp3_meta"],
"result": [
{
"type": "vm_file",
"path": "/home/user/Music/Cheng Xiang - Missing You.mp3",
"dest": "Cheng Xiang - Missing You.mp3"
},
{
"type": "vm_file",
"path": "/home/user/Music/Han Baoyi - Tears of Dancing Girl.mp3",
"dest": "Han Baoyi - Tears of Dancing Girl.mp3"
},
{
"type": "vm_file",
"path": "/home/user/Music/Huang An - I Know Missing is Painful.mp3",
"dest": "Huang An - I Know Missing is Painful.mp3"
},
{
"type": "vm_file",
"path": "/home/user/Music/Chen Shaohua - Red Daughter.mp3",
"dest": "Chen Shaohua - Red Daughter.mp3"
},
{
"type": "vm_file",
"path": "/home/user/Music/Zhou Xuan - Nights in Shanghai.mp3",
"dest": "/Zhou Xuan - Nights in Shanghai.mp3"
}
],
"expected": [
{
"type": "rule",
"rules": {
"title": {
"method": "eq",
"ref": "Missing You"
},
"artist": {
"method": "eq",
"ref": "Cheng Xiang"
}
}
},
{
"type": "rule",
"rules": {
"title": {
"method": "eq",
"ref": "Tears of Dancing Girl"
},
"artist": {
"method": "eq",
"ref": "Han Baoyi"
}
}
},
{
"type": "rule",
"rules": {
"title": {
"method": "eq",
"ref": "I Know Missing is Painful"
},
"artist": {
"method": "eq",
"ref": "Huang An"
}
}
},
{
"type": "rule",
"rules": {
"title": {
"method": "eq",
"ref": "Red Daughter"
},
"artist": {
"method": "eq",
"ref": "Chen Shaohua"
}
}
},
{
"type": "rule",
"rules": {
"title": {
"method": "eq",
"ref": "Nights in Shanghai"
},
"artist": {
"method": "eq",
"ref": "Zhou Xuan"
}
}
}
]
}
}

98
evaluation_examples/examples/multi_apps/42d25c08-fb87-4927-8b65-93631280a26f.json Normal file
View File

@@ -0,0 +1,98 @@
{
"id": "42d25c08-fb87-4927-8b65-93631280a26f",
"snapshot": "chrome",
"instruction": "Hey, my friend has just sent me a web novel, but in txt files. Could you please help me convert it to epub format, so that I can easily read it on my mobile and kindle? I have found several tools to obtain an epub file, like https://github.com/kevinboone/txt2epub and https://github.com/potatoeggy/noveldown. But I'm not sure which one helps. Please help me to do this. Remember to name the file with novel's title.",
"source": "authors",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://github.com/potatoeggy/noveldown",
"https://github.com/kevinboone/txt2epub"
]
}
},
{
"type": "execute",
"parameters": {
"command": ["mkdir", "-p", "/home/user/Documents/Finance/receipts", "/home/user/Documents/Projects/OSWorld", "/home/user/Documents/Novels/4th Year in Tsinghua", "/home/user/Documents/Novels/Pass Through"]
}
},
{
"type": "download",
"parameters": {
"files": [
{"path": "/home/user/Documents/Finance/receipts/aws-invoice-2308.pdf", "url": "https://drive.google.com/uc?id=1azRFXf4A7fvW0S7r9upHvleMEi-92hHM&export=download"},
{"path": "/home/user/Documents/Finance/receipts/aws-invoice-2309.pdf", "url": "https://drive.google.com/uc?id=1x-lpHm8U4U7uRPZ74-9wq9KzW2R55ln1&export=download"},
{"path": "/home/user/Documents/Finance/receipts/aws-invoice-2310.pdf", "url": "https://drive.google.com/uc?id=1pcrgV9G6NO4ekMEQBiupwXtq6mmke7b_&export=download"},
{"path": "/home/user/Documents/Finance/receipts/aws-invoice-2311.pdf", "url": "https://drive.google.com/uc?id=1JzbCK_nIY8X_3QZjnkzTtb-cRoq9zNT-&export=download"},
{"path": "/home/user/Documents/Finance/receipts/aws-invoice-2312.pdf", "url": "https://drive.google.com/uc?id=1RqbulzKG_HeYb1GZmLABOzlohlFg02UU&export=download"},
{"path": "/home/user/Documents/Finance/receipts/X-receipt-2312.pdf", "url": "https://drive.google.com/uc?id=1QzWjNzvNosG_yQr7VVonvYb3cUYF5f3u&export=download"},
{"path": "/home/user/Documents/Finance/tally_book.xlsx", "url": "https://drive.google.com/uc?id=13yuLhBPmouoWR-DybfgaIbWUOxbY_jhL&export=download"},
{"path": "/home/user/.projects.tar.xz", "url": "https://drive.google.com/uc?id=1oJcxpjqF474Wm16i1aZc8DlCEfAvc4t_&export=download"},
{"path": "/home/user/.old-chapters.tar.xz", "url": "https://drive.google.com/uc?id=11y-kLI83sQz6ncmP1-tkIR6mQJ7SfX5-&export=download"},
{"path": "/home/user/.pass-through.tar.xz", "url": "https://drive.google.com/uc?id=1z7QGTWMHhoZCAd7O1O6CWWKy49oKUowf&export=download"}
]
}
},
{
"type": "execute",
"parameters": {
"command": ["tar", "-xJvf", "/home/user/.projects.tar.xz", "-C", "/home/user/Documents/Projects/OSWorld"]
}
},
{
"type": "execute",
"parameters": {
"command": ["tar", "-xJvf", "/home/user/.old-chapters.tar.xz", "-C", "/home/user/Documents/Novels/4th Year in Tsinghua"]
}
},
{
"type": "execute",
"parameters": {
"command": ["tar", "-xJvf", "/home/user/.pass-through.tar.xz", "-C", "/home/user/Documents/Novels/Pass Through"]
}
},
{
"type": "launch",
"parameters": {
"command": ["nautilus", "/home/user/Documents/Novels/Pass Through"]
}
}
],
"trajectory": "trajectories/42d25c08-fb87-4927-8b65-93631280a26f",
"related_apps": ["chrome", "os"],
"evaluator": {
"func": "compare_epub",
"result": {
"type": "vm_file",
"dest": "Pass Through.epub",
"path": "/home/user/Documents/Novels/Pass Through/Pass Through.epub"
},
"expected": {
"type": "cloud_file",
"path": "https://drive.google.com/uc?id=14W0R5s0f6jMuDW9kjJMBmJsgDkyLl-aZ&export=download",
"dest": "Pass Through Gold.epub"
}
}
}

110
evaluation_examples/examples/multi_apps/48c46dc7-fe04-4505-ade7-723cba1aa6f6.json Normal file
View File

@@ -0,0 +1,110 @@
{
"id": "48c46dc7-fe04-4505-ade7-723cba1aa6f6",
"snapshot": "chrome",
"instruction": "I'm currently working on OSWorld project. Every time I need to open the terminal and change to the project directory, open the file manager and change to the project directory, and open relative webpages in chrome browser. This is tedious. Help me to do this automatically. Open project directory in terminal and file manager and open github and online document in chrome browser. That's enough.",
"source": "authors",
"config": [
{
"type": "execute",
"parameters": {
"command": ["mkdir", "-p", "/home/user/Documents/Finance/receipts", "/home/user/Documents/Projects/OSWorld", "/home/user/.local/share/applications"]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "execute",
"parameters": {
"command": ["cp", "/usr/share/applications/google-chrome.desktop", "/home/user/.local/share/applications"]
}
},
{
"type": "execute",
"parameters": {
"command": [
"sed",
"-i.bak",
"-e",
"s#Exec=/usr/bin/google-chrome-stable#& --remote-debugging-port=1337#g",
"/home/user/.local/share/applications/google-chrome.desktop"
]
}
},
{
"type": "execute",
"parameters": {
"command": ["update-desktop-database"]
}
},
{
"type": "download",
"parameters": {
"files": [
{"path": "/home/user/Documents/Finance/receipts/aws-invoice-2308.pdf", "url": "https://drive.google.com/uc?id=1azRFXf4A7fvW0S7r9upHvleMEi-92hHM&export=download"},
{"path": "/home/user/Documents/Finance/receipts/aws-invoice-2309.pdf", "url": "https://drive.google.com/uc?id=1x-lpHm8U4U7uRPZ74-9wq9KzW2R55ln1&export=download"},
{"path": "/home/user/Documents/Finance/receipts/aws-invoice-2310.pdf", "url": "https://drive.google.com/uc?id=1pcrgV9G6NO4ekMEQBiupwXtq6mmke7b_&export=download"},
{"path": "/home/user/Documents/Finance/receipts/aws-invoice-2311.pdf", "url": "https://drive.google.com/uc?id=1JzbCK_nIY8X_3QZjnkzTtb-cRoq9zNT-&export=download"},
{"path": "/home/user/Documents/Finance/receipts/aws-invoice-2312.pdf", "url": "https://drive.google.com/uc?id=1RqbulzKG_HeYb1GZmLABOzlohlFg02UU&export=download"},
{"path": "/home/user/Documents/Finance/receipts/X-receipt-2312.pdf", "url": "https://drive.google.com/uc?id=1QzWjNzvNosG_yQr7VVonvYb3cUYF5f3u&export=download"},
{"path": "/home/user/Documents/Finance/tally_book.xlsx", "url": "https://drive.google.com/uc?id=13yuLhBPmouoWR-DybfgaIbWUOxbY_jhL&export=download"},
{"path": "/home/user/.projects.tar.xz", "url": "https://drive.google.com/uc?id=1oJcxpjqF474Wm16i1aZc8DlCEfAvc4t_&export=download"}
]
}
},
{
"type": "execute",
"parameters": {
"command": ["tar", "-xJvf", "/home/user/.projects.tar.xz", "-C", "/home/user/Documents/Projects/OSWorld"]
}
}
],
"trajectory": "trajectories/48c46dc7-fe04-4505-ade7-723cba1aa6f6",
"related_apps": ["chrome", "os"],
"evaluator": {
"postconfig": [
{
"type": "execute",
"parameters": {
"command": ["wmctrl", "-lx"],
"stdout": "wmctrl.out"
}
}
],
"func": ["check_list", "is_expected_tabs"],
"result": [
{
"type": "cache_file",
"path": "wmctrl.out"
},
{
"type": "open_tabs_info"
}
],
"expected": [
{
"type": "rule",
"rules": {
"expect": [
"\\borg\\.gnome\\.Nautilus\\.Org\\.gnome\\.Nautilus\\b.*\\bOSWorld\\b",
"gnome-terminal-server\\.Gnome-terminal\\b.*~/Documents/Projects/OSWorld\\b"
]
}
},
{
"type": "rule",
"rules": {
"type": "url",
"urls": ["https://github.com", "https://docs.python.org/3/"]
}
}
]
}
}

57
evaluation_examples/examples/multi_apps/788b3701-3ec9-4b67-b679-418bfa726c22.json Normal file
View File

@@ -0,0 +1,57 @@
{
"id": "788b3701-3ec9-4b67-b679-418bfa726c22",
"snapshot": "chrome",
"instruction": "I'm following a small tale set updating on https://github.com/4th-year-in-tsinghua/4th-year-in-tsinghua. I have already downloaded several chapters to my computer for reading and archiving. There should be some updates and help me to download the next chapter I haven't downloaded yet. Save it to my novel collection folder.",
"source": "authors",
"config": [
{
"type": "execute",
"parameters": {
"command": ["mkdir", "-p", "/home/user/Documents/Finance/receipts", "/home/user/Documents/Projects/OSWorld", "/home/user/Documents/Novels/4th Year in Tsinghua"]
}
},
{
"type": "download",
"parameters": {
"files": [
{"path": "/home/user/Documents/Finance/receipts/aws-invoice-2308.pdf", "url": "https://drive.google.com/uc?id=1azRFXf4A7fvW0S7r9upHvleMEi-92hHM&export=download"},
{"path": "/home/user/Documents/Finance/receipts/aws-invoice-2309.pdf", "url": "https://drive.google.com/uc?id=1x-lpHm8U4U7uRPZ74-9wq9KzW2R55ln1&export=download"},
{"path": "/home/user/Documents/Finance/receipts/aws-invoice-2310.pdf", "url": "https://drive.google.com/uc?id=1pcrgV9G6NO4ekMEQBiupwXtq6mmke7b_&export=download"},
{"path": "/home/user/Documents/Finance/receipts/aws-invoice-2311.pdf", "url": "https://drive.google.com/uc?id=1JzbCK_nIY8X_3QZjnkzTtb-cRoq9zNT-&export=download"},
{"path": "/home/user/Documents/Finance/receipts/aws-invoice-2312.pdf", "url": "https://drive.google.com/uc?id=1RqbulzKG_HeYb1GZmLABOzlohlFg02UU&export=download"},
{"path": "/home/user/Documents/Finance/receipts/X-receipt-2312.pdf", "url": "https://drive.google.com/uc?id=1QzWjNzvNosG_yQr7VVonvYb3cUYF5f3u&export=download"},
{"path": "/home/user/Documents/Finance/tally_book.xlsx", "url": "https://drive.google.com/uc?id=13yuLhBPmouoWR-DybfgaIbWUOxbY_jhL&export=download"},
{"path": "/home/user/.projects.tar.xz", "url": "https://drive.google.com/uc?id=1oJcxpjqF474Wm16i1aZc8DlCEfAvc4t_&export=download"},
{"path": "/home/user/.old-chapters.tar.xz", "url": "https://drive.google.com/uc?id=11y-kLI83sQz6ncmP1-tkIR6mQJ7SfX5-&export=download"}
]
}
},
{
"type": "execute",
"parameters": {
"command": ["tar", "-xJvf", "/home/user/.projects.tar.xz", "-C", "/home/user/Documents/Projects/OSWorld"]
}
},
{
"type": "execute",
"parameters": {
"command": ["tar", "-xJvf", "/home/user/.old-chapters.tar.xz", "-C", "/home/user/Documents/Novels/4th Year in Tsinghua"]
}
}
],
"trajectory": "trajectories/788b3701-3ec9-4b67-b679-418bfa726c22",
"related_apps": ["chrome", "os"],
"evaluator": {
"func": "diff_text_file",
"result": {
"type": "vm_file",
"path": "/home/user/Documents/Novels/4th Year in Tsinghua/早期建筑群.tex",
"dest": "download.tex"
},
"expected": {
"type": "cloud_file",
"path": "https://drive.google.com/uc?id=1kiCDbTn7uYfgqbymddC5IKMsvpkihW-s&export=download",
"dest": "real.tex"
}
}
}