lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ver Jan4th

Browse Source 
updated interfaces for thunderbird evaluation, not tested
This commit is contained in:
David Chang
2024-01-04 22:41:57 +08:00
parent f831aa93df
commit 5fedf5b891
10 changed files with 361 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

91
evaluation_examples/examples/thunderbird/7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3.json
View File

@@ -3,10 +3,97 @@
"snapshot": "thunderbird",
"instruction": "Help me access my gmail account with address \"xx@gmail.com\" and password \"xxx\"",
"source": "https://www.wikihow.com/Access-Gmail-With-Mozilla-Thunderbird",
"config": [],
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/david/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/david/thunderbird-profile.tar.gz",
"-C",
"$HOME/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"thunderbird"
],
"evaluator": "evaluation_dir"
"evaluator": {
"postconfig": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://raw.githubusercontent.com/unode/firefox_decrypt/main/firefox_decrypt.py",
"path": "/home/david/firefox_decrypt.py"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"python3",
"/home/david/firefox_decrypt.py",
"$HOME/.thunderbird",
"-n",
"-c",
"2",
"-f",
"csv",
"-d",
","
],
"stdout": "thunderbird-accounts.csv"
}
}
],
"func": "check_csv",
"result": {
"type": "cache_file",
"path": "thunderbird-accounts.csv"
},
"expected": {
"rules": {
"expect": [
{
"url": "imap://imap.gmail.com",
"user": "xx@gmail.com",
"password": "xxx"
},
{
"url": "smtp://smtp.gmail.com",
"user": "xx@gmail.com",
"password": "xxx"
}
]
}
}
}
}

89
evaluation_examples/examples/thunderbird/bb5e4c0d-f964-439c-97b6-bdb9747de3f4.json
View File

@@ -1,12 +1,93 @@
{
"id": "bb5e4c0d-f964-439c-97b6-bdb9747de3f4",
"snapshot": "thunderbird",
"instruction": "Help remove the account \"xx@xx\"",
"instruction": "Help me to remove the account \"anonym-x2024@outlook.com\"",
"source": "https://www.wikihow.com/Remove-an-Email-Account-from-Thunderbird",
"config": [],
"trajectory": "trajectories/",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/david/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/david/thunderbird-profile.tar.gz",
"-C",
"$HOME/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
],
"trajectory": "trajectories/bb5e4c0d-f964-439c-97b6-bdb9747de3f4",
"related_apps": [
"thunderbird"
],
"evaluator": "evaluation_dir"
"evaluator": {
"postconfig": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://raw.githubusercontent.com/unode/firefox_decrypt/main/firefox_decrypt.py",
"path": "/home/david/firefox_decrypt.py"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"python3",
"/home/david/firefox_decrypt.py",
"$HOME/.thunderbird",
"-n",
"-c",
"2",
"-f",
"csv",
"-d",
","
],
"stdout": "thunderbird-accounts.csv"
}
}
],
"func": "check_csv",
"result": {
"type": "cache_file",
"path": "thunderbird-accounts.csv"
},
"expected": {
"rules": {
"unexpect": [
{
"url": "imap://outlook.office365.com",
"user": "anonym-x2024@outlook.com"
}
]
}
}
}
}