lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' of https://github.com/xlang-ai/DesktopEnv

Browse Source
This commit is contained in:
tsuky_chen
2024-03-04 11:41:49 +08:00
parent 88cc41cdf2 0eb37d26f3
commit 1aec945fae
11 changed files with 417 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
evaluation_examples/examples/multi_apps/0c825995-5b70-4526-b663-113f4c999dd2.json
View File

@@ -1,9 +1,48 @@
{
"id": "0c825995-5b70-4526-b663-113f4c999dd2",
"snapshot": "libreoffice_calc",
"instruction": "I'm working on a comprehensive report for our environmental policy review meeting next week. I need to integrate key insights from an important document, which is a guidebook on the Green Economy, where I'm particularly interested in the 'Introduction' section. Could you extract this section and compile them into a new Google Doc named 'environment_policy_report (draft)'? This will significantly aid in our discussion on aligning our environmental policies with sustainable and green economic practices. Thanks!",
"instruction": "I'm working on a comprehensive report for our environmental policy review meeting next week. I need to integrate key insights from an important document, which is a guidebook on the Green Economy, where I'm particularly interested in the 'Introduction' section. Could you extract this section and compile them into a new Google Doc named 'environment_policy_report (draft)' under /environment_policy folder? This will significantly aid in our discussion on aligning our environmental policies with sustainable and green economic practices. Thanks!",
"source": "authors",
"config": [
{
"type": "googledrive",
"parameters": {
"settings_file": "evaluation_examples/settings/googledrive/settings.yml",
"operation": ["delete"],
"args": [
{
"query": "title = 'environment_policy_report (draft).doc' or title = 'environment_policy_report (draft).docx' or title = 'environment_policy_report (draft)'",
"trash": false
}
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "login",
"parameters": {
"settings_file": "evaluation_examples/settings/google/settings.json",
"platform": "googledrive"
}
},
{
"type": "command",
"parameters": {
@@ -30,16 +69,26 @@
"os"
],
"evaluator": {
"postconfig": [],
"func": "",
"result": {
"func": "compare_docx_files",
"result": {
"type": "googledrive_file",
"settings_file": "evaluation_examples/settings/googledrive/settings.yml",
"path_list": [
[
"environment_policy_report (draft).docx"
]
],
"dest": [
"environment_policy_report (draft).docx"
]
},
"expected": {
"expected": {
"type": "cloud_file",
"path": "https://drive.google.com/uc?id=1A2ti9JncAfIa6ks7FTJWHtYlZo-68FtM&export=download",
"dest": "environment_policy_report (draft)_gold.docx"
},
"options": {
"options": {
"content_only": true
}
}
}

35
evaluation_examples/examples/multi_apps/26150609-0da3-4a7d-8868-0faf9c5f01bb.json
View File

@@ -1,6 +1,6 @@
{
"id": "26150609-0da3-4a7d-8868-0faf9c5f01bb",
"snapshot": "libreoffice_calc",
"snapshot": "vs_code",
"instruction": "So, I've been dabbling with coding a Snake game in Python, and I finally got it up and running. It's pretty cool, but it's not without its quirks. The biggest issue I'm facing right now is that the snake can't seem to eat the food, no matter what. Could you help me tweak the code so the snake can actually eat the food? Thanks a bunch!",
"source": "authors",
"config": [
@@ -14,6 +14,16 @@
]
}
},
{
"type": "command",
"parameters": {
"command": [
"pip",
"install",
"pygame"
]
}
},
{
"type": "download",
"parameters": {
@@ -49,15 +59,32 @@
],
"trajectory": "trajectories/26150609-0da3-4a7d-8868-0faf9c5f01bb",
"related_apps": [
"vs_code",
"os"
],
"evaluator": {
"postconfig": [],
"func": "",
"func": "check_python_file_by_test_suite",
"result": {
"type": "vm_file",
"path": [
"/home/user/Desktop/snake/food.py",
"/home/user/Desktop/snake/main.py",
"/home/user/Desktop/snake/settings.py",
"/home/user/Desktop/snake/snake.py"
],
"dest": [
"food.py",
"main.py",
"settings.py",
"snake.py"
],
"multi": true
},
"expected": {
},
"options": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1rLhp5cAS93uwCdZlikQse656Tf1Ut0Xc&export=download&authuser=0&confirm=t&uuid=6bb19b05-6832-4093-9441-634b9d0ba476&at=APZUnTVBJ1h-GgRmXMW_iagvZV1A:1709447052264",
"dest": "test_suite.py"
}
}
}

186
evaluation_examples/examples/multi_apps/881deb30-9549-4583-a841-8270c65f2a17.json Normal file
View File

@@ -0,0 +1,186 @@
{
"id": "881deb30-9549-4583-a841-8270c65f2a17",
"snapshot": "libreoffice_calc",
"instruction": "I want to find a faculty job in Hong Kong, so I am more curious about the \"Early Career Scheme\" of those schools is better to apply, please help me to count all the documents in the ecs pdf files in my hand, and organize the pass rate of each school by year into table!",
"source": "authors",
"config": [
{
"type": "command",
"parameters": {
"command": [
"mkdir",
"-p",
"/home/user/Documents/Fundings/ecs",
"/home/user/Documents/Fundings/grf"
]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"path": "/home/user/Documents/Fundings/ecs/ecs15.pdf",
"url": "https://drive.google.com/uc?id=1FTiT3mLlkehe2yWVdSMWr1w4ltLtQZUy&export=download"
},
{
"path": "/home/user/Documents/Fundings/ecs/ecs16.pdf",
"url": "https://drive.google.com/uc?id=1DMzZyhDey3lDsQ7fcPiJm9AYGECZms3q&export=download"
},
{
"path": "/home/user/Documents/Fundings/ecs/ecs17.pdf",
"url": "https://drive.google.com/uc?id=1TgAWk7FiV8fNrG2L3-Eu7BDccqtXebXY&export=download"
},
{
"path": "/home/user/Documents/Fundings/ecs/ecs23.pdf",
"url": "https://drive.google.com/uc?id=11DVxH4eRjECUxZNbUhYNJhgD0Y5WoN8r&export=download"
},
{
"path": "/home/user/Documents/Fundings/ecs/ecs22.pdf",
"url": "https://drive.google.com/uc?id=17IgyJADA65F40kH79S90QgEzPa7IERXx&export=download"
},
{
"path": "/home/user/Documents/Fundings/ecs/ecs21.pdf",
"url": "https://drive.google.com/uc?id=1kB4eFfLisPXKOirGUHbbcOyf73t7MVqL&export=download"
},
{
"path": "/home/user/Documents/Fundings/ecs/ecs20.pdf",
"url": "https://drive.google.com/uc?id=179j9tD1xRSgd9COM7rzErO6FLYO2sc_a&export=download"
},
{
"path": "/home/user/Documents/Fundings/ecs/ecs19.pdf",
"url": "https://drive.google.com/uc?id=1ogZUTfKjvQhg58GXtVUxe1U8VvHU-3ap&export=download"
},
{
"path": "/home/user/Documents/Fundings/ecs/ecs18.pdf",
"url": "https://drive.google.com/uc?id=1MfuvRhAnhMEMbxn5js2ffWqMHJx2fsd2&export=download"
},
{
"path": "/home/user/Documents/Fundings/grf/customer-information-sheet-for-inward-payments-to-hong-kong.pdf",
"url": "https://drive.google.com/uc?id=1s-H3an7HLBM9ku6d6Hcdj1qkSwKAHngU&export=download"
},
{
"path": "/home/user/Documents/Fundings/grf/grf15.pdf",
"url": "https://drive.google.com/uc?id=1rRQXo9XHnCVTG8XqNAv0SJwPTW36MMbm&export=download"
},
{
"path": "/home/user/Documents/Fundings/grf/grf16.pdf",
"url": "https://drive.google.com/uc?id=18ljRqkdyXEZ464E0dpKjaEa2NFexyw3I&export=download"
},
{
"path": "/home/user/Documents/Fundings/grf/grf17.pdf",
"url": "https://drive.google.com/uc?id=1VrqOnyhpOkMpyIJ6YMrAhixpahjYonOd&export=download"
},
{
"path": "/home/user/Documents/Fundings/grf/grf18.pdf",
"url": "https://drive.google.com/uc?id=182CLDUr372-jpAiY4YvSbGNXF9TsWxzA&export=download"
},
{
"path": "/home/user/Documents/Fundings/grf/grf19.pdf",
"url": "https://drive.google.com/uc?id=1YkJtjlklKN0NmLiI2Hi4f_dKtTm5SPxT&export=download"
},
{
"path": "/home/user/Documents/Fundings/grf/grf20.pdf",
"url": "https://drive.google.com/uc?id=1a7Uc7VCMlEX6fy-5oqE6i1YLitBe7gaf&export=download"
},
{
"path": "/home/user/Documents/Fundings/grf/grf21.pdf",
"url": "https://drive.google.com/uc?id=1s8km4Wle4lc5PkbUQfivBFK0IJQgxMiB&export=download"
},
{
"path": "/home/user/Documents/Fundings/grf/grf22.pdf",
"url": "https://drive.google.com/uc?id=1HTEb1uK7LNvbVyeXgO8WemCPPgiASKiy&export=download"
},
{
"path": "/home/user/Documents/Fundings/grf/grf23.pdf",
"url": "https://drive.google.com/uc?id=1XM-jZlfu_i4waDZHb8Z6Vr5b3LgULTtP&export=download"
},
{
"path": "/home/user/Documents/Fundings/supported_rate.xlsx",
"url": "https://drive.google.com/uc?id=1jkC80XAlm4V_zZF4m75MGDqnVi9Pq-Ft&export=download"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/Documents/Fundings/supported_rate.xlsx"
}
},
{
"type": "sleep",
"parameters": {
"seconds": 5
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/Documents/Fundings/grf"
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/Documents/Fundings/ecs"
}
}
],
"trajectory": "trajectories/881deb30-9549-4583-a841-8270c65f2a17",
"related_apps": [
"libreoffice_calc",
"os"
],
"evaluator": {
"postconfig": [
{
"type": "execute",
"parameters": {
"command": [
"libreoffice",
"--convert-to",
"csv:Text - txt - csv (StarCalc):44,34,UTF-8,,,,false,true,true,false,false,1",
"--outdir",
"/home/user/Documents/Fundings",
"/home/user/Documents/Fundings/supported_rate.xlsx"
]
}
}
],
"func": "compare_table",
"result": {
"type": "vm_file",
"path": [
"/home/user/Documents/Fundings/supported_rate.xlsx",
"/home/user/Documents/Fundings/supported_rate-Sheet1.csv"
],
"dest": [
"supported_rate.xlsx",
"supported_rate-Sheet1.csv"
],
"multi": true
},
"expected": {
"type": "cloud_file",
"path": [
"https://drive.google.com/uc?id=11mBEe1TnJhtHD5QolFuQhPvRCOSpm8ko&export=download",
"https://drive.google.com/uc?id=1len3Jz4cAztVC0t9dUyECCGecRrCsIHj&export=download"
],
"dest": [
"supported_rate-gt.xlsx",
"supported_rate-gt-Sheet1.csv"
],
"multi": true
},
"options": {
"rules": [
{
"type": "sheet_print",
"sheet_idx0": "RNSheet1",
"sheet_idx1": "ENSheet1"
}
]
}
}
}

55
evaluation_examples/examples/multi_apps/9219480b-3aed-47fc-8bac-d2cffc5849f7.json
View File

@@ -1,7 +1,7 @@
{
"id": "9219480b-3aed-47fc-8bac-d2cffc5849f7",
"snapshot": "libreoffice_calc",
"instruction": "Hi, I recently playing with developing a small python-based teris game. While I have finished most of the part, something is wrong under some cases when I press up to rotate, the whole program will crash, please run the code for me and fix the bugs of code.",
"snapshot": "vs_code",
"instruction": "Hi, I recently playing with developing a small python-based tetris game. While I have finished most of the part, something is wrong under some cases when I press up to rotate, the whole program will crash, please run the code for me and fix the bugs of code.",
"source": "authors",
"config": [
{
@@ -10,7 +10,7 @@
"command": [
"mkdir",
"-p",
"/home/user/Desktop/teris"
"/home/user/Desktop/tetris"
]
}
},
@@ -19,19 +19,19 @@
"parameters": {
"files": [
{
"path": "/home/user/Desktop/teris/block.py",
"path": "/home/user/Desktop/tetris/block.py",
"url": "https://drive.usercontent.google.com/download?id=1txPwaWML0e8cjpDW-nw5N6HOC6fYwfI1&export=download&authuser=0&confirm=t&uuid=0b69795a-3600-4ec4-be9b-533deeb76e94&at=APZUnTVK-VGsZfTufLSh_3eRigYb:1709303077428"
},
{
"path": "/home/user/Desktop/teris/main.py",
"path": "/home/user/Desktop/tetris/main.py",
"url": "https://drive.usercontent.google.com/download?id=1vJ2FEw7RNfRr0KBynHM36_nNAc4jTArl&export=download&authuser=0&confirm=t&uuid=11f697bc-1414-46a0-bc2d-b2c49686d85e&at=APZUnTXCtKDpqTAOstIioRrJzx67:1709303181059"
},
{
"path": "/home/user/Desktop/teris/settings.py",
"path": "/home/user/Desktop/tetris/settings.py",
"url": "https://drive.usercontent.google.com/download?id=1tYI8QZz-T-sNFRAMaYnzFv80upIveUT-&export=download&authuser=0&confirm=t&uuid=50c64eed-ab7e-4dbe-82ce-d01b25364556&at=APZUnTXUok8Cos6rUo-fnKLwnPmn:1709303184763"
},
{
"path": "/home/user/Desktop/teris/teris.py",
"path": "/home/user/Desktop/tetris/tetris.py",
"url": "https://drive.usercontent.google.com/download?id=1UOJp5Y6JLmaXmZXHc_GkM3nMfKbS6Dyu&export=download&authuser=0&confirm=t&uuid=3189e5d6-5984-45c7-9dd8-6f8ec3a0845e&at=APZUnTWYTtsu27Ds3aHrZpIAnNcA:1709303187967"
}
]
@@ -42,26 +42,39 @@
"parameters": {
"command": [
"code",
"/home/user/Desktop/teris"
"/home/user/Desktop/tetris"
]
}
}
],
"trajectory": "trajectories/9219480b-3aed-47fc-8bac-d2cffc5849f7",
"related_apps": [
"vs_code",
"os"
],
"evaluator": {
"postconfig": [],
"func": "",
"result": {
},
"expected": {
},
"options": {
}
}
"postconfig": [],
"func": "check_python_file_by_test_suite",
"result": {
"type": "vm_file",
"path": [
"/home/user/Desktop/tetris/block.py",
"/home/user/Desktop/tetris/main.py",
"/home/user/Desktop/tetris/settings.py",
"/home/user/Desktop/tetris/tetris.py"
],
"dest": [
"block.py",
"main.py",
"settings.py",
"tetris.py"
],
"multi": true
},
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1eDEWLCswia1ihoh5kx5GGGDmFLt6GH_A&export=download&authuser=0&confirm=t&uuid=30330a2f-ae74-4299-bf22-0d8a6f814e1f&at=APZUnTUbxsypC6Blx3nJPFfCiaeV:1709449044563",
"dest": "test_suite.py"
}
}
}

47
evaluation_examples/examples/multi_apps/d1acdb87-bb67-4f30-84aa-990e56a09c92.json
View File

@@ -1,7 +1,7 @@
{
"id": "d1acdb87-bb67-4f30-84aa-990e56a09c92",
"snapshot": "libreoffice_calc",
"instruction": "Can you help me find these restaurants on Google Maps? I have a list of places I've been wanting to visit, and I need to gather some detailed information about them, including the address, what people have to say about them (ratings), and the hours of operation. It would be great if you could search for each of them on Google Maps and help me jot down this information into my sheet file, thanks!",
"instruction": "Hello! I'm eagerly planning a culinary adventure to Hong Kong and have curated a list of must-visit restaurants that I've been longing to explore. However, I could use some assistance in compiling a few essential details about these establishments. Would you be so kind as to help me out? It would be fantastic if you could search for these restaurants on Google Maps. I'm particularly interested in obtaining their addresses, any available websites, and contact phone numbers. If you could gather this information and input it into my form file, I would be immensely grateful. Many thanks in advance!",
"source": "authors",
"config": [
{
@@ -52,13 +52,54 @@
"libreoffice_calc"
],
"evaluator": {
"postconfig": [],
"func": "",
"postconfig": [
{
"type": "execute",
"parameters": {
"command": [
"libreoffice",
"--convert-to",
"csv:Text - txt - csv (StarCalc):44,34,UTF-8,,,,false,true,true,false,false,1",
"--outdir",
"/home/user/Desktop",
"/home/user/Desktop/MUST_VISIT.xlsx"
]
}
}
],
"func": "compare_table",
"result": {
"type": "vm_file",
"path": [
"/home/user/Desktop/MUST_VISIT.xlsx",
"/home/user/Desktop/MUST_VISIT-Sheet1.csv"
],
"dest": [
"MUST_VISIT.xlsx",
"MUST_VISIT-Sheet1.csv"
],
"multi": true
},
"expected": {
"type": "cloud_file",
"path": [
"https://drive.google.com/uc?id=1MV6jBvRbbYwPqeFTd_nX40xzyltNhphl&export=download",
"https://drive.google.com/uc?id=1CGoRQDLw9-Ai7daq3qCz0o9kYSZB2WNn&export=download"
],
"dest": [
"MUST_VISIT-gt.xlsx",
"MUST_VISIT-gt-Sheet1.csv"
],
"multi": true
},
"options": {
"rules": [
{
"type": "sheet_print",
"sheet_idx0": "RNSheet1",
"sheet_idx1": "ENSheet1"
}
]
}
}
}

2
evaluation_examples/settings/googledrive/client_secrets.json
View File

@@ -1 +1 @@
{"installed":{"client_id":"786888752612-rgng5v9hcq4as7pn0b40gt9r5lekmht9.apps.googleusercontent.com","project_id":"xlang-2024-benchmarking","auth_uri":"https://accounts.google.com/o/oauth2/auth","token_uri":"https://oauth2.googleapis.com/token","auth_provider_x509_cert_url":"https://www.googleapis.com/oauth2/v1/certs","client_secret":"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","redirect_uris":["http://localhost"]}}
{"installed":{"client_id":"786888752612-rgng5v9hcq4as7pn0b40gt9r5lekmht9.apps.googleusercontent.com","project_id":"xlang-2024-benchmarking","auth_uri":"https://accounts.google.com/o/oauth2/auth","token_uri":"https://oauth2.googleapis.com/token","auth_provider_x509_cert_url":"https://www.googleapis.com/oauth2/v1/certs","client_secret":"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","redirect_uris":["http://localhost"]}}

2
evaluation_examples/settings/googledrive/credentials.json
View File

@@ -1 +1 @@
{"access_token": "ya29.a0AfB_byD2_A8RPm6KzfjaNifjfgZ2M-D9G16GAPIj1ANxM3AMq4DLUXxj76CalsByOsqTUgEvADd-FEKL0FkBBc4ow-EuaLUEOm4yw2LNEOFMhVD_k4PvEIf4767fYU5o__GtyrGt5pNJy0MaBukDY2ui7GQwDuFFGt2q_AaCgYKAYMSARISFQHGX2MidvFUuBpNsy4fkC5DP2k8Aw0173", "client_id": "786888752612-rgng5v9hcq4as7pn0b40gt9r5lekmht9.apps.googleusercontent.com", "client_secret": "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", "refresh_token": "1//0ehtafHmucszRCgYIARAAGA4SNwF-L9IrpDBsnzdHKAlRfrkvzNFw1cpdnRY8rhM5gy4flsPYdysMav27yHamJx39BBGq-LLw40s", "token_expiry": "2024-02-01T08:29:08Z", "token_uri": "https://oauth2.googleapis.com/token", "user_agent": null, "revoke_uri": "https://oauth2.googleapis.com/revoke", "id_token": null, "id_token_jwt": null, "token_response": {"access_token": "ya29.a0AfB_byD2_A8RPm6KzfjaNifjfgZ2M-D9G16GAPIj1ANxM3AMq4DLUXxj76CalsByOsqTUgEvADd-FEKL0FkBBc4ow-EuaLUEOm4yw2LNEOFMhVD_k4PvEIf4767fYU5o__GtyrGt5pNJy0MaBukDY2ui7GQwDuFFGt2q_AaCgYKAYMSARISFQHGX2MidvFUuBpNsy4fkC5DP2k8Aw0173", "expires_in": 3599, "scope": "https://www.googleapis.com/auth/drive", "token_type": "Bearer"}, "scopes": ["https://www.googleapis.com/auth/drive"], "token_info_uri": "https://oauth2.googleapis.com/tokeninfo", "invalid": false, "_class": "OAuth2Credentials", "_module": "oauth2client.client"}
{"access_token": "ya29.a0AfB_byCmR_-BUvipM02LYvRdhSIsjxhdK4a1tpDABrZOjTOWPkPhs4gy070hbrq0tZBeld813_HqZ4q5GTeLzddfVVTWfiHdDSLlT4Bf5_f4ZURSB53XJAdAyYeI7LcT4RwF1WoAQhtldbIBJ4rizVk7L6O3486u6e9OaCgYKARQSARISFQHGX2Mi1U-dePZ0efbg8nydUEsP9Q0171", "client_id": "786888752612-rgng5v9hcq4as7pn0b40gt9r5lekmht9.apps.googleusercontent.com", "client_secret": "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", "refresh_token": "1//0edHIvL2N4T8ICgYIARAAGA4SNwF-L9IrKhoX-pALW0nJ18niS1Gy3Lg9wF_G1joCoGHLM8v4-WJsibIB04KjWYCp_40Cs1WS7es", "token_expiry": "2024-03-02T13:59:28Z", "token_uri": "https://oauth2.googleapis.com/token", "user_agent": null, "revoke_uri": "https://oauth2.googleapis.com/revoke", "id_token": null, "id_token_jwt": null, "token_response": {"access_token": "ya29.a0AfB_byCmR_-BUvipM02LYvRdhSIsjxhdK4a1tpDABrZOjTOWPkPhs4gy070hbrq0tZBeld813_HqZ4q5GTeLzddfVVTWfiHdDSLlT4Bf5_f4ZURSB53XJAdAyYeI7LcT4RwF1WoAQhtldbIBJ4rizVk7L6O3486u6e9OaCgYKARQSARISFQHGX2Mi1U-dePZ0efbg8nydUEsP9Q0171", "expires_in": 3599, "refresh_token": "1//0edHIvL2N4T8ICgYIARAAGA4SNwF-L9IrKhoX-pALW0nJ18niS1Gy3Lg9wF_G1joCoGHLM8v4-WJsibIB04KjWYCp_40Cs1WS7es", "scope": "https://www.googleapis.com/auth/drive", "token_type": "Bearer"}, "scopes": ["https://www.googleapis.com/auth/drive"], "token_info_uri": "https://oauth2.googleapis.com/tokeninfo", "invalid": false, "_class": "OAuth2Credentials", "_module": "oauth2client.client"}