lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main'

Browse Source 
# Conflicts:
#	mm_agents/gpt_4v_agent.py
This commit is contained in:
Timothyxxx
2024-02-01 16:57:01 +08:00
parent 59e2417a08 193557bea4
commit c31c9f4e7d
16 changed files with 32 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
evaluation_examples/examples/libreoffice_calc/0cecd4f3-74de-457b-ba94-29ad6b5dafb6.json
View File

@@ -1,7 +1,7 @@
{
"id": "0cecd4f3-74de-457b-ba94-29ad6b5dafb6",
"snapshot": "libreoffice_calc",
"instruction": "Rename \"Sheet 1\" to \"LARS Resources\". Then make a copy of it. Place the copy before \"Sheet 2\" and rename it by appending a suffix \"(Backup)\", concatenated by a which space. And Also rename \"Sheet2\" by appending the suffix \"(Offline)\".",
"instruction": "Rename \"Sheet 1\" to \"LARS Resources\". Then make a copy of it. Place the copy before \"Sheet 2\" and rename it by appending a suffix \"(Backup)\", concatenated by a white space. And Also rename \"Sheet2\" by appending the suffix \"(Offline)\".",
"source": "https://www.libreofficehelp.com/add-insert-delete-copy-move-rename-a-worksheet-in-libreoffice-calc/",
"config": [
{

2
evaluation_examples/examples/libreoffice_calc/7a4e4bc8-922c-4c84-865c-25ba34136be1.json
View File

@@ -1,7 +1,7 @@
{
"id": "7a4e4bc8-922c-4c84-865c-25ba34136be1",
"snapshot": "libreoffice_calc",
"instruction": "Reorder the columns to be \"Data\", \"First Name\", \"Last Name\", \"Order ID\", \"Sales\"",
"instruction": "Reorder the columns to be \"Date\", \"First Name\", \"Last Name\", \"Order ID\", \"Sales\"",
"source": "https://www.youtube.com/shorts/bvUhr1AHs44",
"config": [
{

2
evaluation_examples/examples/libreoffice_impress/21760ecb-8f62-40d2-8d85-0cee5725cb72.json
View File

@@ -1,7 +1,7 @@
{
"id": "21760ecb-8f62-40d2-8d85-0cee5725cb72",
"snapshot": "libreoffice_impress",
"instruction": "Could you help me add silde transition \"dissolve\" to my first page?",
"instruction": "Could you help me add slide transition \"dissolve\" to my first page?",
"source": "https://www.libreofficehelp.com/add-animations-transitions-libreoffice-impress-slides/",
"config": [
{

2
evaluation_examples/examples/libreoffice_impress/5d901039-a89c-4bfb-967b-bf66f4df075e.json
View File

@@ -1,7 +1,7 @@
{
"id": "5d901039-a89c-4bfb-967b-bf66f4df075e",
"snapshot": "libreoffice_impress",
"instruction": "I want to make this page my cover page. Could you help me stretch this image to fill the entire page, keeping its proportion and centering the image.",
"instruction": "I want to turn the rectangular image of Columbus on the first page into a cover page. Could you help me stretch this image to fill the entire page, keeping its proportion and centering the image?",
"source": "https://superuser.com/questions/986776/how-can-i-stretch-an-image-in-a-libreoffice-impress-presentation-to-fill-the-pag",
"config": [
{

2
evaluation_examples/examples/libreoffice_impress/9ec204e4-f0a3-42f8-8458-b772a6797cab.json
View File

@@ -62,7 +62,7 @@
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1otbzscpOZ0tCXMvsMC0MmNWUC7Pv71of&export=download&authuser=0&confirm=t&uuid=faa0b0c1-6b14-4bce-a1fd-ccf824ee1e60&at=APZUnTXw6TlBOlrPPZ2OhfGnNPf0:1705338135842",
"dest": "MLA_Workshop_061X_Works_Cited_Gold.docx"
"dest": "MLA_Workshop_061X_Works_Cited_Gold.pptx"
},
"result": {
"type": "vm_file",

4
evaluation_examples/examples/libreoffice_impress/af23762e-2bfd-4a1d-aada-20fa8de9ce07.json
View File

@@ -2,7 +2,7 @@
"id": "af23762e-2bfd-4a1d-aada-20fa8de9ce07",
"snapshot": "libreoffice_impress",
"instruction": "I am making PPT on LibreOffice Impress for presentation tomorrow. I need to summarize contents on one slide use Impress \"Summary Slide\" feature. Could you make that for me?",
"source": "https://www.libreofficehelp.com/export-libreoffice-impress-slides-images/#:~:text=Exporting%20a%20single%20slide%20as.jpg%2C.png%2C%20etc%20image%20is,on%20the%20checkbox%20Selection.%20Provide%20jpg%20quality%20options.",
"source": "https://superuser.com/questions/1059080/how-to-make-a-summary-slide-in-impress-listing-the-titles-of-all-slides-autom",
"config": [
{
"type": "download",
@@ -62,7 +62,7 @@
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1nRwmFgYdskv3EiriZZFoT8TzM9CsG5B0&export=download&authuser=0&confirm=t&uuid=f2f919df-2867-4bc3-8bb9-dabd51108ebb&at=APZUnTWzw9LJWWXvH0cvdaWL-Ij-:1705319339474",
"dest": "Forests_Gold.docx"
"dest": "Forests_Gold.pptx"
},
"result": {
"type": "vm_file",

2
evaluation_examples/examples/multi_apps/46407397-a7d5-4c6b-92c6-dbe038b1457b.json
View File

@@ -68,7 +68,7 @@
"parameters": {
"command": [
"tar",
"-xzv",
"-xz",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",

2
evaluation_examples/examples/multi_apps/897e3b53-5d4d-444b-85cb-2cdc8a97d903.json
View File

@@ -1,7 +1,7 @@
{
"id": "897e3b53-5d4d-444b-85cb-2cdc8a97d903",
"snapshot": "chrome",
"instruction": "I have a LibreOffice Writer file form.docx on the desktop. Help me convert it to PDF format and store it in the forms/ folder in my Google Drive.",
"instruction": "I have a LibreOffice Writer file form.docx on the desktop. Help me convert it to PDF format and store the PDF in the forms/ folder in my Google Drive.",
"source": "https://marketplace.uipath.com/listings/convert-word-file-to-pdf-and-store-in-onedrive",
"config": [
{

14
evaluation_examples/examples/multi_apps/b52b40a5-ad70-4c53-b5b0-5650a8387052.json
View File

@@ -68,13 +68,13 @@
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
"tar",
"-xz",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},

2
evaluation_examples/examples/os/7688b85f-87a4-4e4a-b2f8-f3d6c3f29b82.json
View File

@@ -1,7 +1,7 @@
{
"id": "7688b85f-87a4-4e4a-b2f8-f3d6c3f29b82",
"snapshot": "os",
"instruction": "Can you move the file with the path 'todo.txt' on the Desktop to the directory with the path 'done' on the Desktop?",
"instruction": "Can you move the file 'todo.txt' on the Desktop to the directory 'done/' on the Desktop?",
"source": "https://ubuntu.com/tutorials/command-line-for-beginners#5-moving-and-manipulating-files",
"config": [
{

2
evaluation_examples/examples/os/bedcedc4-4d72-425e-ad62-21960b11fe0d.json
View File

@@ -1,7 +1,7 @@
{
"id": "bedcedc4-4d72-425e-ad62-21960b11fe0d",
"snapshot": "os",
"instruction": "Could you set the 'Dim screen when inactive' to on in setting?",
"instruction": "Could you set the 'Dim screen when inactive' to off in setting?",
"source": "https://www.youtube.com/watch?v=D4WyNjt_hbQ&t=2s",
"trajectory": "trajectories/",
"config": [

2
evaluation_examples/examples/thunderbird/030eeff7-b492-4218-b312-701ec99ee0cc.json
View File

@@ -68,7 +68,7 @@
"expected": {
"type": "rule",
"rules": {
"unexpect": {
"expect": {
"mail.identity.id1.auto_quote": {
"method": "eq",
"ref": false

2
evaluation_examples/examples/thunderbird/06fe7178-4491-4589-810f-2e2bc9502122.json
View File

@@ -1,7 +1,7 @@
{
"id": "06fe7178-4491-4589-810f-2e2bc9502122",
"snapshot": "thunderbird",
"instruction": "Could you help me back up all the email files in my profile to ~/emails.bak? Please save them separately in eml format.",
"instruction": "Could you help me back up all the email files in my inbox to ~/emails.bak? Please save them separately in eml format.",
"source": "https://www.quora.com/How-do-I-backup-email-files-in-Mozilla-Thunderbird",
"config": [
{

2
evaluation_examples/examples/thunderbird/2ad9387a-65d8-4e33-ad5b-7580065a27ca.json
View File

@@ -1,7 +1,7 @@
{
"id": "2ad9387a-65d8-4e33-ad5b-7580065a27ca",
"snapshot": "thunderbird",
"instruction": "Create two local folders for me: COMPANY and UNIVERSITY.",
"instruction": "Create two local folders in Thunderbird for me: COMPANY and UNIVERSITY.",
"source": "https://support.mozilla.org/bm/questions/1027435",
"config": [
{

2
evaluation_examples/settings/googledrive/credentials.json
View File

@@ -1 +1 @@
{"access_token": "ya29.a0AfB_byAZmDTDsYds_iatV8a30PUPWcDHVW4Cyg71pTlD0f3eBBwAjV4WpVL8LdAle8sT4j_rX4rWH8iCt3QI2YdrQLFPlaVdBk0zRGGtAEcebIDuQy_VKD6j5c3IGxok9PDON-Mft0ZVJjUVEopgLYA4fYwctbQZ8nyl4AaCgYKAX4SARISFQHGX2Mim-LRNXCfACmecJH94-D09A0173", "client_id": "786888752612-rgng5v9hcq4as7pn0b40gt9r5lekmht9.apps.googleusercontent.com", "client_secret": "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", "refresh_token": "1//0ehtafHmucszRCgYIARAAGA4SNwF-L9IrpDBsnzdHKAlRfrkvzNFw1cpdnRY8rhM5gy4flsPYdysMav27yHamJx39BBGq-LLw40s", "token_expiry": "2024-01-31T14:41:25Z", "token_uri": "https://oauth2.googleapis.com/token", "user_agent": null, "revoke_uri": "https://oauth2.googleapis.com/revoke", "id_token": null, "id_token_jwt": null, "token_response": {"access_token": "ya29.a0AfB_byAZmDTDsYds_iatV8a30PUPWcDHVW4Cyg71pTlD0f3eBBwAjV4WpVL8LdAle8sT4j_rX4rWH8iCt3QI2YdrQLFPlaVdBk0zRGGtAEcebIDuQy_VKD6j5c3IGxok9PDON-Mft0ZVJjUVEopgLYA4fYwctbQZ8nyl4AaCgYKAX4SARISFQHGX2Mim-LRNXCfACmecJH94-D09A0173", "expires_in": 3599, "scope": "https://www.googleapis.com/auth/drive", "token_type": "Bearer"}, "scopes": ["https://www.googleapis.com/auth/drive"], "token_info_uri": "https://oauth2.googleapis.com/tokeninfo", "invalid": false, "_class": "OAuth2Credentials", "_module": "oauth2client.client"}
{"access_token": "ya29.a0AfB_byD2_A8RPm6KzfjaNifjfgZ2M-D9G16GAPIj1ANxM3AMq4DLUXxj76CalsByOsqTUgEvADd-FEKL0FkBBc4ow-EuaLUEOm4yw2LNEOFMhVD_k4PvEIf4767fYU5o__GtyrGt5pNJy0MaBukDY2ui7GQwDuFFGt2q_AaCgYKAYMSARISFQHGX2MidvFUuBpNsy4fkC5DP2k8Aw0173", "client_id": "786888752612-rgng5v9hcq4as7pn0b40gt9r5lekmht9.apps.googleusercontent.com", "client_secret": "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", "refresh_token": "1//0ehtafHmucszRCgYIARAAGA4SNwF-L9IrpDBsnzdHKAlRfrkvzNFw1cpdnRY8rhM5gy4flsPYdysMav27yHamJx39BBGq-LLw40s", "token_expiry": "2024-02-01T08:29:08Z", "token_uri": "https://oauth2.googleapis.com/token", "user_agent": null, "revoke_uri": "https://oauth2.googleapis.com/revoke", "id_token": null, "id_token_jwt": null, "token_response": {"access_token": "ya29.a0AfB_byD2_A8RPm6KzfjaNifjfgZ2M-D9G16GAPIj1ANxM3AMq4DLUXxj76CalsByOsqTUgEvADd-FEKL0FkBBc4ow-EuaLUEOm4yw2LNEOFMhVD_k4PvEIf4767fYU5o__GtyrGt5pNJy0MaBukDY2ui7GQwDuFFGt2q_AaCgYKAYMSARISFQHGX2MidvFUuBpNsy4fkC5DP2k8Aw0173", "expires_in": 3599, "scope": "https://www.googleapis.com/auth/drive", "token_type": "Bearer"}, "scopes": ["https://www.googleapis.com/auth/drive"], "token_info_uri": "https://oauth2.googleapis.com/tokeninfo", "invalid": false, "_class": "OAuth2Credentials", "_module": "oauth2client.client"}

18
mm_agents/gpt_4v_agent.py
View File

@@ -30,6 +30,8 @@ from mm_agents.prompts import SYS_PROMPT_IN_SCREENSHOT_OUT_CODE, SYS_PROMPT_IN_S
SYS_PROMPT_IN_SOM_A11Y_OUT_TAG, \
SYS_PROMPT_SEEACT, ACTION_DESCRIPTION_PROMPT_SEEACT, ACTION_GROUNDING_PROMPT_SEEACT
import logging
logger = logging.getLogger("desktopenv.agent")
@@ -165,9 +167,6 @@ def parse_code_from_som_string(input_string, masks):
logger.error("Predicting the tag with index {} failed.".format(match))
return ""
return text
input_string = replace_tags_with_mappings(input_string, mappings)
actions = parse_code_from_string(input_string)
return actions
@@ -479,11 +478,14 @@ class GPT4v_Agent:
with open("messages.json", "w") as f:
f.write(json.dumps(messages, indent=4))
response = self.call_llm({
"model": self.model,
"messages": messages,
"max_tokens": self.max_tokens
})
try:
response = self.call_llm({
"model": self.model,
"messages": messages,
"max_tokens": self.max_tokens
})
except:
response = ""
logger.debug("RESPONSE: %s", response)