lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'zdy'

Browse Source
This commit is contained in:
David Chang
2024-01-31 23:52:41 +08:00
parent 369ead1024 4897211a46
commit c7fb35de50
27 changed files with 215 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
desktop_env/evaluators/metrics/table.py
View File

@@ -392,17 +392,13 @@ if __name__ == '__main__':
logger.addHandler(stdout_handler)
logger.addHandler(sdebug_handler)
path1 = "../../任务数据/LibreOffice Calc/Multiply_Time_Number.xlsx"
path2 = "../../任务数据/LibreOffice Calc/Multiply_Time_Number_gold.xlsx"
rules = [{"type": "check_cell"
, "sheet_idx": 0
, "coordinate": "E3"
, "props": {"value": {"method": "approx:0.001"
, "ref": 191.6667
}
}
path1 = "snapshots/test/cache/4e6fcf72-daf3-439f-a232-c434ce416af6/Employee_Age_By_Birthday.xlsx"
path2 = "snapshots/test/cache/4e6fcf72-daf3-439f-a232-c434ce416af6/Employee_Age_By_Birthday_gold.xlsx"
rules = [ { "type": "sheet_data"
, "sheet_idx0": 0
, "sheet_idx1": "EI0"
}
]
]
print(compare_table(path1, path2
, rules=rules
)

6
evaluation_examples/examples/libreoffice_calc/01b269ae-2111-4a07-81fd-3fcd711993b0-win.json
View File

@@ -50,6 +50,12 @@
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

6
evaluation_examples/examples/libreoffice_calc/01b269ae-2111-4a07-81fd-3fcd711993b0.json
View File

@@ -50,6 +50,12 @@
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

6
evaluation_examples/examples/libreoffice_calc/0cecd4f3-74de-457b-ba94-29ad6b5dafb6.json
View File

@@ -50,6 +50,12 @@
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

6
evaluation_examples/examples/libreoffice_calc/1334ca3e-f9e3-4db8-9ca7-b4c653be7d17.json
View File

@@ -50,6 +50,12 @@
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

6
evaluation_examples/examples/libreoffice_calc/21df9241-f8d7-4509-b7f1-37e501a823f7.json
View File

@@ -69,6 +69,12 @@
"/home/user/Represent_in_millions_billions.xlsx"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

6
evaluation_examples/examples/libreoffice_calc/347ef137-7eeb-4c80-a3bb-0951f26a8aff.json
View File

@@ -50,6 +50,12 @@
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

6
evaluation_examples/examples/libreoffice_calc/357ef137-7eeb-4c80-a3bb-0951f26a8aff.json
View File

@@ -50,6 +50,12 @@
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

6
evaluation_examples/examples/libreoffice_calc/37608790-6147-45d0-9f20-1137bb35703d.json
View File

@@ -50,6 +50,12 @@
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

6
evaluation_examples/examples/libreoffice_calc/4188d3a4-077d-46b7-9c86-23e1a036f6c1.json
View File

@@ -50,6 +50,12 @@
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

62
evaluation_examples/examples/libreoffice_calc/4e6fcf72-daf3-439f-a232-c434ce416af6.json
View File

@@ -50,12 +50,70 @@
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1sRI72UGfHiVBRFuV4uwhr173u3Sf46Y6&export=download&authuser=0&confirm=t&uuid=90da5e2b-39c0-449d-b753-09dfed73b509&at=APZUnTVFInccKo2QB9JNnIidFfG3:1705909465173",
"path": "/home/user/Employee_Age_By_Birthday_gold.xlsx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/Employee_Age_By_Birthday_gold.xlsx"
}
},
{
"type": "sleep",
"parameters": {
"seconds": 2
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Employee_Age_By_Birthday_gold.xlsx - LibreOffice Calc",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1sRI72UGfHiVBRFuV4uwhr173u3Sf46Y6&export=download&authuser=0&confirm=t&uuid=90da5e2b-39c0-449d-b753-09dfed73b509&at=APZUnTVFInccKo2QB9JNnIidFfG3:1705909465173",
"type": "vm_file",
"path": "/home/user/Employee_Age_By_Birthday_gold.xlsx",
"dest": "Employee_Age_By_Birthday_gold.xlsx"
},
"result": {

6
evaluation_examples/examples/libreoffice_calc/4f07fbe9-70de-4927-a4d5-bb28bc12c52c.json
View File

@@ -69,6 +69,12 @@
"/home/user/Padding_Decimals_In_Formular.xlsx"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

6
evaluation_examples/examples/libreoffice_calc/51b11269-2ca8-4b2a-9163-f21758420e78.json
View File

@@ -50,6 +50,12 @@
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

6
evaluation_examples/examples/libreoffice_calc/6054afcb-5bab-4702-90a0-b259b5d3217c.json
View File

@@ -50,6 +50,12 @@
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

6
evaluation_examples/examples/libreoffice_calc/6e99a1ad-07d2-4b66-a1ce-ece6d99c20a5.json
View File

@@ -69,6 +69,12 @@
"/home/user/Keep_Two_decimal_points.xlsx"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

6
evaluation_examples/examples/libreoffice_calc/7a4e4bc8-922c-4c84-865c-25ba34136be1.json
View File

@@ -50,6 +50,12 @@
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

6
evaluation_examples/examples/libreoffice_calc/7e429b8d-a3f0-4ed0-9b58-08957d00b127.json
View File

@@ -50,6 +50,12 @@
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

6
evaluation_examples/examples/libreoffice_calc/7efeb4b1-3d19-4762-b163-63328d66303b.json
View File

@@ -50,6 +50,12 @@
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

6
evaluation_examples/examples/libreoffice_calc/8b1ce5f2-59d2-4dcc-b0b0-666a714b9a14.json
View File

@@ -50,6 +50,12 @@
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

8
evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json
View File

@@ -1,7 +1,7 @@
{
"id": "a01fbce3-2793-461f-ab86-43680ccbae25",
"snapshot": "libreoffice_calc",
"instruction": "I need to set the decimal separator as a comma (,) for localized data representation and clarity in visualization. Can you assist with this?",
"instruction": "I need to set the decimal separator as a comma (,) for localized data representation and clarity in visualization. Can you help me to update all the numbers in the sheet? Also please keep the decimal numbers as-is.",
"source": "https://superuser.com/questions/1250677/how-to-set-decimal-separator-in-libre-office-calc",
"config": [
{
@@ -69,6 +69,12 @@
"/home/user/Set_Decimal_Separator_Dot.xlsx"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

6
evaluation_examples/examples/libreoffice_calc/a9f325aa-8c05-4e4f-8341-9e4358565f4f.json
View File

@@ -50,6 +50,12 @@
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

6
evaluation_examples/examples/libreoffice_calc/abed40dc-063f-4598-8ba5-9fe749c0615d.json
View File

@@ -50,6 +50,12 @@
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

6
evaluation_examples/examples/libreoffice_calc/d681960f-7bc3-4286-9913-a8812ba3261a.json
View File

@@ -50,6 +50,12 @@
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

6
evaluation_examples/examples/libreoffice_calc/eb03d19a-b88d-4de4-8a64-ca0ac66f426b.json
View File

@@ -50,6 +50,12 @@
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

6
evaluation_examples/examples/libreoffice_calc/ecb0df7a-4e8d-4a03-b162-053391d3afaf.json
View File

@@ -50,6 +50,12 @@
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

6
evaluation_examples/examples/libreoffice_calc/f9584479-3d0d-4c79-affa-9ad7afdd8850.json
View File

@@ -50,6 +50,12 @@
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

6
experiment_screenshot_som.py
View File

@@ -1,4 +1,4 @@
import ctypes
#import ctypes
import datetime
import json
import logging
@@ -131,8 +131,10 @@ def main(example_class, example_id):
example = json.load(f)
example["snapshot"] = "exp_v1"
logger.info("TASK: %s/%s", example_class, example_id)
api_key = os.environ.get("OPENAI_API_KEY")
agent = GPT4v_Agent(api_key=api_key, model=gpt4_model, instruction=example['instruction'],
agent = GPT4v_Agent(api_key=api_key, model=gpt4_model, max_tokens=1000, instruction=example['instruction'],
action_space=action_space, exp="som")
# api_key = os.environ.get("GENAI_API_KEY")