lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ver Jan31stv6

Browse Source 
finished calc human evaluation
updated calc configs with an extra sleep to guarantee the integrity of
downloaded xlsx file
This commit is contained in:
David Chang
2024-01-31 22:55:47 +08:00
parent 3dce3ffe63
commit 4897211a46
28 changed files with 216 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
branch_flag
View File

@@ -1 +1 @@
exp_som
test

16
desktop_env/evaluators/metrics/table.py
View File

@@ -392,17 +392,13 @@ if __name__ == '__main__':
logger.addHandler(stdout_handler)
logger.addHandler(sdebug_handler)
path1 = "../../任务数据/LibreOffice Calc/Multiply_Time_Number.xlsx"
path2 = "../../任务数据/LibreOffice Calc/Multiply_Time_Number_gold.xlsx"
rules = [{"type": "check_cell"
, "sheet_idx": 0
, "coordinate": "E3"
, "props": {"value": {"method": "approx:0.001"
, "ref": 191.6667
}
}
path1 = "snapshots/test/cache/4e6fcf72-daf3-439f-a232-c434ce416af6/Employee_Age_By_Birthday.xlsx"
path2 = "snapshots/test/cache/4e6fcf72-daf3-439f-a232-c434ce416af6/Employee_Age_By_Birthday_gold.xlsx"
rules = [ { "type": "sheet_data"
, "sheet_idx0": 0
, "sheet_idx1": "EI0"
}
]
]
print(compare_table(path1, path2
, rules=rules
)

6
evaluation_examples/examples/libreoffice_calc/01b269ae-2111-4a07-81fd-3fcd711993b0-win.json
View File

@@ -50,6 +50,12 @@
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

6
evaluation_examples/examples/libreoffice_calc/01b269ae-2111-4a07-81fd-3fcd711993b0.json
View File

@@ -50,6 +50,12 @@
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

6
evaluation_examples/examples/libreoffice_calc/0cecd4f3-74de-457b-ba94-29ad6b5dafb6.json
View File

@@ -50,6 +50,12 @@
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

6
evaluation_examples/examples/libreoffice_calc/1334ca3e-f9e3-4db8-9ca7-b4c653be7d17.json
View File

@@ -50,6 +50,12 @@
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

6
evaluation_examples/examples/libreoffice_calc/21df9241-f8d7-4509-b7f1-37e501a823f7.json
View File

@@ -69,6 +69,12 @@
"/home/user/Represent_in_millions_billions.xlsx"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

6
evaluation_examples/examples/libreoffice_calc/347ef137-7eeb-4c80-a3bb-0951f26a8aff.json
View File

@@ -50,6 +50,12 @@
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

6
evaluation_examples/examples/libreoffice_calc/357ef137-7eeb-4c80-a3bb-0951f26a8aff.json
View File

@@ -50,6 +50,12 @@
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

6
evaluation_examples/examples/libreoffice_calc/37608790-6147-45d0-9f20-1137bb35703d.json
View File

@@ -50,6 +50,12 @@
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

6
evaluation_examples/examples/libreoffice_calc/4188d3a4-077d-46b7-9c86-23e1a036f6c1.json
View File

@@ -50,6 +50,12 @@
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

62
evaluation_examples/examples/libreoffice_calc/4e6fcf72-daf3-439f-a232-c434ce416af6.json
View File

@@ -50,12 +50,70 @@
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1sRI72UGfHiVBRFuV4uwhr173u3Sf46Y6&export=download&authuser=0&confirm=t&uuid=90da5e2b-39c0-449d-b753-09dfed73b509&at=APZUnTVFInccKo2QB9JNnIidFfG3:1705909465173",
"path": "/home/user/Employee_Age_By_Birthday_gold.xlsx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/Employee_Age_By_Birthday_gold.xlsx"
}
},
{
"type": "sleep",
"parameters": {
"seconds": 2
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Employee_Age_By_Birthday_gold.xlsx - LibreOffice Calc",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1sRI72UGfHiVBRFuV4uwhr173u3Sf46Y6&export=download&authuser=0&confirm=t&uuid=90da5e2b-39c0-449d-b753-09dfed73b509&at=APZUnTVFInccKo2QB9JNnIidFfG3:1705909465173",
"type": "vm_file",
"path": "/home/user/Employee_Age_By_Birthday_gold.xlsx",
"dest": "Employee_Age_By_Birthday_gold.xlsx"
},
"result": {

6
evaluation_examples/examples/libreoffice_calc/4f07fbe9-70de-4927-a4d5-bb28bc12c52c.json
View File

@@ -69,6 +69,12 @@
"/home/user/Padding_Decimals_In_Formular.xlsx"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

6
evaluation_examples/examples/libreoffice_calc/51b11269-2ca8-4b2a-9163-f21758420e78.json
View File

@@ -50,6 +50,12 @@
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

6
evaluation_examples/examples/libreoffice_calc/6054afcb-5bab-4702-90a0-b259b5d3217c.json
View File

@@ -50,6 +50,12 @@
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

6
evaluation_examples/examples/libreoffice_calc/6e99a1ad-07d2-4b66-a1ce-ece6d99c20a5.json
View File

@@ -69,6 +69,12 @@
"/home/user/Keep_Two_decimal_points.xlsx"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

6
evaluation_examples/examples/libreoffice_calc/7a4e4bc8-922c-4c84-865c-25ba34136be1.json
View File

@@ -50,6 +50,12 @@
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

6
evaluation_examples/examples/libreoffice_calc/7e429b8d-a3f0-4ed0-9b58-08957d00b127.json
View File

@@ -50,6 +50,12 @@
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

6
evaluation_examples/examples/libreoffice_calc/7efeb4b1-3d19-4762-b163-63328d66303b.json
View File

@@ -50,6 +50,12 @@
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

6
evaluation_examples/examples/libreoffice_calc/8b1ce5f2-59d2-4dcc-b0b0-666a714b9a14.json
View File

@@ -50,6 +50,12 @@
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

8
evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json
View File

@@ -1,7 +1,7 @@
{
"id": "a01fbce3-2793-461f-ab86-43680ccbae25",
"snapshot": "libreoffice_calc",
"instruction": "I need to set the decimal separator as a comma (,) for localized data representation and clarity in visualization. Can you assist with this?",
"instruction": "I need to set the decimal separator as a comma (,) for localized data representation and clarity in visualization. Can you help me to update all the numbers in the sheet? Also please keep the decimal numbers as-is.",
"source": "https://superuser.com/questions/1250677/how-to-set-decimal-separator-in-libre-office-calc",
"config": [
{
@@ -69,6 +69,12 @@
"/home/user/Set_Decimal_Separator_Dot.xlsx"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

6
evaluation_examples/examples/libreoffice_calc/a9f325aa-8c05-4e4f-8341-9e4358565f4f.json
View File

@@ -50,6 +50,12 @@
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

6
evaluation_examples/examples/libreoffice_calc/abed40dc-063f-4598-8ba5-9fe749c0615d.json
View File

@@ -50,6 +50,12 @@
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

6
evaluation_examples/examples/libreoffice_calc/d681960f-7bc3-4286-9913-a8812ba3261a.json
View File

@@ -50,6 +50,12 @@
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

6
evaluation_examples/examples/libreoffice_calc/eb03d19a-b88d-4de4-8a64-ca0ac66f426b.json
View File

@@ -50,6 +50,12 @@
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

6
evaluation_examples/examples/libreoffice_calc/ecb0df7a-4e8d-4a03-b162-053391d3afaf.json
View File

@@ -50,6 +50,12 @@
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

6
evaluation_examples/examples/libreoffice_calc/f9584479-3d0d-4c79-affa-9ad7afdd8850.json
View File

@@ -50,6 +50,12 @@
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",

7
main.py
View File

@@ -47,11 +47,12 @@ def human_agent():
Runs the Gym environment with human input.
"""
with open("evaluation_examples/examples/libreoffice_calc/01b269ae-2111-4a07-81fd-3fcd711993b0-win.json", "r") as f:
with open("evaluation_examples/examples/libreoffice_calc/4f07fbe9-70de-4927-a4d5-bb28bc12c52c.json", "r") as f:
example = json.load(f)
example["snapshot"] = "Snapshot 1"
example["snapshot"] = "Snapshot 30"
env = DesktopEnv( path_to_vm="~/vmware/Windows 10 x64/Windows 10 x64.vmx"
#env = DesktopEnv( path_to_vm="~/vmware/Windows 10 x64/Windows 10 x64.vmx"
env = DesktopEnv( path_to_vm="/mnt/data1/david/os-images/Ubuntu-1218/Ubuntu.vmx"
, action_space="computer_13"
, task_config=example
)