lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main'

Browse Source
This commit is contained in:
Timothyxxx
2024-03-21 22:05:40 +08:00
parent 3ce7636abd ca03baacf5
commit 92760b29e1
56 changed files with 1652 additions and 1019 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
desktop_env/evaluators/metrics/slides.py
View File

@@ -54,10 +54,10 @@ def check_image_stretch_and_center(modified_ppt, original_ppt):
if the_image.image.blob == modified_image.image.blob: if the_image.image.blob == modified_image.image.blob:
the_modified_image = modified_image the_modified_image = modified_image
if (abs(the_modified_image.width - original_pres.slide_width) > Inches(0.1) or if (abs(the_modified_image.width - original_pres.slide_width) > Inches(0.5) or
abs(the_modified_image.height - original_pres.slide_height) > Inches(0.1) or abs(the_modified_image.height - original_pres.slide_height) > Inches(0.5) or
abs(the_modified_image.left - (original_pres.slide_width - the_modified_image.width) / 2) > Inches(0.1) or abs(the_modified_image.left - (original_pres.slide_width - the_modified_image.width) / 2) > Inches(0.5) or
abs(the_modified_image.top - (original_pres.slide_height - the_modified_image.height) / 2) > Inches(0.1)): abs(the_modified_image.top - (original_pres.slide_height - the_modified_image.height) / 2) > Inches(0.5)):
return 0. return 0.
return 1. return 1.

82
evaluation_examples/examples/Windows/excel/01b269ae-2111-4a07-81fd-3fcd711993b0.json
View File

@@ -1,82 +0,0 @@
{
"id": "01b269ae-2111-4a07-81fd-3fcd711993b0",
"snapshot": "libreoffice_calc",
"instruction": "Fill all the blank cells with the value in the cell above it",
"source": "https://www.youtube.com/shorts/VrUzPTIwQ04",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1FuOZ-5YoKgLLwl_oZd4R3D8pZACf_ukS&export=download&authuser=0&confirm=t&uuid=2051e7a6-5930-4cef-8d77-20ebf66ec6e6&at=APZUnTX1fXqlxy6rluq-Kw-LUhS5:1705919461032",
"path": "C:\\Users\\chenj\\Student_Level_Fill_Blank.xlsx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "C:\\Users\\chenj\\Student_Level_Fill_Blank.xlsx"
}
}
],
"trajectory": "trajectories/01b269ae-2111-4a07-81fd-3fcd711993b0",
"related_apps": [
"msoffice_excel"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "Student_Level_Fill_Blank - Excel",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5); pyautogui.press(\"enter\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1HTle3vgdZSjJIK_wjXyjtWwbiYJeguwv&export=download&authuser=0&confirm=t&uuid=c5d0868b-bed2-48fb-949b-8a9f3f61e8cf&at=APZUnTVqS9CTZFJ1rPqCGQPDCv3p:1705919542916",
"dest": "Student_Level_Fill_Blank_gold.xlsx"
},
"result": {
"type": "vm_file",
"path": "C:\\Users\\chenj\\Student_Level_Fill_Blank.xlsx",
"dest": "Student_Level_Fill_Blank.xlsx"
},
"options": {
"rules": [
{
"type": "sheet_data",
"sheet_idx0": 0,
"sheet_idx1": "EI0"
}
]
}
}
}

109
evaluation_examples/examples/Windows/excel/0bf05a7d-b28b-44d2-955a-50b41e24012a.json
View File

@@ -1,109 +0,0 @@
{
"id": "0bf05a7d-b28b-44d2-955a-50b41e24012a",
"snapshot": "libreoffice_calc",
"instruction": "I would like to pad all the numbers in the 'Old ID' column with zeros in front, to fill them up to seven digits in the 'New 7 Digit ID' column.",
"source": "https://www.youtube.com/shorts/FPAQaDTS8VY",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1DqGy5JRKOuZMRJ8O76d4Cds4WaRyz8V1&export=download&authuser=0&confirm=t&uuid=fa0694d1-2a77-4fd2-89d3-d9b854317823&at=APZUnTU9BxqG7E8tLZ104c0E8BEL:1705501029016",
"path": "C:\\Users\\chenj\\Customers_New_7digit_Id.xlsx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "C:\\Users\\chenj\\Customers_New_7digit_Id.xlsx"
}
}
],
"trajectory": "trajectories/0bf05a7d-b28b-44d2-955a-50b41e24012a",
"related_apps": [
"libreoffice calc"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "Customers_New_7digit_Id - Excel",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5); pyautogui.press(\"enter\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"libreoffice",
"--convert-to",
"csv:Text - txt - csv (StarCalc):44,34,UTF-8,,,,false,true,true,false,false,1",
"--outdir",
"/home/user",
"C:\\Users\\chenj\\Customers_New_7digit_Id.xlsx"
]
}
}
],
"func": "compare_table",
"result": {
"type": "vm_file",
"path": [
"C:\\Users\\chenj\\Customers_New_7digit_Id.xlsx",
"C:\\Users\\chenj\\Customers_New_7digit_Id-Sheet1.csv"
],
"dest": [
"Customers_New_7digit_Id.xlsx",
"Customers_New_7digit_Id-Sheet1.csv"
],
"multi": true
},
"expected": {
"type": "cloud_file",
"path": [
"https://drive.usercontent.google.com/download?id=1zXz5k5A403IR0GE6DRRXRgQZrSIoVFSz&export=download&authuser=0&confirm=t&uuid=ba70b841-969c-4d91-9288-0011aeecf251&at=APZUnTWx3LL9udCgJAh-VMFzzfod:1705501007861",
"https://drive.usercontent.google.com/download?id=1h1GnUpyj92K7FXiHJ1xVaUYW_UYMDLPM&export=download&authuser=0&confirm=t&uuid=98de75bd-ba32-4ceb-97a7-b8d303a8dc96&at=APZUnTUewdfFIsyC3UlSlMcmmRbo:1705500978790"
],
"dest": [
"Customers_New_7digit_Id_gold.xlsx",
"Customers_New_7digit_Id_gold-Sheet1.csv"
],
"multi": true
},
"options": {
"rules": [
{
"type": "sheet_print",
"sheet_idx0": 0,
"sheet_idx1": "EI0"
}
]
}
}
}

95
evaluation_examples/examples/Windows/excel/0cecd4f3-74de-457b-ba94-29ad6b5dafb6.json
View File

@@ -1,95 +0,0 @@
{
"id": "0cecd4f3-74de-457b-ba94-29ad6b5dafb6",
"snapshot": "libreoffice_calc",
"instruction": "Rename \"Sheet 1\" to \"LARS Resources\". Then make a copy of it. Place the copy before \"Sheet 2\" and rename it by appending a suffix \"(Backup)\", concatenated by a white space. And Also rename \"Sheet2\" by appending the suffix \"(Offline)\".",
"source": "https://www.libreofficehelp.com/add-insert-delete-copy-move-rename-a-worksheet-in-libreoffice-calc/",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1emE6hfKYJpsKNoJ7fmSKCug9pZ9Rv0fx&export=download&authuser=0&confirm=t&uuid=b84ee976-0dab-4a32-a99d-56873f016eec&at=APZUnTVs-ZH77mNQ9-YCLqj2H5xh:1706261078154",
"path": "c:\\users\\chenj\\copy_sheet_insert.xlsx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "c:\\users\\chenj\\copy_sheet_insert.xlsx"
}
}
],
"trajectory": "trajectories/0cecd4f3-74de-457b-ba94-29ad6b5dafb6",
"related_apps": [
"libreoffice_calc"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "copy_sheet_insert.xlsx - LibreOffice Calc",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",
"result": {
"type": "vm_file",
"path": "c:\\users\\chenj\\copy_sheet_insert.xlsx",
"dest": "copy_sheet_insert.xlsx"
},
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1u0mYkDejNIFWfSOvQCi2z6nXTinUqhGG&export=download&authuser=0&confirm=t&uuid=d9036b53-dc50-4451-9e81-a5327b846c29&at=APZUnTU6PV42c83Ug6CGUQ6zx0mA:1706261109913",
"dest": "copy_sheet_insert_gold.xlsx"
},
"options": {
"rules": [
{
"type": "sheet_name"
},
{
"type": "sheet_data",
"sheet_idx0": 0,
"sheet_idx1": 1
},
{
"type": "sheet_data",
"sheet_idx0": 1,
"sheet_idx1": "EI1"
},
{
"type": "sheet_data",
"sheet_idx0": 2,
"sheet_idx1": "EI2"
}
]
}
}
}

6
evaluation_examples/examples/Windows/excel/3aaa4e37-dc91-482e-99af-132a612d40f3.json
View File

@@ -1,6 +1,6 @@
{ {
"id": "3aaa4e37-dc91-482e-99af-132a612d40f3", "id": "3aaa4e37-dc91-482e-99af-132a612d40f3",
"snapshot": "libreoffice_calc", "snapshot": "excel",
"instruction": "Could you help me to export the current sheet to a csv file? Export the contents just as they are shown on the screen. Just keep the other options untouched. A default csv format is ok. The csv should share the file name with the original xlsx.", "instruction": "Could you help me to export the current sheet to a csv file? Export the contents just as they are shown on the screen. Just keep the other options untouched. A default csv format is ok. The csv should share the file name with the original xlsx.",
"source": "https://www.quora.com/How-can-you-import-export-CSV-files-with-LibreOffice-Calc-or-OpenOffice", "source": "https://www.quora.com/How-can-you-import-export-CSV-files-with-LibreOffice-Calc-or-OpenOffice",
"config": [ "config": [
@@ -24,7 +24,7 @@
], ],
"trajectory": "trajectories/3aaa4e37-dc91-482e-99af-132a612d40f3", "trajectory": "trajectories/3aaa4e37-dc91-482e-99af-132a612d40f3",
"related_apps": [ "related_apps": [
"libreoffice_calc" "excel"
], ],
"evaluator": { "evaluator": {
"func": "compare_csv", "func": "compare_csv",
@@ -35,7 +35,7 @@
}, },
"expected": { "expected": {
"type": "cloud_file", "type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1cjyinycSAxi5oBb-dcpC85CMX9i-ggg-&export=download&authuser=0&confirm=t&uuid=b18a6fbd-8318-49ee-a62b-af22523df9e7&at=APZUnTV6dhR5_TcDg002RQ5CHmoC:1706012321261", "path": "https://drive.usercontent.google.com/download?id=1xdQVCACVX2_q1hwy50-5TiNmWXk9jOeM&export=download&authuser=0&confirm=t&uuid=24791e76-40ff-4779-a2f1-baa5f9a7979b&at=APZUnTWshc91kgINGTP-woDuJK_i:1710911663001",
"dest": "Export_Calc_to_CSV_gold.csv" "dest": "Export_Calc_to_CSV_gold.csv"
} }
} }

10
evaluation_examples/examples/Windows/excel/4188d3a4-077d-46b7-9c86-23e1a036f6c1.json
View File

@@ -1,6 +1,6 @@
{ {
"id": "4188d3a4-077d-46b7-9c86-23e1a036f6c1", "id": "4188d3a4-077d-46b7-9c86-23e1a036f6c1",
"snapshot": "libreoffice_calc", "snapshot": "excel",
"instruction": "Help me freeze the range A1:B1 on this sheet to keep the headers always visible", "instruction": "Help me freeze the range A1:B1 on this sheet to keep the headers always visible",
"source": "https://www.libreofficehelp.com/freeze-unfreeze-rows-columns-ranges-calc/", "source": "https://www.libreofficehelp.com/freeze-unfreeze-rows-columns-ranges-calc/",
"config": [ "config": [
@@ -9,7 +9,7 @@
"parameters": { "parameters": {
"files": [ "files": [
{ {
"url": "https://drive.usercontent.google.com/download?id=1pmTn-32BIwXOu2NXgQG7-PCRPaNL5bYN&export=download&authuser=0&confirm=t&uuid=13f1a8fd-997e-4364-a182-ad8768eaac6a&at=APZUnTUOM1rcgxy87qxR1Rkdwdor:1705895562093", "url": "https://drive.usercontent.google.com/download?id=1pHuXBQicCKbPEiPk3PNlF0zacNwZ18B2&export=download&authuser=0&confirm=t&uuid=32e06ec7-ea8b-4e49-98ae-594b095dec9a&at=APZUnTW7mV2ifx2jLKi4-2wHMPIW:1710921676387",
"path": "c:\\users\\chenj\\Freeze_row_column.xlsx" "path": "c:\\users\\chenj\\Freeze_row_column.xlsx"
} }
] ]
@@ -24,14 +24,14 @@
], ],
"trajectory": "trajectories/4188d3a4-077d-46b7-9c86-23e1a036f6c1", "trajectory": "trajectories/4188d3a4-077d-46b7-9c86-23e1a036f6c1",
"related_apps": [ "related_apps": [
"libreoffice_calc" "excel"
], ],
"evaluator": { "evaluator": {
"postconfig": [ "postconfig": [
{ {
"type": "activate_window", "type": "activate_window",
"parameters": { "parameters": {
"window_name": "Freeze_row_column.xlsx - LibreOffice Calc", "window_name": "Freeze_row_column - Excel",
"strict": true "strict": true
} }
}, },
@@ -66,7 +66,7 @@
}, },
"expected": { "expected": {
"type": "cloud_file", "type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1oTsiIJFiAIFaTo320izMP7ml1RDCctgD&export=download&authuser=0&confirm=t&uuid=e3ec92c6-c2bf-4a7e-8e10-c8287350c9c6&at=APZUnTXEp-KV7bido8GcENfWIxME:1705895589811", "path": "https://drive.usercontent.google.com/download?id=1_bqrHZuowhYMtfO7mZLwOJ8SZwzluwng&export=download&authuser=0&confirm=t&uuid=f0d030f2-cc0e-42e0-ab72-ed4e14f7dafd&at=APZUnTXjt4eBKQothOWJoAZFkZK-:1710921727198",
"dest": "Freeze_row_column_gold.xlsx" "dest": "Freeze_row_column_gold.xlsx"
}, },
"options": { "options": {

134
evaluation_examples/examples/Windows/excel/4e6fcf72-daf3-439f-a232-c434ce416af6.json
View File

@@ -1,134 +0,0 @@
{
"id": "4e6fcf72-daf3-439f-a232-c434ce416af6",
"snapshot": "libreoffice_calc",
"instruction": "Please calculate the ages of the employees according to their birthday.",
"source": "https://www.youtube.com/shorts/0uxJccNCKcE",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1WIyJbssCCscQ96be2hF9N7tXPz23JoBT&export=download&authuser=0&confirm=t&uuid=503cdbf3-2fe3-4019-bfd1-5d1faab8d049&at=APZUnTV-XLlF8KEx7zMjtX2kYSuM:1705909207212",
"path": "C:\\Users\\chenj\\Employee_Age_By_Birthday.xlsx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "C:\\Users\\chenj\\Employee_Age_By_Birthday.xlsx"
}
}
],
"trajectory": "trajectories/4e6fcf72-daf3-439f-a232-c434ce416af6",
"related_apps": [
"libreoffice calc"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "Employee_Age_By_Birthday - Excel",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5); pyautogui.press(\"enter\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1sRI72UGfHiVBRFuV4uwhr173u3Sf46Y6&export=download&authuser=0&confirm=t&uuid=90da5e2b-39c0-449d-b753-09dfed73b509&at=APZUnTVFInccKo2QB9JNnIidFfG3:1705909465173",
"path": "C:\\Users\\chenj\\Employee_Age_By_Birthday_gold.xlsx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "C:\\Users\\chenj\\Employee_Age_By_Birthday_gold.xlsx"
}
},
{
"type": "sleep",
"parameters": {
"seconds": 2
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Employee_Age_By_Birthday_gold.xlsx - LibreOffice Calc",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5); pyautogui.press(\"enter\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",
"expected": {
"type": "vm_file",
"path": "C:\\Users\\chenj\\Employee_Age_By_Birthday_gold.xlsx",
"dest": "Employee_Age_By_Birthday_gold.xlsx"
},
"result": {
"type": "vm_file",
"path": "C:\\Users\\chenj\\Employee_Age_By_Birthday.xlsx",
"dest": "Employee_Age_By_Birthday.xlsx"
},
"options": {
"rules": [
{
"type": "sheet_data",
"sheet_idx0": 0,
"sheet_idx1": "EI0"
}
]
}
}
}

10
evaluation_examples/examples/Windows/excel/51b11269-2ca8-4b2a-9163-f21758420e78.json
View File

@@ -1,7 +1,7 @@
{ {
"id": "51b11269-2ca8-4b2a-9163-f21758420e78", "id": "51b11269-2ca8-4b2a-9163-f21758420e78",
"snapshot": "libreoffice_calc", "snapshot": "excel",
"instruction": "Could you help me to sort the records accoring to the amounts ascendingly?", "instruction": "Could you help me to sort the records according to the amounts ascendingly?",
"source": "https://www.reddit.com/r/LibreOfficeCalc/comments/186pcc6/how_to_arrange_numbers_in_a_column_from_minimum/", "source": "https://www.reddit.com/r/LibreOfficeCalc/comments/186pcc6/how_to_arrange_numbers_in_a_column_from_minimum/",
"config": [ "config": [
{ {
@@ -9,7 +9,7 @@
"parameters": { "parameters": {
"files": [ "files": [
{ {
"url": "https://drive.usercontent.google.com/download?id=1lMPR8itmrjesLPh8dqxrssNNnlQUSF8Y&export=download&authuser=0&confirm=t&uuid=f79c8ece-f42d-42dc-b320-9ee4d9a6b5d4&at=APZUnTUQAGyFOhO9mMS4r_Ap-cFy:1706001993533", "url": "https://drive.usercontent.google.com/download?id=1txrttkNQK0Dw3YU8TvmA39BNOqy4F0Tz&export=download&authuser=0&confirm=t&uuid=400efe20-6e4c-4c62-940f-3042077ca0ac&at=APZUnTUfihTMm-AIlXxqQOh6mjGk:1710921074190",
"path": "c:\\users\\chenj\\Arrang_Value_min_to_max.xlsx" "path": "c:\\users\\chenj\\Arrang_Value_min_to_max.xlsx"
} }
] ]
@@ -24,7 +24,7 @@
], ],
"trajectory": "trajectories/51b11269-2ca8-4b2a-9163-f21758420e78", "trajectory": "trajectories/51b11269-2ca8-4b2a-9163-f21758420e78",
"related_apps": [ "related_apps": [
"libreoffice calc" "excel"
], ],
"evaluator": { "evaluator": {
"postconfig": [ "postconfig": [
@@ -61,7 +61,7 @@
"func": "compare_table", "func": "compare_table",
"expected": { "expected": {
"type": "cloud_file", "type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1gZq47ls5eD-YpRH8MIeEYOpYwAAUR_Ua&export=download&authuser=0&confirm=t&uuid=085c84a0-eb3f-4fe8-9fa0-6d2495985cc0&at=APZUnTVtt_9I5I8emixgM2hbnjMh:1706002085970", "path": "https://drive.usercontent.google.com/download?id=1xv9kJFytAjmjvFslFV-0FhZb_lueW5pG&export=download&authuser=0&confirm=t&uuid=dc0b3dbb-50da-461f-915d-62da33bbda9d&at=APZUnTU6B71ryKRAp0-pBjdc8gMO:1710921122626",
"dest": "Arrang_Value_min_to_max_gold.xlsx" "dest": "Arrang_Value_min_to_max_gold.xlsx"
}, },
"result": { "result": {

18
evaluation_examples/examples/Windows/excel/6054afcb-5bab-4702-90a0-b259b5d3217c.json
View File

@@ -1,6 +1,6 @@
{ {
"id": "6054afcb-5bab-4702-90a0-b259b5d3217c", "id": "6054afcb-5bab-4702-90a0-b259b5d3217c",
"snapshot": "libreoffice_calc", "snapshot": "excel",
"instruction": "Some data are missed by now and are filled by 'N/A' temporarily. Please hide them in the table for now. Do not delete them and filter is no needed.", "instruction": "Some data are missed by now and are filled by 'N/A' temporarily. Please hide them in the table for now. Do not delete them and filter is no needed.",
"source": "https://www.youtube.com/shorts/JTbZ8sRxkdU", "source": "https://www.youtube.com/shorts/JTbZ8sRxkdU",
"config": [ "config": [
@@ -9,8 +9,8 @@
"parameters": { "parameters": {
"files": [ "files": [
{ {
"url": "https://drive.usercontent.google.com/download?id=1e1Ccsr_CQau9-boF92GxzZ0RtEHPtfdX&export=download&authuser=0&confirm=t&uuid=a1d4518d-e085-4bfa-ae6f-2514ed48efba&at=APZUnTU_ng4YNBQO7u6Dsuj21Gmq:1705911243359", "url": "https://drive.usercontent.google.com/download?id=1f5q8mCPawX2B8Tk67B1jqTSltv1Mon1V&export=download&authuser=0&confirm=t&uuid=fa148576-4559-4fbb-8ef0-5ccd6a5b8d64&at=APZUnTV6crOIW6tkR5gLj7K_yfta:1710922243040",
"path": "C:\\Users\\user\\Date_Budget_Variance_HideNA.xlsx" "path": "C:\\Users\\chenj\\Date_Budget_Variance_HideNA.xlsx"
} }
] ]
} }
@@ -18,20 +18,20 @@
{ {
"type": "open", "type": "open",
"parameters": { "parameters": {
"path": "C:\\Users\\user\\Date_Budget_Variance_HideNA.xlsx" "path": "C:\\Users\\chenj\\Date_Budget_Variance_HideNA.xlsx"
} }
} }
], ],
"trajectory": "trajectories/6054afcb-5bab-4702-90a0-b259b5d3217c", "trajectory": "trajectories/6054afcb-5bab-4702-90a0-b259b5d3217c",
"related_apps": [ "related_apps": [
"libreoffice calc" "excel"
], ],
"evaluator": { "evaluator": {
"postconfig": [ "postconfig": [
{ {
"type": "activate_window", "type": "activate_window",
"parameters": { "parameters": {
"window_name": "Date_Budget_Variance_HideNA.xlsx - LibreOffice Calc", "window_name": "Date_Budget_Variance_HideNA - Excel",
"strict": true "strict": true
} }
}, },
@@ -47,7 +47,7 @@
"command": [ "command": [
"python", "python",
"-c", "-c",
"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5); pyautogui.press(\"enter\");" "import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5);"
] ]
} }
}, },
@@ -61,12 +61,12 @@
"func": "compare_table", "func": "compare_table",
"expected": { "expected": {
"type": "cloud_file", "type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1ReZexJAvbAAUng0JD3lEHN70J0WcS0_i&export=download&authuser=0&confirm=t&uuid=a11148b1-93e8-4634-a413-26e0e433c2c9&at=APZUnTV6KulVQf6LpHl4IVNqE5hA:1705914637572", "path": "https://drive.usercontent.google.com/download?id=1JyBfMfwj3nh7Rdiv7S6f4u29rHsEVLpB&export=download&authuser=0&confirm=t&uuid=b3a00e64-cde9-4577-a020-8a56cbd4f6a1&at=APZUnTVI2QWmb8AVu8kvtfjhlPpu:1710922301750",
"dest": "Date_Budget_Variance_HideNA_gold.xlsx" "dest": "Date_Budget_Variance_HideNA_gold.xlsx"
}, },
"result": { "result": {
"type": "vm_file", "type": "vm_file",
"path": "C:\\Users\\user\\Date_Budget_Variance_HideNA.xlsx", "path": "C:\\Users\\chenj\\Date_Budget_Variance_HideNA.xlsx",
"dest": "Date_Budget_Variance_HideNA.xlsx" "dest": "Date_Budget_Variance_HideNA.xlsx"
}, },
"options": { "options": {

115
evaluation_examples/examples/Windows/excel/6e99a1ad-07d2-4b66-a1ce-ece6d99c20a5.json
View File

@@ -1,115 +0,0 @@
{
"id": "6e99a1ad-07d2-4b66-a1ce-ece6d99c20a5",
"snapshot": "libreoffice_calc",
"instruction": "Help me format column \"spent\" by keeping two decimal points. ",
"source": "https://www.youtube.com/watch?v=nl-nXjJurhQ",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1FG_e6DhoYybUMtkhAOJrJWCctIcP1E1I&export=download&authuser=0&confirm=t&uuid=2ee528f2-b86d-415d-a93b-350a1fa3820d&at=APZUnTXTQiGRl6DlkMFts81FT-En:1706005475837",
"path": "c:\\users\\chenj\\Keep_Two_decimal_points.xlsx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "c:\\users\\chenj\\Keep_Two_decimal_points.xlsx"
}
}
],
"trajectory": "trajectories/6e99a1ad-07d2-4b66-a1ce-ece6d99c20a5",
"related_apps": [
"libreoffice calc"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "Keep_Two_decimal_points.xlsx - LibreOffice Calc",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"libreoffice",
"--convert-to",
"csv:Text - txt - csv (StarCalc):44,34,UTF-8,,,,false,true,true,false,false,1",
"--outdir",
"/home/user",
"c:\\users\\chenj\\Keep_Two_decimal_points.xlsx"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_table",
"expected": {
"type": "cloud_file",
"path": [
"https://drive.usercontent.google.com/download?id=150XJSpqU-HuYSjPi7_LgrlQFUjKhKCU8&export=download&authuser=0&confirm=t&uuid=9b9fd955-ee06-4c55-a240-290dcd658d84&at=APZUnTW87ODEUqUzDpP2tah8tQCo:1706005816811",
"https://drive.usercontent.google.com/download?id=1-cxAMcFdKRUl0bFHr1nwI1XudvGV__s1&export=download&authuser=0&confirm=t&uuid=2aa40a50-3081-4d88-8d1d-e3adcfa2050e&at=APZUnTXeohokhdZeJzu3XT-ZyUwB:1706005770752"
],
"dest": [
"Keep_Two_decimal_points_gold.xlsx",
"Keep_Two_decimal_points_gold-Sheet1.csv"
],
"multi": true
},
"result": {
"type": "vm_file",
"path": [
"c:\\users\\chenj\\Keep_Two_decimal_points.xlsx",
"c:\\users\\chenj\\Keep_Two_decimal_points-Sheet1.csv"
],
"dest": [
"Keep_Two_decimal_points.xlsx",
"Keep_Two_decimal_points-Sheet1.csv"
],
"multi": true
},
"options": {
"rules": [
{
"type": "sheet_print",
"sheet_idx0": "RNSheet1",
"sheet_idx1": "ENSheet1"
}
]
}
}
}

10
evaluation_examples/examples/Windows/excel/7a4e4bc8-922c-4c84-865c-25ba34136be1.json
View File

@@ -1,6 +1,6 @@
{ {
"id": "7a4e4bc8-922c-4c84-865c-25ba34136be1", "id": "7a4e4bc8-922c-4c84-865c-25ba34136be1",
"snapshot": "libreoffice_calc", "snapshot": "excel",
"instruction": "Reorder the columns to be \"Date\", \"First Name\", \"Last Name\", \"Order ID\", \"Sales\"", "instruction": "Reorder the columns to be \"Date\", \"First Name\", \"Last Name\", \"Order ID\", \"Sales\"",
"source": "https://www.youtube.com/shorts/bvUhr1AHs44", "source": "https://www.youtube.com/shorts/bvUhr1AHs44",
"config": [ "config": [
@@ -9,7 +9,7 @@
"parameters": { "parameters": {
"files": [ "files": [
{ {
"url": "https://drive.usercontent.google.com/download?id=1jS159dHRdeZh54A8WgqJn_somQpKP1K_&export=download&authuser=0&confirm=t&uuid=06aa5391-5f3e-499b-9073-9ce24109f91f&at=APZUnTWFCuiN8b5ilNmxx3aPYGUF:1705551767314", "url": "https://drive.usercontent.google.com/download?id=1sXz6rch9XOzUgdBk6dvdDEyIR3vZ5gkl&export=download&authuser=0&confirm=t&uuid=b0e711ca-5e88-4eb1-8073-4972fbdc697e&at=APZUnTVi-YMJJEO4DALr4pHeB3o5:1710917585083",
"path": "C:\\Users\\chenj\\Name_Order_Id_move_column.xlsx" "path": "C:\\Users\\chenj\\Name_Order_Id_move_column.xlsx"
} }
] ]
@@ -24,7 +24,7 @@
], ],
"trajectory": "trajectories/7a4e4bc8-922c-4c84-865c-25ba34136be1", "trajectory": "trajectories/7a4e4bc8-922c-4c84-865c-25ba34136be1",
"related_apps": [ "related_apps": [
"libreoffice calc" "excel"
], ],
"evaluator": { "evaluator": {
"postconfig": [ "postconfig": [
@@ -47,7 +47,7 @@
"command": [ "command": [
"python", "python",
"-c", "-c",
"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5); pyautogui.press(\"enter\");" "import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5);"
] ]
} }
}, },
@@ -61,7 +61,7 @@
"func": "compare_table", "func": "compare_table",
"expected": { "expected": {
"type": "cloud_file", "type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1jAK4--qLRdUOA57MBsmERpR205GGIMij&export=download&authuser=0&confirm=t&uuid=4ca454c9-d9d5-4e5f-95e1-cb0e849932e5&at=APZUnTVgkUbmLJLC4O7ACKmzfFbq:1705551965635", "path": "https://drive.usercontent.google.com/download?id=1DeT0kFcjrLapgYgE7hNZWvZp3r0NLMrN&export=download&authuser=0&confirm=t&uuid=6651550d-ec4b-4dc7-ba82-5da4cc407b27&at=APZUnTWT75zWKM9orjpbn-pUsnHM:1710917648164",
"dest": "Name_Order_Id_move_column_gold.xlsx" "dest": "Name_Order_Id_move_column_gold.xlsx"
}, },
"result": { "result": {

12
evaluation_examples/examples/Windows/excel/7efeb4b1-3d19-4762-b163-63328d66303b.json
View File

@@ -1,6 +1,6 @@
{ {
"id": "7efeb4b1-3d19-4762-b163-63328d66303b", "id": "7efeb4b1-3d19-4762-b163-63328d66303b",
"snapshot": "libreoffice_calc", "snapshot": "excel",
"instruction": "Fill the Sequence Numbers as \"No. #\" in the \"Seq No.\" column", "instruction": "Fill the Sequence Numbers as \"No. #\" in the \"Seq No.\" column",
"source": "https://www.youtube.com/shorts/4jzXfZNhfmk", "source": "https://www.youtube.com/shorts/4jzXfZNhfmk",
"config": [ "config": [
@@ -9,7 +9,7 @@
"parameters": { "parameters": {
"files": [ "files": [
{ {
"url": "https://drive.usercontent.google.com/download?id=1muvWVg44C6EtBpBQkZ6_ylg3M8r3jjOC&export=download&authuser=0&confirm=t&uuid=41354a7c-d199-4044-960e-0d146fe6f12a&at=APZUnTW3nj5RV3SBJt5tdeVxM4mM:1705553238027", "url": "https://drive.usercontent.google.com/download?id=15skGu4MsPh9nqjfvYQQvPmy0PMEkgiYX&export=download&authuser=0&confirm=t&uuid=f8e99595-f619-424a-906a-799137c0d7bc&at=APZUnTUXxvhco6R8vtSqH6ohFIDX:1710919410207",
"path": "C:\\Users\\chenj\\Order_Sales_Serial#.xlsx" "path": "C:\\Users\\chenj\\Order_Sales_Serial#.xlsx"
} }
] ]
@@ -24,14 +24,14 @@
], ],
"trajectory": "trajectories/", "trajectory": "trajectories/",
"related_apps": [ "related_apps": [
"libreoffice calc" "excel"
], ],
"evaluator": { "evaluator": {
"postconfig": [ "postconfig": [
{ {
"type": "activate_window", "type": "activate_window",
"parameters": { "parameters": {
"window_name": "Order_Sales_Serial#.xlsx - LibreOffice Calc", "window_name": "Order_Sales_Serial# - Excel",
"strict": true "strict": true
} }
}, },
@@ -47,7 +47,7 @@
"command": [ "command": [
"python", "python",
"-c", "-c",
"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5); pyautogui.press(\"enter\");" "import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5);"
] ]
} }
}, },
@@ -61,7 +61,7 @@
"func": "compare_table", "func": "compare_table",
"expected": { "expected": {
"type": "cloud_file", "type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1K37qrxab2cib1SkKPthYkb83fzaj9ETa&export=download&authuser=0&confirm=t&uuid=66b763e6-2fb1-46fc-9753-7f8533366b82&at=APZUnTU1nCKKXBwCbyI8aKpdhE5W:1705560437314", "path": "https://drive.usercontent.google.com/download?id=1cNtCHxuKy_9Pjtnt5g3yA99DsFhX1vl3&export=download&authuser=0&confirm=t&uuid=1a635c58-2083-4972-85c6-da0cc1976ecc&at=APZUnTVoYAeaqTTHT6evt8rCpQbU:1710919479990",
"dest": "Order_Sales_Serial#_gold.xlsx" "dest": "Order_Sales_Serial#_gold.xlsx"
}, },
"result": { "result": {

12
evaluation_examples/examples/Windows/excel/8b1ce5f2-59d2-4dcc-b0b0-666a714b9a14.json
View File

@@ -1,6 +1,6 @@
{ {
"id": "8b1ce5f2-59d2-4dcc-b0b0-666a714b9a14", "id": "8b1ce5f2-59d2-4dcc-b0b0-666a714b9a14",
"snapshot": "libreoffice_calc", "snapshot": "excel",
"instruction": "Given a partial calendar, please highlight all the weekends (Satureday & Sunday) by setting the cell background as red (#ff0000).", "instruction": "Given a partial calendar, please highlight all the weekends (Satureday & Sunday) by setting the cell background as red (#ff0000).",
"source": "https://www.youtube.com/shorts/Hbcwu6IQ1ns", "source": "https://www.youtube.com/shorts/Hbcwu6IQ1ns",
"config": [ "config": [
@@ -9,7 +9,7 @@
"parameters": { "parameters": {
"files": [ "files": [
{ {
"url": "https://drive.usercontent.google.com/download?id=1_gyig5Vs3VOuvkjRoLt2ZpXBIyCZfUmV&export=download&authuser=0&confirm=t&uuid=ed113cdd-4279-454b-a66d-07447e31c818&at=APZUnTVztf5DcbF0DjLJitkpUUxt:1705920417565", "url": "https://drive.usercontent.google.com/download?id=1c55moKFHcD4Kw6jhB0HBOBfKw3dMhjeN&export=download&authuser=0&confirm=t&uuid=4be83beb-098c-4d4d-83b6-47fa32d733bf&at=APZUnTX__VvEkeKpNauUDdEkLGwW:1710920293517",
"path": "C:\\Users\\chenj\\Calendar_Highlight_Weekend_Days.xlsx" "path": "C:\\Users\\chenj\\Calendar_Highlight_Weekend_Days.xlsx"
} }
] ]
@@ -24,14 +24,14 @@
], ],
"trajectory": "trajectories/8b1ce5f2-59d2-4dcc-b0b0-666a714b9a14", "trajectory": "trajectories/8b1ce5f2-59d2-4dcc-b0b0-666a714b9a14",
"related_apps": [ "related_apps": [
"libreoffice calc" "excel"
], ],
"evaluator": { "evaluator": {
"postconfig": [ "postconfig": [
{ {
"type": "activate_window", "type": "activate_window",
"parameters": { "parameters": {
"window_name": "Calendar_Highlight_Weekend_Days.xlsx - LibreOffice Calc", "window_name": "Calendar_Highlight_Weekend_Days - Excel",
"strict": true "strict": true
} }
}, },
@@ -47,7 +47,7 @@
"command": [ "command": [
"python", "python",
"-c", "-c",
"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5); pyautogui.press(\"enter\");" "import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5);"
] ]
} }
}, },
@@ -61,7 +61,7 @@
"func": "compare_table", "func": "compare_table",
"expected": { "expected": {
"type": "cloud_file", "type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1URKsHvPdWDvB-qwsIZ-SqHAmiXaosXKW&export=download&authuser=0&confirm=t&uuid=849064c9-7402-48c5-87f6-e5c290e4bd24&at=APZUnTXarmqM0cO4I0z-Lv7MElzX:1705920495794", "path": "https://drive.usercontent.google.com/download?id=10olq6kYly1dFcDYcuDUyKZ7Aiub2DQ8Y&export=download&authuser=0&confirm=t&uuid=f4c79d78-bf2e-47ae-861b-caff487a6363&at=APZUnTXbyHCxDYsZ9h-CvRxjDREh:1710920367505",
"dest": "Calendar_Highlight_Weekend_Days_gold.xlsx" "dest": "Calendar_Highlight_Weekend_Days_gold.xlsx"
}, },
"result": { "result": {

12
evaluation_examples/examples/Windows/excel/a9f325aa-8c05-4e4f-8341-9e4358565f4f.json
View File

@@ -1,6 +1,6 @@
{ {
"id": "a9f325aa-8c05-4e4f-8341-9e4358565f4f", "id": "a9f325aa-8c05-4e4f-8341-9e4358565f4f",
"snapshot": "libreoffice_calc", "snapshot": "excel",
"instruction": "Remove the adundant whitespaces and canonicalize the letter cases by capitalizing the first letter of each words and leave other letters as lower case.", "instruction": "Remove the adundant whitespaces and canonicalize the letter cases by capitalizing the first letter of each words and leave other letters as lower case.",
"source": "https://www.youtube.com/shorts/A0gmEBRKXWs", "source": "https://www.youtube.com/shorts/A0gmEBRKXWs",
"config": [ "config": [
@@ -9,7 +9,7 @@
"parameters": { "parameters": {
"files": [ "files": [
{ {
"url": "https://drive.usercontent.google.com/download?id=1PGJxDM5QglZFdLd7yLQnA2MASf2so14S&export=download&authuser=0&confirm=t&uuid=725f7d90-1e30-4579-b946-7c8932aac440&at=APZUnTUpha_2nl3sAOmM1AeCCir_:1705561280883", "url": "https://drive.usercontent.google.com/download?id=10ppI3NrLPTyWvDG2UMtVHmNnkeGQ4W8K&export=download&authuser=0&confirm=t&uuid=18aa33f3-947b-4a10-ac45-08c44c8f8432&at=APZUnTUL0XEOEshMLwmEebqH2XAS:1710922810055",
"path": "C:\\Users\\chenj\\Movie_title_garbage_clean.xlsx" "path": "C:\\Users\\chenj\\Movie_title_garbage_clean.xlsx"
} }
] ]
@@ -24,14 +24,14 @@
], ],
"trajectory": "trajectories/a9f325aa-8c05-4e4f-8341-9e4358565f4f", "trajectory": "trajectories/a9f325aa-8c05-4e4f-8341-9e4358565f4f",
"related_apps": [ "related_apps": [
"libreoffice calc" "excel"
], ],
"evaluator": { "evaluator": {
"postconfig": [ "postconfig": [
{ {
"type": "activate_window", "type": "activate_window",
"parameters": { "parameters": {
"window_name": "Movie_title_garbage_clean.xlsx - LibreOffice Calc", "window_name": "Movie_title_garbage_clean - Excel",
"strict": true "strict": true
} }
}, },
@@ -47,7 +47,7 @@
"command": [ "command": [
"python", "python",
"-c", "-c",
"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5); pyautogui.press(\"enter\");" "import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5);"
] ]
} }
}, },
@@ -61,7 +61,7 @@
"func": "compare_table", "func": "compare_table",
"expected": { "expected": {
"type": "cloud_file", "type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1hrmRQig53UW069MEN_V23xJXFrIooylm&export=download&authuser=0&confirm=t&uuid=82a01881-c72c-4463-b258-c8c66f8d72af&at=APZUnTVZVuQj91twmMdH1plMcABA:1705561377705", "path": "https://drive.usercontent.google.com/download?id=1ibmMQuzqt_UgVzPhHs4ZUOezvLVMgT0d&export=download&authuser=0&confirm=t&uuid=16fc1d11-9927-4edf-8fb7-e0218c3718a6&at=APZUnTUcZEZD6JFUz9C_gjtVlzJK:1710922977254",
"dest": "Movie_title_garbage_clean_gold.xlsx" "dest": "Movie_title_garbage_clean_gold.xlsx"
}, },
"result": { "result": {

18
evaluation_examples/examples/Windows/excel/abed40dc-063f-4598-8ba5-9fe749c0615d.json
View File

@@ -1,6 +1,6 @@
{ {
"id": "abed40dc-063f-4598-8ba5-9fe749c0615d", "id": "abed40dc-063f-4598-8ba5-9fe749c0615d",
"snapshot": "libreoffice_calc", "snapshot": "excel",
"instruction": "Check the names in column \"Names with duplicates\" and put the unique ones in column \"Unique Names\". Keep the original order.", "instruction": "Check the names in column \"Names with duplicates\" and put the unique ones in column \"Unique Names\". Keep the original order.",
"source": "https://help.libreoffice.org/7.6/ro/text/scalc/guide/remove_duplicates.html?&DbPAR=SHARED&System=UNIX", "source": "https://help.libreoffice.org/7.6/ro/text/scalc/guide/remove_duplicates.html?&DbPAR=SHARED&System=UNIX",
"config": [ "config": [
@@ -9,8 +9,8 @@
"parameters": { "parameters": {
"files": [ "files": [
{ {
"url": "https://drive.usercontent.google.com/download?id=1A3O37a2M_tkmXHUn6G8kYu73cUMRUZnt&export=download&authuser=0&confirm=t&uuid=9a44147f-15e4-426c-9235-74fdda7439dc&at=APZUnTU4MAD7rODyryb9r0YolrrN:1705918712764", "url": "https://drive.usercontent.google.com/download?id=1_tLwdDWhm87eL7AmHE3lOTfQ2rszQKoC&export=download&authuser=0&confirm=t&uuid=0af8ae3f-e88a-486d-b259-3dff9d5698b5&at=APZUnTXFCuaUaUzrc85z3AJzC55K:1710923300199",
"path": "C:\\Users\\user\\Names_Duplicate_Unique.xlsx" "path": "C:\\Users\\chenj\\Names_Duplicate_Unique.xlsx"
} }
] ]
} }
@@ -18,20 +18,20 @@
{ {
"type": "open", "type": "open",
"parameters": { "parameters": {
"path": "C:\\Users\\user\\Names_Duplicate_Unique.xlsx" "path": "C:\\Users\\chenj\\Names_Duplicate_Unique.xlsx"
} }
} }
], ],
"trajectory": "trajectories/abed40dc-063f-4598-8ba5-9fe749c0615d", "trajectory": "trajectories/abed40dc-063f-4598-8ba5-9fe749c0615d",
"related_apps": [ "related_apps": [
"libreoffice calc" "excel"
], ],
"evaluator": { "evaluator": {
"postconfig": [ "postconfig": [
{ {
"type": "activate_window", "type": "activate_window",
"parameters": { "parameters": {
"window_name": "Names_Duplicate_Unique.xlsx - LibreOffice Calc", "window_name": "Names_Duplicate_Unique - Excel",
"strict": true "strict": true
} }
}, },
@@ -47,7 +47,7 @@
"command": [ "command": [
"python", "python",
"-c", "-c",
"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5); pyautogui.press(\"enter\");" "import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5);"
] ]
} }
}, },
@@ -61,12 +61,12 @@
"func": "compare_table", "func": "compare_table",
"expected": { "expected": {
"type": "cloud_file", "type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1GYG97VdmPG9mlhSBjMlMpjsuDsEDWXNB&export=download&authuser=0&confirm=t&uuid=6dd49f77-6a87-4f99-9027-0c74bad23d6d&at=APZUnTWzHV6JFiTPuo2ICUSEZqq8:1705918802025", "path": "https://drive.usercontent.google.com/download?id=1cDQh2Jy6mfrlx0qqfsgxib6TrdkGjX2X&export=download&authuser=0&confirm=t&uuid=9f1f07bd-5d53-466b-808a-132c24a8f14e&at=APZUnTVkEC3tIW7qngHFkDZa16MY:1710923492481",
"dest": "Names_Duplicate_Unique_gold.xlsx" "dest": "Names_Duplicate_Unique_gold.xlsx"
}, },
"result": { "result": {
"type": "vm_file", "type": "vm_file",
"path": "C:\\Users\\user\\Names_Duplicate_Unique.xlsx", "path": "C:\\Users\\chenj\\Names_Duplicate_Unique.xlsx",
"dest": "Names_Duplicate_Unique.xlsx" "dest": "Names_Duplicate_Unique.xlsx"
}, },
"options": { "options": {

20
evaluation_examples/examples/Windows/excel/eb03d19a-b88d-4de4-8a64-ca0ac66f426b.json
View File

@@ -1,7 +1,7 @@
{ {
"id": "eb03d19a-b88d-4de4-8a64-ca0ac66f426b", "id": "eb03d19a-b88d-4de4-8a64-ca0ac66f426b",
"snapshot": "libreoffice_calc", "snapshot": "excel",
"instruction": "Transpose the table and paste it starting from B8", "instruction": "Transpose the table and paste it starting from B8.",
"source": "https://www.youtube.com/shorts/t9JLUaT55UQ", "source": "https://www.youtube.com/shorts/t9JLUaT55UQ",
"config": [ "config": [
{ {
@@ -9,8 +9,8 @@
"parameters": { "parameters": {
"files": [ "files": [
{ {
"url": "https://drive.usercontent.google.com/download?id=1Zl6ZUkbiV9sISjJRfracd8-xKrW2G3yv&export=download&authuser=0&confirm=t&uuid=6799cca6-62d2-4cfa-b28a-b8da486b5d01&at=APZUnTVy7-CQMjvuFyu7ZMxztfT6:1705563660974", "url": "https://drive.usercontent.google.com/download?id=1sMI-1qKOTOCuIx89viIewfIUj9H9Xs7X&export=download&authuser=0&confirm=t&uuid=380b2a40-94b1-45fa-a1b2-55cea12c86c4&at=APZUnTW2BR0106cDVNrxr84AdsXK:1710923673262",
"path": "C:\\Users\\user\\Students_Class_Subject_Marks.xlsx" "path": "C:\\Users\\chenj\\Students_Class_Subject_Marks.xlsx"
} }
] ]
} }
@@ -18,20 +18,20 @@
{ {
"type": "open", "type": "open",
"parameters": { "parameters": {
"path": "C:\\Users\\user\\Students_Class_Subject_Marks.xlsx" "path": "C:\\Users\\chenj\\Students_Class_Subject_Marks.xlsx"
} }
} }
], ],
"trajectory": "trajectories/eb03d19a-b88d-4de4-8a64-ca0ac66f426b", "trajectory": "trajectories/eb03d19a-b88d-4de4-8a64-ca0ac66f426b",
"related_apps": [ "related_apps": [
"libreoffice calc" "excel"
], ],
"evaluator": { "evaluator": {
"postconfig": [ "postconfig": [
{ {
"type": "activate_window", "type": "activate_window",
"parameters": { "parameters": {
"window_name": "Students_Class_Subject_Marks.xlsx - LibreOffice Calc", "window_name": "Students_Class_Subject_Marks - Excel",
"strict": true "strict": true
} }
}, },
@@ -47,7 +47,7 @@
"command": [ "command": [
"python", "python",
"-c", "-c",
"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5); pyautogui.press(\"enter\");" "import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5);"
] ]
} }
}, },
@@ -61,12 +61,12 @@
"func": "compare_table", "func": "compare_table",
"expected": { "expected": {
"type": "cloud_file", "type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1GwqF52n6cvmQ-ivhmfNCep9mZboQ706-&export=download&authuser=0&confirm=t&uuid=4c750a20-73b8-4072-a4bf-3a4aa77b314a&at=APZUnTXYm-3NVdfneACqfDLYCWko:1705563733494", "path": "https://drive.usercontent.google.com/download?id=1fMOSUpjmnRyzJXZ5_Dfe8t2vEL7mavI5&export=download&authuser=0&confirm=t&uuid=59d7b8a3-01d1-41dd-a9cb-6551953f0d75&at=APZUnTX8cbi1YEtPjyXXD3yJMC7C:1710923745832",
"dest": "Students_Class_Subject_Marks_gold.xlsx" "dest": "Students_Class_Subject_Marks_gold.xlsx"
}, },
"result": { "result": {
"type": "vm_file", "type": "vm_file",
"path": "C:\\Users\\user\\Students_Class_Subject_Marks.xlsx", "path": "C:\\Users\\chenj\\Students_Class_Subject_Marks.xlsx",
"dest": "Students_Class_Subject_Marks.xlsx" "dest": "Students_Class_Subject_Marks.xlsx"
}, },
"options": { "options": {

18
evaluation_examples/examples/Windows/excel/ecb0df7a-4e8d-4a03-b162-053391d3afaf.json
View File

@@ -1,6 +1,6 @@
{ {
"id": "ecb0df7a-4e8d-4a03-b162-053391d3afaf", "id": "ecb0df7a-4e8d-4a03-b162-053391d3afaf",
"snapshot": "libreoffice_calc", "snapshot": "excel",
"instruction": "Enable each cell in the column\"Pass/Fail/Held\" is a drop down list", "instruction": "Enable each cell in the column\"Pass/Fail/Held\" is a drop down list",
"source": "https://www.youtube.com/shorts/tXOovKn0H68", "source": "https://www.youtube.com/shorts/tXOovKn0H68",
"config": [ "config": [
@@ -9,8 +9,8 @@
"parameters": { "parameters": {
"files": [ "files": [
{ {
"url": "https://drive.usercontent.google.com/download?id=1vKtnFG-sL7Ba0UCcUh4dWJDfZeAHAP6l&export=download&authuser=0&confirm=t&uuid=70a267fa-dc71-4893-8fe3-a7254e50c567&at=APZUnTVfnhipGIdrD39159Eqv9lf:1705567650653", "url": "https://drive.usercontent.google.com/download?id=1e0WA__QPxdOaRGExO9yZQjHLnMAD3ox9&export=download&authuser=0&confirm=t&uuid=206da40e-98f1-4e37-88d3-558e95cc3e96&at=APZUnTXTjDwHIZ6utAsff0P1d_6d:1710924265156",
"path": "C:\\Users\\user\\Order_Id_Mark_Pass_Fail.xlsx" "path": "C:\\Users\\chenj\\Order_Id_Mark_Pass_Fail.xlsx"
} }
] ]
} }
@@ -18,20 +18,20 @@
{ {
"type": "open", "type": "open",
"parameters": { "parameters": {
"path": "C:\\Users\\user\\Order_Id_Mark_Pass_Fail.xlsx" "path": "C:\\Users\\chenj\\Order_Id_Mark_Pass_Fail.xlsx"
} }
} }
], ],
"trajectory": "trajectories/ecb0df7a-4e8d-4a03-b162-053391d3afaf", "trajectory": "trajectories/ecb0df7a-4e8d-4a03-b162-053391d3afaf",
"related_apps": [ "related_apps": [
"libreoffice calc" "excel"
], ],
"evaluator": { "evaluator": {
"postconfig": [ "postconfig": [
{ {
"type": "activate_window", "type": "activate_window",
"parameters": { "parameters": {
"window_name": "Order_Id_Mark_Pass_Fail.xlsx - LibreOffice Calc", "window_name": "Order_Id_Mark_Pass_Fail - Excel",
"strict": true "strict": true
} }
}, },
@@ -47,7 +47,7 @@
"command": [ "command": [
"python", "python",
"-c", "-c",
"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5); pyautogui.press(\"enter\");" "import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5);"
] ]
} }
}, },
@@ -97,12 +97,12 @@
}, },
"result": { "result": {
"type": "vm_file", "type": "vm_file",
"path": "C:\\Users\\user\\Order_Id_Mark_Pass_Fail.xlsx", "path": "C:\\Users\\chenj\\Order_Id_Mark_Pass_Fail.xlsx",
"dest": "Order_Id_Mark_Pass_Fail.xlsx" "dest": "Order_Id_Mark_Pass_Fail.xlsx"
}, },
"expected": { "expected": {
"type": "cloud_file", "type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=125u7hc0LOcHXtJhOBd_Z5vt__7kwYOTQ&export=download&authuser=0&confirm=t&uuid=17e5da1b-fb1d-45d8-a9b1-6cd146ebaeee&at=APZUnTVqGyk6n5NPKzrq4KSOe871:1705898482101", "path": "https://drive.usercontent.google.com/download?id=11iY78aGztKWikebXZY1577Ptlf3ZLs9q&export=download&authuser=0&confirm=t&uuid=bd88dfc8-503b-4777-af17-64e4903908ff&at=APZUnTUy5IFEQejlUm9-kHpMfg00:1710926523841",
"dest": "Order_Id_Mark_Pass_Fail_gold.xlsx" "dest": "Order_Id_Mark_Pass_Fail_gold.xlsx"
} }
} }

96
evaluation_examples/examples/Windows/multi_app/185f29bd-5da0-40a6-b69c-ba7f4e0324ef.json Normal file
View File

@@ -0,0 +1,96 @@
{
"id": "185f29bd-5da0-40a6-b69c-ba7f4e0324ef",
"snapshot": "libreoffice_calc",
"instruction": "Transfer the data from our 'Employee Performance Evaluation Summary' Excel sheet into our standardized PDF evaluation forms. Each employee's evaluation data should be accurately filled into the designated fields of the PDF form. It's crucial that the final PDF documents retain a uniform and professional look, ready for distribution to our staff or for filing purposes. Furthermore, please ensure that each PDF file is named according to the employee's name as it appears in the Excel document. This will greatly streamline our evaluation process and enhance our efficiency in managing employee performance records. Oh, use \"√\" as mark on characters.",
"source": "authors",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"path": "C:\\Users\\chenj\\Desktop\\Employee Performance Evaluation Summary.xlsx",
"url": "https://drive.google.com/uc?id=1uOzi66bzO_WUnoS4Oqsodrd7_YPLatEk&export=download"
},
{
"path": "C:\\Users\\chenj\\Desktop\\review_template.pdf",
"url": "https://drive.google.com/uc?id=1YJ4RPGFUuS48tBh31gBerA16JSMw498w&export=download"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "C:\\Users\\chenj\\Desktop\\Employee Performance Evaluation Summary.xlsx"
}
},
{
"type": "sleep",
"parameters": {
"seconds": 2
}
},
{
"type": "open",
"parameters": {
"path": "C:\\Users\\chenj\\Desktop\\review_template.pdf"
}
}
],
"trajectory": "trajectories/185f29bd-5da0-40a6-b69c-ba7f4e0324ef",
"related_apps": [
"libreoffice_calc",
"os",
"pdf"
],
"evaluator": {
"func": "compare_pdfs",
"result": {
"type": "cloud_file",
"path": [
"https://drive.google.com/uc?id=1kZM90nA1krRmV9ug5_BBe8VlrZRVLiLK&export=download",
"https://drive.google.com/uc?id=1zyLzYYThwyit9ciXpfNfPFlBomolOauY&export=download",
"https://drive.google.com/uc?id=1gMT7JBftuymajMAO5rwksORpeVq3uGmH&export=download",
"https://drive.google.com/uc?id=1x0DdtUSZyBifl1tGIWlWKn255WusJeR4&export=download",
"https://drive.google.com/uc?id=1UAcG32WO8XCXElcanjGwbSpJwFuyOkts&export=download",
"https://drive.google.com/uc?id=1PRgryg7Y5evKnDG2LPtAttVp9qAf5VyZ&export=download",
"https://drive.google.com/uc?id=1JxEDriCS2W7BQLdkIgxu_WFCRa9ib4D7&export=download"
],
"dest": [
"Alex Lee_Gold.pdf",
"David Wilson_Gold.pdf",
"Emily Johnson_Gold.pdf",
"John Doe_Gold.pdf",
"Linda Green_Gold.pdf",
"Michael Brown_Gold.pdf",
"Sophia Carter_Gold.pdf"
],
"multi": true,
"gives": [0,1,2,3,4,5,6]
},
"expected": {
"type": "vm_file",
"path": [
"C:\\Users\\chenj\\Desktop\\Alex Lee.pdf",
"C:\\Users\\chenj\\Desktop\\David Wilson.pdf",
"C:\\Users\\chenj\\Desktop\\Emily Johnson.pdf",
"C:\\Users\\chenj\\Desktop\\John Doe.pdf",
"C:\\Users\\chenj\\Desktop\\Linda Green.pdf",
"C:\\Users\\chenj\\Desktop\\Michael Brown.pdf",
"C:\\Users\\chenj\\Desktop\\Sophia Carter.pdf"
],
"dest": [
"Alex Lee.pdf",
"David Wilson.pdf",
"Emily Johnson.pdf",
"John Doe.pdf",
"Linda Green.pdf",
"Michael Brown.pdf",
"Sophia Carter.pdf"
],
"multi": true,
"gives": [0,1,2,3,4,5,6]
}
}
}

109
evaluation_examples/examples/Windows/multi_app/1f18aa87-af6f-41ef-9853-cdb8f32ebdea.json Normal file
View File

@@ -0,0 +1,109 @@
{
"id": "1f18aa87-af6f-41ef-9853-cdb8f32ebdea",
"snapshot": "libreoffice_calc",
"instruction": "I've prepared some grammar tests and placed them in the 'Grammar test' folder. I've already provided the multiple-choice answers for Test 1 in the 'answer doc' file. Could you please follow the same format to write out the answers for the remaining two tests in the doc file? This way, I can distribute them to the students as a reference. Thank you.",
"source": "authors",
"config": [
{
"type": "command",
"parameters": {
"command": "mkdir \"C:\\Users\\chenj\\Desktop\\students work\" \"C:\\Users\\chenj\\Desktop\\Lec powerpoint\" \"C:\\Users\\chenj\\Desktop\\Grammar test\" \"C:\\Users\\chenj\\Desktop\\Grammar rules PDF\" C:\\Users\\chenj\\Desktop\\FDI",
"shell": true
}
},
{
"type": "download",
"parameters": {
"files": [
{
"path": "C:\\Users\\chenj\\Desktop\\Grammer test 1.docx",
"url": "https://drive.google.com/uc?id=1VaXQ9XdzMv079xKFs0Y2XrwdmwFHIvBK&export=download"
},
{
"path": "C:\\Users\\chenj\\Desktop\\Grammer test 2.docx",
"url": "https://drive.google.com/uc?id=1k2T88WreTwi-Yyp9mEJnreEQC3DdkJ2x&export=download"
},
{
"path": "C:\\Users\\chenj\\Desktop\\Grammer test 3.docx",
"url": "https://drive.google.com/uc?id=1QgyQWVOcAJuPaSlrywb9nuFiQDySsTb2&export=download"
},
{
"path": "C:\\Users\\chenj\\Desktop\\Answer.docx",
"url": "https://drive.google.com/uc?id=1BC2DuWJuZggmf6fXl6Ys9xQMZzU6a1br&export=download"
},
{
"path": "C:\\Users\\chenj\\Desktop\\Grammar rules PDF\\irregularrules02.pdf",
"url": "https://drive.google.com/uc?id=1Eln9ehX6y6Df2-S_Hp7Ao1teKRu6I1Tg&export=download"
},
{
"path": "C:\\Users\\chenj\\Desktop\\Grammar rules PDF\\irregularrules01.pdf",
"url": "https://drive.google.com/uc?id=1krdEEdNWvTwMKZU14QtI_xc2lCFVeVcl&export=download"
},
{
"path": "C:\\Users\\chenj\\Desktop\\Grammar rules PDF\\fragrules.pdf",
"url": "https://drive.google.com/uc?id=1IXyI2KeiXsuh6XV2LelcmhZ2PDh_dBQf&export=download"
},
{
"path": "C:\\Users\\chenj\\Desktop\\Grammar rules PDF\\csfsrules.pdf",
"url": "https://drive.google.com/uc?id=1ernwGGrjhYNoHVNAevdb2qNKQ0I5n3RP&export=download"
},
{
"path": "C:\\Users\\chenj\\Desktop\\Public Lecture Teaching Plan.docx",
"url": "https://drive.google.com/uc?id=1ywfVFTEbiSkypZpzLjLmq_ppSbQIC8s8&export=download"
},
{
"path": "C:\\Users\\chenj\\Desktop\\Course Timetable.xlsx",
"url": "https://drive.google.com/uc?id=1NGtahknRq_kXsXlw0tRQ1_CZp9SljoVg&export=download"
}
]
}
}
],
"trajectory": "trajectories/1f18aa87-af6f-41ef-9853-cdb8f32ebdea",
"related_apps": [
"os",
"libreoffice_writer"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "Answer - Word",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5); pyautogui.press(\"enter\");"
]
}
}
],
"func": "compare_docx_files",
"expected": {
"type": "cloud_file",
"path": "https://drive.google.com/uc?id=1TOMGWC3OFuP6yEGQuRJMEFWdg2NcBPSs&export=download",
"dest": "Answer gold.docx"
},
"result": {
"type": "vm_file",
"path": "C:\\Users\\chenj\\Desktop\\Answer.docx",
"dest": "Answer.docx"
},
"options": {
"ignore_case": true,
"ignore_blanks": true
}
}
}

68
evaluation_examples/examples/Windows/multi_app/26660ad1-6ebb-4f59-8cba-a8432dfe8d38.json Normal file
View File

@@ -0,0 +1,68 @@
{
"id": "26660ad1-6ebb-4f59-8cba-a8432dfe8d38",
"snapshot": "multiapps",
"instruction": "I want to test the quality of the network environment my laptop is currently in. Please measure my network situation through speedtest.net, export the measurement results, and save them to Documents\\Test\\Speed (if the dir does not exist, create it).",
"source": "https://www.speedtest.net/",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"ncat.exe", "-k", "-l", "0.0.0.0", "9222",
"--sh-exec", "ncat.exe 127.0.0.1 1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://www.speedtest.net/"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Google Chrome"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",
"related_apps":[
"os",
"browser"
],
"evaluator":{
"func": "compare_time_in_speedtest_results",
"result":{
"type": "vm_file",
"path": "C:\\Users\\chenj\\Documents\\Test\\Speed\\Speedtest Results Export-.csv",
"dest": "Speedtest Results Export-.csv",
"time_suffix": true
},
"expected":{
"type": "time_diff_range",
"diff_range_in_minutes": "60"
}
}
}

127
evaluation_examples/examples/Windows/multi_app/2c1ebcd7-9c6d-4c9a-afad-900e381ecd5e.json Normal file
View File

@@ -0,0 +1,127 @@
{
"id": "2c1ebcd7-9c6d-4c9a-afad-900e381ecd5e",
"snapshot": "libreoffice_calc",
"instruction": "Could you please take a moment to review the 'case study' file located within the 'student work' folder? I'm particularly interested in ensuring that the references section at the end of the document adheres to the APA 7th edition formatting guidelines. Making the necessary adjustments if it turns out that the current formatting does not align with APA 7 standards or exists some errors.",
"source": "authors",
"config": [
{
"type": "command",
"parameters": {
"command": "mkdir \"C:\\Users\\chenj\\Desktop\\students work\" \"C:\\Users\\chenj\\Desktop\\Lec powerpoint\" \"C:\\Users\\chenj\\Desktop\\Grammar test\" \"C:\\Users\\chenj\\Desktop\\Grammar rules PDF\" C:\\Users\\chenj\\Desktop\\FDI",
"shell": true
}
},
{
"type": "download",
"parameters": {
"files": [
{
"path": "C:\\Users\\chenj\\Desktop\\students work\\Zheng He .docx",
"url": "https://drive.google.com/uc?id=1wI4141LAthnY5m6qcCUaGgDooe4wiTgz&export=download"
},
{
"path": "C:\\Users\\chenj\\Desktop\\students work\\The literature reviews of weekly readings.docx",
"url": "https://drive.google.com/uc?id=18zoZCNtP-wTkxXp2FhH3O_NdLZKVMPIr&export=download"
},
{
"path": "C:\\Users\\chenj\\Desktop\\students work\\The British Justice System.docx",
"url": "https://drive.google.com/uc?id=1z3YHSN4CvC5kN1AwTWB_-plRS4p5GAch&export=download"
},
{
"path": "C:\\Users\\chenj\\Desktop\\students work\\quiz2.docx",
"url": "https://drive.google.com/uc?id=1R5Bii_kvnv_fZVXV-6DMt6Hgq-1gXMo1&export=download"
},
{
"path": "C:\\Users\\chenj\\Desktop\\students work\\quiz.docx",
"url": "https://drive.google.com/uc?id=1PvlGMVX7YkricrjoPRe0e5VQlHeozRPD&export=download"
},
{
"path": "C:\\Users\\chenj\\Desktop\\students work\\Q1&2&3.docx",
"url": "https://drive.google.com/uc?id=1kLQ3lnba6p9lqikHhKDdbqrYagHnZWU_&export=download"
},
{
"path": "C:\\Users\\chenj\\Desktop\\students work\\Photo Ethics in Journalism.docx",
"url": "https://drive.google.com/uc?id=1V6nG6HP_9Kb5KBCRTpaGsRTdPxnJSmRm&export=download"
},
{
"path": "C:\\Users\\chenj\\Desktop\\students work\\cassie.docx",
"url": "https://drive.google.com/uc?id=1cW9TGJy56vossXxDsdnutPyCbR70af7M&export=download"
},
{
"path": "C:\\Users\\chenj\\Desktop\\students work\\case study.docx",
"url": "https://drive.google.com/uc?id=11GzpoZvp4qnL2ukXdpbhH-a3zOIHhtDx&export=download"
},
{
"path": "C:\\Users\\chenj\\Desktop\\Grammar rules PDF\\irregularrules02.pdf",
"url": "https://drive.google.com/uc?id=1Eln9ehX6y6Df2-S_Hp7Ao1teKRu6I1Tg&export=download"
},
{
"path": "C:\\Users\\chenj\\Desktop\\Grammar rules PDF\\irregularrules01.pdf",
"url": "https://drive.google.com/uc?id=1krdEEdNWvTwMKZU14QtI_xc2lCFVeVcl&export=download"
},
{
"path": "C:\\Users\\chenj\\Desktop\\Grammar rules PDF\\fragrules.pdf",
"url": "https://drive.google.com/uc?id=1IXyI2KeiXsuh6XV2LelcmhZ2PDh_dBQf&export=download"
},
{
"path": "C:\\Users\\chenj\\Desktop\\Grammar rules PDF\\csfsrules.pdf",
"url": "https://drive.google.com/uc?id=1ernwGGrjhYNoHVNAevdb2qNKQ0I5n3RP&export=download"
},
{
"path": "C:\\Users\\chenj\\Desktop\\Public Lecture Teaching Plan.docx",
"url": "https://drive.google.com/uc?id=1ywfVFTEbiSkypZpzLjLmq_ppSbQIC8s8&export=download"
},
{
"path": "C:\\Users\\chenj\\Desktop\\Course Timetable.xlsx",
"url": "https://drive.google.com/uc?id=1NGtahknRq_kXsXlw0tRQ1_CZp9SljoVg&export=download"
}
]
}
}
],
"trajectory": "trajectories/2c1ebcd7-9c6d-4c9a-afad-900e381ecd5e",
"related_apps": [
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "case study - Word",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5); pyautogui.press(\"enter\");"
]
}
}
],
"func": "compare_references",
"expected": {
"type": "cloud_file",
"path": "https://drive.google.com/uc?id=1325Qfch0JaJ_wJ20ICxMoHeW8KLpK8v0&export=download",
"dest": "case study gold.docx"
},
"result": {
"type": "vm_file",
"path": "C:\\Users\\chenj\\Desktop\\students work\\case study.docx",
"dest": "case study.docx"
},
"options": {
"content_only": true,
"reference_base_result": 0.92
}
}
}

162
evaluation_examples/examples/Windows/multi_app/3a93cae4-ad3e-403e-8c12-65303b271818.json Normal file
View File

@@ -0,0 +1,162 @@
{
"id": "3a93cae4-ad3e-403e-8c12-65303b271818",
"snapshot": "libreoffice_calc",
"instruction": "Could you please add a two-hour lecture slot to my weekly course timetable, scheduled for every Wednesday at 12 PM? It seems I accidentally omitted that when setting up my schedule. I'd appreciate you taking care of that for me. Thanks!",
"source": "authors",
"config": [
{
"type": "command",
"parameters": {
"command": "mkdir \"C:\\Users\\chenj\\Desktop\\students work\" \"C:\\Users\\chenj\\Desktop\\Lec powerpoint\" \"C:\\Users\\chenj\\Desktop\\Grammar test\" \"C:\\Users\\chenj\\Desktop\\Grammar rules PDF\" C:\\Users\\chenj\\Desktop\\FDI",
"shell": true
}
},
{
"type": "download",
"parameters": {
"files": [
{
"path": "C:\\Users\\chenj\\Desktop\\students work\\Zheng He .docx",
"url": "https://drive.google.com/uc?id=1wI4141LAthnY5m6qcCUaGgDooe4wiTgz&export=download"
},
{
"path": "C:\\Users\\chenj\\Desktop\\students work\\cassie.docx",
"url": "https://drive.google.com/uc?id=1cW9TGJy56vossXxDsdnutPyCbR70af7M&export=download"
},
{
"path": "C:\\Users\\chenj\\Desktop\\students work\\case study.docx",
"url": "https://drive.google.com/uc?id=11GzpoZvp4qnL2ukXdpbhH-a3zOIHhtDx&export=download"
},
{
"path": "C:\\Users\\chenj\\Desktop\\Grammar rules PDF\\irregularrules02.pdf",
"url": "https://drive.google.com/uc?id=1Eln9ehX6y6Df2-S_Hp7Ao1teKRu6I1Tg&export=download"
},
{
"path": "C:\\Users\\chenj\\Desktop\\Grammar rules PDF\\irregularrules01.pdf",
"url": "https://drive.google.com/uc?id=1krdEEdNWvTwMKZU14QtI_xc2lCFVeVcl&export=download"
},
{
"path": "C:\\Users\\chenj\\Desktop\\Grammar rules PDF\\fragrules.pdf",
"url": "https://drive.google.com/uc?id=1IXyI2KeiXsuh6XV2LelcmhZ2PDh_dBQf&export=download"
},
{
"path": "C:\\Users\\chenj\\Desktop\\Grammar rules PDF\\csfsrules.pdf",
"url": "https://drive.google.com/uc?id=1ernwGGrjhYNoHVNAevdb2qNKQ0I5n3RP&export=download"
},
{
"path": "C:\\Users\\chenj\\Desktop\\Public Lecture Teaching Plan.docx",
"url": "https://drive.google.com/uc?id=1ywfVFTEbiSkypZpzLjLmq_ppSbQIC8s8&export=download"
},
{
"path": "C:\\Users\\chenj\\Desktop\\Course Timetable.xlsx",
"url": "https://drive.google.com/uc?id=1NGtahknRq_kXsXlw0tRQ1_CZp9SljoVg&export=download"
}
]
}
}
],
"trajectory": "trajectories/3a93cae4-ad3e-403e-8c12-65303b271818",
"related_apps": [
"os",
"libreoffice_calc"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "Course Timetable - Excel",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5); pyautogui.press(\"enter\");"
]
}
}
],
"func": [
"compare_table",
"compare_table",
"compare_table"
],
"result": [
{
"type": "vm_file",
"path": "C:\\Users\\chenj\\Desktop\\Course Timetable.xlsx",
"dest": "Course Timetable.xlsx"
},
{
"type": "vm_file",
"path": "C:\\Users\\chenj\\Desktop\\Course Timetable.xlsx",
"dest": "Course Timetable.xlsx"
},
{
"type": "vm_file",
"path": "C:\\Users\\chenj\\Desktop\\Course Timetable.xlsx",
"dest": "Course Timetable.xlsx"
}
],
"expected": [
{
"type": "cloud_file",
"path": "https://drive.google.com/uc?id=1VMOon8byWuoCW2Uk5etGMJLMzAfwFVyB&export=download",
"dest": "Course Timetable gold.xlsx"
},
{
"type": "cloud_file",
"path": "https://drive.google.com/uc?id=1jAThiIqILZ5t-RFPHVniSvAL8ZJO1H3P&export=download",
"dest": "Course Timetable gold 2.xlsx"
},
{
"type": "cloud_file",
"path": "https://drive.google.com/uc?id=1U0THDtPCgsw-Rb0N9fjF8DeOepPeUajP&export=download",
"dest": "Course Timetable gold 3.xlsx"
}
],
"options": [
{
"rules": [
{
"type": "sheet_data",
"sheet_idx0": "RNSheet1",
"sheet_idx1": "ENSheet1",
"ignore_case": true
}
]
},
{
"rules": [
{
"type": "sheet_data",
"sheet_idx0": "RNSheet1",
"sheet_idx1": "ENSheet1",
"ignore_case": true
}
]
},
{
"rules": [
{
"type": "sheet_data",
"sheet_idx0": "RNSheet1",
"sheet_idx1": "ENSheet1",
"ignore_case": true
}
]
}
],
"conj": "or"
}
}

4
evaluation_examples/examples/Windows/multi_app/46407397-a7d5-4c6b-92c6-dbe038b1457b.json
View File

@@ -74,14 +74,14 @@
{ {
"type": "execute", "type": "execute",
"parameters": { "parameters": {
"command": "rm -r C:\\Users\\chenj\\AppData\\Roaming\\Thunderbird", "command": "rd /s /q C:\\Users\\chenj\\AppData\\Roaming\\Thunderbird",
"shell": true "shell": true
} }
}, },
{ {
"type": "execute", "type": "execute",
"parameters": { "parameters": {
"command": "mv C:\\Users\\chenj\\Thunderbird C:\\Users\\chenj\\AppData\\Roaming\\Thunderbird", "command": "move C:\\Users\\chenj\\Thunderbird C:\\Users\\chenj\\AppData\\Roaming\\Thunderbird",
"shell": true "shell": true
} }
}, },

72
evaluation_examples/examples/Windows/multi_app/6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a.json Normal file
View File

@@ -0,0 +1,72 @@
{
"id": "6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a",
"snapshot": "multiapps",
"instruction": "I now want to count the meeting cities of the three machine learning conferences in the past ten years from 2013 to 2019(including 2013 and 2019). I have listed the names and years of the conferences in excel. Please fill in the vacant locations.",
"source": "author",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=19wUxTQeoKr6ihJWJ_9cu2tzKQH0cnxWH",
"path": "C:\\Users\\chenj\\Desktop\\ConferenceCity.xlsx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "C:\\Users\\chenj\\Desktop\\ConferenceCity.xlsx"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"calc", "chrome", "os"
],
"evaluator": {
"postconfig":[
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1ZcITkIOs2Z86S5L6MShSohFs3_xVfeCP",
"path": "C:\\Users\\chenj\\Desktop\\ConferenceCity_Gold.xlsx"
}
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "ConferenceCity - Excel"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5); pyautogui.press(\"enter\");"
]
}
}
],
"func": "compare_conference_city_in_order",
"expected": {
"type": "rule",
"rules":{
"expected": ["Scottsdale","Atlanta","Lake Tahoe","Banff","Beijing",["Montreal", "Montréal"],"San Diego","Lille",["Montreal", "Montréal"],"San Juan",["New York", "New York City", "NYC"],"Barcelona","Toulon","Sydney","Long Beach","Vancouver","Stockholm",["Montreal", "Montréal"],"New Orleans","Long Beach","Vancouver"]
}
},
"result": {
"type": "vm_file",
"path": "C:\\Users\\chenj\\Desktop\\ConferenceCity.xlsx",
"dest": "ConferenceCity.xlsx"
}
}
}

30
evaluation_examples/examples/Windows/multi_app/74d5859f-ed66-4d3e-aa0e-93d7a592ce41.json
View File

@@ -1,14 +1,14 @@
{ {
"id": "74d5859f-ed66-4d3e-aa0e-93d7a592ce41", "id": "74d5859f-ed66-4d3e-aa0e-93d7a592ce41",
"snapshot": "chrome", "snapshot": "chrome",
"instruction": "Help me to set up an initial web extension project with help of the web tool, tagging it \"happy-extension v0.0.1\". Leave description blank for now. Include a background script and browser action, while other features are not required. Remember to unzip the auto-generated folder into \"~/Projects\".", "instruction": "Help me to set up an initial web extension project with help of the web tool, tagging it \"happy-extension v0.0.1\". Leave description blank for now. Include a background script and browser action, while other features are not required. Remember to unzip the auto-generated folder into \"Documents\\Projects\".",
"source": "authors", "source": "authors",
"config": [ "config": [
{ {
"type": "launch", "type": "launch",
"parameters": { "parameters": {
"command": [ "command": [
"google-chrome", "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe",
"--remote-debugging-port=1337" "--remote-debugging-port=1337"
] ]
} }
@@ -17,9 +17,8 @@
"type": "launch", "type": "launch",
"parameters": { "parameters": {
"command": [ "command": [
"socat", "ncat.exe", "-k", "-l", "0.0.0.0", "9222",
"tcp-listen:9222,fork", "--sh-exec", "ncat.exe 127.0.0.1 1337"
"tcp:localhost:1337"
] ]
} }
}, },
@@ -34,19 +33,16 @@
{ {
"type": "execute", "type": "execute",
"parameters": { "parameters": {
"command": [ "command": "mkdir C:\\Users\\chenj\\Documents\\Projects",
"mkdir", "shell": "true"
"-p",
"/home/user/Projects"
]
} }
}, },
{ {
"type": "launch", "type": "launch",
"parameters": { "parameters": {
"command": [ "command": [
"nautilus", "explorer.exe",
"/home/user/Projects" "C:\\Users\\chenj\\Documents\\Projects"
] ]
} }
} }
@@ -67,27 +63,27 @@
"result": [ "result": [
{ {
"type": "vm_file", "type": "vm_file",
"path": "/home/user/Projects/happy-extension/manifest.json", "path": "C:\\Users\\chenj\\Documents\\Projects\\happy-extension\\manifest.json",
"dest": "manifest.json" "dest": "manifest.json"
}, },
{ {
"type": "vm_file", "type": "vm_file",
"path": "/home/user/Projects/happy-extension/background_script.js", "path": "C:\\Users\\chenj\\Documents\\Projects\\happy-extension\\background_script.js",
"dest": "background_script.js" "dest": "background_script.js"
}, },
{ {
"type": "vm_file", "type": "vm_file",
"path": "/home/user/Projects/happy-extension/browserAction/index.html", "path": "C:\\Users\\chenj\\Documents\\Projects\\happy-extension\\browserAction\\index.html",
"dest": "index.html" "dest": "index.html"
}, },
{ {
"type": "vm_file", "type": "vm_file",
"path": "/home/user/Projects/happy-extension/browserAction/style.css", "path": "C:\\Users\\chenj\\Documents\\Projects\\happy-extension\\browserAction\\style.css",
"dest": "style.css" "dest": "style.css"
}, },
{ {
"type": "vm_file", "type": "vm_file",
"path": "/home/user/Projects/happy-extension/browserAction/script.js", "path": "C:\\Users\\chenj\\Documents\\Projects\\happy-extension\\browserAction\\script.js",
"dest": "script.js" "dest": "script.js"
} }
], ],

241
evaluation_examples/examples/Windows/multi_app/8e116af7-7db7-4e35-a68b-b0939c066c78.json Normal file
View File

@@ -0,0 +1,241 @@
{
"id": "8e116af7-7db7-4e35-a68b-b0939c066c78",
"snapshot": "libreoffice_calc",
"instruction": "Please update my bookkeeping sheet with the recent transactions from the provided folder, detailing my expenses over the past few days.",
"source": "authors",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"path": "C:\\Users\\chenj\\Desktop\\my_bookkeeping.xlsx",
"url": "https://drive.google.com/uc?id=1QOSpTZPFzFZeC0tng4Gfws544LFln836&export=download"
},
{
"path": "C:\\Users\\chenj\\Desktop\\receipt_0.jpeg",
"url": "https://drive.google.com/uc?id=1b0BRc-BzXObVCUEonJfRbDsrgxZugj3U&export=download"
},
{
"path": "C:\\Users\\chenj\\Desktop\\receipt_1.jpg",
"url": "https://drive.google.com/uc?id=1S-JBDqwEf7Z_JXDItK_F4BOHgScTjlyN&export=download"
},
{
"path": "C:\\Users\\chenj\\Desktop\\receipt_2.jpg",
"url": "https://drive.google.com/uc?id=1Ys2abZi9_0y8sxuj2vCbC0OhjC6YdrC-&export=download"
},
{
"path": "C:\\Users\\chenj\\Desktop\\receipt_3.pdf",
"url": "https://drive.google.com/uc?id=1sKvBbGDpmUkv891xTqX7w5dtEvchQahd&export=download"
},
{
"path": "C:\\Users\\chenj\\Desktop\\receipt_4.jpg",
"url": "https://drive.google.com/uc?id=1kW7xH5bc2jRaKGDKHDrgSehTrPgkyzkc&export=download"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "C:\\Users\\chenj\\Desktop\\my_bookkeeping.xlsx"
}
}
],
"trajectory": "trajectories/8e116af7-7db7-4e35-a68b-b0939c066c78",
"related_apps": [
"libreoffice_calc",
"os",
"image",
"pdf"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "my_bookkeeping - Excel",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5); pyautogui.press(\"enter\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 1.0
}
}
],
"func": "compare_table",
"result": {
"type": "vm_file",
"path": "C:\\Users\\chenj\\Desktop\\my_bookkeeping.xlsx",
"dest": "my_bookkeeping.xlsx"
},
"expected": {
"type": "cloud_file",
"path": "https://drive.google.com/uc?id=1ygEDdVlkf2ZyqMxJ_ktqo9G_g--rc6co&export=download",
"dest": "my_bookkeeping_gold.xlsx"
},
"options": {
"rules": [
{
"type": "sheet_fuzzy",
"sheet_idx0": "RNSheet1",
"sheet_idx1": "ENSheet1",
"rules": [
{
"range": [
"A1:A8",
"B1:B8",
"C1:C8",
"D1:D8",
"E1:E8"
],
"type": "exact_match"
}
]
},
{
"type": "sheet_fuzzy",
"sheet_idx0": "RNSheet1",
"sheet_idx1": "ENSheet1",
"rules": [
{
"range": [
"C9:C13"
],
"type": "exact_match",
"ignore_case": true
}
]
},
{
"type": "check_cell",
"sheet_idx": 0,
"coordinate": "D9",
"props": {
"value": {
"method": "approx:0.1",
"ref": -186.93
}
}
},
{
"type": "check_cell",
"sheet_idx": 0,
"coordinate": "D10",
"props": {
"value": {
"method": "approx:0.1",
"ref": -3670
}
}
},
{
"type": "check_cell",
"sheet_idx": 0,
"coordinate": "D11",
"props": {
"value": {
"method": "approx:0.1",
"ref": -5.7
}
}
},
{
"type": "check_cell",
"sheet_idx": 0,
"coordinate": "D12",
"props": {
"value": {
"method": "approx:0.1",
"ref": -154.06
}
}
},
{
"type": "check_cell",
"sheet_idx": 0,
"coordinate": "D13",
"props": {
"value": {
"method": "approx:0.1",
"ref": -8.1
}
}
},
{
"type": "check_cell",
"sheet_idx": 0,
"coordinate": "E9",
"props": {
"value": {
"method": "approx:0.1",
"ref": 603.07
}
}
},
{
"type": "check_cell",
"sheet_idx": 0,
"coordinate": "E10",
"props": {
"value": {
"method": "approx:0.1",
"ref": -3066.93
}
}
},
{
"type": "check_cell",
"sheet_idx": 0,
"coordinate": "E11",
"props": {
"value": {
"method": "approx:0.1",
"ref": -3072.63
}
}
},
{
"type": "check_cell",
"sheet_idx": 0,
"coordinate": "E12",
"props": {
"value": {
"method": "approx:0.1",
"ref": -3226.69
}
}
},
{
"type": "check_cell",
"sheet_idx": 0,
"coordinate": "E13",
"props": {
"value": {
"method": "approx:0.1",
"ref": -3234.79
}
}
}
]
}
}
}

91
evaluation_examples/examples/Windows/multi_app/a82b78bb-7fde-4cb3-94a4-035baf10bcf0.json Normal file
View File

@@ -0,0 +1,91 @@
{
"id": "a82b78bb-7fde-4cb3-94a4-035baf10bcf0",
"snapshot": "libreoffice_calc",
"instruction": "I'm really enjoying this paper. Could you please locate the personal webpages of the initial author and the last three authors? Please include them in a browser bookmark folder titled 'Liked Authors.'",
"source": "authors",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"ncat.exe", "-k", "-l", "0.0.0.0", "9222",
"--sh-exec", "ncat.exe 127.0.0.1 1337"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 2
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1IlJ8kU5MlR6OqJHchsSUJzLCmcrG-8N7&export=download&authuser=0&confirm=t&uuid=d2a1810f-edea-4bfd-9d79-e668b9f11876&at=APZUnTVv_eqtC86YzkEU8_jIhC9W:1709522229162",
"path": "C:\\Users\\chenj\\Desktop\\2206.08853.pdf"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "C:\\Users\\chenj\\Desktop\\2206.08853.pdf"
}
},
{
"type": "sleep",
"parameters": {
"seconds": 2
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('f11'); time.sleep(0.5); pyautogui.click(960, 540); time.sleep(0.5); pyautogui.scroll(-20)"
]
}
}
],
"trajectory": "trajectories/a82b78bb-7fde-4cb3-94a4-035baf10bcf0",
"related_apps": [
"chrome",
"pdf"
],
"evaluator": {
"func": "is_expected_bookmarks",
"result": {
"type": "bookmarks"
},
"expected": {
"type": "rule",
"rules": {
"type": "liked_authors_websites_urls",
"names": [
"Liked Authors"
],
"urls": [
["https://jimfan.me/", "https://research.nvidia.com/person/linxi-jim-fan"],
["https://research.nvidia.com/person/de-an-huang", "https://ai.stanford.edu/~dahuang/"],
["https://yukezhu.me/", "https://www.cs.utexas.edu/people/faculty-researchers/yuke-zhu", "https://experts.utexas.edu/yuke_zhu", "https://research.nvidia.com/person/yuke-zhu"],
["http://tensorlab.cms.caltech.edu/users/anima/", "https://www.eas.caltech.edu/people/anima"]
]
}
}
}
}

38
evaluation_examples/examples/Windows/multi_app/b5062e3e-641c-4e3a-907b-ac864d2e7652.json
View File

@@ -1,17 +1,14 @@
{ {
"id": "b5062e3e-641c-4e3a-907b-ac864d2e7652", "id": "b5062e3e-641c-4e3a-907b-ac864d2e7652",
"snapshot": "libreoffice_calc", "snapshot": "libreoffice_calc",
"instruction": "Please help me to extract the name, e-mail, and affiliation of the first author from each paper in the folder and organize them in an Excel table. Include headers for each field. Sort the authors by their full names alphabetically and save the table as \"~/authors.xlsx\".", "instruction": "Please help me to extract the name, e-mail, and affiliation of the first author from each paper in the folder and organize them in an Excel table. Include headers for each field. Sort the authors by their full names alphabetically and save the table as \"Documents\\authors.xlsx\".",
"source": "authors", "source": "authors",
"config": [ "config": [
{ {
"type": "command", "type": "command",
"parameters": { "parameters": {
"command": [ "command": "mkdir C:\\Users\\chenj\\Documents\\Papers",
"mkdir", "shell": true
"-p",
"/home/user/Documents/Papers"
]
} }
}, },
{ {
@@ -19,19 +16,19 @@
"parameters": { "parameters": {
"files": [ "files": [
{ {
"path": "/home/user/Documents/Papers/zhang_appagent.pdf", "path": "C:\\Users\\chenj\\Documents\\Papers\\zhang_appagent.pdf",
"url": "https://arxiv.org/pdf/2312.13771.pdf" "url": "https://arxiv.org/pdf/2312.13771.pdf"
}, },
{ {
"path": "/home/user/Documents/Papers/niu_screenagent.pdf", "path": "C:\\Users\\chenj\\Documents\\Papers\\niu_screenagent.pdf",
"url": "https://arxiv.org/pdf/2402.07945.pdf" "url": "https://arxiv.org/pdf/2402.07945.pdf"
}, },
{ {
"path": "/home/user/Documents/Papers/koh_visualwebarena.pdf", "path": "C:\\Users\\chenj\\Documents\\Papers\\koh_visualwebarena.pdf",
"url": "https://arxiv.org/pdf/2401.13649.pdf" "url": "https://arxiv.org/pdf/2401.13649.pdf"
}, },
{ {
"path": "/home/user/Documents/Papers/deng_mind2web.pdf", "path": "C:\\Users\\chenj\\Documents\\Papers\\deng_mind2web.pdf",
"url": "https://papers.nips.cc/paper_files/paper/2023/file/5950bf290a1570ea401bf98882128160-Paper-Datasets_and_Benchmarks.pdf" "url": "https://papers.nips.cc/paper_files/paper/2023/file/5950bf290a1570ea401bf98882128160-Paper-Datasets_and_Benchmarks.pdf"
} }
] ]
@@ -41,8 +38,8 @@
"type": "launch", "type": "launch",
"parameters": { "parameters": {
"command": [ "command": [
"nautilus", "explorer.exe",
"/home/user/Documents/Papers" "C:\\Users\\chenj\\Documents\\Papers"
] ]
} }
} }
@@ -53,25 +50,10 @@
"os" "os"
], ],
"evaluator": { "evaluator": {
"postconfig": [
{
"type": "execute",
"parameters": {
"command": [
"libreoffice",
"--convert-to",
"csv:Text - txt - csv (StarCalc):44,34,UTF-8,,,,false,true,true,false,false,1",
"--outdir",
"/home/user",
"/home/user/authors.xlsx"
]
}
}
],
"func": "compare_table", "func": "compare_table",
"result": { "result": {
"type": "vm_file", "type": "vm_file",
"path": "/home/user/authors.xlsx", "path": "C:\\Users\\chenj\\authors.xlsx",
"dest": "authors.xlsx" "dest": "authors.xlsx"
}, },
"expected": { "expected": {

4
evaluation_examples/examples/Windows/multi_app/c867c42d-a52d-4a24-8ae3-f75d256b5618.json
View File

@@ -35,14 +35,14 @@
{ {
"type": "execute", "type": "execute",
"parameters": { "parameters": {
"command": "rm -r C:\\Users\\chenj\\AppData\\Roaming\\Thunderbird", "command": "rd /s /q C:\\Users\\chenj\\AppData\\Roaming\\Thunderbird",
"shell": true "shell": true
} }
}, },
{ {
"type": "execute", "type": "execute",
"parameters": { "parameters": {
"command": "mv C:\\Users\\chenj\\Thunderbird C:\\Users\\chenj\\AppData\\Roaming\\Thunderbird", "command": "move C:\\Users\\chenj\\Thunderbird C:\\Users\\chenj\\AppData\\Roaming\\Thunderbird",
"shell": true "shell": true
} }
}, },

128
evaluation_examples/examples/Windows/multi_app/d1acdb87-bb67-4f30-84aa-990e56a09c92.json Normal file
View File

@@ -0,0 +1,128 @@
{
"id": "d1acdb87-bb67-4f30-84aa-990e56a09c92",
"snapshot": "libreoffice_calc",
"instruction": "Hello! I'm eagerly planning a culinary adventure to Hong Kong and have curated a list of must-visit restaurants that I've been longing to explore. However, I could use some assistance in compiling a few essential details about these establishments. Would you be so kind as to help me out? It would be fantastic if you could search for these restaurants on Google Maps. I'm particularly interested in obtaining their addresses, any available websites, and contact phone numbers. If you could gather this information and input it into my form file, I would be immensely grateful. Many thanks in advance!",
"source": "authors",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"path": "C:\\Users\\chenj\\Desktop\\restaurants.txt",
"url": "https://drive.google.com/uc?id=1IehFLJPZcFv8Ujk31ExbyGLji9AylmmJ&export=download"
},
{
"path": "C:\\Users\\chenj\\Desktop\\MUST_VISIT.xlsx",
"url": "https://drive.google.com/uc?id=1fXmjvZcwkIcckMIAXi3Hv_JAbVWpgs_l&export=download"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "C:\\Users\\chenj\\Desktop\\MUST_VISIT.xlsx"
}
},
{
"type": "open",
"parameters": {
"path": "C:\\Users\\chenj\\Desktop\\restaurants.txt"
}
},
{
"type": "sleep",
"parameters": {
"seconds": 5
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "restaurants.txt"
}
}
],
"trajectory": "trajectories/d1acdb87-bb67-4f30-84aa-990e56a09c92",
"related_apps": [
"os",
"chrome",
"libreoffice_calc"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "MUST_VISIT - Excel",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5); pyautogui.press(\"enter\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 1.0
}
}
],
"func": "compare_table",
"result": {
"type": "vm_file",
"path": "C:\\Users\\chenj\\Desktop\\MUST_VISIT.xlsx",
"dest": "MUST_VISIT.xlsx"
},
"expected": {
"type": "cloud_file",
"path": "https://drive.google.com/uc?id=1MV6jBvRbbYwPqeFTd_nX40xzyltNhphl&export=download",
"dest": "MUST_VISIT-gt.xlsx"
},
"options": {
"rules": [
{
"type": "sheet_fuzzy",
"sheet_idx0": "RNSheet1",
"sheet_idx1": "ENSheet1",
"rules": [
{
"range": ["A1:A6", "D1:D6"],
"type": "exact_match"
},
{
"range": ["B1:B6"],
"type": "fuzzy_match",
"threshold": 85,
"normalization": [
["Rd", "Road"],
["St", "Street"]
],
"ignore_case": true
},
{
"range": ["C1:C6"],
"type": "includes",
"trim_leadings": "+ ",
"ignore_chars": " ()-"
}
]
}
]
}
}
}

101
evaluation_examples/examples/Windows/multi_app/deec51c9-3b1e-4b9e-993c-4776f20e8bb2.json Normal file
View File

@@ -0,0 +1,101 @@
{
"id": "deec51c9-3b1e-4b9e-993c-4776f20e8bb2",
"snapshot": "libreoffice_calc",
"instruction": "Find a paper list of all the new foundation language models issued on 11st Oct. 2023 via arxiv daily, and organize it into the sheet I opened.",
"source": "authors",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"path": "C:\\Users\\chenj\\Desktop\\New Large Language Models.xlsx",
"url": "https://drive.google.com/uc?id=1NJFAUDzatd5TbBqXeCy3-ok4BWj-xayT&export=download"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "C:\\Users\\chenj\\Desktop\\New Large Language Models.xlsx"
}
}
],
"trajectory": "trajectories/deec51c9-3b1e-4b9e-993c-4776f20e8bb2",
"related_apps": [
"libreoffice_calc",
"chrome",
"os"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "New Large Language Models - Excel",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5); pyautogui.press(\"enter\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 1.0
}
}
],
"func": "compare_table",
"result": {
"type": "vm_file",
"path": "C:\\Users\\chenj\\Desktop\\New Large Language Models.xlsx",
"dest": "New Large Language Models.xlsx"
},
"expected": {
"type": "cloud_file",
"path": "https://drive.google.com/uc?id=1BHOyjFo72b74YKWTqPMaoNvCzICkos-G&export=download",
"dest": "New Large Language Models Gold.xlsx"
},
"options": {
"rules": [
{
"type": "sheet_fuzzy",
"sheet_idx0": "RNSheet1",
"sheet_idx1": "ENSheet1",
"rules": [
{
"range": [
"B2:B5",
"C2:C5"
],
"type": "exact_match"
},
{
"range": [
"A2:A5"
],
"type": "fuzzy_match",
"threshold": 90,
"ignore_case": true
}
]
}
]
}
}
}

121
evaluation_examples/examples/Windows/multi_app/e2392362-125e-4f76-a2ee-524b183a3412.json Normal file
View File

@@ -0,0 +1,121 @@
{
"id": "e2392362-125e-4f76-a2ee-524b183a3412",
"snapshot": "chrome",
"instruction": "I recently started using the famous personal academic homepage template from academicpages.github.io to build my own personal homepage, and I have cloned it to my local Documents\\Code\\Website folder. According to an online tutorial, I can configure my name and contact information in the _config.yaml file. However, I am not familiar with the YAML file format. Please help me find the sections related to the name and contact information in this file and change them to \"Test Account\" and \"Test@gmail.com\".",
"source": "authors",
"config": [
{
"type": "command",
"parameters": {
"command": "mkdir C:\\Users\\chenj\\Documents\\Code\\Website",
"shell": true
}
},
{
"type": "download",
"parameters": {
"files": [
{
"path": "C:\\Users\\chenj\\.tmp.7z",
"url": "https://drive.google.com/uc?id=1LYc6rBSuCNBtTQIg-m9zP6KmlEB_Zfdo&export=download"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"C:\\Program Files\\7-Zip\\7z.exe",
"x", "-oC:\\Users\\chenj\\Documents\\Code\\Website",
"C:\\Users\\chenj\\.tmp.7z"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"ncat.exe", "-k", "-l", "0.0.0.0", "9222",
"--sh-exec", "ncat.exe 127.0.0.1 1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://academicpages.github.io/"
]
}
}
],
"trajectory": "trajectories/e2392362-125e-4f76-a2ee-524b183a3412",
"related_apps": [
"chrome",
"os",
"vscode"
],
"evaluator": {
"postconfig": [
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);"
]
}
}
],
"func": "check_json",
"options": {
"is_yaml": true
},
"expected": {
"type": "rule",
"rules": {
"expect": [
{
"key": [
"name"
],
"method": "eq",
"ref": "Test Account"
},
{
"key": [
"author",
"name"
],
"method": "eq",
"ref": "Test Account"
},
{
"key": [
"author",
"email"
],
"method": "eq",
"ref": "Test@gmail.com"
}
]
}
},
"result": {
"type": "vm_file",
"path": "C:\\Users\\chenj\\Documents\\Code\\Website\\academicpages.github.io\\_config.yml",
"dest": "_config.yaml"
}
}
}

82
evaluation_examples/examples/Windows/multi_app/eb303e01-261e-4972-8c07-c9b4e7a4922a.json Normal file
View File

@@ -0,0 +1,82 @@
{
"id": "eb303e01-261e-4972-8c07-c9b4e7a4922a",
"snapshot": "libreoffice_impress",
"instruction": "Tomorrow, I'm scheduled to deliver a talk, and my PowerPoint slides and speaking notes are saved on the desktop. Help me insert my planned remarks for each slide into the \"note\" section of the PowerPoint as a reminder. I've completed this task for some slides; assist me in completing the remaining part.",
"source": "authors",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1MdgN8ECxzLgHgjq8eKqrNQt3MPDjnKwa&export=download&authuser=0&confirm=t&uuid=ed5c37db-c565-4ca1-bbd1-bbdba13e9306&at=APZUnTUNi8YTLlZqMZ0r--bBpBEG:1709449877819",
"path": "C:\\Users\\chenj\\Desktop\\lecture1-2021-with-ink.pptx"
},
{
"url": "https://drive.usercontent.google.com/download?id=1FkPOcsWpsjUXSUld1NblwyVzcsE19uIe&export=download&authuser=0&confirm=t&uuid=27501bc0-732b-4ff7-abf4-a52427aea264&at=APZUnTWleaafIVF2iZkiuHo0vQ66:1709449873140",
"path": "C:\\Users\\chenj\\Desktop\\notes.docx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "C:\\Users\\chenj\\Desktop\\lecture1-2021-with-ink.pptx"
}
}
],
"trajectory": "trajectories/eb303e01-261e-4972-8c07-c9b4e7a4922a",
"related_apps": [
"libreoffice_impress",
"libreoffice_writer"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "lecture1-2021-with-ink - PowerPoint",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5); pyautogui.press(\"enter\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_pptx_files",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=18orj_0q6N4w7ijADOJeU5ZkDDw-RdFUl&export=download&authuser=0&confirm=t&uuid=c05d2bce-bccb-4504-8fe4-7c409788d727&at=APZUnTVlCicnIm0cMdJ9FrZg4MSN:1709453015475",
"dest": "lecture1-2021-with-ink_Gold.pptx"
},
"result": {
"type": "vm_file",
"path": "C:\\Users\\chenj\\Desktop\\lecture1-2021-with-ink.pptx",
"dest": "lecture1-2021-with-ink.pptx"
},
"options": {
"examine_shape": false,
"examine_bullets": false
}
}
}

4
evaluation_examples/examples/Windows/multi_app/f918266a-b3e0-4914-865d-4faa564f1aef.json
View File

@@ -27,7 +27,7 @@
{ {
"type": "execute", "type": "execute",
"parameters": { "parameters": {
"command": "rm C:\\Users\\chenj\\Desktop\\calculator.zip", "command": "del C:\\Users\\chenj\\Desktop\\calculator.zip",
"shell": true "shell": true
} }
} }
@@ -46,7 +46,7 @@
}, },
"result": { "result": {
"type": "vm_file", "type": "vm_file",
"path": "/home/user/Desktop/log.txt", "path": "C:\\Users\\chenj\\Desktop\\log.txt",
"dest": "log.txt" "dest": "log.txt"
} }
} }

8
evaluation_examples/examples/Windows/ppt/3b27600c-3668-4abd-8f84-7bcdebbccbdb.json
View File

@@ -1,6 +1,6 @@
{ {
"id": "3b27600c-3668-4abd-8f84-7bcdebbccbdb", "id": "3b27600c-3668-4abd-8f84-7bcdebbccbdb",
"snapshot": "libreoffice_impress", "snapshot": "ppt",
"instruction": "Please make the background blue on all my slides. I was stuck by finding the entrance to do that for a while...", "instruction": "Please make the background blue on all my slides. I was stuck by finding the entrance to do that for a while...",
"source": "https://www.libreofficehelp.com/change-slide-background-impress/#All_Slides", "source": "https://www.libreofficehelp.com/change-slide-background-impress/#All_Slides",
"config": [ "config": [
@@ -9,7 +9,7 @@
"parameters": { "parameters": {
"files": [ "files": [
{ {
"url": "https://drive.usercontent.google.com/download?id=1aHMJzk2G8B_EqDlTAZLEiJ4h-ZsgA9UE&export=download&authuser=0&confirm=t&uuid=196a082d-5f08-4b3e-a64f-c021351f9cd8&at=APZUnTUXH4gvLvElvm9TtFhUJlIn:1705481007789", "url": "https://drive.usercontent.google.com/download?id=13aX_EkeTbuyFVeIkwVU_YB2f7ZbvRqR8&export=download&authuser=0&confirm=t&uuid=1be9020d-d0ab-4426-8fb9-e337d7bbaac1&at=APZUnTW6wfDLvzjHRfsh9cikK35W:1710926958131",
"path": "C:\\Users\\chenj\\lec17-gui-events.pptx" "path": "C:\\Users\\chenj\\lec17-gui-events.pptx"
} }
] ]
@@ -24,14 +24,14 @@
], ],
"trajectory": "trajectories/", "trajectory": "trajectories/",
"related_apps": [ "related_apps": [
"libreoffice_impress" "ppt"
], ],
"evaluator": { "evaluator": {
"postconfig": [ "postconfig": [
{ {
"type": "activate_window", "type": "activate_window",
"parameters": { "parameters": {
"window_name": "lec17-gui-events.pptx - LibreOffice Impress", "window_name": "lec17-gui-events - PowerPoint",
"strict": true "strict": true
} }
}, },

12
evaluation_examples/examples/Windows/ppt/455d3c66-7dc6-4537-a39a-36d3e9119df7.json
View File

@@ -1,7 +1,7 @@
{ {
"id": "455d3c66-7dc6-4537-a39a-36d3e9119df7", "id": "455d3c66-7dc6-4537-a39a-36d3e9119df7",
"snapshot": "libreoffice_impress", "snapshot": "ppt",
"instruction": "Could you help me export an Impress file to a .png image file and save it as res.png at the same position as the file? ", "instruction": "Could you help me export the PowerPoint file to a .png image file and save it as res.png at the same position as the file? ",
"source": "https://stackoverflow.com/questions/75626383/how-export-libreoffice-impress-to-image", "source": "https://stackoverflow.com/questions/75626383/how-export-libreoffice-impress-to-image",
"config": [ "config": [
{ {
@@ -9,7 +9,7 @@
"parameters": { "parameters": {
"files": [ "files": [
{ {
"url": "https://drive.usercontent.google.com/download?id=12MxMjw28_t1nTLihlDpToKebjsSDsjwx&export=download&authuser=0&confirm=t&uuid=1ccc1da0-d7c7-494f-a0e3-59eb55f54e3f&at=APZUnTXvNIRMsF2cjZuFxmQzByhC:1705253210291", "url": "https://drive.usercontent.google.com/download?id=1jrzNIGxiasIXHSc0rNlrdKr4nASGiolw&export=download&authuser=0&confirm=t&uuid=a00d2846-65fb-4db0-9763-c750e0c336c6&at=APZUnTXpm9eCM4cFu09MkeVQns7b:1710936041868",
"path": "C:\\Users\\chenj\\wssf-project-plan-on-a-page.pptx" "path": "C:\\Users\\chenj\\wssf-project-plan-on-a-page.pptx"
} }
] ]
@@ -22,9 +22,9 @@
} }
} }
], ],
"trajectory": "trajectories/", "trajectory": "trajectories/455d3c66-7dc6-4537-a39a-36d3e9119df7",
"related_apps": [ "related_apps": [
"libreoffice_impress" "ppt"
], ],
"evaluator": { "evaluator": {
"postconfig": [ "postconfig": [
@@ -61,7 +61,7 @@
"func": "compare_images", "func": "compare_images",
"expected": { "expected": {
"type": "cloud_file", "type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1XTDaQ2NlovrusKkuY6udi_BQfLwSP9th&export=download&authuser=0&confirm=t&uuid=d3c7883e-3cea-4bf3-8f83-d878622ee76d&at=APZUnTXQEnT0Gi4rB0oegvVGheyn:1705859805154", "path": "https://drive.usercontent.google.com/download?id=1C5-lKdzpC3Y8dvdXj84fymPtlqLN0FR-&export=download&authuser=0&confirm=t&uuid=a12f0635-839e-4196-bbae-7ec754f95c94&at=APZUnTVsrVGYT7WvkhkMUej5v9fz:1710942950188",
"dest": "res_gold.png" "dest": "res_gold.png"
}, },
"result": { "result": {

10
evaluation_examples/examples/Windows/ppt/550ce7e7-747b-495f-b122-acdc4d0b8e54.json
View File

@@ -1,7 +1,7 @@
{ {
"id": "550ce7e7-747b-495f-b122-acdc4d0b8e54", "id": "550ce7e7-747b-495f-b122-acdc4d0b8e54",
"snapshot": "libreoffice_impress", "snapshot": "ppt",
"instruction": "I am checking our soccer club's to-do list for the last semester and adding strike-through sign on the line we have already accomplished. Could you help me add a strike-through on the first and second line?", "instruction": "I am checking our soccer club's to-do list for the last semester and adding strike-through sign on the line we have already accomplished. Could you help me add a strike-through on the first and second line on Page 5?",
"source": "https://technical-tips.com/blog/software/text-in-libreoffice-strikethrough--6948#:~:text=To%20strikethrough%20Text%20in%20LibreOffice%201%20In%20your,effect%22%20can%20your%20additionally%2C%20for%20example%2C%20double%20underline.", "source": "https://technical-tips.com/blog/software/text-in-libreoffice-strikethrough--6948#:~:text=To%20strikethrough%20Text%20in%20LibreOffice%201%20In%20your,effect%22%20can%20your%20additionally%2C%20for%20example%2C%20double%20underline.",
"config": [ "config": [
{ {
@@ -9,7 +9,7 @@
"parameters": { "parameters": {
"files": [ "files": [
{ {
"url": "https://drive.usercontent.google.com/download?id=126TZ3vOBN2XAfdEmvJVBaa4qs_MZlgN7&export=download&authuser=0&confirm=t&uuid=acea6223-64ff-44cf-9d83-fe1a1640d374&at=APZUnTVBY9Kkm1Xo3ZU-Fe8hoOqC:1706521734182", "url": "https://drive.usercontent.google.com/download?id=1uPIsYoecxaADGDuWA-EbzH77oTiYaPx_&export=download&authuser=0&confirm=t&uuid=5925548f-2d57-457d-b107-2ecea35668bf&at=APZUnTUc1rS7g-u5EbBsiWl9hj03:1711004778606",
"path": "C:\\Users\\chenj\\New_Club_Spring_2018_Training.pptx" "path": "C:\\Users\\chenj\\New_Club_Spring_2018_Training.pptx"
} }
] ]
@@ -40,7 +40,7 @@
], ],
"trajectory": "trajectories/", "trajectory": "trajectories/",
"related_apps": [ "related_apps": [
"libreoffice_impress" "ppt"
], ],
"evaluator": { "evaluator": {
"postconfig": [ "postconfig": [
@@ -77,7 +77,7 @@
"func": "compare_pptx_files", "func": "compare_pptx_files",
"expected": { "expected": {
"type": "cloud_file", "type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1FNoJ8nGSNbsf9rhj9He1Lw17RCe2LJHD&export=download&authuser=0&confirm=t&uuid=8b876496-91c1-4209-be53-34437901b613&at=APZUnTUpOTXFXdPMATBQXSgJHsk7:1706521731233", "path": "https://drive.usercontent.google.com/download?id=1-0-7a8E0-Rye6QbuOaj0UwIqcc3gEM6J&export=download&authuser=0&confirm=t&uuid=64e43c6e-ee8a-45fb-aaf2-8636a8ae422b&at=APZUnTXRYH6X5sEXW_kbuN204uWt:1711004798314",
"dest": "New_Club_Spring_2018_Training_Gold.pptx" "dest": "New_Club_Spring_2018_Training_Gold.pptx"
}, },
"result": { "result": {

12
evaluation_examples/examples/Windows/ppt/5d901039-a89c-4bfb-967b-bf66f4df075e.json
View File

@@ -1,6 +1,6 @@
{ {
"id": "5d901039-a89c-4bfb-967b-bf66f4df075e", "id": "5d901039-a89c-4bfb-967b-bf66f4df075e",
"snapshot": "libreoffice_impress", "snapshot": "ppt",
"instruction": "I want to turn the rectangular image of Columbus on the first page into a cover page. Could you help me stretch this image to fill the entire page, keeping its proportion and centering the image?", "instruction": "I want to turn the rectangular image of Columbus on the first page into a cover page. Could you help me stretch this image to fill the entire page, keeping its proportion and centering the image?",
"source": "https://superuser.com/questions/986776/how-can-i-stretch-an-image-in-a-libreoffice-impress-presentation-to-fill-the-pag", "source": "https://superuser.com/questions/986776/how-can-i-stretch-an-image-in-a-libreoffice-impress-presentation-to-fill-the-pag",
"config": [ "config": [
@@ -9,7 +9,7 @@
"parameters": { "parameters": {
"files": [ "files": [
{ {
"url": "https://drive.usercontent.google.com/download?id=16K6TpGIRZpqOJUu-mtJQ_78kIwLcn-4D&export=download&authuser=0&confirm=t&uuid=41509e5c-eb95-453a-baad-4e12a839a120&at=APZUnTVygE_LL27vx1l6OEg_FRj0:1705849959413", "url": "https://drive.usercontent.google.com/download?id=1Y-MapQ2WrJoz0xcpLe3VZeK9VdN6hQ_1&export=download&authuser=0&confirm=t&uuid=dd374dbf-aa2d-4794-ab6a-822b0e759050&at=APZUnTX2Qh2kU65JKWhD0BsVtowm:1710927428289",
"path": "C:\\Users\\chenj\\CPD_Background_Investigation_Process.pptx" "path": "C:\\Users\\chenj\\CPD_Background_Investigation_Process.pptx"
} }
] ]
@@ -22,16 +22,16 @@
} }
} }
], ],
"trajectory": "trajectories/", "trajectory": "trajectories/5d901039-a89c-4bfb-967b-bf66f4df075e",
"related_apps": [ "related_apps": [
"libreoffice_impress" "ppt"
], ],
"evaluator": { "evaluator": {
"postconfig": [ "postconfig": [
{ {
"type": "activate_window", "type": "activate_window",
"parameters": { "parameters": {
"window_name": "CPD_Background_Investigation_Process.pptx - LibreOffice Impress", "window_name": "CPD_Background_Investigation_Process - PowerPoint",
"strict": true "strict": true
} }
}, },
@@ -61,7 +61,7 @@
"func": "check_image_stretch_and_center", "func": "check_image_stretch_and_center",
"expected": { "expected": {
"type": "cloud_file", "type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=16K6TpGIRZpqOJUu-mtJQ_78kIwLcn-4D&export=download&authuser=0&confirm=t&uuid=41509e5c-eb95-453a-baad-4e12a839a120&at=APZUnTVygE_LL27vx1l6OEg_FRj0:1705849959413", "path": "https://drive.usercontent.google.com/download?id=1Y-MapQ2WrJoz0xcpLe3VZeK9VdN6hQ_1&export=download&authuser=0&confirm=t&uuid=dd374dbf-aa2d-4794-ab6a-822b0e759050&at=APZUnTX2Qh2kU65JKWhD0BsVtowm:1710927428289",
"dest": "CPD_Background_Investigation_Process_Original.pptx" "dest": "CPD_Background_Investigation_Process_Original.pptx"
}, },
"result": { "result": {

13
evaluation_examples/examples/Windows/ppt/9ec204e4-f0a3-42f8-8458-b772a6797cab.json
View File

@@ -1,6 +1,6 @@
{ {
"id": "9ec204e4-f0a3-42f8-8458-b772a6797cab", "id": "9ec204e4-f0a3-42f8-8458-b772a6797cab",
"snapshot": "libreoffice_impress", "snapshot": "ppt",
"instruction": "Make a duplicate of the last two slides for me, please.", "instruction": "Make a duplicate of the last two slides for me, please.",
"source": "https://www.tiktok.com/@lil.d1rt_/video/7247574148887629083", "source": "https://www.tiktok.com/@lil.d1rt_/video/7247574148887629083",
"config": [ "config": [
@@ -9,7 +9,7 @@
"parameters": { "parameters": {
"files": [ "files": [
{ {
"url": "https://drive.usercontent.google.com/download?id=1ad5vUXasdN2MypNap-pBUmgPg5FaxmDA&export=download&authuser=0&confirm=t&uuid=9dc069bb-edd3-4ae9-b356-4c6543778584&at=APZUnTXbgLHjv1MhMFy1IfZL3fQI:1705338128337", "url": "https://drive.usercontent.google.com/download?id=14mkSm2Dv0jA-qyLm65knVegEABUiM41o&export=download&authuser=0&confirm=t&uuid=2d461911-0ced-465d-a721-7f0bcca49a92&at=APZUnTXaHJSNDXL2dGZa4jvaTmgj:1710940747817",
"path": "C:\\Users\\chenj\\MLA_Workshop_061X_Works_Cited.pptx" "path": "C:\\Users\\chenj\\MLA_Workshop_061X_Works_Cited.pptx"
} }
] ]
@@ -22,9 +22,9 @@
} }
} }
], ],
"trajectory": "trajectories/", "trajectory": "trajectories/9ec204e4-f0a3-42f8-8458-b772a6797cab",
"related_apps": [ "related_apps": [
"libreoffice_impress" "ppt"
], ],
"evaluator": { "evaluator": {
"postconfig": [ "postconfig": [
@@ -61,13 +61,16 @@
"func": "compare_pptx_files", "func": "compare_pptx_files",
"expected": { "expected": {
"type": "cloud_file", "type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1otbzscpOZ0tCXMvsMC0MmNWUC7Pv71of&export=download&authuser=0&confirm=t&uuid=faa0b0c1-6b14-4bce-a1fd-ccf824ee1e60&at=APZUnTXw6TlBOlrPPZ2OhfGnNPf0:1705338135842", "path": "https://drive.usercontent.google.com/download?id=16v7zubD_Hu8XFPtp8SiH60Azg5hvJCs4&export=download&authuser=0&confirm=t&uuid=94547bc1-2764-46bc-974f-1b31abebaa2d&at=APZUnTVTpKFXWpv2kHg_KW2mXT5U:1710940711249",
"dest": "MLA_Workshop_061X_Works_Cited_Gold.pptx" "dest": "MLA_Workshop_061X_Works_Cited_Gold.pptx"
}, },
"result": { "result": {
"type": "vm_file", "type": "vm_file",
"path": "C:\\Users\\chenj\\MLA_Workshop_061X_Works_Cited.pptx", "path": "C:\\Users\\chenj\\MLA_Workshop_061X_Works_Cited.pptx",
"dest": "MLA_Workshop_061X_Works_Cited.pptx" "dest": "MLA_Workshop_061X_Works_Cited.pptx"
},
"options": {
"examine_shape": false
} }
} }
} }

8
evaluation_examples/examples/Windows/ppt/a097acff-6266-4291-9fbd-137af7ecd439.json
View File

@@ -1,6 +1,6 @@
{ {
"id": "a097acff-6266-4291-9fbd-137af7ecd439", "id": "a097acff-6266-4291-9fbd-137af7ecd439",
"snapshot": "libreoffice_impress", "snapshot": "ppt",
"instruction": "Could you help me save my slides as pre.pptx on the Desktop?", "instruction": "Could you help me save my slides as pre.pptx on the Desktop?",
"source": "https://www.youtube.com/watch?v=DDmEvjs4iBw", "source": "https://www.youtube.com/watch?v=DDmEvjs4iBw",
"config": [ "config": [
@@ -9,7 +9,7 @@
"parameters": { "parameters": {
"files": [ "files": [
{ {
"url": "https://drive.usercontent.google.com/download?id=1i_-m6mVrdesGJ392bulH5lveHarMwKk_&export=download&authuser=0&confirm=t&uuid=686367a5-9383-4d60-8beb-108ef7119ffa&at=APZUnTXpMsCkSdM7fb5zdVz3qZUX:1705916772791", "url": "https://drive.usercontent.google.com/download?id=1c2QgyKsNpBvjw2zLGyk4lxM9L-Mr_986&export=download&authuser=0&confirm=t&uuid=31e05bc4-37c8-45c6-8ae9-15ede6202f54&at=APZUnTVMRuXMqUARayy64cvAgKUN:1710944701562",
"path": "C:\\Users\\chenj\\Downloads\\Secrets-of-Monetizing-Video.pptx" "path": "C:\\Users\\chenj\\Downloads\\Secrets-of-Monetizing-Video.pptx"
} }
] ]
@@ -24,13 +24,13 @@
], ],
"trajectory": "trajectories/", "trajectory": "trajectories/",
"related_apps": [ "related_apps": [
"libreoffice_impress" "ppt"
], ],
"evaluator": { "evaluator": {
"func": "compare_pptx_files", "func": "compare_pptx_files",
"expected": { "expected": {
"type": "cloud_file", "type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1i_-m6mVrdesGJ392bulH5lveHarMwKk_&export=download&authuser=0&confirm=t&uuid=686367a5-9383-4d60-8beb-108ef7119ffa&at=APZUnTXpMsCkSdM7fb5zdVz3qZUX:1705916772791", "path": "https://drive.usercontent.google.com/download?id=1c2QgyKsNpBvjw2zLGyk4lxM9L-Mr_986&export=download&authuser=0&confirm=t&uuid=31e05bc4-37c8-45c6-8ae9-15ede6202f54&at=APZUnTVMRuXMqUARayy64cvAgKUN:1710944701562",
"dest": "Secrets-of-Monetizing-Video.pptx" "dest": "Secrets-of-Monetizing-Video.pptx"
}, },
"result": { "result": {

8
evaluation_examples/examples/Windows/ppt/ce88f674-ab7a-43da-9201-468d38539e4a.json
View File

@@ -1,6 +1,6 @@
{ {
"id": "ce88f674-ab7a-43da-9201-468d38539e4a", "id": "ce88f674-ab7a-43da-9201-468d38539e4a",
"snapshot": "libreoffice_impress", "snapshot": "ppt",
"instruction": "Please set my slides upright instead of sideways.", "instruction": "Please set my slides upright instead of sideways.",
"source": "https://justclickhere.co.uk/resources/change-slides-in-impress-to-portrait/", "source": "https://justclickhere.co.uk/resources/change-slides-in-impress-to-portrait/",
"config": [ "config": [
@@ -9,7 +9,7 @@
"parameters": { "parameters": {
"files": [ "files": [
{ {
"url": "https://drive.usercontent.google.com/download?id=1LErTnC_w_YPQVo84QK5sifww9xZ-Cq0X&export=download&authuser=0&confirm=t&uuid=81ff0aaf-9c2e-4342-b7ce-36e65dd2218e&at=APZUnTUmQKCTp2HUP0dOqYqD10G3:1705479016156", "url": "https://drive.usercontent.google.com/download?id=1Vpv15scw1zbRAzFx_1nkgG9ROKzCV_1R&export=download&authuser=0&confirm=t&uuid=fb81a042-cc55-4b8c-89c0-51e30c872749&at=APZUnTWufEjoMu2mj1AKkcgkfc4J:1710945229686",
"path": "C:\\Users\\chenj\\AM_Last_Page_Template.pptx" "path": "C:\\Users\\chenj\\AM_Last_Page_Template.pptx"
} }
] ]
@@ -24,14 +24,14 @@
], ],
"trajectory": "trajectories/", "trajectory": "trajectories/",
"related_apps": [ "related_apps": [
"libreoffice_impress" "ppt"
], ],
"evaluator": { "evaluator": {
"postconfig": [ "postconfig": [
{ {
"type": "activate_window", "type": "activate_window",
"parameters": { "parameters": {
"window_name": "AM_Last_Page_Template.pptx - LibreOffice Impress", "window_name": "AM_Last_Page_Template - PowerPoint",
"strict": true "strict": true
} }
}, },

8
evaluation_examples/examples/Windows/word/0810415c-bde4-4443-9047-d5f70165a697.json
View File

@@ -1,6 +1,6 @@
{ {
"id": "0810415c-bde4-4443-9047-d5f70165a697", "id": "0810415c-bde4-4443-9047-d5f70165a697",
"snapshot": "libreoffice_writer", "snapshot": "word",
"instruction": "Make the line spacing of first two paragraph into double line spacing", "instruction": "Make the line spacing of first two paragraph into double line spacing",
"source": "https://www.youtube.com/watch?v=Q_AaL6ljudU", "source": "https://www.youtube.com/watch?v=Q_AaL6ljudU",
"config": [ "config": [
@@ -24,14 +24,14 @@
], ],
"trajectory": "trajectories/", "trajectory": "trajectories/",
"related_apps": [ "related_apps": [
"libreoffice_writer" "word"
], ],
"evaluator": { "evaluator": {
"postconfig": [ "postconfig": [
{ {
"type": "activate_window", "type": "activate_window",
"parameters": { "parameters": {
"window_name": "Novels_Intro_Packet.docx - LibreOffice Writer", "window_name": "Novels_Intro_Packet - Word",
"strict": true "strict": true
} }
}, },
@@ -47,7 +47,7 @@
"command": [ "command": [
"python", "python",
"-c", "-c",
"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5); pyautogui.press(\"enter\");" "import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5);"
] ]
} }
} }

8
evaluation_examples/examples/Windows/word/0b17a146-2934-46c7-8727-73ff6b6483e8.json
View File

@@ -1,6 +1,6 @@
{ {
"id": "0b17a146-2934-46c7-8727-73ff6b6483e8", "id": "0b17a146-2934-46c7-8727-73ff6b6483e8",
"snapshot": "libreoffice_writer", "snapshot": "word",
"instruction": "Help me change the 2 in \"H2O\" to a subscript.", "instruction": "Help me change the 2 in \"H2O\" to a subscript.",
"source": "https://askubuntu.com/questions/245695/how-do-you-insert-subscripts-and-superscripts-into-ordinary-non-formula-text-i", "source": "https://askubuntu.com/questions/245695/how-do-you-insert-subscripts-and-superscripts-into-ordinary-non-formula-text-i",
"config": [ "config": [
@@ -24,14 +24,14 @@
], ],
"trajectory": "trajectories/", "trajectory": "trajectories/",
"related_apps": [ "related_apps": [
"libreoffice_writer" "word"
], ],
"evaluator": { "evaluator": {
"postconfig": [ "postconfig": [
{ {
"type": "activate_window", "type": "activate_window",
"parameters": { "parameters": {
"window_name": "H2O_Factsheet_WA.docx - LibreOffice Writer", "window_name": "H2O_Factsheet_WA - Word",
"strict": true "strict": true
} }
}, },
@@ -47,7 +47,7 @@
"command": [ "command": [
"python", "python",
"-c", "-c",
"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5); pyautogui.press(\"enter\");" "import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5);"
] ]
} }
} }

8
evaluation_examples/examples/Windows/word/0e763496-b6bb-4508-a427-fad0b6c3e195.json
View File

@@ -1,6 +1,6 @@
{ {
"id": "0e763496-b6bb-4508-a427-fad0b6c3e195", "id": "0e763496-b6bb-4508-a427-fad0b6c3e195",
"snapshot": "libreoffice_writer", "snapshot": "word",
"instruction": "Change the font to \"Times New Roman\" throughout the text.", "instruction": "Change the font to \"Times New Roman\" throughout the text.",
"source": "https://ask.libreoffice.org/t/how-do-i-change-the-font-for-the-whole-document-in-writer/9220", "source": "https://ask.libreoffice.org/t/how-do-i-change-the-font-for-the-whole-document-in-writer/9220",
"config": [ "config": [
@@ -24,14 +24,14 @@
], ],
"trajectory": "trajectories/", "trajectory": "trajectories/",
"related_apps": [ "related_apps": [
"libreoffice_writer" "word"
], ],
"evaluator": { "evaluator": {
"postconfig": [ "postconfig": [
{ {
"type": "activate_window", "type": "activate_window",
"parameters": { "parameters": {
"window_name": "Dublin_Zoo_Intro.docx - LibreOffice Writer", "window_name": "Dublin_Zoo_Intro - Word",
"strict": true "strict": true
} }
}, },
@@ -47,7 +47,7 @@
"command": [ "command": [
"python", "python",
"-c", "-c",
"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5); pyautogui.press(\"enter\");" "import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(5);"
] ]
} }
} }

8
evaluation_examples/examples/Windows/word/3ef2b351-8a84-4ff2-8724-d86eae9b842e.json
View File

@@ -1,6 +1,6 @@
{ {
"id": "3ef2b351-8a84-4ff2-8724-d86eae9b842e", "id": "3ef2b351-8a84-4ff2-8724-d86eae9b842e",
"snapshot": "libreoffice_writer", "snapshot": "word",
"instruction": "Help me center align the heading in LibreOffice.", "instruction": "Help me center align the heading in LibreOffice.",
"source": "https://askubuntu.com/questions/1066351/how-do-you-center-align-in-libreoffice#:~:text=Ctrl%20%2B%20e%20will%20Center%20align%20the%20cursor%20for%20you.", "source": "https://askubuntu.com/questions/1066351/how-do-you-center-align-in-libreoffice#:~:text=Ctrl%20%2B%20e%20will%20Center%20align%20the%20cursor%20for%20you.",
"config": [ "config": [
@@ -24,14 +24,14 @@
], ],
"trajectory": "trajectories/", "trajectory": "trajectories/",
"related_apps": [ "related_apps": [
"libreoffice_writer" "word"
], ],
"evaluator": { "evaluator": {
"postconfig": [ "postconfig": [
{ {
"type": "activate_window", "type": "activate_window",
"parameters": { "parameters": {
"window_name": "Constitution_Template_With_Guidelines.docx - LibreOffice Writer", "window_name": "Constitution_Template_With_Guidelines - Word",
"strict": true "strict": true
} }
}, },
@@ -47,7 +47,7 @@
"command": [ "command": [
"python", "python",
"-c", "-c",
"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5); pyautogui.press(\"enter\");" "import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(5);"
] ]
} }
} }

67
evaluation_examples/examples/Windows/word/45d61a06-6545-4422-97b7-bc76cfa964c1.json
View File

@@ -1,67 +0,0 @@
{
"id": "45d61a06-6545-4422-97b7-bc76cfa964c1",
"snapshot": "libreoffice_writer",
"instruction": "Replace all newlines with paragraph marks in LibreOffice Writer",
"source": "https://stackoverflow.com/questions/71685737/how-to-replace-all-newlines-with-paragraph-marks-in-libreoffice-write",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=16lQcSkw-JQ_v8Sg0HkCtnOzyK-4cok8N&export=download&authuser=0&confirm=t&uuid=a3f833ae-2572-4cf3-8a21-6d250e689415&at=APZUnTUfn24NGMtXEzz2Nf7cFLjt:1704975857398",
"path": "C:\\Users\\chenj\\NOVEL_Submission_Guidelines.docx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "C:\\Users\\chenj\\NOVEL_Submission_Guidelines.docx"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"libreoffice_writer"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "NOVEL_Submission_Guidelines.docx - LibreOffice Writer",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
]
}
}
],
"func": "compare_line_spacing",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1LaS5ObaOsbyKX1M17vi8ZseZwawgvNmf&export=download&authuser=0&confirm=t&uuid=db523e81-fa22-4002-97ff-e5dff92106a7&at=APZUnTVnT3ZYOGW7ZQdeW4SZP7mX:1704975993684",
"dest": "NOVEL_Submission_Guidelines_Gold.docx"
},
"result": {
"type": "vm_file",
"path": "C:\\Users\\chenj\\NOVEL_Submission_Guidelines.docx",
"dest": "NOVEL_Submission_Guidelines.docx"
}
}
}

4
evaluation_examples/examples/Windows/word/4bcb1253-a636-4df4-8cb0-a35c04dfef31.json
View File

@@ -1,6 +1,6 @@
{ {
"id": "4bcb1253-a636-4df4-8cb0-a35c04dfef31", "id": "4bcb1253-a636-4df4-8cb0-a35c04dfef31",
"snapshot": "libreoffice_writer", "snapshot": "word",
"instruction": "Export the current document into PDF, keep the file name", "instruction": "Export the current document into PDF, keep the file name",
"source": "https://www.libreofficehelp.com/save-export-writer-documents-in-pdf-epub-format/", "source": "https://www.libreofficehelp.com/save-export-writer-documents-in-pdf-epub-format/",
"config": [ "config": [
@@ -24,7 +24,7 @@
], ],
"trajectory": "trajectories/", "trajectory": "trajectories/",
"related_apps": [ "related_apps": [
"libreoffice_writer" "word"
], ],
"evaluator": { "evaluator": {
"func": "compare_pdfs", "func": "compare_pdfs",

77
evaluation_examples/examples/Windows/word/66399b0d-8fda-4618-95c4-bfc6191617e9.json
View File

@@ -1,77 +0,0 @@
{
"id": "66399b0d-8fda-4618-95c4-bfc6191617e9",
"snapshot": "libreoffice_writer",
"instruction": "Could you help me insert a 7(columns)*5(rows) empty table at the point of cursor?",
"source": "https://www.youtube.com/watch?v=l25Evu4ohKg",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1yeD0YIFCSEAdi6MPiiqLe6F4a19I5wZP&export=download&authuser=0&confirm=t&uuid=b6693633-00d7-4a8b-b35e-77ec452dd6a3&at=APZUnTVHRdoysZFBTmGmX6rCJLHK:1706330570587",
"path": "C:\\Users\\chenj\\Table_Of_Work_Effort_Instructions.docx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "C:\\Users\\chenj\\Table_Of_Work_Effort_Instructions.docx"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.press(\"down\", presses=40, interval=0.01); time.sleep(1); pyautogui.scroll(-2)"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"libreoffice_writer"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "Table_Of_Work_Effort_Instructions.docx - LibreOffice Writer",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
]
}
}
],
"func": "compare_docx_tables",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=14JfHsW9GvbhORdtVAtvEbOi00MqEyHfb&export=download&authuser=0&confirm=t&uuid=3dba2459-ac37-4cad-a982-adecd406382a&at=APZUnTVQUqUPq_WacgY2xu4PvAKB:1704181465512",
"dest": "Table_Of_Work_Effort_Instructions_Gold.docx"
},
"result": {
"type": "vm_file",
"path": "C:\\Users\\chenj\\Table_Of_Work_Effort_Instructions.docx",
"dest": "Table_Of_Work_Effort_Instructions.docx"
}
}
}

6
evaluation_examples/examples/Windows/word/6f81754e-285d-4ce0-b59e-af7edb02d108.json
View File

@@ -1,6 +1,6 @@
{ {
"id": "6f81754e-285d-4ce0-b59e-af7edb02d108", "id": "6f81754e-285d-4ce0-b59e-af7edb02d108",
"snapshot": "libreoffice_writer", "snapshot": "word",
"instruction": "A certain railway company in Hong Kong uses a signaling system to keep track of trains in its railway system. Each line in the docx file represents a train calling at a station from 0600 to 1200 on 2022-09-22, and has the following format: time_HH:MM:SS, train_id, station_id, platform_no.. I want to remove duplicated train ids in order to know how many different trains are running from 0600 to 1200. Could you help me on this? I am doing it manually and it is very inefficient.", "instruction": "A certain railway company in Hong Kong uses a signaling system to keep track of trains in its railway system. Each line in the docx file represents a train calling at a station from 0600 to 1200 on 2022-09-22, and has the following format: time_HH:MM:SS, train_id, station_id, platform_no.. I want to remove duplicated train ids in order to know how many different trains are running from 0600 to 1200. Could you help me on this? I am doing it manually and it is very inefficient.",
"source": "https://superuser.com/questions/789473/remove-duplicate-lines-in-libreoffice-openoffice-writer", "source": "https://superuser.com/questions/789473/remove-duplicate-lines-in-libreoffice-openoffice-writer",
"config": [ "config": [
@@ -24,14 +24,14 @@
], ],
"trajectory": "trajectories/", "trajectory": "trajectories/",
"related_apps": [ "related_apps": [
"libreoffice_writer" "word"
], ],
"evaluator": { "evaluator": {
"postconfig": [ "postconfig": [
{ {
"type": "activate_window", "type": "activate_window",
"parameters": { "parameters": {
"window_name": "HK_train_record.docx - LibreOffice Writer", "window_name": "HK_train_record - Word",
"strict": true "strict": true
} }
}, },

67
evaluation_examples/examples/Windows/word/72b810ef-4156-4d09-8f08-a0cf57e7cefe.json
View File

@@ -1,67 +0,0 @@
{
"id": "72b810ef-4156-4d09-8f08-a0cf57e7cefe",
"snapshot": "libreoffice_writer",
"instruction": "I am peer-reviewing my friend's course outline. I think the last paragraph is redundant so I want to add strike-through on words in the last paragraph. Can you do this for me?",
"source": "https://superuser.com/questions/657792/libreoffice-writer-how-to-apply-strikethrough-text-formatting?rq=1",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?id=1Uqgr9Y_kjoMoDoUwt80hv1EtFaisyztU&export=download",
"path": "C:\\Users\\chenj\\GEOG2169_Course_Outline_2022-23.docx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "C:\\Users\\chenj\\GEOG2169_Course_Outline_2022-23.docx"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"libreoffice_writer"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "GEOG2169_Course_Outline_2022-23.docx - LibreOffice Writer",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
]
}
}
],
"func": "evaluate_strike_through_last_paragraph",
"expected": {
"type": "cloud_file",
"path": "https://drive.google.com/uc?id=1IpAnQRYo1whrnzIGyo8UldZf4Tli-yVT&export=download",
"dest": "GEOG2169_Course_Outline_2022-23_Gold.docx"
},
"result": {
"type": "vm_file",
"path": "C:\\Users\\chenj\\GEOG2169_Course_Outline_2022-23.docx",
"dest": "GEOG2169_Course_Outline_2022-23.docx"
}
}
}

8
evaluation_examples/examples/Windows/word/b21acd93-60fd-4127-8a43-2f5178f4a830.json
View File

@@ -1,7 +1,7 @@
{ {
"id": "b21acd93-60fd-4127-8a43-2f5178f4a830", "id": "b21acd93-60fd-4127-8a43-2f5178f4a830",
"snapshot": "libreoffice_writer", "snapshot": "word",
"instruction": "I have been practicing professional writing lately. Now I am writing essay which requires one paragraph each for introduction, body and conclusion with single-space for introduction, double-space for body then one-and-a-half-space for conclusion. The font size of this essay is 12. Could you help me on this?", "instruction": "I have been practicing professional writing lately. Now I am writing essay which requires one paragraph each for introduction, body and conclusion with single-space for introduction, double-space for body then one-and-a-half-space for conclusion. Could you help me on this?",
"source": "https://superuser.com/questions/1097199/how-can-i-double-space-a-document-in-libreoffice?rq=1", "source": "https://superuser.com/questions/1097199/how-can-i-double-space-a-document-in-libreoffice?rq=1",
"config": [ "config": [
{ {
@@ -24,14 +24,14 @@
], ],
"trajectory": "trajectories/", "trajectory": "trajectories/",
"related_apps": [ "related_apps": [
"libreoffice_writer" "word"
], ],
"evaluator": { "evaluator": {
"postconfig": [ "postconfig": [
{ {
"type": "activate_window", "type": "activate_window",
"parameters": { "parameters": {
"window_name": "CCHU9045_Course_Outline_2019-20.docx - LibreOffice Writer", "window_name": "CCHU9045_Course_Outline_2019-20 - Word",
"strict": true "strict": true
} }
}, },

12
evaluation_examples/examples/Windows/word/e528b65e-1107-4b8c-8988-490e4fece599.json
View File

@@ -1,6 +1,6 @@
{ {
"id": "e528b65e-1107-4b8c-8988-490e4fece599", "id": "e528b65e-1107-4b8c-8988-490e4fece599",
"snapshot": "libreoffice_writer", "snapshot": "word",
"instruction": "Capitalize the first letter of all words.", "instruction": "Capitalize the first letter of all words.",
"source": "https://www.youtube.com/watch?v=l25Evu4ohKg", "source": "https://www.youtube.com/watch?v=l25Evu4ohKg",
"config": [ "config": [
@@ -9,7 +9,7 @@
"parameters": { "parameters": {
"files": [ "files": [
{ {
"url": "https://drive.usercontent.google.com/download?id=181hyG_NZUJaUp7kUdeFoUpNGpOcjzYFV&export=download&authuser=0&confirm=t&uuid=fdb8aee4-ea1f-43c1-bc66-9d44099512e1&at=APZUnTVUrypBeVnTfkJsriv7S2GO:1704969486759", "url": "https://drive.usercontent.google.com/download?id=1rcwueBf1TtBJu6NftUpzqFDVRSZzRCGT&export=download&authuser=0&confirm=t&uuid=a71c0649-c1b5-468a-8bad-aa59b11b510c&at=APZUnTVQ0F2KgX7sa-DqAXDhQ3Vu:1710854515082",
"path": "C:\\Users\\chenj\\Geography_And_Magical_Realism.docx" "path": "C:\\Users\\chenj\\Geography_And_Magical_Realism.docx"
} }
] ]
@@ -24,14 +24,14 @@
], ],
"trajectory": "trajectories/", "trajectory": "trajectories/",
"related_apps": [ "related_apps": [
"libreoffice_writer" "word"
], ],
"evaluator": { "evaluator": {
"postconfig": [ "postconfig": [
{ {
"type": "activate_window", "type": "activate_window",
"parameters": { "parameters": {
"window_name": "Geography_And_Magical_Realism.docx - LibreOffice Writer", "window_name": "Geography_And_Magical_Realism - Word",
"strict": true "strict": true
} }
}, },
@@ -47,7 +47,7 @@
"command": [ "command": [
"python", "python",
"-c", "-c",
"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5); pyautogui.press(\"enter\");" "import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5);"
] ]
} }
} }
@@ -55,7 +55,7 @@
"func": "compare_docx_files", "func": "compare_docx_files",
"expected": { "expected": {
"type": "cloud_file", "type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1qwoJMuMhLxWwONRLmxwMdUuVIFBAhhNT&export=download&authuser=0&confirm=t&uuid=8fa4d503-ec55-419f-ac90-f5915a5f67b5&at=APZUnTVEntT4oMiEfOuWcUKaInBs:1704969347952", "path": "https://drive.usercontent.google.com/download?id=1QM2Ql81_Rex-qa7xTjUvikSb6OGtc4JC&export=download&authuser=0&confirm=t&uuid=6d24454d-3526-4694-9bf6-adc6bc0afd78&at=APZUnTWptHSOEAum1ZUXQQYED2Lb:1710855675026",
"dest": "Geography_And_Magical_Realism_Gold.docx" "dest": "Geography_And_Magical_Realism_Gold.docx"
}, },
"result": { "result": {

6
evaluation_examples/examples/Windows/word/ecc2413d-8a48-416e-a3a2-d30106ca36cb.json
View File

@@ -1,6 +1,6 @@
{ {
"id": "ecc2413d-8a48-416e-a3a2-d30106ca36cb", "id": "ecc2413d-8a48-416e-a3a2-d30106ca36cb",
"snapshot": "libreoffice_writer", "snapshot": "word",
"instruction": "Hey, can you throw in a blank page right after this one?", "instruction": "Hey, can you throw in a blank page right after this one?",
"source": "https://www.quora.com/How-can-I-insert-a-blank-page-on-libreoffice", "source": "https://www.quora.com/How-can-I-insert-a-blank-page-on-libreoffice",
"config": [ "config": [
@@ -24,14 +24,14 @@
], ],
"trajectory": "trajectories/", "trajectory": "trajectories/",
"related_apps": [ "related_apps": [
"libreoffice_writer" "word"
], ],
"evaluator": { "evaluator": {
"postconfig": [ "postconfig": [
{ {
"type": "activate_window", "type": "activate_window",
"parameters": { "parameters": {
"window_name": "Sample_Statutory_Declaration.docx - LibreOffice Writer", "window_name": "Sample_Statutory_Declaration - Word",
"strict": true "strict": true
} }
}, },

28
evaluation_examples/examples/multi_apps/2b9493d7-49b8-493a-a71b-56cd1f4d6908.json
View File

@@ -7,12 +7,12 @@
{ {
"type": "download", "type": "download",
"parameters": { "parameters": {
"files": [ "files": [
{ {
"url": "https://drive.usercontent.google.com/download?id=1gqqY56robX1tb4YPa3Yk1d72T_k-Rgz3&export=download&authuser=0&confirm=t", "url": "https://drive.usercontent.google.com/download?id=1gqqY56robX1tb4YPa3Yk1d72T_k-Rgz3&export=download&authuser=0&confirm=t&uuid=aae50df0-7f7f-49f2-b17c-a81dd89a6c83&at=APZUnTXSb3-yc2zdtGVkeKbK6aQC:1710658340540",
"path": "/home/user/Desktop/15-MB-docx-file-download.docx" "path": "/home/user/Desktop/15-MB-docx-file-download.docx"
} }
] ]
} }
}, },
{ {
@@ -50,14 +50,14 @@
"evaluator": { "evaluator": {
"postconfig": [ "postconfig": [
{ {
"type": "execute", "type": "execute",
"parameters": { "parameters": {
"command": [ "command": [
"/bin/bash", "/bin/bash",
"-c", "-c",
"killall gnome-terminal-server" "killall gnome-terminal-server"
] ]
} }
} }
], ],
"func": "check_include_exclude", "func": "check_include_exclude",

122
evaluation_examples/examples/multi_apps/6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a.json
View File

@@ -1,50 +1,50 @@
{ {
"id": "6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a", "id": "6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a",
"snapshot": "multiapps", "snapshot": "multiapps",
"instruction": "I now want to count the meeting cities of the three machine learning conferences in the past ten years from 2013 to 2019(including 2013 and 2019). I have listed the names and years of the conferences in excel. Please fill in the vacant locations.", "instruction": "I now want to count the meeting cities of the three machine learning conferences in the past ten years from 2013 to 2019(including 2013 and 2019). I have listed the names and years of the conferences in excel. Please fill in the vacant locations.",
"source": "author", "source": "author",
"config": [ "config": [
{ {
"type": "download", "type": "download",
"parameters": { "parameters": {
"files": [ "files": [
{ {
"url": "https://drive.google.com/uc?export=download&id=19wUxTQeoKr6ihJWJ_9cu2tzKQH0cnxWH", "url": "https://drive.google.com/uc?export=download&id=19wUxTQeoKr6ihJWJ_9cu2tzKQH0cnxWH",
"path": "/home/user/Desktop/ConferenceCity.xlsx" "path": "/home/user/Desktop/ConferenceCity.xlsx"
} }
] ]
} }
}, },
{ {
"type": "open", "type": "open",
"parameters": { "parameters": {
"path": "/home/user/Desktop/ConferenceCity.xlsx" "path": "/home/user/Desktop/ConferenceCity.xlsx"
} }
} }
], ],
"trajectory": "trajectories/", "trajectory": "trajectories/",
"related_apps": [ "related_apps": [
"calc", "chrome", "os" "calc", "chrome", "os"
], ],
"evaluator": { "evaluator": {
"postconfig":[ "postconfig":[
{ {
"type": "download", "type": "download",
"parameters": { "parameters": {
"files": [ "files": [
{ {
"url": "https://drive.google.com/uc?export=download&id=1ZcITkIOs2Z86S5L6MShSohFs3_xVfeCP", "url": "https://drive.google.com/uc?export=download&id=1ZcITkIOs2Z86S5L6MShSohFs3_xVfeCP",
"path": "/home/user/Desktop/ConferenceCity_Gold.xlsx" "path": "/home/user/Desktop/ConferenceCity_Gold.xlsx"
} }
] ]
} }
}, },
{ {
"type": "activate_window", "type": "activate_window",
"parameters": { "parameters": {
"window_name": "ConferenceCity.xlsx - LibreOffice Calc" "window_name": "ConferenceCity.xlsx - LibreOffice Calc"
} }
}, },
{ {
"type": "execute", "type": "execute",
"parameters": { "parameters": {
@@ -55,18 +55,18 @@
] ]
} }
} }
], ],
"func": "compare_conference_city_in_order", "func": "compare_conference_city_in_order",
"expected": { "expected": {
"type": "rule", "type": "rule",
"rules":{ "rules":{
"expected": ["Scottsdale","Atlanta","Lake Tahoe","Banff","Beijing",["Montreal", "Montréal"],"San Diego","Lille",["Montreal", "Montréal"],"San Juan",["New York", "New York City", "NYC"],"Barcelona","Toulon","Sydney","Long Beach","Vancouver","Stockholm",["Montreal", "Montréal"],"New Orleans","Long Beach","Vancouver"] "expected": ["Scottsdale","Atlanta","Lake Tahoe","Banff","Beijing",["Montreal", "Montréal"],"San Diego","Lille",["Montreal", "Montréal"],"San Juan",["New York", "New York City", "NYC"],"Barcelona","Toulon","Sydney","Long Beach","Vancouver","Stockholm",["Montreal", "Montréal"],"New Orleans","Long Beach","Vancouver"]
} }
}, },
"result": { "result": {
"type": "vm_file", "type": "vm_file",
"path": "/home/user/Desktop/ConferenceCity.xlsx", "path": "/home/user/Desktop/ConferenceCity.xlsx",
"dest": "ConferenceCity.xlsx" "dest": "ConferenceCity.xlsx"
} }
} }
} }

4
evaluation_examples/examples/multi_apps/e2392362-125e-4f76-a2ee-524b183a3412.json
View File

@@ -31,7 +31,7 @@
"command": [ "command": [
"tar", "tar",
"-xJvf", "-xJvf",
".tmp.tar.xz", "/home/user/.tmp.tar.xz",
"-C", "-C",
"/home/user/Code/Website/" "/home/user/Code/Website/"
] ]
@@ -124,4 +124,4 @@
"dest": "_config.yaml" "dest": "_config.yaml"
} }
} }
} }