lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into zdy

Browse Source
This commit is contained in:
David Chang
2024-01-11 23:02:00 +08:00
parent 3c04872dcf b30d87bba8
commit 127a101994
39 changed files with 580 additions and 282 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
evaluation_examples/examples/chrome/06fe7178-4491-4589-810f-2e2bc9502122.json
View File

@@ -1,7 +1,7 @@
{
"id": "06fe7178-4491-4589-810f-2e2bc9502122",
"snapshot": "chrome",
"instruction": "Could you help me open the previously closed tab?",
"instruction": "Can you make my computer bring back the last tab I shut down?",
"source": "https://www.wikihow.com/Switch-Tabs-in-Chrome",
"config": [],
"trajectory": "trajectories/",

2
evaluation_examples/examples/chrome/12086550-11c0-466b-b367-1d9e75b3910e.json
View File

@@ -1,7 +1,7 @@
{
"id": "12086550-11c0-466b-b367-1d9e75b3910e",
"snapshot": "chrome",
"instruction": "Help me find the hidden password of xx website.",
"instruction": "Computer, please navigate to the area in my browser settings where my passwords are stored. I want to check my login information for Etsy without revealing it just yet.",
"source": "https://www.quora.com/What-are-the-cool-tricks-to-use-Google-Chrome",
"config": [],
"trajectory": "trajectories/",

2
evaluation_examples/examples/chrome/7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3.json
View File

@@ -1,7 +1,7 @@
{
"id": "7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3",
"snapshot": "chrome",
"instruction": "Could you help me delete cookies from openai website?",
"instruction": "Can you help me clean up my computer by getting rid of all the tracking things that websites like Amazon or eBay might have saved? I want to make sure my browsing is private and those sites don't remember me.",
"source": "https://support.google.com/chrome/answer/95647?hl=en&ref_topic=7438325&sjid=16867045591165135686-AP#zippy=%2Cdelete-cookies-from-a-site",
"config": [],
"trajectory": "trajectories/",

21
evaluation_examples/examples/chrome/bb5e4c0d-f964-439c-97b6-bdb9747de3f4.json
View File

@@ -1,18 +1,33 @@
{
"id": "bb5e4c0d-f964-439c-97b6-bdb9747de3f4",
"snapshot": "chrome",
"instruction": "Could you help me set my default search engine to be Google?",
"instruction": "Can you make Bing the main search thingy when I look stuff up on the internet?",
"source": "https://support.google.com/chrome/answer/95426?sjid=16867045591165135686-AP",
"config": [],
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=9222"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "",
"func": "exact_match",
"result": {
"type": "default_search_engine"
},
"expected": {
"type": "rule",
"rules": {
"expected": "Bing"
}
}
}
}

10
evaluation_examples/examples/libreoffice_writer/0810415c-bde4-4443-9047-d5f70165a697.json
View File

@@ -10,7 +10,7 @@
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1-svVsH-l2ofufEKuN-cYrIrvXNobtATE&export=download&authuser=0&confirm=t&uuid=be7f891a-f858-48f5-a72d-4e42bbfb8b65&at=APZUnTXzBnaeSJjmxeh4zG03pzA0:1704179807785",
"path": "Desktop/Double_Line_Spacing.docx"
"path": "Desktop/Novels_Intro_Packet.docx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
"path": "Desktop/Double_Line_Spacing.docx"
"path": "Desktop/Novels_Intro_Packet.docx"
}
}
],
@@ -31,12 +31,12 @@
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1-svVsH-l2ofufEKuN-cYrIrvXNobtATE&export=download&authuser=0&confirm=t&uuid=be7f891a-f858-48f5-a72d-4e42bbfb8b65&at=APZUnTXzBnaeSJjmxeh4zG03pzA0:1704179807785",
"dest": "Double_Line_Spacing_Gold.docx"
"dest": "Novels_Intro_Packet_Gold.docx"
},
"result": {
"type": "vm_file",
"path": "Desktop/Double_Line_Spacing.docx",
"dest": "Double_Line_Spacing.docx"
"path": "Desktop/Novels_Intro_Packet.docx",
"dest": "Novels_Intro_Packet.docx"
}
}
}

16
evaluation_examples/examples/libreoffice_writer/0b17a146-2934-46c7-8727-73ff6b6483e8.json
View File

@@ -1,7 +1,7 @@
{
"id": "0b17a146-2934-46c7-8727-73ff6b6483e8",
"snapshot": "libreoffice_writer",
"instruction": "Change the 2 in H2O to a subscript.",
"instruction": "Help me change the 2 in \"H2O\" to a subscript.",
"source": "https://askubuntu.com/questions/245695/how-do-you-insert-subscripts-and-superscripts-into-ordinary-non-formula-text-i",
"config": [
{
@@ -9,8 +9,8 @@
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1Nx5AoKNM7tcDRE6y_qjNIDrPOKqhNyfm&export=download&authuser=0&confirm=t&uuid=bb4de348-3bbf-46a2-95b2-e2719c67547a&at=APZUnTUeA-BW7mkQsEw7NGm272zx:1704172916742",
"path": "Desktop/Enter_Subscript.docx"
"url": "https://drive.usercontent.google.com/download?id=1FkorQBeTJ5L2jLuvu4YxHSlBMK4VEEG6&export=download&authuser=0&confirm=t&uuid=cc63dc0b-bae7-4ef6-a40d-e2da721976ef&at=APZUnTWyPZlZPFlqGTWAWXWmS04c:1704976667765",
"path": "Desktop/H2O_Factsheet_WA.docx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
"path": "Desktop/Enter_Subscript.docx"
"path": "Desktop/H2O_Factsheet_WA.docx"
}
}
],
@@ -30,13 +30,13 @@
"func": "compare_docx_files",
"result": {
"type": "vm_file",
"path": "Desktop/Enter_Subscript.docx",
"dest": "Enter_Subscript.docx"
"path": "Desktop/H2O_Factsheet_WA.docx",
"dest": "H2O_Factsheet_WA.docx"
},
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1AaKXeD9ZgfMykgijZ4G8MEzUjmMJElkq&export=download&authuser=0&confirm=t&uuid=5e347f0d-4efc-4478-878e-d89455d1593b&at=APZUnTWCYWfsD4eCeG52VJiK8-xB:1704172886196",
"dest": "Enter_Subscript_Gold.docx"
"path": "https://drive.usercontent.google.com/download?id=1dM_FSTGDWxSW64VEth_wKMYNkvw0y_tq&export=download&authuser=0&confirm=t&uuid=342f41e2-f48f-41ff-8942-f7dfe5de1dba&at=APZUnTXHfskcX3tvmrSbzCOyQIgb:1704976694506",
"dest": "H2O_Factsheet_WA_Gold.docx"
}
}
}

8
evaluation_examples/examples/libreoffice_writer/0e47de2a-32e0-456c-a366-8c607ef7a9d2.json
View File

@@ -10,7 +10,7 @@
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1aDWe-vAmcfQSgtPjFfrncq8ZFnCy4uUK&export=download&authuser=0&confirm=t&uuid=788af72a-ddaf-4ba3-aedb-96f34cc4d815&at=APZUnTVSRSSfMGcjXqLzvMixnkp6:1704179663299",
"path": "Desktop/Add_Page_Number_Bottom_Left.docx"
"path": "Desktop/LibreOffice_Open_Source_Word_Processing.docx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
"path": "Desktop/Add_Page_Number_Bottom_Left.docx"
"path": "Desktop/LibreOffice_Open_Source_Word_Processing.docx"
}
}
],
@@ -30,8 +30,8 @@
"func": "has_page_numbers_in_footers",
"result": {
"type": "vm_file",
"path": "Desktop/Add_Page_Number_Bottom_Left.docx",
"dest": "Add_Page_Number_Bottom_Left.docx"
"path": "Desktop/LibreOffice_Open_Source_Word_Processing.docx",
"dest": "LibreOffice_Open_Source_Word_Processing.docx"
}
}
}

8
evaluation_examples/examples/libreoffice_writer/0e763496-b6bb-4508-a427-fad0b6c3e195.json
View File

@@ -10,7 +10,7 @@
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1fIHNzFm8JabWoLKOnxrFM722fQ1d_huK&export=download&authuser=0&confirm=t&uuid=d11a8dda-1e4e-4dc9-b05c-e6b47624dbf0&at=APZUnTVG0ViFnKJa00314wVr3uP9:1704185871014",
"path": "Desktop/Change_Font_Through_File.docx"
"path": "Desktop/Dublin_Zoo_Intro.docx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
"path": "Desktop/Change_Font_Through_File.docx"
"path": "Desktop/Dublin_Zoo_Intro.docx"
}
}
],
@@ -36,8 +36,8 @@
},
"result": {
"type": "vm_file",
"path": "Desktop/Change_Font_Through_File.docx",
"dest": "Change_Font_Through_File.docx"
"path": "Desktop/Dublin_Zoo_Intro.docx",
"dest": "Dublin_Zoo_Intro.docx"
}
}
}

12
evaluation_examples/examples/libreoffice_writer/3ef2b351-8a84-4ff2-8724-d86eae9b842e.json
View File

@@ -1,7 +1,7 @@
{
"id": "3ef2b351-8a84-4ff2-8724-d86eae9b842e",
"snapshot": "libreoffice_writer",
"instruction": "center-justify the first line",
"instruction": "Help me center align the heading in LibreOffice.",
"source": "https://askubuntu.com/questions/1066351/how-do-you-center-align-in-libreoffice#:~:text=Ctrl%20%2B%20e%20will%20Center%20align%20the%20cursor%20for%20you.",
"config": [
{
@@ -9,8 +9,8 @@
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1IQ4rKyHMOui71YlyL7huggpLYFYtj923&export=download&authuser=0&confirm=t&uuid=014c2335-c0c6-4712-9d5a-ca8d3217e07f&at=APZUnTVrM698NQgSh4hqYXR8cjDc:1704185072996",
"path": "Desktop/Centering_First_Line.docx"
"url": "https://drive.usercontent.google.com/download?id=1P8QodvDF-3S50rx6UmW4M2D4Kr-p_Q-h&export=download&authuser=0&confirm=t&uuid=eea70a33-4c3f-4e90-885d-dd3df0d605bc&at=APZUnTX7ISvBhOICNrPLoqK0m3G-:1704971931660",
"path": "Desktop/Constitution_Template_With_Guidelines.docx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
"path": "Desktop/Centering_First_Line.docx"
"path": "Desktop/Constitution_Template_With_Guidelines.docx"
}
}
],
@@ -30,8 +30,8 @@
"func": "is_first_line_centered",
"result": {
"type": "vm_file",
"path": "Desktop/Centering_First_Line.docx",
"dest": "Centering_First_Line.docx"
"path": "Desktop/Constitution_Template_With_Guidelines.docx",
"dest": "Constitution_Template_With_Guidelines.docx"
}
}
}

18
evaluation_examples/examples/libreoffice_writer/45d61a06-6545-4422-97b7-bc76cfa964c1.json
View File

@@ -1,7 +1,7 @@
{
"id": "45d61a06-6545-4422-97b7-bc76cfa964c1",
"snapshot": "libreoffice_writer",
"instruction": "Replace all newlines with paragraph marks in LibreOffice Write",
"instruction": "Replace all newlines with paragraph marks in LibreOffice Writer",
"source": "https://stackoverflow.com/questions/71685737/how-to-replace-all-newlines-with-paragraph-marks-in-libreoffice-write",
"config": [
{
@@ -9,8 +9,8 @@
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=18XFjPVUnLG_-KOM5sn-Sk74HP_JHivMy&export=download&authuser=0&confirm=t&uuid=d23041bc-2ddd-42c4-84ae-481b953f021c&at=APZUnTVYh0AK0245qsDOCol7SdMB:1704185512767",
"path": "Desktop/Replace_Newlines_with_Paragraph_Marks.docx"
"url": "https://drive.usercontent.google.com/download?id=16lQcSkw-JQ_v8Sg0HkCtnOzyK-4cok8N&export=download&authuser=0&confirm=t&uuid=a3f833ae-2572-4cf3-8a21-6d250e689415&at=APZUnTUfn24NGMtXEzz2Nf7cFLjt:1704975857398",
"path": "Desktop/NOVEL_Submission_Guidelines.docx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
"path": "Desktop/Replace_Newlines_with_Paragraph_Marks.docx"
"path": "Desktop/NOVEL_Submission_Guidelines.docx"
}
}
],
@@ -30,13 +30,13 @@
"func": "compare_line_spacing",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1bP_noic02MuzrM8CdJIQN7F1gN4N8sel&export=download&authuser=0&confirm=t&uuid=657e0e4f-7b96-4d7e-83f4-99b79c68708f&at=APZUnTX7HsmefsMlzQaCGK2fg5Em:1704185514197",
"dest": "Replace_Newlines_with_Paragraph_Marks_Gold.docx"
"path": "https://drive.usercontent.google.com/download?id=1LaS5ObaOsbyKX1M17vi8ZseZwawgvNmf&export=download&authuser=0&confirm=t&uuid=db523e81-fa22-4002-97ff-e5dff92106a7&at=APZUnTVnT3ZYOGW7ZQdeW4SZP7mX:1704975993684",
"dest": "NOVEL_Submission_Guidelines_Gold.docx"
},
"result": {
"type": "vm_file",
"path": "Desktop/Double_Line_Spacing.docx",
"dest": "Replace_Newlines_with_Paragraph_Marks.docx"
"path": "Desktop/NOVEL_Submission_Guidelines.docx",
"dest": "NOVEL_Submission_Guidelines.docx"
}
}
}
}

8
evaluation_examples/examples/libreoffice_writer/4bcb1253-a636-4df4-8cb0-a35c04dfef31.json
View File

@@ -9,8 +9,8 @@
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1czb_13MshoiM-yCxvUGYD8OnIIrf-3VX&export=download&authuser=0&confirm=t&uuid=e7c30b67-7fac-4b64-a222-d04bc7c82842&at=APZUnTUA1te5vt7L__zJ7xuMs48e:1704177347643",
"path": "Desktop/Save_Writer_PDF.docx"
"url": "https://drive.usercontent.google.com/download?id=1cn3CaA_4ds1WY8SgBT3TvjRunxocCBiu&export=download&authuser=0&confirm=t&uuid=622f5ae1-6f21-4f31-8a3e-e4ead6ea6bc3&at=APZUnTVKOV40Ww5PovU7at2ELzb9:1704949558060",
"path": "Desktop/View_Person_Organizational_Summary.docx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
"path": "Desktop/Save_Writer_PDF.docx"
"path": "Desktop/View_Person_Organizational_Summary.docx"
}
}
],
@@ -28,7 +28,7 @@
],
"evaluator": {
"func": "check_file_exists",
"file_name": "Save_Writer_PDF.pdf",
"file_name": "View_Person_Organizational_Summary.pdf",
"directory": "/home/user/Downloads/"
}
}

16
evaluation_examples/examples/libreoffice_writer/663876c7-3471-43db-ba51-f410b13d9d7d.json
View File

@@ -1,7 +1,7 @@
{
"id": "663876c7-3471-43db-ba51-f410b13d9d7d",
"snapshot": "libreoffice_writer",
"instruction": "Insert the equation \"(a + b)^2 = a^2 + 2 a b + b^2\"",
"instruction": "Insert the equation \"(a + b)^2 = a^2 + 2 a b + b^2\" at the position of the cursor.",
"source": "https://askubuntu.com/questions/319593/how-to-type-science-equations-in-libre-office",
"config": [
{
@@ -9,8 +9,8 @@
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1FgMp7Ny63eXzeF23qHYhqQux31djlkah&export=download&authuser=0&confirm=t&uuid=d6b5208d-3b3a-4972-a641-ed738a419fdb&at=APZUnTX16Fz8Qg-B0NWpWgC-3Dyu:1704184410221",
"path": "Desktop/Insert_Equation.docx"
"url": "https://drive.usercontent.google.com/download?id=1J3--srr-Fmt2z-wOAyPDmO5GoiJMkUvZ&export=download&authuser=0&confirm=t&uuid=5f275f3d-ab7d-4d27-848a-8012af4bcf7f&at=APZUnTVe5ucwe1Q0oVi9jwiY5PF7:1704971494071",
"path": "Desktop/Factoring_Perfect_Square_Trinomials.docx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
"path": "Desktop/Insert_Equation.docx"
"path": "Desktop/Factoring_Perfect_Square_Trinomials.docx"
}
}
],
@@ -30,13 +30,13 @@
"func": "compare_insert_equation",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1hMFJnwHs7Iaexz3b9O2LJQUfsJ2wiwZ9&export=download&authuser=0&confirm=t&uuid=2abb49fb-d9c7-46cf-bc21-e69ecb9cefc6&at=APZUnTVzEZjChcUb4MIoxuq4cGea:1704184411805",
"dest": "Insert_Equation_Gold.docx"
"path": "https://drive.usercontent.google.com/download?id=1QGfyQUPEjOX8osycnjR8E5yHeIEGccKI&export=download&authuser=0&confirm=t&uuid=174491d6-7855-4323-9fa5-5146fde48c4a&at=APZUnTXA8bIcIbtw9mUN-K-IU9bT:1704971524974",
"dest": "Factoring_Perfect_Square_Trinomials_Gold.docx"
},
"result": {
"type": "vm_file",
"path": "Desktop/Insert_Equation.docx",
"dest": "Insert_Equation.docx"
"path": "Desktop/Factoring_Perfect_Square_Trinomials.docx",
"dest": "Factoring_Perfect_Square_Trinomials.docx"
}
}
}

12
evaluation_examples/examples/libreoffice_writer/66399b0d-8fda-4618-95c4-bfc6191617e9.json
View File

@@ -1,7 +1,7 @@
{
"id": "66399b0d-8fda-4618-95c4-bfc6191617e9",
"snapshot": "libreoffice_writer",
"instruction": "Insert a 7*5 empty table",
"instruction": "Could you help me insert a 7*5 empty table at the point of cursor?",
"source": "https://www.youtube.com/watch?v=l25Evu4ohKg",
"config": [
{
@@ -10,7 +10,7 @@
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1kXBP0jMxTeVahzFLYbYHJtjjgmuzrA8R&export=download&authuser=0&confirm=t&uuid=f8b9bad3-415d-4d39-a4fb-05a4cf881cf0&at=APZUnTXaohwzl8_2RDF_tgUsP9cH:1704181463579",
"path": "Desktop/Insert_Empty_Table.docx"
"path": "Desktop/Table_Of_Work_Effort_Instructions.docx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
"path": "Desktop/Insert_Empty_Table.docx"
"path": "Desktop/Table_Of_Work_Effort_Instructions.docx"
}
}
],
@@ -31,12 +31,12 @@
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=14JfHsW9GvbhORdtVAtvEbOi00MqEyHfb&export=download&authuser=0&confirm=t&uuid=3dba2459-ac37-4cad-a982-adecd406382a&at=APZUnTVQUqUPq_WacgY2xu4PvAKB:1704181465512",
"dest": "Insert_Empty_Table_Gold.docx"
"dest": "Table_Of_Work_Effort_Instructions_Gold.docx"
},
"result": {
"type": "vm_file",
"path": "Desktop/Insert_Empty_Table.docx",
"dest": "Insert_Empty_Table.docx"
"path": "Desktop/Table_Of_Work_Effort_Instructions.docx",
"dest": "Table_Of_Work_Effort_Instructions.docx"
}
}
}

16
evaluation_examples/examples/libreoffice_writer/6ada715d-3aae-4a32-a6a7-429b2e43fb93.json
View File

@@ -1,7 +1,7 @@
{
"id": "6ada715d-3aae-4a32-a6a7-429b2e43fb93",
"snapshot": "libreoffice_writer",
"instruction": "Copy the screenshot 1.jpg from the desktop to where my cursor is locatedInsert the image which is in IMAGE_PATH where my cursor is",
"instruction": "Copy the screenshot 1.png from the desktop to where my cursor is located",
"source": "https://www.quora.com/How-do-you-insert-images-into-a-LibreOffice-Writer-document",
"config": [
{
@@ -10,7 +10,7 @@
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1ebLG0gmqYv41ek4UmKWhFsxBnoUSGjKp&export=download&authuser=0&confirm=t&uuid=8f7d7bee-1fe4-4c4c-8b69-8aaf47199c57&at=APZUnTVYUvYTopUXCVs69QWWwPbq:1704173993139",
"path": "Desktop/Insert_Image_At_Cursor.docx"
"path": "Desktop/Viewing_Your_Class_Schedule_and_Textbooks.docx"
}
]
}
@@ -20,8 +20,8 @@
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1QfjQ4SKtjKDXWpqa2u6mC_KtB3ASEK5O&export=download&authuser=0&confirm=t&uuid=06af00b9-58f3-4691-a6a3-34309c80cbbb&at=APZUnTVZpE1lMxcvGG0cdt5zuxZ_:1704174003198",
"path": "Desktop/1.jpg"
"url": "https://drive.usercontent.google.com/download?id=1-wYHrgPMZl5Y9Losd4AGkMVHWNU7Uemg&export=download&authuser=0&confirm=t&uuid=3bc52ab7-1739-407d-9b14-36c9305c973e&at=APZUnTW95CwED3zm5KRREFmO2mc2:1704952228530",
"path": "Desktop/1.png"
}
]
}
@@ -29,7 +29,7 @@
{
"type": "open",
"parameters": {
"path": "Desktop/Insert_Image_At_Cursor.docx"
"path": "Desktop/Viewing_Your_Class_Schedule_and_Textbooks.docx"
}
}
],
@@ -41,13 +41,13 @@
"func": "compare_contains_image",
"result": {
"type": "vm_file",
"path": "Desktop/Insert_Image_At_Cursor.docx",
"dest": "Insert_Image_At_Cursor.docx"
"path": "Desktop/Viewing_Your_Class_Schedule_and_Textbooks.docx",
"dest": "Viewing_Your_Class_Schedule_and_Textbooks.docx"
},
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1xbhlfqGrPutHHi2aHg66jwXD-yaZpe9j&export=download&authuser=0&confirm=t&uuid=427765e0-3f97-4a72-92db-a1fe7cdde73b&at=APZUnTUhNLh2PDu4OGkCVQW-LPCd:1704173991269",
"dest": "Insert_Image_At_Cursor_Gold.docx"
"dest": "Viewing_Your_Class_Schedule_and_Textbooks_Gold.docx"
}
}
}

16
evaluation_examples/examples/libreoffice_writer/936321ce-5236-426a-9a20-e0e3c5dc536f.json
View File

@@ -1,7 +1,7 @@
{
"id": "936321ce-5236-426a-9a20-e0e3c5dc536f",
"snapshot": "libreoffice_writer",
"instruction": "Convert the content seperated by commas to a table",
"instruction": "Could you help me convert the text seperated by commas to a table?",
"source": "https://www.youtube.com/watch?v=l25Evu4ohKg",
"config": [
{
@@ -9,8 +9,8 @@
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=139-NmslBR_9qlD7hUO08xj_VFffV95eM&export=download&authuser=0&confirm=t&uuid=64a6c35d-f3ce-4c25-9f83-4a952e24c5ad&at=APZUnTUL1GMR_QbpFQnC9fPwkdqa:1704183959196",
"path": "Desktop/Convert_Text_To_Table.docx"
"url": "https://drive.usercontent.google.com/download?id=1WcHqnDy0OPFiaGtvPSvKXuaLc_rtGzYC&export=download&authuser=0&confirm=t&uuid=0c552800-5dfe-4a3f-a960-8d384481fa7f&at=APZUnTWYjmvuXkU-pb77dy7EgCeD:1704970003044",
"path": "Desktop/Graphemes_Sound_Letter_Patterns.docx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
"path": "Desktop/Convert_Text_To_Table.docx"
"path": "Desktop/Graphemes_Sound_Letter_Patterns.docx"
}
}
],
@@ -30,13 +30,13 @@
"func": "compare_docx_tables",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1fs2msYaxnEZL9XASENQMZIag2MTxIBJs&export=download&authuser=0&confirm=t&uuid=6c71f008-082c-4f0c-9ffc-0a802f5cbfe6&at=APZUnTVDpucMDfk5P2T-0dZx_KVV:1704183960360",
"dest": "Convert_Text_To_Table_Gold.docx"
"path": "https://drive.usercontent.google.com/download?id=1og3gCBRhAeHk02E1wAsbfau_1ywdoEJL&export=download&authuser=0&confirm=t&uuid=264de3b4-b07c-4382-8a73-a766b1d1fc58&at=APZUnTX8yRMoDXCHi1y1H380GaSX:1704969967989",
"dest": "Graphemes_Sound_Letter_Patterns_Gold.docx"
},
"result": {
"type": "vm_file",
"path": "Desktop/Convert_Text_To_Table.docx",
"dest": "Convert_Text_To_Table.docx"
"path": "Desktop/Graphemes_Sound_Letter_Patterns.docx",
"dest": "Graphemes_Sound_Letter_Patterns.docx"
}
}
}

16
evaluation_examples/examples/libreoffice_writer/adf5e2c3-64c7-4644-b7b6-d2f0167927e7.json
View File

@@ -1,7 +1,7 @@
{
"id": "adf5e2c3-64c7-4644-b7b6-d2f0167927e7",
"snapshot": "libreoffice_writer",
"instruction": "Helping me adding \"C. Luo and M. J. Carey, \"LSM-based storage techniques: a survey,\" The VLDB Journal, vol. 29, no. 1, pp. 393–418, 2020.\" to my reference list, and add a cross reference at the end of the first paragraph",
"instruction": "Help me adding \"Steinberg, F. M., Bearden, M. M., & Keen, C. L. (2003). Cocoa and chocolate flavonoids: Implications for cardiovascular health. Journal of the American Dietetic Association, 103(2), 215-223. doi: 10.1053/jada.2003.50028\" to my reference list, and add a cross reference where my cursor is located (in the fourth paragraph).",
"source": "https://seekstar.github.io/2022/04/11/libreoffice%E5%BC%95%E7%94%A8%E6%96%87%E7%8C%AE/",
"config": [
{
@@ -9,8 +9,8 @@
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1xOfwImgkPzdmjQomj-MCFd8nQS75OjaH&export=download&authuser=0&confirm=t&uuid=7eb91c26-dad5-4480-b1ec-35e506cde1e4&at=APZUnTW01MvBI_gkC8yoiyAVs7yi:1704188254979",
"path": "Desktop/Add_Citation_Cross_Reference.docx"
"url": "https://drive.usercontent.google.com/download?id=1ShGL4gWSV7nzamAb0V2KqjoCOhyodcKU&export=download&authuser=0&confirm=t&uuid=5f67edb8-cbbf-4a83-b46e-f193ad55e1e8&at=APZUnTVRJenYCM--vETagQ5ACTT5:1704979226579",
"path": "Desktop/Essay_Writing_English_for_uni.docx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
"path": "Desktop/Add_Citation_Cross_Reference.docx"
"path": "Desktop/Essay_Writing_English_for_uni.docx"
}
}
],
@@ -30,13 +30,13 @@
"func": "compare_docx_files",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1wFQU7hkAT2wmSHTgM22F9Ep4WXmymEMW&export=download&authuser=0&confirm=t&uuid=a7ea1eec-678b-4407-b023-df13cc6f8c54&at=APZUnTW3WoqOfS9A1BW79XfV8jKh:1704188260410",
"dest": "Add_Citation_Cross_Reference_Gold.docx"
"path": "https://drive.usercontent.google.com/download?id=1bf7IKjjxGAQfbOlq9rLPzjTtkOZ3dREj&export=download&authuser=0&confirm=t&uuid=0420cfa5-7e51-4688-8c93-74748914ce52&at=APZUnTWIShIleccvgVBIr6ZsSnOw:1704979256798",
"dest": "Essay_Writing_English_for_uni_Gold.docx"
},
"result": {
"type": "vm_file",
"path": "Desktop/Add_Citation_Cross_Reference.docx",
"dest": "Add_Citation_Cross_Reference.docx"
"path": "Desktop/Essay_Writing_English_for_uni.docx",
"dest": "Essay_Writing_English_for_uni.docx"
}
}
}

14
evaluation_examples/examples/libreoffice_writer/e528b65e-1107-4b8c-8988-490e4fece599.json
View File

@@ -9,8 +9,8 @@
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1LZ1_U9CyR8oOkqfbY6HMLWr4wBVYhCLR&export=download&authuser=0&confirm=t&uuid=574d533a-8df9-4f33-bebd-689b623f27a9&at=APZUnTVruCDRxY661_PVT9BA3839:1704180420603",
"path": "Desktop/Capitalize_First_Letter.docx"
"url": "https://drive.usercontent.google.com/download?id=181hyG_NZUJaUp7kUdeFoUpNGpOcjzYFV&export=download&authuser=0&confirm=t&uuid=fdb8aee4-ea1f-43c1-bc66-9d44099512e1&at=APZUnTVUrypBeVnTfkJsriv7S2GO:1704969486759",
"path": "Geography_And_Magical_Realism.docx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
"path": "Desktop/Capitalize_First_Letter.docx"
"path": "Desktop/Geography_And_Magical_Realism.docx"
}
}
],
@@ -30,13 +30,13 @@
"func": "compare_docx_files",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1ykIGg48GjYUOSw8t44evShJE2HX7R5w3&export=download&authuser=0&confirm=t&uuid=ed81d6cd-6044-49f1-be86-35adeaeeea00&at=APZUnTUxW8WLyPr-_smA2Mnwpuuv:1704180422490https://drive.usercontent.google.com/download?id=1ykIGg48GjYUOSw8t44evShJE2HX7R5w3&export=download&authuser=0&confirm=t&uuid=ed81d6cd-6044-49f1-be86-35adeaeeea00&at=APZUnTUxW8WLyPr-_smA2Mnwpuuv:1704180422490",
"dest": "Capitalize_First_Letter_Gold.docx"
"path": "https://drive.usercontent.google.com/download?id=1qwoJMuMhLxWwONRLmxwMdUuVIFBAhhNT&export=download&authuser=0&confirm=t&uuid=8fa4d503-ec55-419f-ac90-f5915a5f67b5&at=APZUnTVEntT4oMiEfOuWcUKaInBs:1704969347952",
"dest": "Geography_And_Magical_Realism_Gold.docx"
},
"result": {
"type": "vm_file",
"path": "Desktop/Capitalize_First_Letter.docx",
"dest": "Capitalize_First_Letter.docx"
"path": "Desktop/Geography_And_Magical_Realism.docx",
"dest": "Geography_And_Magical_Realism.docx"
}
}
}

10
evaluation_examples/examples/libreoffice_writer/ecc2413d-8a48-416e-a3a2-d30106ca36cb.json
View File

@@ -1,7 +1,7 @@
{
"id": "ecc2413d-8a48-416e-a3a2-d30106ca36cb",
"snapshot": "libreoffice_writer",
"instruction": "Insert a blank page",
"instruction": "Help me insert a blank page where my cursor is located.",
"source": "https://www.quora.com/How-can-I-insert-a-blank-page-on-libreoffice",
"config": [
{
@@ -10,7 +10,7 @@
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1sDufDSC4foI379-Jikya9WK7FBUSqgrt&export=download&authuser=0&confirm=t&uuid=0abd82d6-2b2c-49bc-af5e-49bfe1c99278&at=APZUnTURIqTNJcIHBcMP2BxEaGXr:1704174850900",
"path": "Desktop/Insert_Blank_Page.docx"
"path": "Desktop/Sample_Statutory_Declaration.docx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
"path": "Desktop/Insert_Blank_Page.docx"
"path": "Desktop/Sample_Statutory_Declaration.docx"
}
}
],
@@ -30,8 +30,8 @@
"func": "contains_page_break",
"result": {
"type": "vm_file",
"path": "Desktop/Insert_Blank_Page.docx",
"dest": "Insert_Blank_Page.docx"
"path": "Desktop/Sample_Statutory_Declaration.docx",
"dest": "Sample_Statutory_Declaration.docx"
}
}
}

6
evaluation_examples/examples/libreoffice_writer/f178a4a9-d090-4b56-bc4c-4b72a61a035d.json
View File

@@ -9,8 +9,8 @@
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1X2XTU2ZFuMXOhm7T400e6AOe6eBYxWzD&export=download&authuser=0&confirm=t&uuid=1318923f-6d54-4148-aa80-a454b9963cec&at=APZUnTU-h1nmcjBO_ytWVxXuh8l9:1704187013730",
"path": "Desktop/Set_Default_Font.docx"
"url": "https://drive.usercontent.google.com/download?id=1xREbNAu_2wLTs8EQT0NnHLpIkAVAfpyk&export=download&authuser=0&confirm=t&uuid=dd5cb525-ff4b-41a2-8123-d488f2f21fad&at=APZUnTXaYBqLT9fRtGYZHOedq-PG:1704977194647",
"path": "Desktop/loa-one-time-submission-sealand.docx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
"path": "Desktop/Set_Default_Font.docx"
"path": "Desktop/loa-one-time-submission-sealand.docx"
}
}
],

2
evaluation_examples/examples/vlc/59f21cfb-0120-4326-b255-a5b827b38967.json
View File

@@ -1,7 +1,7 @@
{
"id": "59f21cfb-0120-4326-b255-a5b827b38967",
"snapshot": "base_setup",
"instruction": "Play the music video on my desktop",
"instruction": "Could you play the music video that's saved on my desktop for me?",
"source": "https://docs.videolan.me/vlc-user/desktop/3.0/en/basic/media.html#playing-a-file",
"config": [
{

24
evaluation_examples/examples/vlc/8f080098-ddb1-424c-b438-4e96e5e4786e.json
View File

@@ -1,7 +1,7 @@
{
"id": "8f080098-ddb1-424c-b438-4e96e5e4786e",
"snapshot": "base_setup",
"instruction": "Could you help me extract MP3 Audio to AUDIO_PATH from Video at VIDEO_PATH using VLC Media Player?",
"instruction": "Could you download the song from this music video and save it as an MP3 file? I'd like to have it on my device to play whenever I want. Please title the file \"Baby Justin Bieber.mp3.\" I really appreciate your help!",
"source": "https://medium.com/@jetscribe_ai/how-to-extract-mp3-audio-from-videos-using-vlc-media-player-beeef644ebfb",
"config": [
{
@@ -10,7 +10,7 @@
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=19jBiz8sb0M7KHHATO9qeTPr17aWm4me-&export=download&authuser=0&confirm=t&uuid=7a2261f4-3905-433f-b53f-a52dd0845651&at=APZUnTU1nmXSa1ObrA5NHYt8t1-p:1704710908141",
"path": "Baby Justin Bieber.mp4"
"path": "Desktop/Baby Justin Bieber.mp4"
}
]
}
@@ -18,7 +18,11 @@
{
"type": "launch",
"parameters": {
"command": "vlc"
"command": [
"vlc",
"--start-time=73",
"Desktop/Baby Justin Bieber.mp4"
]
}
}
],
@@ -27,16 +31,16 @@
"vlc"
],
"evaluator": {
"func": "is_vlc_recordings_folder",
"func": "compare_audios",
"expected": {
"type": "rule",
"rules": {
"recording_file_path": "/home/user/Desktop"
}
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1eRuuiUAOmckyn2zQebx1nWQ40kpjSJ_N&export=download&authuser=0&confirm=t&uuid=8012703a-9e45-4d91-9bda-90c119e42254&at=APZUnTW0uyCSeg7FClSc50UJVRz-:1704898635455",
"dest": "baby_gold.mp3"
},
"result": {
"type": "vlc_config",
"dest": "vlcrc"
"type": "vm_file",
"path": "Desktop/Baby Justin Bieber.mp3",
"dest": "baby.mp3"
}
}
}

2
evaluation_examples/examples/vlc/a1c3ab35-02de-4999-a7ed-2fd12c972c6e.json
View File

@@ -1,7 +1,7 @@
{
"id": "a1c3ab35-02de-4999-a7ed-2fd12c972c6e",
"snapshot": "base_setup",
"instruction": "Could you help me compress the video to DIR_PATH?",
"instruction": "Could you help me compress the video to MPEG-4 format and save with name with prefix _?",
"source": "https://www.quora.com/How-do-I-compress-a-video-with-VLC",
"config": [],
"trajectory": "trajectories/",

2
evaluation_examples/examples/vlc/bba3381f-b5eb-4439-bd9e-80c22218d5a7.json
View File

@@ -1,7 +1,7 @@
{
"id": "bba3381f-b5eb-4439-bd9e-80c22218d5a7",
"snapshot": "base_setup",
"instruction": "Help me play the online video at https://www.youtube.com/watch?v=pgBsyTKAwLw",
"instruction": "Can you start streaming the video from this link for me? https://www.youtube.com/watch?v=pgBsyTKAwLw",
"source": "https://www.quora.com/How-do-I-play-online-videos-using-the-VLC-media-player",
"config": [
{

33
evaluation_examples/examples/vlc/efcf0d81-0835-4880-b2fd-d866e8bc2294.json
View File

@@ -1,30 +1,47 @@
{
"id": "efcf0d81-0835-4880-b2fd-d866e8bc2294",
"snapshot": "base_setup",
"instruction": "Set this frame of the current video as my wallpaper",
"source": "https://www.youtube.com/watch?v=XHprwDJ0-fU&t=436s",
"instruction": "Make this part of the video my computer's background picture",
"source": "https://www.youtube.com/watch?v=XHprwDJ0-fU&t=436s, https://help.ubuntu.com/stable/ubuntu-help/look-background.html.en",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "",
"path": ""
"url": "https://drive.usercontent.google.com/download?id=1H9D0jSkzpzEWsJZG0HdNNNHDMi3fnGli&export=download&authuser=0&confirm=t&uuid=a6d03223-db7a-48bd-b5f8-50b51f4d3d68&at=APZUnTVi3cqgvRV49c6sluV-nMo_:1704879367756",
"path": "Desktop/Interstellar Movie - Official Trailer.mp4"
}
]
}
},
{
"type": "launch",
"parameters": {
"command": ["vlc", "/path/to/your/video.mp4", "--start-time=0", "--run-time=10", "vlc://quit", "&&", "vlc", "/path/to/your/video.mp4", "--start-time=10"]
}
"parameters": {
"command": [
"vlc",
"--start-time=120.5",
"--stop-time=121",
"--play-and-pause",
"Desktop/Interstellar Movie - Official Trailer.mp4"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"vlc"
],
"evaluator": "evaluation_dir"
"evaluator": {
"func": "compare_images",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=10P8kLkWeYhtK3Gl15nB-gH7VLO4S7Xal&export=download&authuser=0&confirm=t&uuid=d5c5db29-435d-404f-8900-683bf56fbc66&at=APZUnTXm1ZHS7CbGiWaB04WySH-t:1704891263184",
"dest": "interstellar_wallpaper_gold.png"
},
"result": {
"type": "vm_wallpaper",
"dest": "result_wallpaper.png"
}
}
}

42
evaluation_examples/examples/vlc/fba2c100-79e8-42df-ae74-b592418d54f4.json
View File

@@ -1,12 +1,48 @@
{
"id": "fba2c100-79e8-42df-ae74-b592418d54f4",
"snapshot": "base_setup",
"instruction": "Screenshot the current frame of the video",
"instruction": "Snap a photo of the current video scene, save it as 'interstellar.png', and put it on the Desktop, please.",
"source": "https://www.youtube.com/watch?v=XHprwDJ0-fU&t=436s",
"config": [],
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1u7CFVr5bWti7OnhvyjtSdQolzy0lYicm&export=download&authuser=0&confirm=t&uuid=18498a7c-9b52-4c00-a2cc-2e8b521c4d9c&at=APZUnTX0R_TRccU_UmtJWXZ6On3x:1704890439561",
"path": "Desktop/Interstellar Movie - Official Trailer.mp4"
}
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"vlc",
"--start-time=120.5",
"--stop-time=121",
"--play-and-pause",
"Desktop/Interstellar Movie - Official Trailer.mp4"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"vlc"
],
"evaluator": "evaluation_dir"
"evaluator": {
"func": "compare_images",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1shU1TQ3ao9QWfhmC63vbD10YFbhv7PYv&export=download&authuser=0&confirm=t&uuid=6a22a47a-6d35-4996-b02e-b24b33a1c449&at=APZUnTVlvDKpxW7QKmBBmdNq2jRw:1704891215314",
"dest": "interstellar_gold.png"
},
"result": {
"type": "vm_file",
"path": "Desktop/interstellar.png",
"dest": "interstellar.png"
}
}
}