From 88cc41cdf239bb6d27190b4bf5e53e38e346be06 Mon Sep 17 00:00:00 2001 From: tsuky_chen <3107760494@qq.com> Date: Mon, 4 Mar 2024 11:40:02 +0800 Subject: [PATCH] update multi apps --- .../69acbb55-d945-4927-a87b-8480e1a5bb7e.json | 50 +++++++++++ .../acb0f96b-e27c-44d8-b55f-7cb76609dfcd.json | 49 +++++++++++ .../eb303e01-261e-4972-8c07-c9b4e7a4922a.json | 82 +++++++++++++++++++ 3 files changed, 181 insertions(+) create mode 100644 evaluation_examples/examples/multi_apps/69acbb55-d945-4927-a87b-8480e1a5bb7e.json create mode 100644 evaluation_examples/examples/multi_apps/acb0f96b-e27c-44d8-b55f-7cb76609dfcd.json create mode 100644 evaluation_examples/examples/multi_apps/eb303e01-261e-4972-8c07-c9b4e7a4922a.json diff --git a/evaluation_examples/examples/multi_apps/69acbb55-d945-4927-a87b-8480e1a5bb7e.json b/evaluation_examples/examples/multi_apps/69acbb55-d945-4927-a87b-8480e1a5bb7e.json new file mode 100644 index 0000000..8e75c5f --- /dev/null +++ b/evaluation_examples/examples/multi_apps/69acbb55-d945-4927-a87b-8480e1a5bb7e.json @@ -0,0 +1,50 @@ +{ + "id": "69acbb55-d945-4927-a87b-8480e1a5bb7e", + "snapshot": "os", + "instruction": "Set up the environment for the project in /home/user according to the guidance on \"https: //github.com/xlang-ai/instructor-embedding\".", + "source": "authors", + "config": [ + { + "type": "launch", + "parameters": { + "command": [ + "google-chrome", + "--remote-debugging-port=1337" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "socat", + "tcp-listen:9222,fork", + "tcp:localhost:1337" + ] + } + } + ], + "trajectory": "trajectories/69acbb55-d945-4927-a87b-8480e1a5bb7e", + "related_apps": [ + "os", + "chrome", + "vscode" + ], + "evaluator": { + "func": "exact_match", + "result": { + "type": "vm_command_line", + "command": [ + "python", + "-c", + "from InstructorEmbedding import INSTRUCTOR;" + ] + }, + "expected": { + "type": "rule", + "rules": { + "expected": "" + } + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/acb0f96b-e27c-44d8-b55f-7cb76609dfcd.json b/evaluation_examples/examples/multi_apps/acb0f96b-e27c-44d8-b55f-7cb76609dfcd.json new file mode 100644 index 0000000..a816523 --- /dev/null +++ b/evaluation_examples/examples/multi_apps/acb0f96b-e27c-44d8-b55f-7cb76609dfcd.json @@ -0,0 +1,49 @@ +{ + "id": "acb0f96b-e27c-44d8-b55f-7cb76609dfcd", + "snapshot": "os", + "instruction": "Please help me clone the repo \"https://github.com/xlang-ai/instructor-embedding\" to /home/user.", + "source": "authors", + "config": [ + { + "type": "launch", + "parameters": { + "command": [ + "google-chrome", + "--remote-debugging-port=1337" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "socat", + "tcp-listen:9222,fork", + "tcp:localhost:1337" + ] + } + } + ], + "trajectory": "trajectories/acb0f96b-e27c-44d8-b55f-7cb76609dfcd", + "related_apps": [ + "os", + "chrome" + ], + "evaluator": { + "func": "is_extension_installed", + "result": { + "type": "vm_command_line", + "command": [ + "ls", + "/home/user/" + ] + }, + "expected": { + "type": "rule", + "rules": { + "type": "contain", + "expected": "instructor-embedding" + } + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/eb303e01-261e-4972-8c07-c9b4e7a4922a.json b/evaluation_examples/examples/multi_apps/eb303e01-261e-4972-8c07-c9b4e7a4922a.json new file mode 100644 index 0000000..7ee7e1a --- /dev/null +++ b/evaluation_examples/examples/multi_apps/eb303e01-261e-4972-8c07-c9b4e7a4922a.json @@ -0,0 +1,82 @@ +{ + "id": "eb303e01-261e-4972-8c07-c9b4e7a4922a", + "snapshot": "libreoffice_impress", + "instruction": "I am going to give a talk tomorrow, and my PPT and speaking notes are saved on the desktop. Help me Insert what I'm going to say for each slide in \"note\" of PPT as a reminder. I have done it for some slides. Help me finish the job for the rest.", + "source": "authors", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1MdgN8ECxzLgHgjq8eKqrNQt3MPDjnKwa&export=download&authuser=0&confirm=t&uuid=ed5c37db-c565-4ca1-bbd1-bbdba13e9306&at=APZUnTUNi8YTLlZqMZ0r--bBpBEG:1709449877819", + "path": "/home/user/Desktop/lecture1-2021-with-ink.pptx" + }, + { + "url": "https://drive.usercontent.google.com/download?id=1FkPOcsWpsjUXSUld1NblwyVzcsE19uIe&export=download&authuser=0&confirm=t&uuid=27501bc0-732b-4ff7-abf4-a52427aea264&at=APZUnTWleaafIVF2iZkiuHo0vQ66:1709449873140", + "path": "/home/user/Desktop/notes.docx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/Desktop/lecture1-2021-with-ink.pptx" + } + } + ], + "trajectory": "trajectories/eb303e01-261e-4972-8c07-c9b4e7a4922a", + "related_apps": [ + "libreoffice_impress", + "libreoffice_writer" + ], + "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "lecture1-2021-with-ink.pptx - LibreOffice Impress", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + } + ], + "func": "compare_pptx_files", + "expected": { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=18orj_0q6N4w7ijADOJeU5ZkDDw-RdFUl&export=download&authuser=0&confirm=t&uuid=c05d2bce-bccb-4504-8fe4-7c409788d727&at=APZUnTVlCicnIm0cMdJ9FrZg4MSN:1709453015475", + "dest": "lecture1-2021-with-ink_Gold.pptx" + }, + "result": { + "type": "vm_file", + "path": "/home/user/Desktop/lecture1-2021-with-ink.pptx", + "dest": "lecture1-2021-with-ink.pptx" + }, + "options": { + "examine_shape": false, + "examine_bullets": false + } + } +} \ No newline at end of file