Merge branch 'main' of github.com:ztjhz/DesktopEnv

evaluation_examples/examples/multi_apps/69acbb55-d945-4927-a87b-8480e1a5bb7e.json

@@ -0,0 +1,50 @@
+{
+  "id": "69acbb55-d945-4927-a87b-8480e1a5bb7e",
+  "snapshot": "os",
+  "instruction": "Set up the environment for the project in /home/user according to the guidance on \"https: //github.com/xlang-ai/instructor-embedding\".",
+  "source": "authors",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "google-chrome",
+          "--remote-debugging-port=1337"
+        ]
+      }
+    },
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "socat",
+          "tcp-listen:9222,fork",
+          "tcp:localhost:1337"
+        ]
+      }
+    }
+  ],
+  "trajectory": "trajectories/69acbb55-d945-4927-a87b-8480e1a5bb7e",
+  "related_apps": [
+    "os",
+    "chrome",
+    "vscode"
+  ],
+  "evaluator": {
+    "func": "exact_match",
+    "result": {
+      "type": "vm_command_line",
+      "command": [
+        "python",
+        "-c",
+        "from InstructorEmbedding import INSTRUCTOR;"
+      ]
+    },
+    "expected": {
+      "type": "rule",
+      "rules": {
+        "expected": ""
+      }
+    }
+  }
+}

evaluation_examples/examples/multi_apps/acb0f96b-e27c-44d8-b55f-7cb76609dfcd.json

@@ -0,0 +1,49 @@
+{
+  "id": "acb0f96b-e27c-44d8-b55f-7cb76609dfcd",
+  "snapshot": "os",
+  "instruction": "Please help me clone the repo \"https://github.com/xlang-ai/instructor-embedding\" to /home/user.",
+  "source": "authors",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "google-chrome",
+          "--remote-debugging-port=1337"
+        ]
+      }
+    },
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "socat",
+          "tcp-listen:9222,fork",
+          "tcp:localhost:1337"
+        ]
+      }
+    }
+  ],
+  "trajectory": "trajectories/acb0f96b-e27c-44d8-b55f-7cb76609dfcd",
+  "related_apps": [
+    "os",
+    "chrome"
+  ],
+  "evaluator": {
+    "func": "is_extension_installed",
+    "result": {
+      "type": "vm_command_line",
+      "command": [
+        "ls",
+        "/home/user/"
+      ]
+    },
+    "expected": {
+      "type": "rule",
+      "rules": {
+        "type": "contain",
+        "expected": "instructor-embedding"
+      }
+    }
+  }
+}

evaluation_examples/examples/multi_apps/eb303e01-261e-4972-8c07-c9b4e7a4922a.json

@@ -0,0 +1,82 @@
+{
+    "id": "eb303e01-261e-4972-8c07-c9b4e7a4922a",
+    "snapshot": "libreoffice_impress",
+    "instruction": "I am going to give a talk tomorrow, and my PPT and speaking notes are saved on the desktop. Help me Insert what I'm going to say for each slide in \"note\" of PPT as a reminder. I have done it for some slides. Help me finish the job for the rest.",
+    "source": "authors",
+    "config": [
+        {
+            "type": "download",
+            "parameters": {
+                "files": [
+                    {
+                        "url": "https://drive.usercontent.google.com/download?id=1MdgN8ECxzLgHgjq8eKqrNQt3MPDjnKwa&export=download&authuser=0&confirm=t&uuid=ed5c37db-c565-4ca1-bbd1-bbdba13e9306&at=APZUnTUNi8YTLlZqMZ0r--bBpBEG:1709449877819",
+                        "path": "/home/user/Desktop/lecture1-2021-with-ink.pptx"
+                    },
+                    {
+                        "url": "https://drive.usercontent.google.com/download?id=1FkPOcsWpsjUXSUld1NblwyVzcsE19uIe&export=download&authuser=0&confirm=t&uuid=27501bc0-732b-4ff7-abf4-a52427aea264&at=APZUnTWleaafIVF2iZkiuHo0vQ66:1709449873140",
+                        "path": "/home/user/Desktop/notes.docx"
+                    }
+                ]
+            }
+        },
+        {
+            "type": "open",
+            "parameters": {
+                "path": "/home/user/Desktop/lecture1-2021-with-ink.pptx"
+            }
+        }
+    ],
+    "trajectory": "trajectories/eb303e01-261e-4972-8c07-c9b4e7a4922a",
+    "related_apps": [
+        "libreoffice_impress",
+        "libreoffice_writer"
+    ],
+    "evaluator": {
+        "postconfig": [
+            {
+                "type": "activate_window",
+                "parameters": {
+                    "window_name": "lecture1-2021-with-ink.pptx - LibreOffice Impress",
+                    "strict": true
+                }
+            },
+            {
+                "type": "sleep",
+                "parameters": {
+                    "seconds": 5
+                }
+            },
+            {
+                "type": "execute",
+                "parameters": {
+                    "command": [
+                        "python",
+                        "-c",
+                        "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);"
+                    ]
+                }
+            },
+            {
+                "type": "sleep",
+                "parameters": {
+                    "seconds": 0.5
+                }
+            }
+        ],
+        "func": "compare_pptx_files",
+        "expected": {
+            "type": "cloud_file",
+            "path": "https://drive.usercontent.google.com/download?id=18orj_0q6N4w7ijADOJeU5ZkDDw-RdFUl&export=download&authuser=0&confirm=t&uuid=c05d2bce-bccb-4504-8fe4-7c409788d727&at=APZUnTVlCicnIm0cMdJ9FrZg4MSN:1709453015475",
+            "dest": "lecture1-2021-with-ink_Gold.pptx"
+        },
+        "result": {
+            "type": "vm_file",
+            "path": "/home/user/Desktop/lecture1-2021-with-ink.pptx",
+            "dest": "lecture1-2021-with-ink.pptx"
+        },
+        "options": {
+            "examine_shape": false,
+            "examine_bullets": false
+        }
+    }
+}