update multi apps

2024-03-07 01:24:36 +08:00
parent 69ef653a7c
commit 5b5475094e
8 changed files with 215 additions and 8 deletions
--- a/evaluation_examples/examples/multi_apps/02ce9a50-7af2-47ed-8596-af0c230501f8.json
+++ b/evaluation_examples/examples/multi_apps/02ce9a50-7af2-47ed-8596-af0c230501f8.json
@@ -0,0 +1,61 @@
+{
+  "id": "02ce9a50-7af2-47ed-8596-af0c230501f8",
+  "snapshot": "libreoffice_writer",
+  "instruction": "I'm using libreoffice writer to write a tutorial about linux, and now I want to show the results obtained by using the \"ls\" command in /home/user. Please run this command and save the screenshot as 'ls.png' on Desktop.",
+  "source": "authors",
+  "config": [
+    {
+      "type": "download",
+      "parameters": {
+        "files": [
+          {
+            "url": "https://drive.usercontent.google.com/download?id=1cOYh50n0S3-Q_h8HDTCEPv2v6NsJl07m&export=download&authuser=0&confirm=t&uuid=9741b52a-d998-4875-94ec-ab0a45a40bf9&at=APZUnTX__WVALT4i0EIQoRAWkm-k:1709715620836",
+            "path": "/home/user/Desktop/top-10-linux-commands-for-newbies.docx"
+          }
+        ]
+      }
+    },
+    {
+      "type": "open",
+      "parameters": {
+        "path": "/home/user/Desktop/top-10-linux-commands-for-newbies.docx"
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 2
+      }
+    },
+    {
+      "type": "execute",
+      "parameters": {
+        "command": [
+          "python",
+          "-c",
+          "import pyautogui; import time; pyautogui.hotkey('f11'); time.sleep(0.5); pyautogui.click(960, 540); time.sleep(0.5); pyautogui.scroll(-20)"
+        ]
+      }
+    }
+  ],
+  "trajectory": "trajectories/02ce9a50-7af2-47ed-8596-af0c230501f8",
+  "related_apps": [
+    "libreoffice_writer",
+    "os"
+  ],
+  "evaluator": {
+    "func": "compare_image_text",
+    "result": {
+      "type": "vm_file",
+      "path": "/home/user/Desktop/ls.png",
+      "dest": "ls.png"
+    },
+    "expected": {
+      "type": "rule",
+      "rules": {
+        "type": "text",
+        "text": "$ ls\n"
+      }
+    }
+  }
+}
--- a/evaluation_examples/examples/multi_apps/09a37c51-e625-49f4-a514-20a773797a8a.json
+++ b/evaluation_examples/examples/multi_apps/09a37c51-e625-49f4-a514-20a773797a8a.json
@@ -0,0 +1,48 @@
+{
+  "id": "09a37c51-e625-49f4-a514-20a773797a8a",
+  "snapshot": "libreoffice_writer",
+  "instruction": "I received a request from my friend that he wanted me to help him modify a picture. On the Desktop is the requirement doc and the picture to be adjusted. Modify the image as he said and save modified pic as \"pic.jpg\" on Desktop. Thanks!",
+  "source": "authors",
+  "config": [
+    {
+      "type": "download",
+      "parameters": {
+        "files": [
+          {
+            "url": "https://drive.usercontent.google.com/download?id=1npPOtBdE5t_mzScyA94vDlxB8SCZNubv&export=download&authuser=0&confirm=t&uuid=64ee33b7-66a5-4f8e-9c98-95bea9521504&at=APZUnTW74Ntotdg-xRjaoxuDFju9:1709727407282",
+            "path": "/home/user/Desktop/requirment.docx"
+          },
+          {
+            "url": "https://drive.usercontent.google.com/download?id=1eQHixX0TTExye0lbmHQny5CYhaREB5fQ&export=download&authuser=0&confirm=t&uuid=8546ef19-ea50-4e1a-9296-2db0302afd62&at=APZUnTVEmYOeAIdok_anTWOVHBCp:1709727527585",
+            "path": "/home/user/Desktop/ChMkKV8wsR6IBfEtABYfc0Tgu9cAAA1lQHO_78AFh-L733.jpg"
+          }
+        ]
+      }
+    },
+    {
+      "type": "open",
+      "parameters": {
+        "path": "/home/user/Desktop/requirment.docx"
+      }
+    }
+  ],
+  "trajectory": "trajectories/09a37c51-e625-49f4-a514-20a773797a8a",
+  "related_apps": [
+    "libreoffice_writer",
+    "gimp",
+    "os"
+  ],
+  "evaluator": {
+    "func": "compare_images",
+    "expected": {
+      "type": "cloud_file",
+      "path": "https://drive.usercontent.google.com/download?id=1Ee1vNyG7gGpLKK2VlLfj6PxcmdkMdvqK&export=download&authuser=0&confirm=t&uuid=1f441c5d-b62d-4850-870f-8e8f113a4091&at=APZUnTWEvKSSkuGBWzen0S9L7aHP:1709727474803",
+      "dest": "pic.jpg"
+    },
+    "result": {
+      "type": "vm_file",
+      "path": "/home/user/Desktop/pic.jpg",
+      "dest": "pic.jpg"
+    }
+  }
+}
--- a/evaluation_examples/examples/multi_apps/4c26e3f3-3a14-4d86-b44a-d3cedebbb487.json
+++ b/evaluation_examples/examples/multi_apps/4c26e3f3-3a14-4d86-b44a-d3cedebbb487.json
@@ -0,0 +1,44 @@
+{
+    "id": "4c26e3f3-3a14-4d86-b44a-d3cedebbb487",
+    "snapshot": "libreoffice_impress",
+    "instruction": "I found the image on the second slide is too dark. Could you make it brighter for me? Keep the modified picture on Desktop. Name it \"background\". Thanks!",
+    "source": "https://www.quora.com/How-do-I-edit-a-photo-in-GIMP",
+    "config": [
+        {
+            "type": "download",
+            "parameters": {
+                "files": [
+                    {
+                        "url": "https://drive.usercontent.google.com/download?id=1SD5GlVUuG9RijMCEzuniPq1dIWxfr8xQ&export=download&authuser=0&confirm=t&uuid=67c13e54-9368-4e77-bda9-31776ec37479&at=APZUnTXGR14_2pztP7HoAiELiztJ:1709709598471",
+                        "path": "/home/user/Desktop/PPT-Template_widescreen.pptx"
+                    }
+                ]
+            }
+        },
+        {
+            "type": "open",
+            "parameters": {
+                "path": "/home/user/Desktop/PPT-Template_widescreen.pptx"
+            }
+        }
+    ],
+    "trajectory": "trajectories/4c26e3f3-3a14-4d86-b44a-d3cedebbb487",
+    "related_apps": [
+        "gimp",
+        "libreoffice_impress",
+        "os"
+    ],
+    "evaluator": {
+        "func": "check_brightness_decrease_and_structure_sim",
+        "expected": {
+            "type": "vm_file",
+            "path": "/home/user/Desktop/background.png",
+            "dest": "background.png"
+        },
+        "result": {
+            "type": "cloud_file",
+            "path": "https://drive.usercontent.google.com/download?id=1lpRSXEoZq3ENOG5ekaAsBQSNv5ig0mDr&export=download&authuser=0&confirm=t&uuid=4cb10a33-81b3-4814-a969-f469832e33e5&at=APZUnTWN3pyiVpS003vLOgCcq2gu:1709710047375",
+            "dest": "image_original.png"
+        }
+    }
+}
--- a/evaluation_examples/examples/multi_apps/a82b78bb-7fde-4cb3-94a4-035baf10bcf0.json
+++ b/evaluation_examples/examples/multi_apps/a82b78bb-7fde-4cb3-94a4-035baf10bcf0.json
@@ -4,6 +4,25 @@
  "instruction": "I'm really enjoying this paper. Could you please find the personal webpages of the first author and the last three authors, and add them to a browser bookmark folder named 'Liked Authors'?",
  "source": "authors",
  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "google-chrome",
+          "--remote-debugging-port=1337"
+        ]
+      }
+    },
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "socat",
+          "tcp-listen:9222,fork",
+          "tcp:localhost:1337"
+        ]
+      }
+    },
    {
      "type": "download",
      "parameters": {
@@ -40,15 +59,28 @@
  ],
  "trajectory": "trajectories/a82b78bb-7fde-4cb3-94a4-035baf10bcf0",
  "related_apps": [
+    "chrome",
+    "pdf"
  ],
  "evaluator": {
-    "postconfig": [],
-    "func": "",
+    "func": "is_expected_bookmarks",
    "result": {
+      "type": "bookmarks"
    },
    "expected": {
-    },
-    "options": {
+      "type": "rule",
+      "rules": {
+        "type": "liked_authors_websites_urls",
+        "names": [
+          "Liked Authors"
+        ],
+        "urls": [
+          "https://jimfan.me/",
+          "https://ai.stanford.edu/~dahuang/",
+          "https://yukezhu.me/",
+          "https://www.eas.caltech.edu/people/anima"
+        ]
+      }
    }
  }
-}
+}