update multi apps

desktop_env/evaluators/metrics/__init__.py

@@ -47,7 +47,8 @@ from .docs import (
     check_file_exists,
     check_tabstops,
     compare_contains_image,
-    compare_docx_images
+    compare_docx_images,
+    compare_image_text
 )
 from .general import (
     check_csv,

desktop_env/evaluators/metrics/chrome.py

@@ -91,6 +91,17 @@ def is_expected_bookmarks(bookmarks: List[str], rule: Dict[str, Any]) -> float:
         bookmark_bar_websites_urls = [bookmark['url'] for bookmark in bookmarks['bookmark_bar']['children'] if
                                       bookmark['type'] == 'url']
         return 1. if set(bookmark_bar_websites_urls) == set(rule['urls']) else 0.
+    elif rule['type'] == "liked_authors_websites_urls":
+        # Check if "liked authors" folder exists
+        liked_authors_folder = next((bookmark for bookmark in bookmarks['bookmark_bar']['children'] if
+                                      bookmark['type'] == 'folder' and bookmark['name'] == 'Liked Authors'), None)
+        if liked_authors_folder:
+            # Check if it contains the specified URLs
+            liked_authors_urls = [bookmark['url'] for bookmark in liked_authors_folder['children'] if
+                                  bookmark['type'] == 'url']
+            return 1. if set(liked_authors_urls) == set(rule['urls']) else 0.
+        else:
+            return 0.
     else:
         raise TypeError(f"{rule['type']} not support yet!")
 

desktop_env/evaluators/metrics/docs.py

@@ -183,6 +183,16 @@ def compare_docx_images(docx_file1, docx_file2):
             return 0
     return 1
 
+import pytesseract
+
+def compare_image_text(image_path, rule):
+    img = Image.open(image_path)
+    img_text = pytesseract.image_to_string(img)
+    if rule['type'] == 'text':
+        return 1 if rule['text'] in img_text else 0
+    else:
+        raise ValueError("Unsupported rule type")
+
 def compare_line_spacing(docx_file1, docx_file2):
     if not compare_docx_files(docx_file1, docx_file2):
         return 0

desktop_env/evaluators/metrics/slides.py

@@ -182,7 +182,7 @@ def compare_pptx_files(file1_path, file2_path, **options):
             else:
                 return None
         
-        if get_slide_notes(slide1) != get_slide_notes(slide2) and examine_note:
+        if get_slide_notes(slide1).strip() != get_slide_notes(slide2).strip() and examine_note:
             return 0
         # check if the shapes are the same
         for shape1, shape2 in zip(slide1.shapes, slide2.shapes):
@@ -235,7 +235,7 @@ def compare_pptx_files(file1_path, file2_path, **options):
                     return 0
 
             if hasattr(shape1, "text") and hasattr(shape2, "text"):
-                if shape1.text != shape2.text and examine_text:
+                if shape1.text.strip() != shape2.text.strip() and examine_text:
                     return 0    
                 
                 # check if the paragraphs are the same

evaluation_examples/examples/multi_apps/02ce9a50-7af2-47ed-8596-af0c230501f8.json

@@ -0,0 +1,61 @@
+{
+  "id": "02ce9a50-7af2-47ed-8596-af0c230501f8",
+  "snapshot": "libreoffice_writer",
+  "instruction": "I'm using libreoffice writer to write a tutorial about linux, and now I want to show the results obtained by using the \"ls\" command in /home/user. Please run this command and save the screenshot as 'ls.png' on Desktop.",
+  "source": "authors",
+  "config": [
+    {
+      "type": "download",
+      "parameters": {
+        "files": [
+          {
+            "url": "https://drive.usercontent.google.com/download?id=1cOYh50n0S3-Q_h8HDTCEPv2v6NsJl07m&export=download&authuser=0&confirm=t&uuid=9741b52a-d998-4875-94ec-ab0a45a40bf9&at=APZUnTX__WVALT4i0EIQoRAWkm-k:1709715620836",
+            "path": "/home/user/Desktop/top-10-linux-commands-for-newbies.docx"
+          }
+        ]
+      }
+    },
+    {
+      "type": "open",
+      "parameters": {
+        "path": "/home/user/Desktop/top-10-linux-commands-for-newbies.docx"
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 2
+      }
+    },
+    {
+      "type": "execute",
+      "parameters": {
+        "command": [
+          "python",
+          "-c",
+          "import pyautogui; import time; pyautogui.hotkey('f11'); time.sleep(0.5); pyautogui.click(960, 540); time.sleep(0.5); pyautogui.scroll(-20)"
+        ]
+      }
+    }
+  ],
+  "trajectory": "trajectories/02ce9a50-7af2-47ed-8596-af0c230501f8",
+  "related_apps": [
+    "libreoffice_writer",
+    "os"
+  ],
+  "evaluator": {
+    "func": "compare_image_text",
+    "result": {
+      "type": "vm_file",
+      "path": "/home/user/Desktop/ls.png",
+      "dest": "ls.png"
+    },
+    "expected": {
+      "type": "rule",
+      "rules": {
+        "type": "text",
+        "text": "$ ls\n"
+      }
+    }
+  }
+}

evaluation_examples/examples/multi_apps/09a37c51-e625-49f4-a514-20a773797a8a.json

@@ -0,0 +1,48 @@
+{
+  "id": "09a37c51-e625-49f4-a514-20a773797a8a",
+  "snapshot": "libreoffice_writer",
+  "instruction": "I received a request from my friend that he wanted me to help him modify a picture. On the Desktop is the requirement doc and the picture to be adjusted. Modify the image as he said and save modified pic as \"pic.jpg\" on Desktop. Thanks!",
+  "source": "authors",
+  "config": [
+    {
+      "type": "download",
+      "parameters": {
+        "files": [
+          {
+            "url": "https://drive.usercontent.google.com/download?id=1npPOtBdE5t_mzScyA94vDlxB8SCZNubv&export=download&authuser=0&confirm=t&uuid=64ee33b7-66a5-4f8e-9c98-95bea9521504&at=APZUnTW74Ntotdg-xRjaoxuDFju9:1709727407282",
+            "path": "/home/user/Desktop/requirment.docx"
+          },
+          {
+            "url": "https://drive.usercontent.google.com/download?id=1eQHixX0TTExye0lbmHQny5CYhaREB5fQ&export=download&authuser=0&confirm=t&uuid=8546ef19-ea50-4e1a-9296-2db0302afd62&at=APZUnTVEmYOeAIdok_anTWOVHBCp:1709727527585",
+            "path": "/home/user/Desktop/ChMkKV8wsR6IBfEtABYfc0Tgu9cAAA1lQHO_78AFh-L733.jpg"
+          }
+        ]
+      }
+    },
+    {
+      "type": "open",
+      "parameters": {
+        "path": "/home/user/Desktop/requirment.docx"
+      }
+    }
+  ],
+  "trajectory": "trajectories/09a37c51-e625-49f4-a514-20a773797a8a",
+  "related_apps": [
+    "libreoffice_writer",
+    "gimp",
+    "os"
+  ],
+  "evaluator": {
+    "func": "compare_images",
+    "expected": {
+      "type": "cloud_file",
+      "path": "https://drive.usercontent.google.com/download?id=1Ee1vNyG7gGpLKK2VlLfj6PxcmdkMdvqK&export=download&authuser=0&confirm=t&uuid=1f441c5d-b62d-4850-870f-8e8f113a4091&at=APZUnTWEvKSSkuGBWzen0S9L7aHP:1709727474803",
+      "dest": "pic.jpg"
+    },
+    "result": {
+      "type": "vm_file",
+      "path": "/home/user/Desktop/pic.jpg",
+      "dest": "pic.jpg"
+    }
+  }
+}

evaluation_examples/examples/multi_apps/4c26e3f3-3a14-4d86-b44a-d3cedebbb487.json

@@ -0,0 +1,44 @@
+{
+    "id": "4c26e3f3-3a14-4d86-b44a-d3cedebbb487",
+    "snapshot": "libreoffice_impress",
+    "instruction": "I found the image on the second slide is too dark. Could you make it brighter for me? Keep the modified picture on Desktop. Name it \"background\". Thanks!",
+    "source": "https://www.quora.com/How-do-I-edit-a-photo-in-GIMP",
+    "config": [
+        {
+            "type": "download",
+            "parameters": {
+                "files": [
+                    {
+                        "url": "https://drive.usercontent.google.com/download?id=1SD5GlVUuG9RijMCEzuniPq1dIWxfr8xQ&export=download&authuser=0&confirm=t&uuid=67c13e54-9368-4e77-bda9-31776ec37479&at=APZUnTXGR14_2pztP7HoAiELiztJ:1709709598471",
+                        "path": "/home/user/Desktop/PPT-Template_widescreen.pptx"
+                    }
+                ]
+            }
+        },
+        {
+            "type": "open",
+            "parameters": {
+                "path": "/home/user/Desktop/PPT-Template_widescreen.pptx"
+            }
+        }
+    ],
+    "trajectory": "trajectories/4c26e3f3-3a14-4d86-b44a-d3cedebbb487",
+    "related_apps": [
+        "gimp",
+        "libreoffice_impress",
+        "os"
+    ],
+    "evaluator": {
+        "func": "check_brightness_decrease_and_structure_sim",
+        "expected": {
+            "type": "vm_file",
+            "path": "/home/user/Desktop/background.png",
+            "dest": "background.png"
+        },
+        "result": {
+            "type": "cloud_file",
+            "path": "https://drive.usercontent.google.com/download?id=1lpRSXEoZq3ENOG5ekaAsBQSNv5ig0mDr&export=download&authuser=0&confirm=t&uuid=4cb10a33-81b3-4814-a969-f469832e33e5&at=APZUnTWN3pyiVpS003vLOgCcq2gu:1709710047375",
+            "dest": "image_original.png"
+        }
+    }
+}

evaluation_examples/examples/multi_apps/a82b78bb-7fde-4cb3-94a4-035baf10bcf0.json

@@ -4,6 +4,25 @@
   "instruction": "I'm really enjoying this paper. Could you please find the personal webpages of the first author and the last three authors, and add them to a browser bookmark folder named 'Liked Authors'?",
   "source": "authors",
   "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "google-chrome",
+          "--remote-debugging-port=1337"
+        ]
+      }
+    },
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "socat",
+          "tcp-listen:9222,fork",
+          "tcp:localhost:1337"
+        ]
+      }
+    },
     {
       "type": "download",
       "parameters": {
@@ -40,15 +59,28 @@
   ],
   "trajectory": "trajectories/a82b78bb-7fde-4cb3-94a4-035baf10bcf0",
   "related_apps": [
+    "chrome",
+    "pdf"
   ],
   "evaluator": {
-    "postconfig": [],
+    "func": "is_expected_bookmarks",
-    "func": "",
     "result": {
+      "type": "bookmarks"
     },
     "expected": {
-    },
+      "type": "rule",
-    "options": {
+      "rules": {
+        "type": "liked_authors_websites_urls",
+        "names": [
+          "Liked Authors"
+        ],
+        "urls": [
+          "https://jimfan.me/",
+          "https://ai.stanford.edu/~dahuang/",
+          "https://yukezhu.me/",
+          "https://www.eas.caltech.edu/people/anima"
+        ]
+      }
     }
   }
-}
+}