fix multi apps

desktop_env/evaluators/metrics/vscode.py

@@ -104,6 +104,27 @@ def compare_text_file(actual: str, expected: str, **options) -> float:
     return 0.0
 
 import zipfile
+from difflib import SequenceMatcher
+import PyPDF2
+
+def compare_pdf_content(content1, content2, text_similarity_threshold):
+    def extract_text_from_pdf(content):
+        with open("temp.pdf", "wb") as temp_pdf:
+            temp_pdf.write(content)
+        with open("temp.pdf", "rb") as temp_pdf:
+            pdf_reader = PyPDF2.PdfReader(temp_pdf)
+            text = ''
+            for page_num in range(len(pdf_reader.pages)):
+                page = pdf_reader.pages[page_num]
+                text += page.extract_text()
+        return text
+
+    text1 = extract_text_from_pdf(content1)
+    text2 = extract_text_from_pdf(content2)
+
+    similarity_ratio = SequenceMatcher(None, text1, text2).ratio()
+
+    return similarity_ratio >= text_similarity_threshold
 
 def compare_zip_files(actual: str, expected: str, **options) -> float:
     """
@@ -128,7 +149,12 @@ def compare_zip_files(actual: str, expected: str, **options) -> float:
             content1 = zip_file1.read(file_name)
             content2 = zip_file2.read(file_name)
 
-            if content1 != content2:
+            if file_name.lower().endswith('.pdf'):
+                if compare_pdf_content(content1, content2, 0.95):
+                    continue
+                else:
+                    return 0.0
+            elif content1 != content2:
                 return 0.0
     return 1.0
 

evaluation_examples/examples/multi_apps/02ce9a50-7af2-47ed-8596-af0c230501f8.json

@@ -54,7 +54,7 @@
       "type": "rule",
       "rules": {
         "type": "text",
-        "text": "$ ls\n"
+        "text": " Ls"
       }
     }
   }

evaluation_examples/examples/multi_apps/09a37c51-e625-49f4-a514-20a773797a8a.json

@@ -37,7 +37,7 @@
     "expected": {
       "type": "cloud_file",
       "path": "https://drive.usercontent.google.com/download?id=1Ee1vNyG7gGpLKK2VlLfj6PxcmdkMdvqK&export=download&authuser=0&confirm=t&uuid=1f441c5d-b62d-4850-870f-8e8f113a4091&at=APZUnTWEvKSSkuGBWzen0S9L7aHP:1709727474803",
-      "dest": "pic.jpg"
+      "dest": "pic_Gold.jpg"
     },
     "result": {
       "type": "vm_file",

evaluation_examples/examples/multi_apps/20236825-b5df-46e7-89bf-62e1d640a897.json

@@ -47,7 +47,7 @@
         "expected": {
             "type": "cloud_file",
             "path": "https://drive.usercontent.google.com/download?id=1g2Trt9oxQyW_sx8aIztFA0zNsE4yNw2x&export=download&authuser=0&confirm=t&uuid=342751c4-54f1-4760-9326-e7388845ded0&at=APZUnTV5BcbaxIZrDglWbs84Oxln:1709623697315",
-            "dest": "res.txt"
+            "dest": "res_Gold.txt"
         },
         "result": {
             "type": "vm_file",

evaluation_examples/examples/multi_apps/227d2f97-562b-4ccb-ae47-a5ec9e142fbb.json

@@ -63,7 +63,7 @@
 		"expected": {
 			"type": "cloud_file",
 			"path": "https://drive.usercontent.google.com/download?id=11kWQc1XFEqcIMuW0-NnZRSdv1199OmVI&export=download&authuser=0&confirm=t&uuid=694676fd-1ac9-4501-8acf-f48018494c7f&at=APZUnTV-koL51ka5dHum_HpGywv_:1709618406292",
-			"dest": "image.docx"
+			"dest": "image_Gold.docx"
 		}
 	}
 }

evaluation_examples/examples/multi_apps/4c26e3f3-3a14-4d86-b44a-d3cedebbb487.json

@@ -1,7 +1,7 @@
 {
     "id": "4c26e3f3-3a14-4d86-b44a-d3cedebbb487",
     "snapshot": "libreoffice_impress",
-    "instruction": "I've noticed that the image on the second slide is too dim. Can you please enhance its brightness for me? Save the adjusted image on the Desktop and name it \"background\". Thank you!",
+    "instruction": "I've noticed that the image on the second slide is too dim. Can you please enhance its brightness for me? Save the adjusted image on the Desktop and name it \"background.png\". Thank you!",
     "source": "https://www.quora.com/How-do-I-edit-a-photo-in-GIMP",
     "config": [
         {

evaluation_examples/examples/multi_apps/68a25bd4-59c7-4f4d-975e-da0c8509c848.json

@@ -61,12 +61,12 @@
             {
                 "type": "cloud_file",
                 "path": "https://drive.usercontent.google.com/download?id=1kAp7ulAR_h4snny212yg1xyR1cMy3H2Q&export=download&authuser=0&confirm=t&uuid=3f6cb74c-63cc-4653-9083-00626ef2fc11&at=APZUnTWuXvVM2w1Q9h0hOsuX6thn:1709789680904",
-                "dest": "paper01.pdf"
+                "dest": "paper01_Gold.pdf"
             },
             {
                 "type": "cloud_file",
                 "path": "https://drive.usercontent.google.com/download?id=1UMimItX51tzNXkIMGPpHOdPNF5Dx0Tpy&export=download&authuser=0&confirm=t&uuid=97b668a6-2d0d-4389-ac5e-234e931b4328&at=APZUnTVvuvbAE8r7jpK8AkzGUzyw:1709790384938",
-                "dest": "ans.docx"
+                "dest": "ans_Gold.docx"
             }
         ],
         "result": [

evaluation_examples/examples/multi_apps/81c425f5-78f3-4771-afd6-3d2973825947.json

@@ -63,7 +63,7 @@
 		"expected": {
 			"type": "cloud_file",
 			"path": "https://drive.usercontent.google.com/download?id=1KbdlQC0qSAYewG8QnZgParnSwv3s3dub&export=download&authuser=0&confirm=t&uuid=15dcc25c-8168-425e-96e1-fd27e0d6904b&at=APZUnTVho4ZrREHf9DC4rKwdIi3R:1709557117932",
-			"dest": "price.docx"
+			"dest": "price_Gold.docx"
 		}
 	}
 }

evaluation_examples/examples/multi_apps/8df7e444-8e06-4f93-8a1a-c5c974269d82.json

@@ -36,7 +36,7 @@
         "expected": {
             "type": "cloud_file",
             "path": "https://drive.usercontent.google.com/download?id=1IKRu-dMFP4Aqzq5-4TOmOWVci0qvC27K&export=download&authuser=0&confirm=t&uuid=e2dabad2-5648-4bc3-a40f-f008089cd613&at=APZUnTVh5JD5nT3EvutwHIaSnJAT:1709633945616",
-            "dest": "Recruitment_and_retention_of_health_professionals_across_Europe.zip"
+            "dest": "Recruitment_and_retention_of_health_professionals_across_Europe_Gold.zip"
         },
         "result": {
             "type": "vm_file",

evaluation_examples/examples/multi_apps/aad10cd7-9337-4b62-b704-a857848cedf2.json

@@ -59,7 +59,7 @@
     "expected": {
       "type": "cloud_file",
       "path": "https://drive.usercontent.google.com/download?id=1r2KJv0H3foo1WlWnArxdXnaew-yixNqL&export=download&authuser=0&confirm=t&uuid=633cc27c-d38b-4c45-907d-025341b4af1c&at=APZUnTV8AW5F_aLVooprdfgt-Q-Z:1709547335200",
-      "dest": "notes.docx"
+      "dest": "notes_Gold.docx"
     },
     "result": {
       "type": "vm_file",

evaluation_examples/examples/multi_apps/b337d106-053f-4d37-8da0-7f9c4043a66b.json

@@ -1,7 +1,7 @@
 {
   "id": "b337d106-053f-4d37-8da0-7f9c4043a66b",
   "snapshot": "os",
-  "instruction": "Recently, I've been exploring the use of the Vim editor for code editing. However, the default settings don't display relative line numbers. Please search the internet for a tutorial on enabling relative line numbers and setting it as the default for my local Vim.",
+  "instruction": "Recently, I've been exploring the use of the Vim editor for code editing. However, the default settings don't display line numbers in Vim editor. Please search the internet for a tutorial on adding line numbers in Vim and setting it as default for my local Vim.",
   "source": "authors",
   "config": [
     {

evaluation_examples/examples/multi_apps/bb83cab4-e5c7-42c7-a67b-e46068032b86.json

@@ -54,7 +54,7 @@
 				}
 			}
 		],
-		"func": "compare_docx_tables",
+		"func": "compare_docx_files",
 		"result": {
 			"type": "vm_file",
 			"path": "/home/user/Desktop/script.docx",
@@ -63,7 +63,7 @@
 		"expected": {
 			"type": "cloud_file",
 			"path": "https://drive.usercontent.google.com/download?id=1-Sol1W8S7Ybj-3KBJJarbcYUqS5wAQ1C&export=download&authuser=0&confirm=t&uuid=d967f546-b8f7-4ac2-b8fc-b1635f1cfbc4&at=APZUnTUazmbS2X3BSXDEQtJgobgf:1709559012053",
-			"dest": "script.docx"
+			"dest": "script_Gold.docx"
 		}
 	}
 }

evaluation_examples/examples/multi_apps/f5c13cdd-205c-4719-a562-348ae5cd1d91.json

@@ -7,7 +7,11 @@
     {
       "type": "execute",
       "parameters": {
-        "command": ["mkdir", "-p", "/home/user/Documents/Departments/finance"]
+        "command": [
+          "mkdir",
+          "-p",
+          "/home/user/Documents/Departments/finance"
+        ]
       }
     },
     {
@@ -29,37 +33,52 @@
         ]
       }
     },
-		{
+    {
-			"type": "execute",
+      "type": "execute",
-			"parameters": {
+      "parameters": {
-				"command": [
+        "command": [
-					"tar",
+          "tar",
-					"-xzv",
+          "-xzv",
-					"--recursive-unlink",
+          "--recursive-unlink",
-					"-f",
+          "-f",
-					"/home/user/thunderbird-profile.tar.gz",
+          "/home/user/thunderbird-profile.tar.gz",
-					"-C",
+          "-C",
-					"/home/user/"
+          "/home/user/"
-				]
+        ]
-			}
+      }
-		},
+    },
-		{
-			"type": "launch",
-			"parameters": {
-				"command": "/usr/bin/thunderbird -compose \"from='Anonym Tester <anonym-x2024@outlook.com>',subject='Reminder of Payment',body='$(cat /home/user/.payment-reminder-mail-body.html)'\"",
-				"shell": true
-			}
-		},
     {
       "type": "launch",
       "parameters": {
-        "command": ["nautilus", "/home/user/Documents/Departments/finance"]
+        "command": "/usr/bin/thunderbird -compose \"from='Anonym Tester <anonym-x2024@outlook.com>',subject='Reminder of Payment',body='$(cat /home/user/.payment-reminder-mail-body.html)'\"",
+        "shell": true
+      }
+    },
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "nautilus",
+          "/home/user/Documents/Departments/finance"
+        ]
       }
     }
   ],
   "trajectory": "trajectories/f5c13cdd-205c-4719-a562-348ae5cd1d91",
-  "related_apps": ["thunderbird", "os", "libreoffice_calc"],
+  "related_apps": [
+    "thunderbird",
+    "os",
+    "libreoffice_calc"
+  ],
   "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 10
+        }
+      }
+    ],
     "func": "check_accessibility_tree",
     "result": {
       "type": "accessibility_tree"
@@ -67,11 +86,27 @@
     "expected": {
       "type": "rule",
       "rules": [
-        {"selectors": ["tool-bar[attr|id=MsgHeadersToolbar] label[name=To]~[attr|class=\"address-pill\"]>label[attr|class=\"pill-label\"][name*=\"fox@someuniversity.edu\"]"]},
+        {
-        {"selectors": ["tool-bar[attr|id=MsgHeadersToolbar] label[name=To]~[attr|class=\"address-pill\"]>label[attr|class=\"pill-label\"][name*=\"iron@someuniversity.edu\"]"]},
+          "selectors": [
-        {"selectors": ["tool-bar[attr|id=MsgHeadersToolbar] label[name=To]~[attr|class=\"address-pill\"]>label[attr|class=\"pill-label\"][name*=\"nancy@someuniversity.edu\"]"]},
+            "tool-bar[attr|id=MsgHeadersToolbar] label[name=To]~[attr|class=\"address-pill\"]>label[attr|class=\"pill-label\"][name*=\"fox@someuniversity.edu\"]"
-        {"selectors": ["tool-bar[attr|id=MsgHeadersToolbar] label[name=To]~[attr|class=\"address-pill\"]>label[attr|class=\"pill-label\"][name*=\"stella@someuniversity.edu\"]"]}
+          ]
+        },
+        {
+          "selectors": [
+            "tool-bar[attr|id=MsgHeadersToolbar] label[name=To]~[attr|class=\"address-pill\"]>label[attr|class=\"pill-label\"][name*=\"iron@someuniversity.edu\"]"
+          ]
+        },
+        {
+          "selectors": [
+            "tool-bar[attr|id=MsgHeadersToolbar] label[name=To]~[attr|class=\"address-pill\"]>label[attr|class=\"pill-label\"][name*=\"nancy@someuniversity.edu\"]"
+          ]
+        },
+        {
+          "selectors": [
+            "tool-bar[attr|id=MsgHeadersToolbar] label[name=To]~[attr|class=\"address-pill\"]>label[attr|class=\"pill-label\"][name*=\"stella@someuniversity.edu\"]"
+          ]
+        }
       ]
     }
   }
-}
+}

evaluation_examples/examples/multi_apps/f918266a-b3e0-4914-865d-4faa564f1aef.json

@@ -26,7 +26,7 @@
             }
         }
     ],
-    "trajectory": "trajectories/",
+    "trajectory": "trajectories/f918266a-b3e0-4914-865d-4faa564f1aef",
     "related_apps": [
         "vscode",
         "os"

requirements.txt

@@ -44,5 +44,6 @@ dashscope
 google-generativeai
 PyYaml
 mutagen
-pytesseract
+easyocr
 borb
+pypdf2