Merge branch 'main' of github.com:ztjhz/DesktopEnv

2024-02-15 15:34:15 +08:00
parent 1184cffd5f dd1644b3f4
commit fb2fc2e6a4
8 changed files with 496 additions and 11 deletions
--- a/desktop_env/evaluators/metrics/slides.py
+++ b/desktop_env/evaluators/metrics/slides.py
@@ -147,6 +147,8 @@ def compare_pptx_files(file1_path, file2_path, **options):
    examine_color_rgb = options.get("examine_color_rgb", True)
    examine_font_underline = options.get("examine_font_underline", True)
    examine_strike_through = options.get("examine_strike_through", True)
+    examine_alignment = options.get("examine_alignment", True)
+    examine_bottom_position = options.get("examine_bottom_position", False)
    examine_bullets = options.get("examine_bullets", True)

    # compare the number of slides
@@ -155,19 +157,24 @@ def compare_pptx_files(file1_path, file2_path, **options):

    # compare the content of each slide
    for slide1, slide2 in zip(prs1.slides, prs2.slides):
-
        # check if the shapes are the same
        for shape1, shape2 in zip(slide1.shapes, slide2.shapes):
-            if (
-                    shape1.left != shape2.left or shape1.top != shape2.top or shape1.width != shape2.width or shape1.height != shape2.height) and examine_shape:
+            if examine_bottom_position and shape1.top != shape2.top:
+                if hasattr(shape1, "text") and hasattr(shape2, "text") and shape1.text == shape2.text and shape1.text == "Product Comparison":
+                    if shape1.top >= shape2.top:
+                        return 0
+                
+            if (shape1.left != shape2.left or shape1.top != shape2.top or shape1.width != shape2.width or shape1.height != shape2.height) and examine_shape:
                return 0

            if hasattr(shape1, "text") and hasattr(shape2, "text"):
                if shape1.text != shape2.text and examine_text:
-                    return 0
-
+                    return 0    
+                
                # check if the paragraphs are the same
                for para1, para2 in zip(shape1.text_frame.paragraphs, shape2.text_frame.paragraphs):
+                    if para1.alignment != para2.alignment and examine_alignment:
+                        return 0
                    # check if the runs are the same
                    if para1.text != para2.text and examine_text:
                        return 0
@@ -190,8 +197,9 @@ def compare_pptx_files(file1_path, file2_path, **options):
                        if run1.font.italic != run2.font.italic and examine_font_italic:
                            return 0

-                        if run1.font.color.rgb != run2.font.color.rgb and examine_color_rgb:
-                            return 0
+                        if hasattr(run1.font.color, "rgb") and hasattr(run2.font.color, "rgb"):
+                            if run1.font.color.rgb != run2.font.color.rgb and examine_color_rgb:
+                                return 0

                        if run1.font.underline != run2.font.underline and examine_font_underline:
                            return 0
--- a/evaluation_examples/examples/libreoffice_impress/08aced46-45a2-48d7-993b-ed3fb5b32302.json
+++ b/evaluation_examples/examples/libreoffice_impress/08aced46-45a2-48d7-993b-ed3fb5b32302.json
@@ -0,0 +1,79 @@
+{
+  "id": "08aced46-45a2-48d7-993b-ed3fb5b32302",
+  "snapshot": "libreoffice_impress",
+  "instruction": "Give the slide the right aligned title, \"Note\"",
+  "source": "https://arxiv.org/pdf/2311.01767.pdf",
+  "config": [
+    {
+      "type": "download",
+      "parameters": {
+        "files": [
+          {
+            "url": "https://drive.usercontent.google.com/download?id=15OOk9xJzM4WBJRy5rDT2iMrEmm7b-VXj&export=download&authuser=0&confirm=t&uuid=501abcf6-9d70-4ef7-b1ed-0c6e0a47ae7c&at=APZUnTUn3aTxAlhaMSjmljHvPfrr:1707741990590",
+            "path": "/home/user/Desktop/22_6.pptx"
+          }
+        ]
+      }
+    },
+    {
+      "type": "open",
+      "parameters": {
+        "path": "/home/user/Desktop/22_6.pptx"
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "libreoffice_impress"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "activate_window",
+        "parameters": {
+          "window_name": "22_6.pptx - LibreOffice Impress",
+          "strict": true
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python",
+            "-c",
+            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);pyautogui.press('down');time.sleep(0.5);pyautogui.press('enter');time.sleep(0.5);"
+          ]
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      }
+    ],
+    "func": "compare_pptx_files",
+    "expected": {
+      "type": "cloud_file",
+      "path": "https://drive.usercontent.google.com/download?id=1lIdDVe95j3Hpscn9eXEuaawhSL2QmhEl&export=download&authuser=0&confirm=t&uuid=63a092e2-a4c1-458d-9904-b288f5f8422b&at=APZUnTXEYGXubO9J-0Ybvi5TXOfb:1707741962330",
+      "dest": "22_6_Gold.pptx"
+    },
+    "result": {
+      "type": "vm_file",
+      "path": "/home/user/Desktop/22_6.pptx",
+      "dest": "22_6.pptx"
+    },
+    "options": {
+      "examine_shape": false,
+      "examine_font_size": false,
+      "examine_font_bold": false,
+      "examine_font_name": false
+    }
+  }
+}
--- a/evaluation_examples/examples/libreoffice_impress/15aece23-a215-4579-91b4-69eec72e18da.json
+++ b/evaluation_examples/examples/libreoffice_impress/15aece23-a215-4579-91b4-69eec72e18da.json
@@ -0,0 +1,81 @@
+{
+  "id": "15aece23-a215-4579-91b4-69eec72e18da",
+  "snapshot": "libreoffice_impress",
+  "instruction": "Move the title to the bottom of the slide. ## 1, title, slide, bottom corner",
+  "source": "https://arxiv.org/pdf/2311.01767.pdf",
+  "config": [
+    {
+      "type": "download",
+      "parameters": {
+        "files": [
+          {
+            "url": "https://drive.usercontent.google.com/download?id=1SCyQzGU05PF-BPrXJ7GsFrnyuRcKcbBb&export=download&authuser=0&confirm=t&uuid=a3b90508-2288-48a4-95d5-c8e7bc11d5fd&at=APZUnTUt44ntxFBUaL3I6oqRhfjy:1707917097401",
+            "path": "/home/user/Desktop/134_2.pptx"
+          }
+        ]
+      }
+    },
+    {
+      "type": "open",
+      "parameters": {
+        "path": "/home/user/Desktop/134_2.pptx"
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "libreoffice_impress"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "activate_window",
+        "parameters": {
+          "window_name": "134_2.pptx - LibreOffice Impress",
+          "strict": true
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python",
+            "-c",
+            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);pyautogui.press('down');time.sleep(0.5);pyautogui.press('enter');time.sleep(0.5);"
+          ]
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      }
+    ],
+    "func": "compare_pptx_files",
+    "expected": {
+      "type": "cloud_file",
+      "path": "https://drive.usercontent.google.com/download?id=1OqpF8JpJjaXwHvDANB0E7nJR2m4ssXJO&export=download&authuser=0&confirm=t&uuid=d9eb02df-4b20-4f54-afe1-53437820946a&at=APZUnTUiporlBynLvGUB1-zDS9xm:1707916188451",
+      "dest": "134_2_Gold.pptx"
+    },
+    "result": {
+      "type": "vm_file",
+      "path": "/home/user/Desktop/134_2.pptx",
+      "dest": "134_2.pptx"
+    },
+    "options": {
+      "examine_shape": false,
+      "examine_font_size": false,
+      "examine_font_bold": false,
+      "examine_font_name": false,
+      "examine_alignment": false,
+      "examine_bottom_position": true
+    }
+  }
+}
--- a/evaluation_examples/examples/libreoffice_impress/550ce7e7-747b-495f-b122-acdc4d0b8e54.json
+++ b/evaluation_examples/examples/libreoffice_impress/550ce7e7-747b-495f-b122-acdc4d0b8e54.json
@@ -84,9 +84,6 @@
      "type": "vm_file",
      "path": "/home/user/Desktop/New_Club_Spring_2018_Training.pptx",
      "dest": "New_Club_Spring_2018_Training.pptx"
-    },
-    "options": {
-            "examine_shape": false
-        }
+    }
  }
 }
--- a/evaluation_examples/examples/libreoffice_impress/648de793-7368-4fb7-b16f-a15362da8003.json
+++ b/evaluation_examples/examples/libreoffice_impress/648de793-7368-4fb7-b16f-a15362da8003.json
@@ -0,0 +1,80 @@
+{
+  "id": "648de793-7368-4fb7-b16f-a15362da8003",
+  "snapshot": "libreoffice_impress",
+  "instruction": "In the slide, uh, add a right arrow in the left side",
+  "source": "https://arxiv.org/pdf/2311.01767.pdf",
+  "config": [
+    {
+      "type": "download",
+      "parameters": {
+        "files": [
+          {
+            "url": "https://drive.usercontent.google.com/download?id=1kOQ_PlYlGOKhdm-etxB2D8VvLaA4mqNT&export=download&authuser=0&confirm=t&uuid=9be6111e-53cf-4724-bcef-2ae4e819003b&at=APZUnTXZJ2KPlfmSL9R4ZBU0jAPs:1707932348450",
+            "path": "/home/user/Desktop/19_0.pptx"
+          }
+        ]
+      }
+    },
+    {
+      "type": "open",
+      "parameters": {
+        "path": "/home/user/Desktop/19_0.pptx"
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "libreoffice_impress"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "activate_window",
+        "parameters": {
+          "window_name": "19_0.pptx - LibreOffice Impress",
+          "strict": true
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python",
+            "-c",
+            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);pyautogui.press('down');time.sleep(0.5);pyautogui.press('enter');time.sleep(0.5);"
+          ]
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      }
+    ],
+    "func": "compare_pptx_files",
+    "expected": {
+      "type": "cloud_file",
+      "path": "https://drive.usercontent.google.com/download?id=180atkxCNr-fZrT9YTKV_8qyhpMFqpEUq&export=download&authuser=0&confirm=t&uuid=e5ee127a-93a1-4069-b058-86a714de64ff&at=APZUnTVUuP0snmxIvS92-t4ObLf9:1707932394767",
+      "dest": "19_0_Gold.pptx"
+    },
+    "result": {
+      "type": "vm_file",
+      "path": "/home/user/Desktop/19_0.pptx",
+      "dest": "19_0.pptx"
+    },
+    "options": {
+      "examine_shape": false,
+      "examine_font_size": false,
+      "examine_font_bold": false,
+      "examine_font_name": false,
+      "examine_alignment": false
+    }
+  }
+}
--- a/evaluation_examples/examples/libreoffice_impress/73c99fb9-f828-43ce-b87a-01dc07faa224.json
+++ b/evaluation_examples/examples/libreoffice_impress/73c99fb9-f828-43ce-b87a-01dc07faa224.json
@@ -0,0 +1,80 @@
+{
+  "id": "73c99fb9-f828-43ce-b87a-01dc07faa224",
+  "snapshot": "libreoffice_impress",
+  "instruction": "Add \"Page 1\" into the slide.",
+  "source": "https://arxiv.org/pdf/2311.01767.pdf",
+  "config": [
+    {
+      "type": "download",
+      "parameters": {
+        "files": [
+          {
+            "url": "https://drive.usercontent.google.com/download?id=1KDDJCei7KE1yRWGNUSmUnFnpZBH2gDeE&export=download&authuser=0&confirm=t&uuid=f85aa168-2476-46ab-9ee1-4bb373904650&at=APZUnTVvlubece5OCv6g-ULVovsQ:1707912989623",
+            "path": "/home/user/Desktop/109_4.pptx"
+          }
+        ]
+      }
+    },
+    {
+      "type": "open",
+      "parameters": {
+        "path": "/home/user/Desktop/109_4.pptx"
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "libreoffice_impress"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "activate_window",
+        "parameters": {
+          "window_name": "109_4.pptx - LibreOffice Impress",
+          "strict": true
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python",
+            "-c",
+            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);pyautogui.press('down');time.sleep(0.5);pyautogui.press('enter');time.sleep(0.5);"
+          ]
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      }
+    ],
+    "func": "compare_pptx_files",
+    "expected": {
+      "type": "cloud_file",
+      "path": "https://drive.usercontent.google.com/download?id=1w6DoJnhq8SDJQ76iDB_L2r7mGZNtVFIO&export=download&authuser=0&confirm=t&uuid=1d12f276-0b9c-4f8b-84fa-71d4ae011224&at=APZUnTUJOXjccK3edFKFEyoU6VRA:1707913047283",
+      "dest": "109_4_Gold.pptx"
+    },
+    "result": {
+      "type": "vm_file",
+      "path": "/home/user/Desktop/109_4.pptx",
+      "dest": "109_4.pptx"
+    },
+    "options": {
+      "examine_shape": false,
+      "examine_font_size": false,
+      "examine_font_bold": false,
+      "examine_font_name": false,
+      "examine_alignment": false
+    }
+  }
+}
--- a/evaluation_examples/examples/libreoffice_impress/986fc832-6af2-417c-8845-9272b3a1528b.json
+++ b/evaluation_examples/examples/libreoffice_impress/986fc832-6af2-417c-8845-9272b3a1528b.json
@@ -0,0 +1,81 @@
+{
+  "id": "986fc832-6af2-417c-8845-9272b3a1528b",
+  "snapshot": "libreoffice_impress",
+  "instruction": "underline the content and make the font color on this slide dark red 2.",
+  "source": "https://arxiv.org/pdf/2311.01767.pdf",
+  "config": [
+    {
+      "type": "download",
+      "parameters": {
+        "files": [
+          {
+            "url": "https://drive.usercontent.google.com/download?id=1WgEV25hWlzvOXIImI1ZaDZO1yKC72Qay&export=download&authuser=0&confirm=t&uuid=e94d9f07-feeb-4b7b-82f7-30a90bb15983&at=APZUnTUw8ObGQCHaMY5AZdIMvA-k:1707919959393",
+            "path": "/home/user/Desktop/154_3.pptx"
+          }
+        ]
+      }
+    },
+    {
+      "type": "open",
+      "parameters": {
+        "path": "/home/user/Desktop/154_3.pptx"
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "libreoffice_impress"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "activate_window",
+        "parameters": {
+          "window_name": "154_3.pptx - LibreOffice Impress",
+          "strict": true
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python",
+            "-c",
+            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);pyautogui.press('down');time.sleep(0.5);pyautogui.press('enter');time.sleep(0.5);"
+          ]
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      }
+    ],
+    "func": "compare_pptx_files",
+    "expected": {
+      "type": "cloud_file",
+      "path": "https://drive.usercontent.google.com/download?id=1trYqwEuj2ruF830UGUHF1M3I1at4nW11&export=download&authuser=0&confirm=t&uuid=1d215b22-0d02-46a8-9c4e-64f0c8ce0d0d&at=APZUnTUaJxIF7EgFitSO1Dppy1LT:1707919967130",
+      "dest": "154_3_Gold.pptx"
+    },
+    "result": {
+      "type": "vm_file",
+      "path": "/home/user/Desktop/154_3.pptx",
+      "dest": "154_3.pptx"
+    },
+    "options": {
+      "examine_shape": false,
+      "examine_font_size": false,
+      "examine_font_bold": false,
+      "examine_font_name": false,
+      "examine_alignment": false,
+      "examine_bottom_position": true
+    }
+  }
+}
--- a/evaluation_examples/examples/libreoffice_impress/edb61b14-a854-4bf5-a075-c8075c11293a.json
+++ b/evaluation_examples/examples/libreoffice_impress/edb61b14-a854-4bf5-a075-c8075c11293a.json
@@ -0,0 +1,79 @@
+{
+  "id": "edb61b14-a854-4bf5-a075-c8075c11293a",
+  "snapshot": "libreoffice_impress",
+  "instruction": "In the last slide, make the font style \"Times New Roman\"",
+  "source": "https://arxiv.org/pdf/2311.01767.pdf",
+  "config": [
+    {
+      "type": "download",
+      "parameters": {
+        "files": [
+          {
+            "url": "https://drive.usercontent.google.com/download?id=1SvG7uqJ4nd-YimLiQK1XsrI65iCxOMOw&export=download&authuser=0&confirm=t&uuid=20e98914-c490-4bb6-8a37-1c839e2226fe&at=APZUnTU-4RRKoHat4k60maslvMSZ:1707901107881",
+            "path": "/home/user/Desktop/24_8.pptx"
+          }
+        ]
+      }
+    },
+    {
+      "type": "open",
+      "parameters": {
+        "path": "/home/user/Desktop/24_8.pptx"
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "libreoffice_impress"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "activate_window",
+        "parameters": {
+          "window_name": "24_8.pptx - LibreOffice Impress",
+          "strict": true
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python",
+            "-c",
+            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);pyautogui.press('down');time.sleep(0.5);pyautogui.press('enter');time.sleep(0.5);"
+          ]
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      }
+    ],
+    "func": "compare_pptx_files",
+    "expected": {
+      "type": "cloud_file",
+      "path": "https://drive.usercontent.google.com/download?id=1a7sXQrGQBlPntn64owyl8KWE8eLUXJMv&export=download&authuser=0&confirm=t&uuid=cc02c1a4-7613-484e-9d13-478f57a795a0&at=APZUnTV_gcXjdY6Jz9AAWK-W55l6:1707900948910",
+      "dest": "24_8_Gold.pptx"
+    },
+    "result": {
+      "type": "vm_file",
+      "path": "/home/user/Desktop/24_8.pptx",
+      "dest": "24_8.pptx"
+    },
+    "options": {
+      "examine_shape": false,
+      "examine_font_size": false,
+      "examine_font_bold": false,
+      "examine_alignment": false
+    }
+  }
+}