update multi-app examples

2024-01-30 19:24:29 +08:00
parent d5d9fc56de
commit 4726ea7edb
7 changed files with 175 additions and 9 deletions
--- a/desktop_env/evaluators/metrics/init.py
+++ b/desktop_env/evaluators/metrics/init.py
@@ -66,7 +66,7 @@ from .gimp import (
    check_triangle_position,
    check_structure_sim,
    check_config_status,
-    compare_images,
+    compare_image_list,
    increase_saturation,
    decrease_brightness,
    check_file_exists,
--- a/desktop_env/evaluators/metrics/gimp.py
+++ b/desktop_env/evaluators/metrics/gimp.py
@@ -4,7 +4,7 @@ from skimage.metrics import structural_similarity as ssim
 from PIL import Image, ImageChops, ImageStat


-def compare_images(pred_img_path_list: Union[str, List[str]],
+def compare_image_list(pred_img_path_list: Union[str, List[str]],
                   gold_img_path_list: Union[str, List[str]]) -> float:
    """ Compare two image lists, only if all images are the same, return 1.0, otherwise return 0.0
    """
--- a/evaluation_examples/examples/multi_apps/3680a5ee-6870-426a-a997-eba929a0d25c.json
+++ b/evaluation_examples/examples/multi_apps/3680a5ee-6870-426a-a997-eba929a0d25c.json
@@ -0,0 +1,71 @@
+{
+    "id": "3680a5ee-6870-426a-a997-eba929a0d25c",
+    "snapshot": "libreoffice_calc",
+    "instruction": "I have file1.xlsx and file2.ods on the Desktop and each has one column. Help me use only the command line to merge these two columns into one LibreOffice Calc file called output.csv and open it from terminal.",
+    "source": "https://unix.stackexchange.com/questions/510850/how-to-open-calc-from-terminal-and-insert-files",
+    "config": [
+        {
+            "type": "download",
+            "parameters": {
+                "files": [
+                    {
+                        "url": "https://drive.usercontent.google.com/download?id=1ofMPSBPXtt0h6t0arjYo0GMtKAAhfEPz&export=download",
+                        "path": "/home/user/Desktop/file1.xlsx"
+                    },
+                    {
+                        "url": "https://drive.usercontent.google.com/download?id=15QOxr0_MiHR80ppMX02aHnnsWs25aXgZ&export=download",
+                        "path": "/home/user/Desktop/file2.ods"
+                    }
+                ]
+            }
+        },
+        {
+            "type": "launch",
+            "parameters": {
+                "command": [
+                    "gnome-terminal",
+                    "--maximize",
+                    "--working-directory=/home/user/Desktop"
+                ]
+            }
+        }
+    ],
+    "trajectory": "trajectories/",
+    "related_apps": [
+        "libreoffice_calc",
+        "terminal"
+    ],
+    "evaluator": {
+        "func": ["check_include_exclude", "compare_csv"],
+        "result": [
+            {
+                "type": "vm_command_line",
+                "command": [
+                    "/bin/bash",
+                    "-c",
+                    "output=$(ps aux | grep \"[s]office\" | awk '{print $7}' | grep -E \"pts/|tty\"); if [ -z \"$output\" ]; then echo \"use no terminal\"; else echo \"use terminal\"; fi;"
+                ]
+            },
+            {
+                "type": "vm_file",
+                "path": "/home/user/Desktop/output.csv",
+                "dest": "output.csv"
+            }
+        ],
+        "expected": [
+            {
+                "type": "rule",
+                "rules": {
+                    "include": [
+                        "use terminal"
+                    ]
+                }
+            },
+            {
+                "type": "cloud_file",
+                "path": "https://drive.usercontent.google.com/download?id=1OBQXIJ06HQeodaCwU7z0Kq_-d3YuGNqE&export=download",
+                "dest": "output_gold.csv"
+            }
+        ]
+    }
+}
--- a/evaluation_examples/examples/multi_apps/46407397-a7d5-4c6b-92c6-dbe038b1457b.json
+++ b/evaluation_examples/examples/multi_apps/46407397-a7d5-4c6b-92c6-dbe038b1457b.json
@@ -1,7 +1,7 @@
 {
    "id": "46407397-a7d5-4c6b-92c6-dbe038b1457b",
    "snapshot": "chrome",
-    "instruction": "Help me export charts, graph or other images from docx files received in email xxx in Thunderbird and upload these png files to the figures/ folder in Google Drive for later use (use numbers to name them).",
+    "instruction": "Help me export charts, graph or other images from docx files received in email \"Lecture Document\" in Notes folder and upload these png files to the figures/ folder in Google Drive for later use (use numbers to name them).",
    "source": "https://marketplace.uipath.com/listings/merge-pdfs-from-gmail-email-attachments-and-upload-to-gogle-drive",
    "config": [
        {
@@ -58,7 +58,7 @@
            "parameters": {
                "files": [
                    {
-                        "url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=11b93787-7076-47ba-b04b-b63a4e9aab02&at=APZUnTV_3ASC-R55FmBXgTLcC46e:1706187828620",
+                        "url": "https://drive.usercontent.google.com/download?id=18jdi0OanMtAQenm4ODTivsxTSzdj4HUV&export=download&authuser=0&confirm=t&uuid=e858d3cc-4535-4419-a651-8856ac517d19&at=APZUnTW7g4ygfrkKTPBWCO13twRj:1706611460571",
                        "path": "/home/user/thunderbird-profile.tar.gz"
                    }
                ]
@@ -93,7 +93,7 @@
        "chrome"
    ],
    "evaluator": {
-        "func": "compare_images",
+        "func": "compare_image_list",
        "result": {
            "type": "googledrive_file",
            "settings_file": "evaluation_examples/settings/googledrive/settings.yml",
--- a/evaluation_examples/examples/multi_apps/b52b40a5-ad70-4c53-b5b0-5650a8387052.json
+++ b/evaluation_examples/examples/multi_apps/b52b40a5-ad70-4c53-b5b0-5650a8387052.json
@@ -1,7 +1,7 @@
 {
    "id": "b52b40a5-ad70-4c53-b5b0-5650a8387052",
    "snapshot": "chrome",
-    "instruction": "Could you help me merge all PDF files in the email attachment in Thunderbird into one file and upload it to attachment_full.pdf in Google Drive?",
+    "instruction": "Could you help me merge all PDF files in the \"Paper Recommendation\" email attachment in Thunderbird into one file and upload it to attachment_full.pdf in Google Drive?",
    "source": "https://marketplace.uipath.com/listings/merge-pdfs-from-gmail-email-attachments-and-upload-to-gogle-drive",
    "config": [
        {
@@ -58,7 +58,7 @@
            "parameters": {
                "files": [
                    {
-                        "url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=11b93787-7076-47ba-b04b-b63a4e9aab02&at=APZUnTV_3ASC-R55FmBXgTLcC46e:1706187828620",
+                        "url": "https://drive.usercontent.google.com/download?id=1_SujjgvE6SkrIINB7bd-4quti7ICGaB_&export=download&authuser=0&confirm=t&uuid=aa95822d-55ca-46e0-8b52-14d495b41995&at=APZUnTX7f8cbJ00STW_kV8kMY1KW:1706609819112",
                        "path": "/home/user/thunderbird-profile.tar.gz"
                    }
                ]
@@ -104,7 +104,7 @@
        },
        "expected": {
            "type": "cloud_file",
-            "path": "https://drive.usercontent.google.com/download?id=1-FaONI6f5g9XyJAlx8vPFgTP_SEZ-1vV&export=download&authuser=0&confirm=t&uuid=b0ad08b7-5b4e-4372-99fa-0952cd096144&at=APZUnTVrZK9_alT_gchTKE6ZYeod:1706194464805",
+            "path": "https://drive.usercontent.google.com/download?id=1-FaONI6f5g9XyJAlx8vPFgTP_SEZ-1vV&export=download&authuser=0&confirm=t&uuid=de7b274a-25b8-4361-a614-4910a88ba4bd&at=APZUnTVRw21zrIY4ydp9sagr9ZhZ:1706609799766",
            "dest": "attachment_full_gold.pdf"
        }
    }
--- a/evaluation_examples/examples/multi_apps/ee9a3c83-f437-4879-8918-be5efbb9fac7.json
+++ b/evaluation_examples/examples/multi_apps/ee9a3c83-f437-4879-8918-be5efbb9fac7.json
@@ -0,0 +1,95 @@
+{
+    "id": "ee9a3c83-f437-4879-8918-be5efbb9fac7",
+    "snapshot": "libreoffice_calc",
+    "instruction": "Could you help me convert the opened ods file in the desktop to csv file with the same file name using command line when Libreoffice instance is running?",
+    "source": "https://stackoverflow.com/questions/64589140/convert-ods-to-csv-using-command-line-when-libreoffice-instance-is-running",
+    "config": [
+        {
+            "type": "download",
+            "parameters": {
+                "files": [
+                    {
+                        "url": "https://drive.usercontent.google.com/download?id=1V6e4KB6Sabp6nAHkq5Seun0qbjwEfdaF&export=download",
+                        "path": "/home/user/Desktop/file_example_ODS_5000.ods"
+                    }
+                ]
+            }
+        },
+        {
+            "type": "open",
+            "parameters": {
+                "path": "/home/user/Desktop/file_example_ODS_5000.ods"
+            }
+        },
+        {
+            "type": "execute",
+            "parameters": {
+                "command": [
+                    "/bin/bash",
+                    "-c",
+                    "history -c && echo > ~/.bash_history && sleep 3"
+                ]
+            }
+        },
+        {
+            "type": "launch",
+            "parameters": {
+                "command": [
+                    "gnome-terminal",
+                    "--maximize",
+                    "--working-directory=/home/user/Desktop"
+                ]
+            }
+        }
+    ],
+    "trajectory": "trajectories/",
+    "related_apps": [
+        "libreoffice_calc",
+        "terminal"
+    ],
+    "evaluator": {
+        "postconfig": [
+            {
+                "type": "execute",
+                "parameters": {
+                  "command": [
+                      "/bin/bash",
+                      "-c",
+                      "killall gnome-terminal-server"
+                  ]
+                }
+              }
+        ],
+        "func": ["check_include_exclude", "compare_csv"],
+        "result": [
+            {
+                "type": "vm_command_line",
+                "command": [
+                    "/bin/bash",
+                    "-c",
+                    "use_terminal=$(cat ~/.bash_history | grep \"\\(soffice\\|libreoffice\\).\\+--convert-to\\s\\+csv\"); if [ -z \"$use_terminal\" ]; then echo \"use no terminal\"; else echo \"use terminal\"; fi"
+                ]
+            },
+            {
+                "type": "vm_file",
+                "path": "/home/user/Desktop/file_example_ODS_5000.csv",
+                "dest": "file_example_ODS_5000.csv"
+            }
+        ],
+        "expected": [
+            {
+                "type": "rule",
+                "rules": {
+                    "include": [
+                        "use terminal"
+                    ]
+                }
+            },
+            {
+                "type": "cloud_file",
+                "path": "https://drive.usercontent.google.com/download?id=1L8cJjHYDtZymQp4-QdNZ9i6jzZzsXJm1&export=download",
+                "dest": "file_example_ODS_5000_gold.csv"
+            }
+        ]
+    }
+}
--- a/evaluation_examples/settings/googledrive/credentials.json
+++ b/evaluation_examples/settings/googledrive/credentials.json
@@ -1 +1 @@
-{"access_token": "ya29.a0AfB_byB77Ran1kP3F1FKu9xL-zMeffAX-m3Z8JFvP2UD6iMM8_s4FQoNlOK2gstGSrW0G9seRlOmDG129Qq6XwI5BiwfxB1ZDGUKuikOYl6ZFgS69tzNXXzuKbLUivkQqoBZl28njdWUsVBFKjy_IvFzlDQAC6-YOrPkPAaCgYKAVwSARISFQHGX2Mi8GhWBz1GC2iqsEtbet6ETA0173", "client_id": "786888752612-6cv6lermep9n6704s4kv20h08lotias9.apps.googleusercontent.com", "client_secret": "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", "refresh_token": "1//0e0qXy4xW1Ud5CgYIARAAGA4SNwF-L9IrWfaomed_CK0R7zZffcpT-GIXf3y2ZjqqAD0UP6UkbaMV9F_OEC6pBVaaX4TYnBKx3os", "token_expiry": "2024-01-29T05:13:41Z", "token_uri": "https://oauth2.googleapis.com/token", "user_agent": null, "revoke_uri": "https://oauth2.googleapis.com/revoke", "id_token": null, "id_token_jwt": null, "token_response": {"access_token": "ya29.a0AfB_byB77Ran1kP3F1FKu9xL-zMeffAX-m3Z8JFvP2UD6iMM8_s4FQoNlOK2gstGSrW0G9seRlOmDG129Qq6XwI5BiwfxB1ZDGUKuikOYl6ZFgS69tzNXXzuKbLUivkQqoBZl28njdWUsVBFKjy_IvFzlDQAC6-YOrPkPAaCgYKAVwSARISFQHGX2Mi8GhWBz1GC2iqsEtbet6ETA0173", "expires_in": 3599, "scope": "https://www.googleapis.com/auth/drive", "token_type": "Bearer"}, "scopes": ["https://www.googleapis.com/auth/drive"], "token_info_uri": "https://oauth2.googleapis.com/tokeninfo", "invalid": false, "_class": "OAuth2Credentials", "_module": "oauth2client.client"}
+{"access_token": "ya29.a0AfB_byBdrcgbmWKGyEUOxxuJBhxOs1uu0AqWeLgJKKKLG_dVg4iQKJAdiWD2oetHrKn17p4ZtfX-vt1VQ0BiF2MPD2exX1oESsQkXaO8q4TM1olIpadvlUBqUkqKJcjCqn1dp1oaTVYU-Srf2wQCGnDt3ozjljdkHXN_MQaCgYKAa4SARISFQHGX2MiWnixlrP3Se3vEV73_4fenA0173", "client_id": "786888752612-6cv6lermep9n6704s4kv20h08lotias9.apps.googleusercontent.com", "client_secret": "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", "refresh_token": "1//0e0qXy4xW1Ud5CgYIARAAGA4SNwF-L9IrWfaomed_CK0R7zZffcpT-GIXf3y2ZjqqAD0UP6UkbaMV9F_OEC6pBVaaX4TYnBKx3os", "token_expiry": "2024-01-30T11:20:53Z", "token_uri": "https://oauth2.googleapis.com/token", "user_agent": null, "revoke_uri": "https://oauth2.googleapis.com/revoke", "id_token": null, "id_token_jwt": null, "token_response": {"access_token": "ya29.a0AfB_byBdrcgbmWKGyEUOxxuJBhxOs1uu0AqWeLgJKKKLG_dVg4iQKJAdiWD2oetHrKn17p4ZtfX-vt1VQ0BiF2MPD2exX1oESsQkXaO8q4TM1olIpadvlUBqUkqKJcjCqn1dp1oaTVYU-Srf2wQCGnDt3ozjljdkHXN_MQaCgYKAa4SARISFQHGX2MiWnixlrP3Se3vEV73_4fenA0173", "expires_in": 3599, "scope": "https://www.googleapis.com/auth/drive", "token_type": "Bearer"}, "scopes": ["https://www.googleapis.com/auth/drive"], "token_info_uri": "https://oauth2.googleapis.com/tokeninfo", "invalid": false, "_class": "OAuth2Credentials", "_module": "oauth2client.client"}