ver Mar6thv3

new multi_app tasks and metrics
2024-03-06 23:29:01 +08:00
parent e045eee901
commit 054e016aff
8 changed files with 548 additions and 2 deletions
--- a/evaluation_examples/examples/multi_apps/48c46dc7-fe04-4505-ade7-723cba1aa6f6.json
+++ b/evaluation_examples/examples/multi_apps/48c46dc7-fe04-4505-ade7-723cba1aa6f6.json
@@ -0,0 +1,110 @@
+{
+  "id": "48c46dc7-fe04-4505-ade7-723cba1aa6f6",
+  "snapshot": "chrome",
+  "instruction": "I'm currently working on OSWorld project. Every time I need to open the terminal and change to the project directory, open the file manager and change to the project directory, and open relative webpages in chrome browser. This is tedious. Help me to do this automatically. Open project directory in terminal and file manager and open github and online document in chrome browser. That's enough.",
+  "source": "authors",
+  "config": [
+    {
+      "type": "execute",
+      "parameters": {
+        "command": ["mkdir", "-p", "/home/user/Documents/Finance/receipts", "/home/user/Documents/Projects/OSWorld", "/home/user/.local/share/applications"]
+      }
+    },
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "socat",
+          "tcp-listen:9222,fork",
+          "tcp:localhost:1337"
+        ]
+      }
+    },
+    {
+      "type": "execute",
+      "parameters": {
+        "command": ["cp", "/usr/share/applications/google-chrome.desktop", "/home/user/.local/share/applications"]
+      }
+    },
+    {
+      "type": "execute",
+      "parameters": {
+        "command": [
+          "sed",
+          "-i.bak",
+          "-e",
+          "s#Exec=/usr/bin/google-chrome-stable#& --remote-debugging-port=1337#g",
+          "/home/user/.local/share/applications/google-chrome.desktop"
+        ]
+      }
+    },
+    {
+      "type": "execute",
+      "parameters": {
+        "command": ["update-desktop-database"]
+      }
+    },
+    {
+      "type": "download",
+      "parameters": {
+        "files": [
+          {"path": "/home/user/Documents/Finance/receipts/aws-invoice-2308.pdf", "url": "https://drive.google.com/uc?id=1azRFXf4A7fvW0S7r9upHvleMEi-92hHM&export=download"},
+          {"path": "/home/user/Documents/Finance/receipts/aws-invoice-2309.pdf", "url": "https://drive.google.com/uc?id=1x-lpHm8U4U7uRPZ74-9wq9KzW2R55ln1&export=download"},
+          {"path": "/home/user/Documents/Finance/receipts/aws-invoice-2310.pdf", "url": "https://drive.google.com/uc?id=1pcrgV9G6NO4ekMEQBiupwXtq6mmke7b_&export=download"},
+          {"path": "/home/user/Documents/Finance/receipts/aws-invoice-2311.pdf", "url": "https://drive.google.com/uc?id=1JzbCK_nIY8X_3QZjnkzTtb-cRoq9zNT-&export=download"},
+          {"path": "/home/user/Documents/Finance/receipts/aws-invoice-2312.pdf", "url": "https://drive.google.com/uc?id=1RqbulzKG_HeYb1GZmLABOzlohlFg02UU&export=download"},
+          {"path": "/home/user/Documents/Finance/receipts/X-receipt-2312.pdf", "url": "https://drive.google.com/uc?id=1QzWjNzvNosG_yQr7VVonvYb3cUYF5f3u&export=download"},
+          {"path": "/home/user/Documents/Finance/tally_book.xlsx", "url": "https://drive.google.com/uc?id=13yuLhBPmouoWR-DybfgaIbWUOxbY_jhL&export=download"},
+          {"path": "/home/user/.projects.tar.xz", "url": "https://drive.google.com/uc?id=1oJcxpjqF474Wm16i1aZc8DlCEfAvc4t_&export=download"}
+        ]
+      }
+    },
+    {
+      "type": "execute",
+      "parameters": {
+        "command": ["tar", "-xJvf", "/home/user/.projects.tar.xz", "-C", "/home/user/Documents/Projects/OSWorld"]
+      }
+    }
+  ],
+  "trajectory": "trajectories/48c46dc7-fe04-4505-ade7-723cba1aa6f6",
+  "related_apps": ["chrome", "os"],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "execute",
+        "parameters": {
+          "command": ["wmctrl", "-lx"],
+          "stdout": "wmctrl.out"
+        }
+      }
+    ],
+    "func": ["check_list", "is_expected_tabs"],
+    "result": [
+      {
+        "type": "cache_file",
+        "path": "wmctrl.out"
+      },
+      {
+        "type": "open_tabs_info"
+      }
+    ],
+    "expected": [
+      {
+        "type": "rule",
+        "rules": {
+          "expect": [
+            "\\borg\\.gnome\\.Nautilus\\.Org\\.gnome\\.Nautilus\\b.*\\bOSWorld\\b",
+            "gnome-terminal-server\\.Gnome-terminal\\b.*~/Documents/Projects/OSWorld\\b"
+          ]
+        }
+      },
+      {
+        "type": "rule",
+        "rules": {
+          "type": "url",
+          "urls": ["https://github.com", "https://docs.python.org/3/"]
+        }
+      }
+    ]
+  }
+}