ver Mar6thv3

new multi_app tasks and metrics
2024-03-06 23:29:01 +08:00
parent e045eee901
commit 054e016aff
8 changed files with 548 additions and 2 deletions
--- a/evaluation_examples/examples/multi_apps/788b3701-3ec9-4b67-b679-418bfa726c22.json
+++ b/evaluation_examples/examples/multi_apps/788b3701-3ec9-4b67-b679-418bfa726c22.json
@@ -0,0 +1,57 @@
+{
+  "id": "788b3701-3ec9-4b67-b679-418bfa726c22",
+  "snapshot": "chrome",
+  "instruction": "I'm following a small tale set updating on https://github.com/4th-year-in-tsinghua/4th-year-in-tsinghua. I have already downloaded several chapters to my computer for reading and archiving. There should be some updates and help me to download the next chapter I haven't downloaded yet. Save it to my novel collection folder.",
+  "source": "authors",
+  "config": [
+    {
+      "type": "execute",
+      "parameters": {
+        "command": ["mkdir", "-p", "/home/user/Documents/Finance/receipts", "/home/user/Documents/Projects/OSWorld", "/home/user/Documents/Novels/4th Year in Tsinghua"]
+      }
+    },
+    {
+      "type": "download",
+      "parameters": {
+        "files": [
+          {"path": "/home/user/Documents/Finance/receipts/aws-invoice-2308.pdf", "url": "https://drive.google.com/uc?id=1azRFXf4A7fvW0S7r9upHvleMEi-92hHM&export=download"},
+          {"path": "/home/user/Documents/Finance/receipts/aws-invoice-2309.pdf", "url": "https://drive.google.com/uc?id=1x-lpHm8U4U7uRPZ74-9wq9KzW2R55ln1&export=download"},
+          {"path": "/home/user/Documents/Finance/receipts/aws-invoice-2310.pdf", "url": "https://drive.google.com/uc?id=1pcrgV9G6NO4ekMEQBiupwXtq6mmke7b_&export=download"},
+          {"path": "/home/user/Documents/Finance/receipts/aws-invoice-2311.pdf", "url": "https://drive.google.com/uc?id=1JzbCK_nIY8X_3QZjnkzTtb-cRoq9zNT-&export=download"},
+          {"path": "/home/user/Documents/Finance/receipts/aws-invoice-2312.pdf", "url": "https://drive.google.com/uc?id=1RqbulzKG_HeYb1GZmLABOzlohlFg02UU&export=download"},
+          {"path": "/home/user/Documents/Finance/receipts/X-receipt-2312.pdf", "url": "https://drive.google.com/uc?id=1QzWjNzvNosG_yQr7VVonvYb3cUYF5f3u&export=download"},
+          {"path": "/home/user/Documents/Finance/tally_book.xlsx", "url": "https://drive.google.com/uc?id=13yuLhBPmouoWR-DybfgaIbWUOxbY_jhL&export=download"},
+          {"path": "/home/user/.projects.tar.xz", "url": "https://drive.google.com/uc?id=1oJcxpjqF474Wm16i1aZc8DlCEfAvc4t_&export=download"},
+          {"path": "/home/user/.old-chapters.tar.xz", "url": "https://drive.google.com/uc?id=11y-kLI83sQz6ncmP1-tkIR6mQJ7SfX5-&export=download"}
+        ]
+      }
+    },
+    {
+      "type": "execute",
+      "parameters": {
+        "command": ["tar", "-xJvf", "/home/user/.projects.tar.xz", "-C", "/home/user/Documents/Projects/OSWorld"]
+      }
+    },
+    {
+      "type": "execute",
+      "parameters": {
+        "command": ["tar", "-xJvf", "/home/user/.old-chapters.tar.xz", "-C", "/home/user/Documents/Novels/4th Year in Tsinghua"]
+      }
+    }
+  ],
+  "trajectory": "trajectories/788b3701-3ec9-4b67-b679-418bfa726c22",
+  "related_apps": ["chrome", "os"],
+  "evaluator": {
+    "func": "diff_text_file",
+    "result": {
+      "type": "vm_file",
+      "path": "/home/user/Documents/Novels/4th Year in Tsinghua/早期建筑群.tex",
+      "dest": "download.tex"
+    },
+    "expected": {
+      "type": "cloud_file",
+      "path": "https://drive.google.com/uc?id=1kiCDbTn7uYfgqbymddC5IKMsvpkihW-s&export=download",
+      "dest": "real.tex"
+    }
+  }
+}