Update OS evals

2024-01-25 10:45:51 +08:00
parent 09f3e776ae
commit 93b4ff7d95
10 changed files with 311 additions and 0 deletions
--- a/evaluation_examples/examples/os/ec4e3f68-9ea4-4c18-a5c9-69f89d1178b3.json
+++ b/evaluation_examples/examples/os/ec4e3f68-9ea4-4c18-a5c9-69f89d1178b3.json
@@ -0,0 +1,18 @@
+{
+    "id": "ec4e3f68-9ea4-4c18-a5c9-69f89d1178b3",
+    "snapshot": "os",
+    "instruction": "Can you remove the first favorite app from 'favorites'?",
+    "source": "https://www.youtube.com/watch?v=D4WyNjt_hbQ&t=2s",
+    "config": [],
+    "trajectory": "trajectories/",
+    "related_apps": [
+      "os"
+    ],
+    "evaluator": {
+      "func": "is_first_favorite_app_removed",
+      "result": {
+      },
+      "expected": {
+      }
+    }
+  }