Update OS evals

2024-01-25 10:45:51 +08:00
parent 09f3e776ae
commit 93b4ff7d95
10 changed files with 311 additions and 0 deletions
--- a/evaluation_examples/examples/os/ae039631-2b12-4637-84f6-c67d51511be3.json
+++ b/evaluation_examples/examples/os/ae039631-2b12-4637-84f6-c67d51511be3.json
@@ -0,0 +1,33 @@
+{
+  "id": "ae039631-2b12-4637-84f6-c67d51511be3",
+  "snapshot": "os",
+  "instruction": "Could you please help me check out my current default browser?",
+  "source": "https://ubuntu.com/tutorials/command-line-for-beginners#5-moving-and-manipulating-files",
+  "config": [
+    {
+      "type": "execute",
+      "parameters": {
+        "command": "xdg-settings set default-web-browser firefox.desktop"
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "os"
+  ],
+  "evaluator": {
+    "func": "exact_match",
+    "result": {
+      "type": "vm_command_line",
+      "config": {
+        "command": "xdg-settings get default-web-browser"
+      }
+    },
+    "expected": {
+      "type": "rule",
+      "rules":{
+        "expected": "firefox.desktop"
+      }
+    }
+  }
+}