Update OS evals

2024-01-25 10:45:51 +08:00
parent 09f3e776ae
commit 93b4ff7d95
10 changed files with 311 additions and 0 deletions
--- a/evaluation_examples/examples/os/bedcedc4-4d72-425e-ad62-21960b11fe0d.json
+++ b/evaluation_examples/examples/os/bedcedc4-4d72-425e-ad62-21960b11fe0d.json
@@ -0,0 +1,25 @@
+{
+    "id": "bedcedc4-4d72-425e-ad62-21960b11fe0d",
+    "snapshot": "os",
+    "instruction": "Could you set the 'Dim screen when inactive' to on in setting?",
+    "source": "https://www.youtube.com/watch?v=D4WyNjt_hbQ&t=2s",
+    "trajectory": "trajectories/",
+    "related_apps": [
+      "os"
+    ],
+    "evaluator": {
+      "func": "exact_match",
+      "result": {
+        "type": "vm_command_line",
+        "config": {
+          "command": "gsettings get org.gnome.desktop.session idle-delay"
+        }
+      },
+      "expected": {
+        "type": "rule",
+        "rules":{
+          "expected": "0"
+        }
+      }
+    }
+  }