Add infeasible support

desktop_env/envs/desktop_env.py

@@ -175,10 +175,16 @@ class DesktopEnv(gym.Env):
         if isinstance(self.evaluator["func"], list) \
         else getattr(metrics, self.evaluator["func"])
         self.metric_conj: str = self.evaluator.get("conj", "and")  # take conjunction of multiple metrics
-        self.result_getter: Getter = [getattr(getters, "get_{:}".format(res["type"])) for res in
+        if "result" in self.evaluator:
+            self.result_getter: Getter = [getattr(getters, "get_{:}".format(res["type"])) for res in
                                       self.evaluator["result"]] \
             if isinstance(self.evaluator["result"], list) \
             else getattr(getters, "get_{:}".format(self.evaluator["result"]["type"]))
+        else:
+            self.result_getter = [None] * len(self.metric) \
+                if isinstance(self.metric, list) \
+                else None
+
         if "expected" in self.evaluator:
             self.expected_getter: Getter = [getattr(getters, "get_{:}".format(exp["type"])) if exp else None for exp in
                                             self.evaluator["expected"]] \
@@ -293,6 +299,12 @@ class DesktopEnv(gym.Env):
 
         self.setup_controller.setup(self.evaluator.get("postconfig", []))
 
+        if self.metric == "infeasible":
+            if self.action_history[-1] == "FAIL":
+                return 1
+            else:
+                return 0
+
         if type(self.metric) == list:
             results = []
             for idx, metric in enumerate(self.metric):
@@ -315,7 +327,8 @@ class DesktopEnv(gym.Env):
                     return 0
                 elif self.metric_conj == 'or' and float(metric) == 1.0:
                     return 1
-                else: results.append(metric)
+                else:
+                    results.append(metric)
             return sum(results) / len(results) if self.metric_conj == 'and' else max(results)
         else:
             try:

evaluation_examples/examples/os/b3d4a89c-53f2-4d6b-8b6a-541fb5d205fa.json

@@ -9,7 +9,7 @@
     "os"
   ],
   "evaluator": {
-    "func": "",
+    "func": "infeasible",
     "result": {
     },
     "expected": {

evaluation_examples/examples/os/e2eb4bf1-aa93-4192-b55d-03e2fb6dfd15.json

@@ -8,7 +8,7 @@
       "os"
     ],
     "evaluator": {
-      "func": "",
+      "func": "infeasible",
       "result": {
       },
       "expected": {

evaluation_examples/examples/os/fe41f596-a71b-4c2f-9b2f-9dcd40b568c3.json

@@ -5,32 +5,15 @@
   "source": "https://help.ubuntu.com/lts/ubuntu-help/power-percentage.html.en",
   "trajectory": "trajectories/",
   "config": [
-    {
-      "type": "execute",
-      "parameters": {
-        "command": [
-          "python",
-          "-c",
-          "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
-        ]
-      }
-    }
   ],
   "related_apps": [
     "os"
   ],
   "evaluator": {
-    "func": "exact_match",
+    "func": "infeasible",
     "result": {
-      "type": "vm_command_line",
-      "command": "gsettings get org.gnome.desktop.interface show-battery-percentage",
-      "shell": true
     },
     "expected": {
-      "type": "rule",
-      "rules": {
-        "expected": "true\n"
-      }
     }
   }
 }