Add raw accessibility-tree based prompting method (but the tokens are too large); Minor fix some small bugs

desktop_env/envs/desktop_env.py

@@ -204,7 +204,10 @@ class DesktopEnv(gym.Env):
         time.sleep(5)
         logger.info("Environment setup complete.")
 
-        observation = {"screenshot": self._get_obs()}
+        observation = {
+            "screenshot": self._get_obs(),
+            "accessibility_tree": self.controller.get_accessibility_tree(),
+        }
         return observation
 
     def step(self, action, pause=0.5):

desktop_env/evaluators/metrics/general.py

@@ -4,7 +4,7 @@ import functools
 import operator
 import re
 from numbers import Number
-from typing import Callable, Any
+from typing import Callable, Any, Union
 from typing import Dict, List, Pattern
 
 import lxml.etree