Merge branch 'main' into zdy

desktop_env/controllers/python.py

@@ -83,6 +83,8 @@ class PythonController:
         """
         Executes an action on the server computer.
         """
+        if action in ['WAIT', 'FAIL', 'DONE']:
+            return
 
         action_type = action["action_type"]
         parameters = action["parameters"] if "parameters" in action else {}

desktop_env/envs/desktop_env.py

@@ -204,7 +204,10 @@ class DesktopEnv(gym.Env):
         time.sleep(5)
         logger.info("Environment setup complete.")
 
-        observation = {"screenshot": self._get_obs()}
+        observation = {
+            "screenshot": self._get_obs(),
+            "accessibility_tree": self.controller.get_accessibility_tree(),
+        }
         return observation
 
     def step(self, action, pause=0.5):
@@ -231,8 +234,11 @@ class DesktopEnv(gym.Env):
             # the set of all possible actions defined in the action representation
             self.controller.execute_action(action)
         elif self.action_space == "pyautogui":
-            # the set of all possible python commands insides `pyautogui`
-            self.controller.execute_python_command(action)
+            if action in ['WAIT', 'FAIL', 'DONE']:
+                self.controller.execute_action(action)
+            else:
+                # the set of all possible python commands insides `pyautogui`
+                self.controller.execute_python_command(action)
 
         observation = {
             "screenshot": self._get_obs(),

desktop_env/evaluators/getters/chrome.py

@@ -159,9 +159,19 @@ def get_open_tabs_info(env, config: Dict[str, str]):
         tabs_info = []
         for context in browser.contexts:
             for page in context.pages:
-                title = page.title()
-                url = page.url
-                tabs_info.append({'title': title, 'url': url})
+                try:
+                    # Wait for the page to finish loading, this prevents the "execution context was destroyed" issue
+                    page.wait_for_load_state('load')  # Wait for the 'load' event to complete
+                    title = page.title()
+                    url = page.url
+                    tabs_info.append({'title': title, 'url': url})
+                except TimeoutError:
+                    # If page loading times out, catch the exception and store the current information in the list
+                    tabs_info.append({'title': 'Load timeout', 'url': page.url})
+                except Exception as e:
+                    # Catch other potential exceptions that might occur while reading the page title
+                    print(f'Error: {e}')
+                    tabs_info.append({'title': 'Error encountered', 'url': page.url})
 
         browser.close()
         return tabs_info

desktop_env/evaluators/metrics/__init__.py

@@ -14,4 +14,4 @@ from .gimp import increase_saturation, decrease_brightness, check_file_exists, c
 from .general import check_csv, check_accessibility_tree, check_list, run_sqlite3, check_json
 from .thunderbird import check_thunderbird_prefs, check_thunderbird_filter
 from .vscode import compare_text_file, compare_config, compare_answer, is_extension_installed
-from .impress import check_slide_numbers_color, compare_pptx_files, check_for_two_lines
+from .impress import check_slide_numbers_color, compare_pptx_files, check_for_two_lines, check_for_audio, check_formula_shape, check_file_exists

desktop_env/evaluators/metrics/general.py

@@ -4,7 +4,7 @@ import functools
 import operator
 import re
 from numbers import Number
-from typing import Callable, Any
+from typing import Callable, Any, Union
 from typing import Dict, List, Pattern
 
 import lxml.etree

desktop_env/evaluators/metrics/impress.py

@@ -71,6 +71,29 @@ def check_file_exists(directory, filename):
     file_path = os.path.join(directory, filename)
     return 1 if os.path.isfile(file_path) else 0
 
+def has_audio_on_page(slide):
+    for shape in slide.shapes:
+        if shape.shape_type == 13:
+            return True
+    return False
+
+def check_for_audio(prs):
+    prs = Presentation(prs)
+    for i, slide in enumerate(prs.slides):
+        if has_audio_on_page(slide):
+            return 1
+    return 0
+
+def check_formula_shape(prs):
+    prs = Presentation(prs)
+    slide = prs.slides[13]
+        
+    for shape in slide.shapes:
+
+        if shape.has_text_frame and shape.shape_type == 1:
+            return 1
+    return 0
+
 if __name__ == "__main__":
     path1 = "../../任务数据/LibreOffice Impress/Change_Color_Slide_Number_gold_textbox.pptx"
     presentation = Presentation(path1)