Fix minor bugs of multiple apps examples

desktop_env/controllers/setup.py

@@ -367,7 +367,7 @@ class SetupController:
                     context = browser.contexts[0]
 
                 page = context.new_page()  # Create a new page (tab) within the existing context
-                page.goto(url)
+                page.goto(url, timeout=60000)
                 logger.info(f"Opened tab {i + 1}: {url}")
 
                 if i == 0:

desktop_env/envs/desktop_env.py

@@ -84,7 +84,6 @@ class DesktopEnv(gym.Env):
 
         # Initialize emulator and controller
         logger.info("Initializing...")
-        self._config_screen_size()
         self._start_emulator()
         self.vm_ip = self._get_vm_ip()
         self.controller = PythonController(vm_ip=self.vm_ip)
@@ -105,60 +104,6 @@ class DesktopEnv(gym.Env):
         self._step_no: int = 0
         self.action_history: List[Dict[str, any]] = []
 
-    def _config_screen_size(self):
-        """
-        fixme: Experimental features, will cause unexpected error when system corrupt
-        """
-        def calculate_vram_size(width, height, bits_per_pixel=32):
-            """
-            Calculate VRAM size for given width, height, and color depth.
-            Color depth defaults to 32 bits per pixel.
-            """
-            bytes_per_pixel = bits_per_pixel // 8
-            vram_size = width * height * bytes_per_pixel
-            return vram_size
-
-        if not os.path.isfile(self.path_to_vm):
-            logger.warning(f"The specified vmx file does not exist: {self.path_to_vm}")
-            return False
-
-        width, height = self.vm_screen_size
-        vramSize = calculate_vram_size(width, height)
-
-        try:
-            with open(self.path_to_vm, 'r') as file:
-                lines = file.readlines()
-
-            new_lines = []
-            for line in lines:
-                if "svga.autodetect" in line:
-                    continue
-                elif "svga.vramSize" in line:
-                    continue
-                elif "displayWidth" in line:
-                    continue
-                elif "displayHeight" in line:
-                    continue
-                else:
-                    new_lines.append(line)
-
-            # Append new settings for screen size and VRAM.
-            new_lines.append(f'svga.autodetect = "TRUE"\n')
-            new_lines.append(f'svga.vramSize = "{vramSize}"\n')
-            new_lines.append(f'displayWidth = "{width}"\n')
-            new_lines.append(f'displayHeight = "{height}"\n')
-
-            with open(self.path_to_vm, 'w') as file:
-                file.writelines(new_lines)
-            logger.info(f"Screen size for {self.path_to_vm} set to {width}x{height} with VRAM size {vramSize} bytes")
-            return True
-        except IOError as e:
-            logger.error(f"An IOError occurred: {e}")
-            return False
-        except Exception as e:
-            logger.error(f"An error occurred: {e}")
-            return False
-
     def _start_emulator(self):
         while True:
             try:
@@ -280,7 +225,6 @@ class DesktopEnv(gym.Env):
         _execute_command(["vmrun", "-T", "ws", "revertToSnapshot", self.path_to_vm, self.snapshot_path])
         time.sleep(5)
 
-        self._config_screen_size()
         print(self.vm_screen_size)
         logger.info("Starting emulator...")
         self._start_emulator()

desktop_env/evaluators/metrics/gimp.py

@@ -12,6 +12,8 @@ def compare_image_list(pred_img_path_list: Union[str, List[str]],
         pred_img_path_list = [pred_img_path_list]
         gold_img_path_list = [gold_img_path_list]
     for pred_img_path, gold_img_path in zip(pred_img_path_list, gold_img_path_list):
+        if not pred_img_path or not gold_img_path:
+            return 0.0
         pred_img = Image.open(pred_img_path)
         gold_img = Image.open(gold_img_path)
         diff = ImageChops.difference(pred_img, gold_img)

evaluation_examples/examples/multi_apps/46407397-a7d5-4c6b-92c6-dbe038b1457b.json

@@ -40,9 +40,8 @@
             "type": "chrome_open_tabs",
             "parameters": {
                 "urls_to_open": [
-                    "https://www.freerice.com/",
+                    "https://news.google.com",
-                    "https://www.hku.hk/",
+                    "https://x.com"
-                    "https://about.meta.com/technologies/facebook-app/"
                 ]
             }
         },

evaluation_examples/examples/multi_apps/4e9f0faf-2ecc-4ae8-a804-28c9a75d1ddc.json

@@ -44,9 +44,8 @@
             "type": "chrome_open_tabs",
             "parameters": {
                 "urls_to_open": [
-                    "https://www.freerice.com/",
+                    "https://news.google.com",
-                    "https://www.hku.hk/",
+                    "https://x.com"
-                    "https://about.meta.com/technologies/facebook-app/"
                 ]
             }
         },

evaluation_examples/examples/multi_apps/58565672-7bfe-48ab-b828-db349231de6b.json

@@ -28,7 +28,6 @@
             "parameters": {
                 "urls_to_open": [
                     "https://www.apple.com/",
-                    "https://en.sjtu.edu.cn/",
                     "https://scholar.google.com/"
                 ]
             }
@@ -83,7 +82,6 @@
                 "type": "url",
                 "urls": [
                     "https://www.apple.com/",
-                    "https://en.sjtu.edu.cn/",
                     "https://scholar.google.com/",
                     "https://www.amazon.com/"
                 ]

evaluation_examples/examples/multi_apps/897e3b53-5d4d-444b-85cb-2cdc8a97d903.json

@@ -21,7 +21,7 @@
             "type": "launch",
             "parameters": {
                 "command": [
-                    "chromium-browser",
+                    "google-chrome",
                     "--remote-debugging-port=1337"
                 ]
             }
@@ -42,6 +42,7 @@
                 "urls_to_open": [
                     "https://www.zhihu.com/",
                     "https://www.coursera.org/",
+                    "https://www.deepl.com",
                     "https://www.wikidata.org/wiki/Wikidata:Main_Page"
                 ]
             }

evaluation_examples/examples/multi_apps/b52b40a5-ad70-4c53-b5b0-5650a8387052.json

@@ -40,9 +40,9 @@
             "type": "chrome_open_tabs",
             "parameters": {
                 "urls_to_open": [
-                    "https://www.freerice.com/",
+                    "https://news.google.com",
-                    "https://www.hku.hk/",
+                    "https://x.com",
-                    "https://about.meta.com/technologies/facebook-app/"
+                    "https://www.deepl.com"
                 ]
             }
         },

evaluation_examples/settings/googledrive/credentials.json

@@ -1 +1 @@
-{"access_token": "ya29.a0AfB_byC8lcdjvZcfqITa6l6TTITbtzYsnCFadUEzRmzggkA2riAnfxT9ZqI22QOmKdhdxD-TcSi9o-BpP_thCMTi_Mjaelkm9lyDcW-eb0tTG_0hC_b7Ra3TAmnzBIvnroFytJ3m9UIU7RqxttCSfrNz-zPLCWzivnAyaCgYKAR4SARISFQHGX2MiDgoIUJDl_-NKz3L5dsp2wg0171", "client_id": "786888752612-rgng5v9hcq4as7pn0b40gt9r5lekmht9.apps.googleusercontent.com", "client_secret": "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", "refresh_token": "1//0ehtafHmucszRCgYIARAAGA4SNwF-L9IrpDBsnzdHKAlRfrkvzNFw1cpdnRY8rhM5gy4flsPYdysMav27yHamJx39BBGq-LLw40s", "token_expiry": "2024-01-31T10:36:37Z", "token_uri": "https://oauth2.googleapis.com/token", "user_agent": null, "revoke_uri": "https://oauth2.googleapis.com/revoke", "id_token": null, "id_token_jwt": null, "token_response": {"access_token": "ya29.a0AfB_byC8lcdjvZcfqITa6l6TTITbtzYsnCFadUEzRmzggkA2riAnfxT9ZqI22QOmKdhdxD-TcSi9o-BpP_thCMTi_Mjaelkm9lyDcW-eb0tTG_0hC_b7Ra3TAmnzBIvnroFytJ3m9UIU7RqxttCSfrNz-zPLCWzivnAyaCgYKAR4SARISFQHGX2MiDgoIUJDl_-NKz3L5dsp2wg0171", "expires_in": 3599, "refresh_token": "1//0ehtafHmucszRCgYIARAAGA4SNwF-L9IrpDBsnzdHKAlRfrkvzNFw1cpdnRY8rhM5gy4flsPYdysMav27yHamJx39BBGq-LLw40s", "scope": "https://www.googleapis.com/auth/drive", "token_type": "Bearer"}, "scopes": ["https://www.googleapis.com/auth/drive"], "token_info_uri": "https://oauth2.googleapis.com/tokeninfo", "invalid": false, "_class": "OAuth2Credentials", "_module": "oauth2client.client"}
+{"access_token": "ya29.a0AfB_byDIizbKXOjPLHrhl3tX0xhI_Pv2U1qv_nnhX_V_QWjCKvq9es2Pgc21SY8z8W4zolgSN4RipR2iA5zHYIDkH1sn6ZhcqC_ExxrmJ_OgZt71ljzfhfh19CfGjr_ki7HoXa_UOg4X__N0IPpy5UTFE8aU1T4F3dIMaCgYKAS0SARISFQHGX2MiW_ZTvRlqH2LYf6ylkeSB9Q0171", "client_id": "786888752612-rgng5v9hcq4as7pn0b40gt9r5lekmht9.apps.googleusercontent.com", "client_secret": "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", "refresh_token": "1//0ehtafHmucszRCgYIARAAGA4SNwF-L9IrpDBsnzdHKAlRfrkvzNFw1cpdnRY8rhM5gy4flsPYdysMav27yHamJx39BBGq-LLw40s", "token_expiry": "2024-01-31T11:54:11Z", "token_uri": "https://oauth2.googleapis.com/token", "user_agent": null, "revoke_uri": "https://oauth2.googleapis.com/revoke", "id_token": null, "id_token_jwt": null, "token_response": {"access_token": "ya29.a0AfB_byDIizbKXOjPLHrhl3tX0xhI_Pv2U1qv_nnhX_V_QWjCKvq9es2Pgc21SY8z8W4zolgSN4RipR2iA5zHYIDkH1sn6ZhcqC_ExxrmJ_OgZt71ljzfhfh19CfGjr_ki7HoXa_UOg4X__N0IPpy5UTFE8aU1T4F3dIMaCgYKAS0SARISFQHGX2MiW_ZTvRlqH2LYf6ylkeSB9Q0171", "expires_in": 3599, "scope": "https://www.googleapis.com/auth/drive", "token_type": "Bearer"}, "scopes": ["https://www.googleapis.com/auth/drive"], "token_info_uri": "https://oauth2.googleapis.com/tokeninfo", "invalid": false, "_class": "OAuth2Credentials", "_module": "oauth2client.client"}