From d65b6994d3cef3f82748563103d3dcfdf5d7fad6 Mon Sep 17 00:00:00 2001 From: Timothyxxx <384084775@qq.com> Date: Wed, 31 Jan 2024 19:40:41 +0800 Subject: [PATCH] Fix minor bugs of multiple apps examples --- desktop_env/controllers/setup.py | 2 +- desktop_env/envs/desktop_env.py | 56 ------------------- desktop_env/evaluators/metrics/gimp.py | 2 + .../46407397-a7d5-4c6b-92c6-dbe038b1457b.json | 5 +- .../4e9f0faf-2ecc-4ae8-a804-28c9a75d1ddc.json | 5 +- .../58565672-7bfe-48ab-b828-db349231de6b.json | 2 - .../897e3b53-5d4d-444b-85cb-2cdc8a97d903.json | 3 +- .../b52b40a5-ad70-4c53-b5b0-5650a8387052.json | 6 +- .../settings/googledrive/credentials.json | 2 +- 9 files changed, 13 insertions(+), 70 deletions(-) diff --git a/desktop_env/controllers/setup.py b/desktop_env/controllers/setup.py index 4c637a7..38646be 100644 --- a/desktop_env/controllers/setup.py +++ b/desktop_env/controllers/setup.py @@ -367,7 +367,7 @@ class SetupController: context = browser.contexts[0] page = context.new_page() # Create a new page (tab) within the existing context - page.goto(url) + page.goto(url, timeout=60000) logger.info(f"Opened tab {i + 1}: {url}") if i == 0: diff --git a/desktop_env/envs/desktop_env.py b/desktop_env/envs/desktop_env.py index f8cd082..831a876 100644 --- a/desktop_env/envs/desktop_env.py +++ b/desktop_env/envs/desktop_env.py @@ -84,7 +84,6 @@ class DesktopEnv(gym.Env): # Initialize emulator and controller logger.info("Initializing...") - self._config_screen_size() self._start_emulator() self.vm_ip = self._get_vm_ip() self.controller = PythonController(vm_ip=self.vm_ip) @@ -105,60 +104,6 @@ class DesktopEnv(gym.Env): self._step_no: int = 0 self.action_history: List[Dict[str, any]] = [] - def _config_screen_size(self): - """ - fixme: Experimental features, will cause unexpected error when system corrupt - """ - def calculate_vram_size(width, height, bits_per_pixel=32): - """ - Calculate VRAM size for given width, height, and color depth. - Color depth defaults to 32 bits per pixel. - """ - bytes_per_pixel = bits_per_pixel // 8 - vram_size = width * height * bytes_per_pixel - return vram_size - - if not os.path.isfile(self.path_to_vm): - logger.warning(f"The specified vmx file does not exist: {self.path_to_vm}") - return False - - width, height = self.vm_screen_size - vramSize = calculate_vram_size(width, height) - - try: - with open(self.path_to_vm, 'r') as file: - lines = file.readlines() - - new_lines = [] - for line in lines: - if "svga.autodetect" in line: - continue - elif "svga.vramSize" in line: - continue - elif "displayWidth" in line: - continue - elif "displayHeight" in line: - continue - else: - new_lines.append(line) - - # Append new settings for screen size and VRAM. - new_lines.append(f'svga.autodetect = "TRUE"\n') - new_lines.append(f'svga.vramSize = "{vramSize}"\n') - new_lines.append(f'displayWidth = "{width}"\n') - new_lines.append(f'displayHeight = "{height}"\n') - - with open(self.path_to_vm, 'w') as file: - file.writelines(new_lines) - logger.info(f"Screen size for {self.path_to_vm} set to {width}x{height} with VRAM size {vramSize} bytes") - return True - except IOError as e: - logger.error(f"An IOError occurred: {e}") - return False - except Exception as e: - logger.error(f"An error occurred: {e}") - return False - def _start_emulator(self): while True: try: @@ -280,7 +225,6 @@ class DesktopEnv(gym.Env): _execute_command(["vmrun", "-T", "ws", "revertToSnapshot", self.path_to_vm, self.snapshot_path]) time.sleep(5) - self._config_screen_size() print(self.vm_screen_size) logger.info("Starting emulator...") self._start_emulator() diff --git a/desktop_env/evaluators/metrics/gimp.py b/desktop_env/evaluators/metrics/gimp.py index fadf265..a1d3a82 100644 --- a/desktop_env/evaluators/metrics/gimp.py +++ b/desktop_env/evaluators/metrics/gimp.py @@ -12,6 +12,8 @@ def compare_image_list(pred_img_path_list: Union[str, List[str]], pred_img_path_list = [pred_img_path_list] gold_img_path_list = [gold_img_path_list] for pred_img_path, gold_img_path in zip(pred_img_path_list, gold_img_path_list): + if not pred_img_path or not gold_img_path: + return 0.0 pred_img = Image.open(pred_img_path) gold_img = Image.open(gold_img_path) diff = ImageChops.difference(pred_img, gold_img) diff --git a/evaluation_examples/examples/multi_apps/46407397-a7d5-4c6b-92c6-dbe038b1457b.json b/evaluation_examples/examples/multi_apps/46407397-a7d5-4c6b-92c6-dbe038b1457b.json index 79b1ecc..f4fd1bf 100644 --- a/evaluation_examples/examples/multi_apps/46407397-a7d5-4c6b-92c6-dbe038b1457b.json +++ b/evaluation_examples/examples/multi_apps/46407397-a7d5-4c6b-92c6-dbe038b1457b.json @@ -40,9 +40,8 @@ "type": "chrome_open_tabs", "parameters": { "urls_to_open": [ - "https://www.freerice.com/", - "https://www.hku.hk/", - "https://about.meta.com/technologies/facebook-app/" + "https://news.google.com", + "https://x.com" ] } }, diff --git a/evaluation_examples/examples/multi_apps/4e9f0faf-2ecc-4ae8-a804-28c9a75d1ddc.json b/evaluation_examples/examples/multi_apps/4e9f0faf-2ecc-4ae8-a804-28c9a75d1ddc.json index f0b6a9e..9245e1b 100644 --- a/evaluation_examples/examples/multi_apps/4e9f0faf-2ecc-4ae8-a804-28c9a75d1ddc.json +++ b/evaluation_examples/examples/multi_apps/4e9f0faf-2ecc-4ae8-a804-28c9a75d1ddc.json @@ -44,9 +44,8 @@ "type": "chrome_open_tabs", "parameters": { "urls_to_open": [ - "https://www.freerice.com/", - "https://www.hku.hk/", - "https://about.meta.com/technologies/facebook-app/" + "https://news.google.com", + "https://x.com" ] } }, diff --git a/evaluation_examples/examples/multi_apps/58565672-7bfe-48ab-b828-db349231de6b.json b/evaluation_examples/examples/multi_apps/58565672-7bfe-48ab-b828-db349231de6b.json index 6732dba..98fbb57 100644 --- a/evaluation_examples/examples/multi_apps/58565672-7bfe-48ab-b828-db349231de6b.json +++ b/evaluation_examples/examples/multi_apps/58565672-7bfe-48ab-b828-db349231de6b.json @@ -28,7 +28,6 @@ "parameters": { "urls_to_open": [ "https://www.apple.com/", - "https://en.sjtu.edu.cn/", "https://scholar.google.com/" ] } @@ -83,7 +82,6 @@ "type": "url", "urls": [ "https://www.apple.com/", - "https://en.sjtu.edu.cn/", "https://scholar.google.com/", "https://www.amazon.com/" ] diff --git a/evaluation_examples/examples/multi_apps/897e3b53-5d4d-444b-85cb-2cdc8a97d903.json b/evaluation_examples/examples/multi_apps/897e3b53-5d4d-444b-85cb-2cdc8a97d903.json index 61d8bea..9da5cc0 100644 --- a/evaluation_examples/examples/multi_apps/897e3b53-5d4d-444b-85cb-2cdc8a97d903.json +++ b/evaluation_examples/examples/multi_apps/897e3b53-5d4d-444b-85cb-2cdc8a97d903.json @@ -21,7 +21,7 @@ "type": "launch", "parameters": { "command": [ - "chromium-browser", + "google-chrome", "--remote-debugging-port=1337" ] } @@ -42,6 +42,7 @@ "urls_to_open": [ "https://www.zhihu.com/", "https://www.coursera.org/", + "https://www.deepl.com", "https://www.wikidata.org/wiki/Wikidata:Main_Page" ] } diff --git a/evaluation_examples/examples/multi_apps/b52b40a5-ad70-4c53-b5b0-5650a8387052.json b/evaluation_examples/examples/multi_apps/b52b40a5-ad70-4c53-b5b0-5650a8387052.json index 340927a..25b9024 100644 --- a/evaluation_examples/examples/multi_apps/b52b40a5-ad70-4c53-b5b0-5650a8387052.json +++ b/evaluation_examples/examples/multi_apps/b52b40a5-ad70-4c53-b5b0-5650a8387052.json @@ -40,9 +40,9 @@ "type": "chrome_open_tabs", "parameters": { "urls_to_open": [ - "https://www.freerice.com/", - "https://www.hku.hk/", - "https://about.meta.com/technologies/facebook-app/" + "https://news.google.com", + "https://x.com", + "https://www.deepl.com" ] } }, diff --git a/evaluation_examples/settings/googledrive/credentials.json b/evaluation_examples/settings/googledrive/credentials.json index 45010ca..2c44560 100644 --- a/evaluation_examples/settings/googledrive/credentials.json +++ b/evaluation_examples/settings/googledrive/credentials.json @@ -1 +1 @@ -{"access_token": "ya29.a0AfB_byC8lcdjvZcfqITa6l6TTITbtzYsnCFadUEzRmzggkA2riAnfxT9ZqI22QOmKdhdxD-TcSi9o-BpP_thCMTi_Mjaelkm9lyDcW-eb0tTG_0hC_b7Ra3TAmnzBIvnroFytJ3m9UIU7RqxttCSfrNz-zPLCWzivnAyaCgYKAR4SARISFQHGX2MiDgoIUJDl_-NKz3L5dsp2wg0171", "client_id": "786888752612-rgng5v9hcq4as7pn0b40gt9r5lekmht9.apps.googleusercontent.com", "client_secret": "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", "refresh_token": "1//0ehtafHmucszRCgYIARAAGA4SNwF-L9IrpDBsnzdHKAlRfrkvzNFw1cpdnRY8rhM5gy4flsPYdysMav27yHamJx39BBGq-LLw40s", "token_expiry": "2024-01-31T10:36:37Z", "token_uri": "https://oauth2.googleapis.com/token", "user_agent": null, "revoke_uri": "https://oauth2.googleapis.com/revoke", "id_token": null, "id_token_jwt": null, "token_response": {"access_token": "ya29.a0AfB_byC8lcdjvZcfqITa6l6TTITbtzYsnCFadUEzRmzggkA2riAnfxT9ZqI22QOmKdhdxD-TcSi9o-BpP_thCMTi_Mjaelkm9lyDcW-eb0tTG_0hC_b7Ra3TAmnzBIvnroFytJ3m9UIU7RqxttCSfrNz-zPLCWzivnAyaCgYKAR4SARISFQHGX2MiDgoIUJDl_-NKz3L5dsp2wg0171", "expires_in": 3599, "refresh_token": "1//0ehtafHmucszRCgYIARAAGA4SNwF-L9IrpDBsnzdHKAlRfrkvzNFw1cpdnRY8rhM5gy4flsPYdysMav27yHamJx39BBGq-LLw40s", "scope": "https://www.googleapis.com/auth/drive", "token_type": "Bearer"}, "scopes": ["https://www.googleapis.com/auth/drive"], "token_info_uri": "https://oauth2.googleapis.com/tokeninfo", "invalid": false, "_class": "OAuth2Credentials", "_module": "oauth2client.client"} \ No newline at end of file +{"access_token": "ya29.a0AfB_byDIizbKXOjPLHrhl3tX0xhI_Pv2U1qv_nnhX_V_QWjCKvq9es2Pgc21SY8z8W4zolgSN4RipR2iA5zHYIDkH1sn6ZhcqC_ExxrmJ_OgZt71ljzfhfh19CfGjr_ki7HoXa_UOg4X__N0IPpy5UTFE8aU1T4F3dIMaCgYKAS0SARISFQHGX2MiW_ZTvRlqH2LYf6ylkeSB9Q0171", "client_id": "786888752612-rgng5v9hcq4as7pn0b40gt9r5lekmht9.apps.googleusercontent.com", "client_secret": "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", "refresh_token": "1//0ehtafHmucszRCgYIARAAGA4SNwF-L9IrpDBsnzdHKAlRfrkvzNFw1cpdnRY8rhM5gy4flsPYdysMav27yHamJx39BBGq-LLw40s", "token_expiry": "2024-01-31T11:54:11Z", "token_uri": "https://oauth2.googleapis.com/token", "user_agent": null, "revoke_uri": "https://oauth2.googleapis.com/revoke", "id_token": null, "id_token_jwt": null, "token_response": {"access_token": "ya29.a0AfB_byDIizbKXOjPLHrhl3tX0xhI_Pv2U1qv_nnhX_V_QWjCKvq9es2Pgc21SY8z8W4zolgSN4RipR2iA5zHYIDkH1sn6ZhcqC_ExxrmJ_OgZt71ljzfhfh19CfGjr_ki7HoXa_UOg4X__N0IPpy5UTFE8aU1T4F3dIMaCgYKAS0SARISFQHGX2MiW_ZTvRlqH2LYf6ylkeSB9Q0171", "expires_in": 3599, "scope": "https://www.googleapis.com/auth/drive", "token_type": "Bearer"}, "scopes": ["https://www.googleapis.com/auth/drive"], "token_info_uri": "https://oauth2.googleapis.com/tokeninfo", "invalid": false, "_class": "OAuth2Credentials", "_module": "oauth2client.client"} \ No newline at end of file