Merge branch 'zdy'

desktop_env/controllers/setup.py

@@ -7,6 +7,7 @@ import uuid
 import tempfile
 from typing import Any, Union, Optional
 from typing import Dict, List
+import os
 
 import requests
 from pydrive.auth import GoogleAuth
@@ -114,6 +115,7 @@ class SetupController:
             if not os.path.exists(cache_path):
                 max_retries = 3
                 downloaded = False
+                e = None
                 for i in range(max_retries):
                     try:
                         response = requests.get(url, stream=True)
@@ -128,7 +130,7 @@ class SetupController:
                         break
 
                     except requests.RequestException as e:
-                        logger.error(f"Failed to download {url}. Retrying... ({max_retries - i - 1} attempts left)")
+                        logger.error(f"Failed to download {url} caused by {e}. Retrying... ({max_retries - i - 1} attempts left)")
                 if not downloaded:
                     raise requests.RequestException(f"Failed to download {url}. No retries left. Error: {e}")
 
@@ -344,39 +346,49 @@ class SetupController:
         port = 9222  # fixme: this port is hard-coded, need to be changed from config file
 
         remote_debugging_url = f"http://{host}:{port}"
-        with sync_playwright() as p:
+        logger.info("Connect to Chrome @: %s", remote_debugging_url)
+        logger.debug("PLAYWRIGHT ENV: %s", repr(os.environ))
+        for attempt in range(15):
+            if attempt>0:
+                time.sleep(5)
+
             browser = None
-            for attempt in range(15):
+            with sync_playwright() as p:
                 try:
                     browser = p.chromium.connect_over_cdp(remote_debugging_url)
-                    break
+                    #break
                 except Exception as e:
                     if attempt < 14:
                         logger.error(f"Attempt {attempt + 1}: Failed to connect, retrying. Error: {e}")
-                        time.sleep(1)
+                        #time.sleep(10)
+                        continue
                     else:
                         logger.error(f"Failed to connect after multiple attempts: {e}")
                         raise e
 
-            if not browser:
+                if not browser:
-                return
+                    return
 
-            for i, url in enumerate(urls_to_open):
+                logger.info("Opening %s...", urls_to_open)
-                # Use the first context (which should be the only one if using default profile)
+                for i, url in enumerate(urls_to_open):
-                if i == 0:
+                    # Use the first context (which should be the only one if using default profile)
-                    context = browser.contexts[0]
+                    if i == 0:
+                        context = browser.contexts[0]
 
-                page = context.new_page()  # Create a new page (tab) within the existing context
+                    page = context.new_page()  # Create a new page (tab) within the existing context
-                page.goto(url, timeout=60000)
+                    try:
-                logger.info(f"Opened tab {i + 1}: {url}")
+                        page.goto(url, timeout=60000)
+                    except:
+                        logger.warning("Opening %s exceeds time limit", url) # only for human test
+                    logger.info(f"Opened tab {i + 1}: {url}")
 
-                if i == 0:
+                    if i == 0:
-                    # clear the default tab
+                        # clear the default tab
-                    default_page = context.pages[0]
+                        default_page = context.pages[0]
-                    default_page.close()
+                        default_page.close()
 
-            # Do not close the context or browser; they will remain open after script ends
+                # Do not close the context or browser; they will remain open after script ends
-            return browser, context
+                return browser, context
 
     def _chrome_close_tabs_setup(self, urls_to_close: List[str]):
         time.sleep(5)  # Wait for Chrome to finish launching
@@ -552,4 +564,4 @@ class SetupController:
             else:
                 raise NotImplementedError
 
-            return browser, context
+            return browser, context

desktop_env/evaluators/README.md

@@ -191,7 +191,7 @@ To enable and use the HTTP interface in VLC Media Player for remote control and
 #### 4. Configure Lua HTTP
 
 - Expand the `Main interfaces` node and select `Lua`.
-- Under `Lua HTTP`, set a password in the `Lua HTTP` section. This password will be required to access the HTTP interface.
+- Under `Lua HTTP`, set a password `password` in the `Lua HTTP` section. This password will be required to access the HTTP interface.
 
 #### 5. Save and Restart VLC
 
@@ -217,4 +217,4 @@ pip install opencv-python-headless Pillow imagehash
 - If the port is in use by another application, you may change the port number in VLC's settings.
 
 ## GIMP
-Click on the "Keep" of the image loading pop-up.
+Click on the "Keep" of the image loading pop-up.

evaluation_examples/examples/chrome/bb5e4c0d-f964-439c-97b6-bdb9747de3f4.json

@@ -12,6 +12,16 @@
           "--remote-debugging-port=9222"
         ]
       }
+    },
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "socat",
+          "tcp-listen:9222,fork",
+          "tcp:localhost:1337"
+        ]
+      }
     }
   ],
   "trajectory": "trajectories/",

evaluation_examples/examples/vlc/8f080098-ddb1-424c-b438-4e96e5e4786e.json

@@ -1,7 +1,7 @@
 {
   "id": "8f080098-ddb1-424c-b438-4e96e5e4786e",
   "snapshot": "base_setup",
-  "instruction": "Could you download the song from this music video and save it as an MP3 file? I'd like to have it on my device to play whenever I want. Please title the file \"Baby Justin Bieber.mp3.\" I really appreciate your help!",
+  "instruction": "Could you download the song from this music video and save it as an MP3 file? I'd like to have it on my device to play whenever I want. Please save the file just on the desktop and title the file \"Baby Justin Bieber.mp3.\" I really appreciate your help!",
   "source": "https://medium.com/@jetscribe_ai/how-to-extract-mp3-audio-from-videos-using-vlc-media-player-beeef644ebfb",
   "config": [
     {