Merge branch 'main' into zdy

desktop_env/evaluators/getters/chrome.py

@@ -444,7 +444,7 @@ def get_open_tabs_info(env, config: Dict[str, str]):
             for page in context.pages:
                 try:
                     # Wait for the page to finish loading, this prevents the "execution context was destroyed" issue
-                    page.wait_for_load_state('load')  # Wait for the 'load' event to complete
+                    page.wait_for_load_state('networkidle')  # Wait for the 'load' event to complete
                     title = page.title()
                     url = page.url
                     tabs_info.append({'title': title, 'url': url})
@@ -486,12 +486,17 @@ def get_active_url_from_accessTree(env, config):
     # first, use accessibility API to get the active tab URL
     at: _Element = lxml.etree.fromstring(accessibility_tree)
     arch = platform.machine()
-    if "arm" in arch:
+    print("Your architecture is: {}".format(arch))
-        selector = CSSSelector(", application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]",
+    try:
-                               namespaces=_accessibility_ns_map)
+        if "arm" in arch:
-    else:
+            selector = CSSSelector("application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]",
-        selector = CSSSelector("application[name=Google\\ Chrome] entry[name=Address\\ and\\ search\\ bar]",
+                                namespaces=_accessibility_ns_map)
-                               namespaces=_accessibility_ns_map)
+        else:
+            selector = CSSSelector("application[name=Google\\ Chrome] entry[name=Address\\ and\\ search\\ bar]",
+                                namespaces=_accessibility_ns_map)
+    except:
+        logger.error("Failed to parse the selector for active tab URL")
+        return None
     elements: List[_Element] = selector(at)
     # if "xpath" in config:
     #     elements: List[_Element] = at.xpath(config["xpath"], namespaces=_accessibility_ns_map)
@@ -519,6 +524,9 @@ def get_active_tab_info(env, config: Dict[str, str]):
     }
     """
     active_tab_url = get_active_url_from_accessTree(env, config)
+    if active_tab_url is None:
+        logger.error("Failed to get the url of active tab")
+        return None
     host = env.vm_ip
     port = 9222  # fixme: this port is hard-coded, need to be changed from config file
 
@@ -533,7 +541,11 @@ def get_active_tab_info(env, config: Dict[str, str]):
         active_tab_info = {}
         # go to the target URL page
         page = browser.new_page()
-        page.goto(active_tab_url)
+        try:
+            page.goto(active_tab_url)
+        except:
+            logger.error("Failed to go to the target URL page")
+            return None
         page.wait_for_load_state('load')  # Wait for the 'load' event to complete
         active_tab_info = {
             'title': page.title(),
@@ -972,6 +984,8 @@ def get_active_tab_html_parse(env, config: Dict[str, Any]):
     }
     """
     active_tab_url = get_active_url_from_accessTree(env, config)
+    if not isinstance(active_tab_url, str):
+        return None
     host = env.vm_ip
     port = 9222  # fixme: this port is hard-coded, need to be changed from config file
 
@@ -1002,9 +1016,11 @@ def get_active_tab_html_parse(env, config: Dict[str, Any]):
         target_page = None
         for context in browser.contexts:
             for page in context.pages:
-                page.wait_for_load_state("load")
+                page.wait_for_load_state("networkidle")
                 if page.url == active_tab_url:
                     target_page = page
+                    print("tartget page url: ", target_page.url)
+                    print("tartget page title: ", target_page.title())
                     break
         if target_page is None:
             return {}

desktop_env/evaluators/getters/misc.py

@@ -165,7 +165,10 @@ def get_rule_relativeTime(env, config: R) -> R:
                 next_day = 11
                 to_absoluteDay = datetime(next_year, next_month, next_day)
             elif to_time == "next Friday":
-                to_absoluteDay = now + timedelta(days=((4-now.weekday()) if now.weekday() < 4 else (6-now.weekday()) + 5))
+                if now.weekday() < 4 and from_time in ["next Monday"]:
+                    to_absoluteDay = now + timedelta(days=((4-now.weekday())+7))
+                else:
+                    to_absoluteDay = now + timedelta(days=((4-now.weekday()) if now.weekday() < 4 else (6-now.weekday()) + 5))
             else:
                 pass # more rules here
         regular_to_time = apply_rules_to_timeFormat(relativeRules["expected"]["to"], to_absoluteDay)

evaluation_examples/examples/chrome/b7895e80-f4d1-4648-bee0-4eb45a6f1fa8.json

@@ -65,8 +65,8 @@
                   "to": "this Sunday"
                 },
                 "expected": {
-                   "from": "{DoW}, {Month} {DayD}",
+                   "from": "{DoW}, {Month} {Day0D}",
-                   "to": "{DoW}, {Month} {DayD}",
+                   "to": "{DoW}, {Month} {Day0D}",
                     "city": "New York City Hotels",
                     "adult": "2 adults",
                     "rank": "Price (low to high)"

evaluation_examples/examples/chrome/f3b19d1e-2d48-44e9-b4e1-defcae1a0197.json

@@ -22,6 +22,20 @@
             "tcp:localhost:1337"
           ]
         }
+      },
+      {
+        "type": "chrome_open_tabs",
+        "parameters": {
+          "urls_to_open": [
+            "https://seatgeek.com/"
+          ]
+        }
+      },
+      {
+        "type": "activate_window",
+        "parameters": {
+          "window_name": "Google Chrome"
+        }
       }
     ],
     "trajectory": "trajectories/",
@@ -37,7 +51,7 @@
         "expected":{
             "type": "rule",
             "rules":{
-                "expected": ["https://seatgeek\\.com/help/articles/\\d+-Buying-Tickets"]
+                "expected": ["Buying-Tickets"]
             }
         }
     }

evaluation_examples/examples/gimp/045bf3ff-9077-4b86-b483-a1040a949cff.json

@@ -0,0 +1,39 @@
+{
+    "id": "045bf3ff-9077-4b86-b483-a1040a949cff",
+    "snapshot": "gimp",
+    "instruction": "Could you turn my image into CYMK mode?",
+    "source": "https://www.makeuseof.com/tag/can-photoshop-gimp-cant/",
+    "config": [
+        {
+            "type": "download",
+            "parameters": {
+                "files": [
+                    {
+                        "url": "https://drive.google.com/uc?export=download&id=1YnUq4q9rv7pr1MZPdaD2mVFZRrsgISmn",
+                        "path": "/home/user/Desktop/gate.jpeg"
+                    }
+                ]
+            }
+        },
+        {
+            "type": "launch",
+            "parameters": {
+                "command": [
+                    "gimp",
+                    "/home/user/Desktop/gate.jpeg"
+                ]
+            }
+        }
+    ],
+    "trajectory": "trajectories/",
+    "related_apps": [
+        "gimp"
+    ],
+    "evaluator": {
+        "func": "infeasible",
+        "expected": {
+        },
+        "result": {
+        }
+    }
+}

evaluation_examples/examples/gimp/2e6f678f-472d-4c55-99cc-8e7c5c402a71.json

@@ -0,0 +1,46 @@
+{
+    "id": "2e6f678f-472d-4c55-99cc-8e7c5c402a71",
+    "snapshot": "gimp",
+    "instruction": "Please batch process the images on the desktop by lifting the brightness to 50.",
+    "source": "",
+    "config": [
+        {
+            "type": "download",
+            "parameters": {
+                "files": [
+                    {
+                        "url": "https://drive.google.com/uc?export=download&id=1uOZWtT9E8YW_IOu51meW5a0jAgwS1DoX",
+                        "path": "/home/user/Desktop/squirrel.jpeg"
+                    },
+                    {
+                        "url": "https://drive.google.com/uc?export=download&id=1KCyoqh3bTsbY42r9YSqIvcGuUr6i95GU",
+                        "path": "/home/user/Desktop/panda.jpeg"
+                    },
+                    {
+                        "url": "https://drive.google.com/uc?export=download&id=1xftsr0mRUvqKGPCHOnzUMm7tMnuqdhAA",
+                        "path": "/home/user/Desktop/heron.jpeg"
+                    }
+                ]
+            }
+        },
+        {
+            "type": "launch",
+            "parameters": {
+                "command": [
+                    "gimp"
+                ]
+            }
+        }
+    ],
+    "trajectory": "trajectories/",
+    "related_apps": [
+        "gimp"
+    ],
+    "evaluator": {
+        "func": "infeasible",
+        "expected": {
+        },
+        "result": {
+        }
+    }
+}

evaluation_examples/examples/gimp/58d3eeeb-e9d0-499f-962e-fd0db2a744d8.json

@@ -0,0 +1,39 @@
+{
+    "id": "58d3eeeb-e9d0-499f-962e-fd0db2a744d8",
+    "snapshot": "gimp",
+    "instruction": "Please describe what is happening in the image.",
+    "source": "",
+    "config": [
+        {
+            "type": "download",
+            "parameters": {
+                "files": [
+                    {
+                        "url": "https://drive.google.com/uc?export=download&id=1w9R8RYkbFl88n8_Vgk_zZeD5J4t4yrzC",
+                        "path": "/home/user/Desktop/heron.jpeg"
+                    }
+                ]
+            }
+        },
+        {
+            "type": "launch",
+            "parameters": {
+                "command": [
+                    "gimp",
+                    "/home/user/Desktop/heron.jpeg"
+                ]
+            }
+        }
+    ],
+    "trajectory": "trajectories/",
+    "related_apps": [
+        "gimp"
+    ],
+    "evaluator": {
+        "func": "infeasible",
+        "expected": {
+        },
+        "result": {
+        }
+    }
+}

evaluation_examples/examples/gimp/8ea73f6f-9689-42ad-8c60-195bbf06a7ba.json

@@ -0,0 +1,39 @@
+{
+    "id": "8ea73f6f-9689-42ad-8c60-195bbf06a7ba",
+    "snapshot": "gimp",
+    "instruction": "Can you enhance this low-resolution photo to high-resolution without increasing the file size?",
+    "source": "",
+    "config": [
+        {
+            "type": "download",
+            "parameters": {
+                "files": [
+                    {
+                        "url": "https://drive.google.com/uc?export=download&id=14r8Y0Cq8yTtQ5kfqhwWij5ow3LWApy2n",
+                        "path": "/home/user/Desktop/low_resolution.jpeg"
+                    }
+                ]
+            }
+        },
+        {
+            "type": "launch",
+            "parameters": {
+                "command": [
+                    "gimp",
+                    "/home/user/Desktop/low_resolution.jpeg"
+                ]
+            }
+        }
+    ],
+    "trajectory": "trajectories/",
+    "related_apps": [
+        "gimp"
+    ],
+    "evaluator": {
+        "func": "infeasible",
+        "expected": {
+        },
+        "result": {
+        }
+    }
+}

evaluation_examples/examples/gimp/dbbf4b99-2253-4b10-9274-45f246af2466.json

@@ -0,0 +1,39 @@
+{
+    "id": "dbbf4b99-2253-4b10-9274-45f246af2466",
+    "snapshot": "gimp",
+    "instruction": "Convert my new raw image into jpeg.",
+    "source": "https://www.makeuseof.com/tag/can-photoshop-gimp-cant/",
+    "config": [
+        {
+            "type": "download",
+            "parameters": {
+                "files": [
+                    {
+                        "url": "https://drive.google.com/uc?export=download&id=1zcxzbQ5dp-mgpu0mom-LB8BKJH-V01Kr",
+                        "path": "/home/user/Desktop/yicun.raw"
+                    }
+                ]
+            }
+        },
+        {
+            "type": "launch",
+            "parameters": {
+                "command": [
+                    "gimp",
+                    "/home/user/Desktop/yicun.raw"
+                ]
+            }
+        }
+    ],
+    "trajectory": "trajectories/",
+    "related_apps": [
+        "gimp"
+    ],
+    "evaluator": {
+        "func": "infeasible",
+        "expected": {
+        },
+        "result": {
+        }
+    }
+}