diff --git a/desktop_env/evaluators/getters/chrome.py b/desktop_env/evaluators/getters/chrome.py index f856704..d40de23 100644 --- a/desktop_env/evaluators/getters/chrome.py +++ b/desktop_env/evaluators/getters/chrome.py @@ -444,7 +444,7 @@ def get_open_tabs_info(env, config: Dict[str, str]): for page in context.pages: try: # Wait for the page to finish loading, this prevents the "execution context was destroyed" issue - page.wait_for_load_state('load') # Wait for the 'load' event to complete + page.wait_for_load_state('networkidle') # Wait for the 'load' event to complete title = page.title() url = page.url tabs_info.append({'title': title, 'url': url}) @@ -486,12 +486,17 @@ def get_active_url_from_accessTree(env, config): # first, use accessibility API to get the active tab URL at: _Element = lxml.etree.fromstring(accessibility_tree) arch = platform.machine() - if "arm" in arch: - selector = CSSSelector(", application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]", - namespaces=_accessibility_ns_map) - else: - selector = CSSSelector("application[name=Google\\ Chrome] entry[name=Address\\ and\\ search\\ bar]", - namespaces=_accessibility_ns_map) + print("Your architecture is: {}".format(arch)) + try: + if "arm" in arch: + selector = CSSSelector("application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]", + namespaces=_accessibility_ns_map) + else: + selector = CSSSelector("application[name=Google\\ Chrome] entry[name=Address\\ and\\ search\\ bar]", + namespaces=_accessibility_ns_map) + except: + logger.error("Failed to parse the selector for active tab URL") + return None elements: List[_Element] = selector(at) # if "xpath" in config: # elements: List[_Element] = at.xpath(config["xpath"], namespaces=_accessibility_ns_map) @@ -519,6 +524,9 @@ def get_active_tab_info(env, config: Dict[str, str]): } """ active_tab_url = get_active_url_from_accessTree(env, config) + if active_tab_url is None: + logger.error("Failed to get the url of active tab") + return None host = env.vm_ip port = 9222 # fixme: this port is hard-coded, need to be changed from config file @@ -533,7 +541,11 @@ def get_active_tab_info(env, config: Dict[str, str]): active_tab_info = {} # go to the target URL page page = browser.new_page() - page.goto(active_tab_url) + try: + page.goto(active_tab_url) + except: + logger.error("Failed to go to the target URL page") + return None page.wait_for_load_state('load') # Wait for the 'load' event to complete active_tab_info = { 'title': page.title(), @@ -972,6 +984,8 @@ def get_active_tab_html_parse(env, config: Dict[str, Any]): } """ active_tab_url = get_active_url_from_accessTree(env, config) + if not isinstance(active_tab_url, str): + return None host = env.vm_ip port = 9222 # fixme: this port is hard-coded, need to be changed from config file @@ -1002,9 +1016,11 @@ def get_active_tab_html_parse(env, config: Dict[str, Any]): target_page = None for context in browser.contexts: for page in context.pages: - page.wait_for_load_state("load") + page.wait_for_load_state("networkidle") if page.url == active_tab_url: target_page = page + print("tartget page url: ", target_page.url) + print("tartget page title: ", target_page.title()) break if target_page is None: return {} diff --git a/desktop_env/evaluators/getters/misc.py b/desktop_env/evaluators/getters/misc.py index 99de948..db04aea 100644 --- a/desktop_env/evaluators/getters/misc.py +++ b/desktop_env/evaluators/getters/misc.py @@ -165,7 +165,10 @@ def get_rule_relativeTime(env, config: R) -> R: next_day = 11 to_absoluteDay = datetime(next_year, next_month, next_day) elif to_time == "next Friday": - to_absoluteDay = now + timedelta(days=((4-now.weekday()) if now.weekday() < 4 else (6-now.weekday()) + 5)) + if now.weekday() < 4 and from_time in ["next Monday"]: + to_absoluteDay = now + timedelta(days=((4-now.weekday())+7)) + else: + to_absoluteDay = now + timedelta(days=((4-now.weekday()) if now.weekday() < 4 else (6-now.weekday()) + 5)) else: pass # more rules here regular_to_time = apply_rules_to_timeFormat(relativeRules["expected"]["to"], to_absoluteDay) diff --git a/evaluation_examples/examples/chrome/b7895e80-f4d1-4648-bee0-4eb45a6f1fa8.json b/evaluation_examples/examples/chrome/b7895e80-f4d1-4648-bee0-4eb45a6f1fa8.json index 8ffa341..d42196b 100644 --- a/evaluation_examples/examples/chrome/b7895e80-f4d1-4648-bee0-4eb45a6f1fa8.json +++ b/evaluation_examples/examples/chrome/b7895e80-f4d1-4648-bee0-4eb45a6f1fa8.json @@ -65,8 +65,8 @@ "to": "this Sunday" }, "expected": { - "from": "{DoW}, {Month} {DayD}", - "to": "{DoW}, {Month} {DayD}", + "from": "{DoW}, {Month} {Day0D}", + "to": "{DoW}, {Month} {Day0D}", "city": "New York City Hotels", "adult": "2 adults", "rank": "Price (low to high)" diff --git a/evaluation_examples/examples/chrome/f3b19d1e-2d48-44e9-b4e1-defcae1a0197.json b/evaluation_examples/examples/chrome/f3b19d1e-2d48-44e9-b4e1-defcae1a0197.json index 74bc333..ea19f69 100644 --- a/evaluation_examples/examples/chrome/f3b19d1e-2d48-44e9-b4e1-defcae1a0197.json +++ b/evaluation_examples/examples/chrome/f3b19d1e-2d48-44e9-b4e1-defcae1a0197.json @@ -22,6 +22,20 @@ "tcp:localhost:1337" ] } + }, + { + "type": "chrome_open_tabs", + "parameters": { + "urls_to_open": [ + "https://seatgeek.com/" + ] + } + }, + { + "type": "activate_window", + "parameters": { + "window_name": "Google Chrome" + } } ], "trajectory": "trajectories/", @@ -37,7 +51,7 @@ "expected":{ "type": "rule", "rules":{ - "expected": ["https://seatgeek\\.com/help/articles/\\d+-Buying-Tickets"] + "expected": ["Buying-Tickets"] } } } diff --git a/evaluation_examples/examples/gimp/045bf3ff-9077-4b86-b483-a1040a949cff.json b/evaluation_examples/examples/gimp/045bf3ff-9077-4b86-b483-a1040a949cff.json new file mode 100644 index 0000000..54e5534 --- /dev/null +++ b/evaluation_examples/examples/gimp/045bf3ff-9077-4b86-b483-a1040a949cff.json @@ -0,0 +1,39 @@ +{ + "id": "045bf3ff-9077-4b86-b483-a1040a949cff", + "snapshot": "gimp", + "instruction": "Could you turn my image into CYMK mode?", + "source": "https://www.makeuseof.com/tag/can-photoshop-gimp-cant/", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1YnUq4q9rv7pr1MZPdaD2mVFZRrsgISmn", + "path": "/home/user/Desktop/gate.jpeg" + } + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "gimp", + "/home/user/Desktop/gate.jpeg" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "gimp" + ], + "evaluator": { + "func": "infeasible", + "expected": { + }, + "result": { + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/gimp/2e6f678f-472d-4c55-99cc-8e7c5c402a71.json b/evaluation_examples/examples/gimp/2e6f678f-472d-4c55-99cc-8e7c5c402a71.json new file mode 100644 index 0000000..6f7b55e --- /dev/null +++ b/evaluation_examples/examples/gimp/2e6f678f-472d-4c55-99cc-8e7c5c402a71.json @@ -0,0 +1,46 @@ +{ + "id": "2e6f678f-472d-4c55-99cc-8e7c5c402a71", + "snapshot": "gimp", + "instruction": "Please batch process the images on the desktop by lifting the brightness to 50.", + "source": "", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1uOZWtT9E8YW_IOu51meW5a0jAgwS1DoX", + "path": "/home/user/Desktop/squirrel.jpeg" + }, + { + "url": "https://drive.google.com/uc?export=download&id=1KCyoqh3bTsbY42r9YSqIvcGuUr6i95GU", + "path": "/home/user/Desktop/panda.jpeg" + }, + { + "url": "https://drive.google.com/uc?export=download&id=1xftsr0mRUvqKGPCHOnzUMm7tMnuqdhAA", + "path": "/home/user/Desktop/heron.jpeg" + } + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "gimp" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "gimp" + ], + "evaluator": { + "func": "infeasible", + "expected": { + }, + "result": { + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/gimp/58d3eeeb-e9d0-499f-962e-fd0db2a744d8.json b/evaluation_examples/examples/gimp/58d3eeeb-e9d0-499f-962e-fd0db2a744d8.json new file mode 100644 index 0000000..5fe5537 --- /dev/null +++ b/evaluation_examples/examples/gimp/58d3eeeb-e9d0-499f-962e-fd0db2a744d8.json @@ -0,0 +1,39 @@ +{ + "id": "58d3eeeb-e9d0-499f-962e-fd0db2a744d8", + "snapshot": "gimp", + "instruction": "Please describe what is happening in the image.", + "source": "", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1w9R8RYkbFl88n8_Vgk_zZeD5J4t4yrzC", + "path": "/home/user/Desktop/heron.jpeg" + } + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "gimp", + "/home/user/Desktop/heron.jpeg" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "gimp" + ], + "evaluator": { + "func": "infeasible", + "expected": { + }, + "result": { + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/gimp/8ea73f6f-9689-42ad-8c60-195bbf06a7ba.json b/evaluation_examples/examples/gimp/8ea73f6f-9689-42ad-8c60-195bbf06a7ba.json new file mode 100644 index 0000000..a19c2a5 --- /dev/null +++ b/evaluation_examples/examples/gimp/8ea73f6f-9689-42ad-8c60-195bbf06a7ba.json @@ -0,0 +1,39 @@ +{ + "id": "8ea73f6f-9689-42ad-8c60-195bbf06a7ba", + "snapshot": "gimp", + "instruction": "Can you enhance this low-resolution photo to high-resolution without increasing the file size?", + "source": "", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=14r8Y0Cq8yTtQ5kfqhwWij5ow3LWApy2n", + "path": "/home/user/Desktop/low_resolution.jpeg" + } + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "gimp", + "/home/user/Desktop/low_resolution.jpeg" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "gimp" + ], + "evaluator": { + "func": "infeasible", + "expected": { + }, + "result": { + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/gimp/dbbf4b99-2253-4b10-9274-45f246af2466.json b/evaluation_examples/examples/gimp/dbbf4b99-2253-4b10-9274-45f246af2466.json new file mode 100644 index 0000000..f9087f3 --- /dev/null +++ b/evaluation_examples/examples/gimp/dbbf4b99-2253-4b10-9274-45f246af2466.json @@ -0,0 +1,39 @@ +{ + "id": "dbbf4b99-2253-4b10-9274-45f246af2466", + "snapshot": "gimp", + "instruction": "Convert my new raw image into jpeg.", + "source": "https://www.makeuseof.com/tag/can-photoshop-gimp-cant/", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1zcxzbQ5dp-mgpu0mom-LB8BKJH-V01Kr", + "path": "/home/user/Desktop/yicun.raw" + } + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "gimp", + "/home/user/Desktop/yicun.raw" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "gimp" + ], + "evaluator": { + "func": "infeasible", + "expected": { + }, + "result": { + } + } +} \ No newline at end of file