diff --git a/desktop_env/assets/history_empty.sqlite b/desktop_env/assets/history_empty.sqlite new file mode 100644 index 0000000..c68c17c Binary files /dev/null and b/desktop_env/assets/history_empty.sqlite differ diff --git a/desktop_env/controllers/setup.py b/desktop_env/controllers/setup.py index a29021b..1f081a9 100644 --- a/desktop_env/controllers/setup.py +++ b/desktop_env/controllers/setup.py @@ -1,24 +1,29 @@ import json import logging +import os import os.path +import sqlite3 +import tempfile import time import traceback import uuid -import tempfile +from datetime import datetime, timedelta from typing import Any, Union, Optional from typing import Dict, List -import os +import shutil import requests +from playwright.sync_api import sync_playwright, TimeoutError from pydrive.auth import GoogleAuth from pydrive.drive import GoogleDrive, GoogleDriveFile, GoogleDriveFileList -from playwright.sync_api import sync_playwright, TimeoutError from requests_toolbelt.multipart.encoder import MultipartEncoder +from desktop_env.controllers.python import PythonController from desktop_env.evaluators.metrics.utils import compare_urls logger = logging.getLogger("desktopenv.setup") +FILE_PATH = os.path.dirname(os.path.abspath(__file__)) class SetupController: def __init__(self, vm_ip: str, cache_dir: str): @@ -130,7 +135,8 @@ class SetupController: break except requests.RequestException as e: - logger.error(f"Failed to download {url} caused by {e}. Retrying... ({max_retries - i - 1} attempts left)") + logger.error( + f"Failed to download {url} caused by {e}. Retrying... ({max_retries - i - 1} attempts left)") if not downloaded: raise requests.RequestException(f"Failed to download {url}. No retries left. Error: {e}") @@ -349,18 +355,18 @@ class SetupController: logger.info("Connect to Chrome @: %s", remote_debugging_url) logger.debug("PLAYWRIGHT ENV: %s", repr(os.environ)) for attempt in range(15): - if attempt>0: + if attempt > 0: time.sleep(5) browser = None with sync_playwright() as p: try: browser = p.chromium.connect_over_cdp(remote_debugging_url) - #break + # break except Exception as e: if attempt < 14: logger.error(f"Attempt {attempt + 1}: Failed to connect, retrying. Error: {e}") - #time.sleep(10) + # time.sleep(10) continue else: logger.error(f"Failed to connect after multiple attempts: {e}") @@ -379,7 +385,7 @@ class SetupController: try: page.goto(url, timeout=60000) except: - logger.warning("Opening %s exceeds time limit", url) # only for human test + logger.warning("Opening %s exceeds time limit", url) # only for human test logger.info(f"Opened tab {i + 1}: {url}") if i == 0: @@ -458,16 +464,17 @@ class SetupController: for p in paths: q = f'"{parent_id}" in parents and title = "{p}" and mimeType = "application/vnd.google-apps.folder" and trashed = false' folder = drive.ListFile({'q': q}).GetList() - if len(folder) == 0: # not exists, create it + if len(folder) == 0: # not exists, create it parents = {} if parent_id == 'root' else {'parents': [{'id': parent_id}]} - file = drive.CreateFile({'title': p, 'mimeType':'application/vnd.google-apps.folder', **parents}) + file = drive.CreateFile({'title': p, 'mimeType': 'application/vnd.google-apps.folder', **parents}) file.Upload() parent_id = file['id'] - else: parent_id = folder[0]['id'] + else: + parent_id = folder[0]['id'] return parent_id for oid, operation in enumerate(config['operation']): - if operation == 'delete': # delete a specific file + if operation == 'delete': # delete a specific file # query pattern string, by default, remove all files/folders not in the trash to the trash params = config['args'][oid] q = params.get('query', '') @@ -476,15 +483,19 @@ class SetupController: filelist: GoogleDriveFileList = drive.ListFile({'q': q_file}).GetList() q_folder = f"( {q} ) and mimeType = 'application/vnd.google-apps.folder'" if q.strip() else "mimeType = 'application/vnd.google-apps.folder'" folderlist: GoogleDriveFileList = drive.ListFile({'q': q_folder}).GetList() - for file in filelist: # first delete file, then folder + for file in filelist: # first delete file, then folder file: GoogleDriveFile - if trash: file.Trash() - else: file.Delete() + if trash: + file.Trash() + else: + file.Delete() for folder in folderlist: folder: GoogleDriveFile # note that, if a folder is trashed/deleted, all files and folders in it will be trashed/deleted - if trash: folder.Trash() - else: folder.Delete() + if trash: + folder.Trash() + else: + folder.Delete() elif operation == 'mkdirs': params = config['args'][oid] mkdir_in_googledrive(params['path']) @@ -508,7 +519,6 @@ class SetupController: else: raise ValueError('[ERROR]: not implemented clean type!') - def _login_setup(self, **config): """ Login to a website with account and password information. @args: @@ -537,7 +547,7 @@ class SetupController: raise e if not browser: return - + context = browser.contexts[0] platform = config['platform'] @@ -565,3 +575,82 @@ class SetupController: raise NotImplementedError return browser, context + + def _update_browse_history_setup(self, **config): + db_path = os.path.join("desktop_env", "assets", "history_empty.sqlite") + + # copy a new history file in the tmp folder + cache_path = os.path.join(self.cache_dir, "history_new.sqlite") + shutil.copyfile(db_path, cache_path) + db_path = cache_path + + history = config['history'] + + for history_item in history: + url = history_item['url'] + title = history_item['title'] + visit_time = datetime.now() - timedelta(seconds=history_item['visit_time_from_now_in_seconds']) + + # Chrome use ms from 1601-01-01 as timestamp + epoch_start = datetime(1601, 1, 1) + chrome_timestamp = int((visit_time - epoch_start).total_seconds() * 1000000) + + conn = sqlite3.connect(db_path) + cursor = conn.cursor() + + cursor.execute(''' + INSERT INTO urls (url, title, visit_count, typed_count, last_visit_time, hidden) + VALUES (?, ?, ?, ?, ?, ?) + ''', (url, title, 1, 0, chrome_timestamp, 0)) + + url_id = cursor.lastrowid + + cursor.execute(''' + INSERT INTO visits (url, visit_time, from_visit, transition, segment_id, visit_duration) + VALUES (?, ?, ?, ?, ?, ?) + ''', (url_id, chrome_timestamp, 0, 805306368, 0, 0)) + + conn.commit() + conn.close() + + logger.info('Fake browsing history added successfully.') + + controller = PythonController(self.vm_ip) + + # get the path of the history file according to the platform + os_type = controller.get_vm_platform() + + if os_type == 'Windows': + chrome_history_path = controller.execute_python_command( + """import os; print(os.path.join(os.getenv('USERPROFILE'), "AppData", "Local", "Google", "Chrome", "User Data", "Default", "History"))""")[ + 'output'].strip() + elif os_type == 'Darwin': + chrome_history_path = controller.execute_python_command( + """import os; print(os.path.join(os.getenv('HOME'), "Library", "Application Support", "Google", "Chrome", "Default", "History"))""")[ + 'output'].strip() + elif os_type == 'Linux': + chrome_history_path = controller.execute_python_command( + "import os; print(os.path.join(os.getenv('HOME'), '.config', 'google-chrome', 'Default', 'History'))")[ + 'output'].strip() + else: + raise Exception('Unsupported operating system') + + form = MultipartEncoder({ + "file_path": chrome_history_path, + "file_data": (os.path.basename(chrome_history_path), open(db_path, "rb")) + }) + headers = {"Content-Type": form.content_type} + logger.debug(form.content_type) + + # send request to server to upload file + try: + logger.debug("REQUEST ADDRESS: %s", self.http_server + "/setup" + "/upload") + response = requests.post(self.http_server + "/setup" + "/upload", headers=headers, data=form) + if response.status_code == 200: + logger.info("Command executed successfully: %s", response.text) + else: + logger.error("Failed to upload file. Status code: %s", response.text) + except requests.exceptions.RequestException as e: + logger.error("An error occurred while trying to send the request: %s", e) + + self._execute_setup(["sudo chown -R user:user /home/user/.config/google-chrome/Default/History"], shell=True) diff --git a/desktop_env/envs/desktop_env.py b/desktop_env/envs/desktop_env.py index b12fbca..5a8a03f 100644 --- a/desktop_env/envs/desktop_env.py +++ b/desktop_env/envs/desktop_env.py @@ -175,10 +175,16 @@ class DesktopEnv(gym.Env): if isinstance(self.evaluator["func"], list) \ else getattr(metrics, self.evaluator["func"]) self.metric_conj: str = self.evaluator.get("conj", "and") # take conjunction of multiple metrics - self.result_getter: Getter = [getattr(getters, "get_{:}".format(res["type"])) for res in + if "result" in self.evaluator: + self.result_getter: Getter = [getattr(getters, "get_{:}".format(res["type"])) for res in self.evaluator["result"]] \ if isinstance(self.evaluator["result"], list) \ else getattr(getters, "get_{:}".format(self.evaluator["result"]["type"])) + else: + self.result_getter = [None] * len(self.metric) \ + if isinstance(self.metric, list) \ + else None + if "expected" in self.evaluator: self.expected_getter: Getter = [getattr(getters, "get_{:}".format(exp["type"])) if exp else None for exp in self.evaluator["expected"]] \ @@ -293,6 +299,12 @@ class DesktopEnv(gym.Env): self.setup_controller.setup(self.evaluator.get("postconfig", [])) + if self.metric == "infeasible": + if self.action_history[-1] == "FAIL": + return 1 + else: + return 0 + if type(self.metric) == list: results = [] for idx, metric in enumerate(self.metric): @@ -315,7 +327,8 @@ class DesktopEnv(gym.Env): return 0 elif self.metric_conj == 'or' and float(metric) == 1.0: return 1 - else: results.append(metric) + else: + results.append(metric) return sum(results) / len(results) if self.metric_conj == 'and' else max(results) else: try: diff --git a/desktop_env/evaluators/metrics/slides.py b/desktop_env/evaluators/metrics/slides.py index a025933..33bf2ce 100644 --- a/desktop_env/evaluators/metrics/slides.py +++ b/desktop_env/evaluators/metrics/slides.py @@ -139,6 +139,7 @@ def compare_pptx_files(file1_path, file2_path, **options): examine_number_of_slides = options.get("examine_number_of_slides", True) examine_shape = options.get("examine_shape", True) examine_text = options.get("examine_text", True) + examine_indent = options.get("examine_indent", True) examine_font_name = options.get("examine_font_name", True) examine_font_size = options.get("examine_font_size", True) examine_font_bold = options.get("examine_font_bold", True) @@ -148,6 +149,7 @@ def compare_pptx_files(file1_path, file2_path, **options): examine_strike_through = options.get("examine_strike_through", True) examine_alignment = options.get("examine_alignment", True) examine_bottom_position = options.get("examine_bottom_position", False) + examine_bullets = options.get("examine_bullets", True) # compare the number of slides if len(prs1.slides) != len(prs2.slides) and examine_number_of_slides: @@ -174,6 +176,12 @@ def compare_pptx_files(file1_path, file2_path, **options): if para1.alignment != para2.alignment and examine_alignment: return 0 # check if the runs are the same + if para1.text != para2.text and examine_text: + return 0 + + if para1.level != para2.level and examine_indent: + return 0 + for run1, run2 in zip(para1.runs, para2.runs): # check if the font properties are the same @@ -200,7 +208,40 @@ def compare_pptx_files(file1_path, file2_path, **options): 'strike', 'noStrike') and examine_strike_through: return 0 - # fixme: Actually there are more properties to be compared, but we cannot get them through pptx + def _extract_bullets(xml_data): + root = ET.fromstring(xml_data) + + namespaces = { + 'a': 'http://schemas.openxmlformats.org/drawingml/2006/main', + 'p': 'http://schemas.openxmlformats.org/presentationml/2006/main', + } + + bullets = [] + + for paragraph in root.findall('.//a:p', namespaces): + pPr = paragraph.find('a:pPr', namespaces) + if pPr is not None: + lvl = pPr.get('lvl') + buChar = pPr.find('a:buChar', namespaces) + char = buChar.get('char') if buChar is not None else "No Bullet" + buClr = pPr.find('a:buClr/a:srgbClr', namespaces) + color = buClr.get('val') if buClr is not None else "No Color" + else: + lvl = "No Level" + char = "No Bullet" + color = "No Color" + + text = "".join(t.text for t in paragraph.findall('.//a:t', namespaces)) + + bullets.append((lvl, char, text, color)) + + return bullets + + if _extract_bullets(run1.part.blob.decode('utf-8')) != _extract_bullets( + run2.part.blob.decode('utf-8')) and examine_bullets: + return 0 + + # fixme: Actually there are more properties to be compared, we can add them later via parsing the xml data return 1 @@ -422,6 +463,7 @@ if __name__ == '__main__': # r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\550ce7e7-747b-495f-b122-acdc4d0b8e54\New_Club_Spring_2018_Training_Gold.pptx")) # print(evaluate_presentation_fill_to_rgb_distance(r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\3b27600c-3668-4abd-8f84-7bcdebbccbdb\lec17-gui-events.pptx", {"rgb": (0, 0, 255)})) # print(check_auto_saving_time(r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\2cd43775-7085-45d8-89fa-9e35c0a915cf\registrymodifications.xcu", {"minutes": 3})) - print(compare_pptx_files(r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\a669ef01-ded5-4099-9ea9-25e99b569840\Writing-Outlines.pptx", - r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\a669ef01-ded5-4099-9ea9-25e99b569840\Writing-Outlines_Gold.pptx", - examine_shape=False)) + print(compare_pptx_files( + r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\a669ef01-ded5-4099-9ea9-25e99b569840\Writing-Outlines.pptx", + r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\a669ef01-ded5-4099-9ea9-25e99b569840\Writing-Outlines_Gold.pptx", + examine_shape=False)) diff --git a/evaluation_examples/examples/chrome/3720f614-37fd-4d04-8a6b-76f54f8c222d.json b/evaluation_examples/examples/chrome/3720f614-37fd-4d04-8a6b-76f54f8c222d.json new file mode 100644 index 0000000..ee0b3c5 --- /dev/null +++ b/evaluation_examples/examples/chrome/3720f614-37fd-4d04-8a6b-76f54f8c222d.json @@ -0,0 +1,19 @@ +{ + "id": "3720f614-37fd-4d04-8a6b-76f54f8c222d", + "snapshot": "chrome", + "instruction": "I am more familiar with Korean as I am from Korea. I want to use chrome with my mother tongue. Could you help me change the Chrome interface language to Korean? ", + "source": "https://superuser.com/questions/984668/change-interface-language-of-chrome-to-english", + "config": [ + ], + "trajectory": "trajectories/", + "related_apps": [ + "chrome" + ], + "evaluator": { + "func": "infeasible", + "result": { + }, + "expected": { + } + } +} diff --git a/evaluation_examples/examples/chrome/44ee5668-ecd5-4366-a6ce-c1c9b8d4e938.json b/evaluation_examples/examples/chrome/44ee5668-ecd5-4366-a6ce-c1c9b8d4e938.json index 3a31855..def3f9e 100644 --- a/evaluation_examples/examples/chrome/44ee5668-ecd5-4366-a6ce-c1c9b8d4e938.json +++ b/evaluation_examples/examples/chrome/44ee5668-ecd5-4366-a6ce-c1c9b8d4e938.json @@ -4,6 +4,213 @@ "instruction": "I am looking for an website address I accessed a month ago, but Youtube websites which take almost all of my browsing history are interrupting my search. This is too annoying. I want to remove all my Youtube browsing history first to facilitate my search. Could you help me clear browsing history from Youtube?", "source": "https://superuser.com/questions/1787991/clear-browsing-history-from-specific-site-on-chrome", "config": [ + { + "type": "update_browse_history", + "parameters": { + "history": [ + { + "url": "https://www.youtube.com/watch?v=dQw4w9WgXcQ", + "title": "Rick Astley - Never Gonna Give You Up (Official Music Video)", + "visit_time_from_now_in_seconds": 3600 + }, + { + "url": "https://www.youtube.com/watch?v=9bZkp7q19f0", + "title": "PSY - GANGNAM STYLE(강남스타일) M/V", + "visit_time_from_now_in_seconds": 1631 + }, + { + "url": "https://www.youtube.com/watch?v=3tmd-ClpJxA", + "title": "Maroon 5 - Sugar (Official Music Video)", + "visit_time_from_now_in_seconds": 900 + }, + { + "url": "https://www.nytimes.com/", + "title": "The New York Times", + "visit_time_from_now_in_seconds": 300 + }, + { + "url": "https://www.youtube.com/watch?v=OPf0YbXqDm0", + "title": "Ed Sheeran - Shape of You [Official Music Video]", + "visit_time_from_now_in_seconds": 1200 + }, + { + "url": "https://www.youtube.com/watch?v=JGwWNGJdvx8", + "title": "Taylor Swift - Shake It Off", + "visit_time_from_now_in_seconds": 2400 + }, + { + "url": "https://www.bbc.co.uk/", + "title": "BBC", + "visit_time_from_now_in_seconds": 1500 + }, + { + "url": "https://www.youtube.com/watch?v=2Vv-BfVoq4g", + "title": "Adele - Hello", + "visit_time_from_now_in_seconds": 1800 + }, + { + "url": "https://www.youtube.com/watch?v=YQHsXMglC9A", + "title": "Katy Perry - Roar (Official Music Video)", + "visit_time_from_now_in_seconds": 2100 + }, + { + "url": "https://www.cnn.com/", + "title": "CNN", + "visit_time_from_now_in_seconds": 2700 + }, + { + "url": "https://www.youtube.com/watch?v=ru0K8uYEZWw", + "title": "Justin Bieber - Baby ft. Ludacris (Official Music Video)", + "visit_time_from_now_in_seconds": 3200 + }, + { + "url": "https://www.youtube.com/watch?v=9bZkp7q19f0", + "title": "PSY - GANGNAM STYLE(강남스타일) M/V", + "visit_time_from_now_in_seconds": 3700 + }, + { + "url": "https://www.nationalgeographic.com/", + "title": "National Geographic", + "visit_time_from_now_in_seconds": 4000 + }, + { + "url": "https://www.youtube.com/watch?v=OPf0YbXqDm0", + "title": "Ed Sheeran - Shape of You [Official Music Video]", + "visit_time_from_now_in_seconds": 4300 + }, + { + "url": "https://www.youtube.com/watch?v=JGwWNGJdvx8", + "title": "Taylor Swift - Shake It Off", + "visit_time_from_now_in_seconds": 4700 + }, + { + "url": "https://www.bbc.co.uk/", + "title": "BBC", + "visit_time_from_now_in_seconds": 5000 + }, + { + "url": "https://www.youtube.com/watch?v=2Vv-BfVoq4g", + "title": "Adele - Hello", + "visit_time_from_now_in_seconds": 5300 + }, + { + "url": "https://www.youtube.com/watch?v=YQHsXMglC9A", + "title": "Katy Perry - Roar (Official Music Video)", + "visit_time_from_now_in_seconds": 5600 + }, + { + "url": "https://www.cnn.com/", + "title": "CNN", + "visit_time_from_now_in_seconds": 5900 + }, + { + "url": "https://www.youtube.com/watch?v=ru0K8uYEZWw", + "title": "Justin Bieber - Baby ft. Ludacris (Official Music Video)", + "visit_time_from_now_in_seconds": 6300 + }, + { + "url": "https://www.youtube.com/watch?v=dQw4w9WgXcQ", + "title": "Rick Astley - Never Gonna Give You Up (Official Music Video)", + "visit_time_from_now_in_seconds": 6700 + }, + { + "url": "https://www.nationalgeographic.com/", + "title": "National Geographic", + "visit_time_from_now_in_seconds": 7000 + }, + { + "url": "https://www.youtube.com/watch?v=OPf0YbXqDm0", + "title": "Ed Sheeran - Shape of You [Official Music Video]", + "visit_time_from_now_in_seconds": 7300 + }, + { + "url": "https://www.youtube.com/watch?v=JGwWNGJdvx8", + "title": "Taylor Swift - Shake It Off", + "visit_time_from_now_in_seconds": 7600 + }, + { + "url": "https://www.bbc.co.uk/", + "title": "BBC", + "visit_time_from_now_in_seconds": 7900 + }, + { + "url": "https://www.youtube.com/watch?v=2Vv-BfVoq4g", + "title": "Adele - Hello", + "visit_time_from_now_in_seconds": 8200 + }, + { + "url": "https://www.youtube.com/watch?v=YQHsXMglC9A", + "title": "Katy Perry - Roar (Official Music Video)", + "visit_time_from_now_in_seconds": 8500 + }, + { + "url": "https://www.cnn.com/", + "title": "CNN", + "visit_time_from_now_in_seconds": 8800 + }, + { + "url": "https://www.youtube.com/watch?v=ru0K8uYEZWw", + "title": "Justin Bieber - Baby ft. Ludacris (Official Music Video)", + "visit_time_from_now_in_seconds": 9100 + }, + { + "url": "https://www.youtube.com/watch?v=dQw4w9WgXcQ", + "title": "Rick Astley - Never Gonna Give You Up (Official Music Video)", + "visit_time_from_now_in_seconds": 9400 + }, + { + "url": "https://www.nationalgeographic.com/", + "title": "National Geographic", + "visit_time_from_now_in_seconds": 9700 + }, + { + "url": "https://www.youtube.com/watch?v=OPf0YbXqDm0", + "title": "Ed Sheeran - Shape of You [Official Music Video]", + "visit_time_from_now_in_seconds": 10000 + }, + { + "url": "https://www.youtube.com/watch?v=JGwWNGJdvx8", + "title": "Taylor Swift - Shake It Off", + "visit_time_from_now_in_seconds": 10300 + }, + { + "url": "https://www.bbc.co.uk/", + "title": "BBC", + "visit_time_from_now_in_seconds": 10600 + }, + { + "url": "https://www.youtube.com/watch?v=2Vv-BfVoq4g", + "title": "Adele - Hello", + "visit_time_from_now_in_seconds": 10900 + }, + { + "url": "https://www.youtube.com/watch?v=YQHsXMglC9A", + "title": "Katy Perry - Roar (Official Music Video)", + "visit_time_from_now_in_seconds": 11200 + }, + { + "url": "https://www.cnn.com/", + "title": "CNN", + "visit_time_from_now_in_seconds": 11500 + }, + { + "url": "https://www.youtube.com/watch?v=ru0K8uYEZWw", + "title": "Justin Bieber - Baby ft. Ludacris (Official Music Video)", + "visit_time_from_now_in_seconds": 11800 + }, + { + "url": "https://www.youtube.com/watch?v=dQw4w9WgXcQ", + "title": "Rick Astley - Never Gonna Give You Up (Official Music Video)", + "visit_time_from_now_in_seconds": 12100 + }, + { + "url": "https://www.nationalgeographic.com/", + "title": "National Geographic", + "visit_time_from_now_in_seconds": 12400 + } + ] + } + }, { "type": "launch", "parameters": { diff --git a/evaluation_examples/examples/chrome/93eabf48-6a27-4cb6-b963-7d5fe1e0d3a9.json b/evaluation_examples/examples/chrome/93eabf48-6a27-4cb6-b963-7d5fe1e0d3a9.json new file mode 100644 index 0000000..331a46e --- /dev/null +++ b/evaluation_examples/examples/chrome/93eabf48-6a27-4cb6-b963-7d5fe1e0d3a9.json @@ -0,0 +1,19 @@ +{ + "id": "93eabf48-6a27-4cb6-b963-7d5fe1e0d3a9", + "snapshot": "chrome", + "instruction": "Could you assist me in turning off the dark mode feature in Google Chrome? I've noticed that while dark mode is great for reducing glare, it actually makes it more challenging for me to read text clearly, especially with my astigmatism.", + "source": "https://superuser.com/questions/1417973/how-to-disable-google-chrome-dark-mode", + "config": [ + ], + "trajectory": "trajectories/", + "related_apps": [ + "chrome" + ], + "evaluator": { + "func": "infeasible", + "result": { + }, + "expected": { + } + } +} diff --git a/evaluation_examples/examples/chrome/ae78f875-5b98-4907-bbb5-9c737fc68c03.json b/evaluation_examples/examples/chrome/ae78f875-5b98-4907-bbb5-9c737fc68c03.json index 41c0dea..e19fdc5 100644 --- a/evaluation_examples/examples/chrome/ae78f875-5b98-4907-bbb5-9c737fc68c03.json +++ b/evaluation_examples/examples/chrome/ae78f875-5b98-4907-bbb5-9c737fc68c03.json @@ -4,40 +4,16 @@ "instruction": "Could you please change the number of search results displayed on one page to 50? I find that having more results visible at once significantly enhances my research efficiency, as it reduces the need to constantly click through multiple pages. ", "source": "https://support.google.com/chrome/thread/219988391/increase-search-results-per-page?hl=en", "config": [ - { - "type": "launch", - "parameters": { - "command": [ - "google-chrome", - "--remote-debugging-port=1337" - ] - } - }, - { - "type": "launch", - "parameters": { - "command": [ - "socat", - "tcp-listen:9222,fork", - "tcp:localhost:1337" - ] - } - } ], "trajectory": "trajectories/", "related_apps": [ "chrome" ], "evaluator": { - "func": "exact_match", + "func": "infeasible", "result": { - "type": "number_of_search_results" }, "expected": { - "type": "rule", - "rules": { - "expected": 50 - } } } } diff --git a/evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json b/evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json index 6b92cc1..aba58cd 100644 --- a/evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json +++ b/evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json @@ -4,78 +4,16 @@ "instruction": "Make sparkline chart line by line", "source": "https://www.youtube.com/shorts/L3Z-F1QTQFY", "config": [ - { - "type": "download", - "parameters": { - "files": [ - { - "url": "https://drive.usercontent.google.com/download?id=1uywX5XWMvesnb4-8LPKEzr2HFU7HmoIu&export=download&authuser=0&confirm=t&uuid=267bfe49-a861-4272-ae7c-39c95df35e84&at=APZUnTUbs-FF06hSMv3yWfdXc02l:1703508870351", - "path": "/home/user/OrderId_Month_Chart.xlsx" - } - ] - } - }, - { - "type": "open", - "parameters": { - "path": "/home/user/OrderId_Month_Chart.xlsx" - } - } ], "trajectory": "trajectories/2bd59342-0664-4ccb-ba87-79379096cc08", "related_apps": [ - "libreoffice calc" + "libreoffice_calc" ], "evaluator": { - "postconfig": [ - { - "type": "activate_window", - "parameters": { - "window_name": "OrderId_Month_Chart.xlsx - LibreOffice Calc", - "strict": true - } - }, - { - "type": "sleep", - "parameters": { - "seconds": 0.5 - } - }, - { - "type": "execute", - "parameters": { - "command": [ - "python", - "-c", - "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" - ] - } - } - ], - "func": "compare_table", + "func": "infeasible", "expected": { - "type": "cloud_file", - "path": "https://drive.usercontent.google.com/download?id=1KQJJLVPGtTL_7ArEWvwwbFbJSiA3cgSE&export=download&authuser=0&confirm=t&uuid=6b11c721-caad-439a-b369-4c13c7a485df&at=APZUnTV5-1isKrDKSHV9NeJ6TDeS:1703509054094", - "dest": "OrderId_Month_Chart_gold.xlsx" }, "result": { - "type": "vm_file", - "path": "/home/user/OrderId_Month_Chart.xlsx", - "dest": "OrderId_Month_Chart.xlsx" - }, - "options": { - "rules": [ - { - "type": "sparkline", - "sheet_idx0": 0, - "sheet_idx1": "EI0" - }, - { - "type": "sheet_data", - "sheet_idx0": 0, - "sheet_idx1": "EI0" - } - ] } } } \ No newline at end of file diff --git a/evaluation_examples/examples/libreoffice_calc/7b802dad-6e0f-4204-9815-d4e3f57627d8.json b/evaluation_examples/examples/libreoffice_calc/7b802dad-6e0f-4204-9815-d4e3f57627d8.json new file mode 100644 index 0000000..0ebfeaf --- /dev/null +++ b/evaluation_examples/examples/libreoffice_calc/7b802dad-6e0f-4204-9815-d4e3f57627d8.json @@ -0,0 +1,19 @@ +{ + "id": "7b802dad-6e0f-4204-9815-d4e3f57627d8", + "snapshot": "libreoffice_calc", + "instruction": "I would like to sort this table based on cell color, placing all the rows marked with pink at the beginning, while keeping their order among themselves unchanged.", + "source": "https://www.youtube.com/shorts/Of-lzeP1usE", + "config": [ + ], + "trajectory": "trajectories/7b802dad-6e0f-4204-9815-d4e3f57627d8", + "related_apps": [ + "libreoffice_calc" + ], + "evaluator": { + "func": "infeasible", + "expected": { + }, + "result": { + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/libreoffice_impress/358aa0a7-6677-453f-ae35-e440f004c31e.json b/evaluation_examples/examples/libreoffice_impress/358aa0a7-6677-453f-ae35-e440f004c31e.json index 227002d..b594213 100644 --- a/evaluation_examples/examples/libreoffice_impress/358aa0a7-6677-453f-ae35-e440f004c31e.json +++ b/evaluation_examples/examples/libreoffice_impress/358aa0a7-6677-453f-ae35-e440f004c31e.json @@ -54,7 +54,7 @@ { "type": "sleep", "parameters": { - "seconds": 0.5 + "seconds": 15 } } ], diff --git a/evaluation_examples/examples/libreoffice_impress/a669ef01-ded5-4099-9ea9-25e99b569840.json b/evaluation_examples/examples/libreoffice_impress/a669ef01-ded5-4099-9ea9-25e99b569840.json index 853f01a..25c8780 100644 --- a/evaluation_examples/examples/libreoffice_impress/a669ef01-ded5-4099-9ea9-25e99b569840.json +++ b/evaluation_examples/examples/libreoffice_impress/a669ef01-ded5-4099-9ea9-25e99b569840.json @@ -9,7 +9,7 @@ "parameters": { "files": [ { - "url": "https://drive.usercontent.google.com/download?id=1C0u-Qvvwa6UbJVTzzQHfdNIgp2i051xA&export=download&authuser=0&confirm=t&uuid=5551a43c-3ff7-424f-b82c-50a5c96b5809&at=APZUnTViShb8pJUviOkmVtn7Pums:1707299959829", + "url": "https://drive.usercontent.google.com/download?id=1hr2flq5iSyMYSps6Jd-3pDOEfZoHFCbb&export=download&authuser=0&confirm=t&uuid=02746987-6ea8-4fbb-8817-8051dab152e7&at=APZUnTWaH071WARB_12CQDvjfg6b:1707314868059", "path": "/home/user/Desktop/Writing-Outlines.pptx" } ] @@ -71,16 +71,13 @@ "func": "compare_pptx_files", "expected": { "type": "cloud_file", - "path": "https://drive.usercontent.google.com/download?id=1d4WJwm7KDhDIOJ9r9vKhRTrt_bF3PSln&export=download&authuser=0&confirm=t&uuid=a8ec87de-96cf-49f0-98e4-faa1218354fe&at=APZUnTWiOoiHFjyM4jTunLP4t5wE:1707299961717", + "path": "https://drive.usercontent.google.com/download?id=15mnwwGTDlelIf27C1HdJOiMgfEWNIoLl&export=download&authuser=0&confirm=t&uuid=b53e5fbb-565b-4498-9dc5-071eded307e0&at=APZUnTUdfXCDVFEQPCTYckB-H2Fn:1707314644205", "dest": "Writing-Outlines_Gold.pptx" }, "result": { "type": "vm_file", "path": "/home/user/Desktop/Writing-Outlines.pptx", "dest": "Writing-Outlines.pptx" - }, - "options": { - "examine_shape": false } } } diff --git a/evaluation_examples/examples/os/b3d4a89c-53f2-4d6b-8b6a-541fb5d205fa.json b/evaluation_examples/examples/os/b3d4a89c-53f2-4d6b-8b6a-541fb5d205fa.json index be03893..d7bb963 100644 --- a/evaluation_examples/examples/os/b3d4a89c-53f2-4d6b-8b6a-541fb5d205fa.json +++ b/evaluation_examples/examples/os/b3d4a89c-53f2-4d6b-8b6a-541fb5d205fa.json @@ -9,7 +9,7 @@ "os" ], "evaluator": { - "func": "", + "func": "infeasible", "result": { }, "expected": { diff --git a/evaluation_examples/examples/os/e2eb4bf1-aa93-4192-b55d-03e2fb6dfd15.json b/evaluation_examples/examples/os/e2eb4bf1-aa93-4192-b55d-03e2fb6dfd15.json index fdf0ede..70cb9e7 100644 --- a/evaluation_examples/examples/os/e2eb4bf1-aa93-4192-b55d-03e2fb6dfd15.json +++ b/evaluation_examples/examples/os/e2eb4bf1-aa93-4192-b55d-03e2fb6dfd15.json @@ -8,7 +8,7 @@ "os" ], "evaluator": { - "func": "", + "func": "infeasible", "result": { }, "expected": { diff --git a/evaluation_examples/examples/os/fe41f596-a71b-4c2f-9b2f-9dcd40b568c3.json b/evaluation_examples/examples/os/fe41f596-a71b-4c2f-9b2f-9dcd40b568c3.json index 79d8618..b20f794 100644 --- a/evaluation_examples/examples/os/fe41f596-a71b-4c2f-9b2f-9dcd40b568c3.json +++ b/evaluation_examples/examples/os/fe41f596-a71b-4c2f-9b2f-9dcd40b568c3.json @@ -5,32 +5,15 @@ "source": "https://help.ubuntu.com/lts/ubuntu-help/power-percentage.html.en", "trajectory": "trajectories/", "config": [ - { - "type": "execute", - "parameters": { - "command": [ - "python", - "-c", - "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);" - ] - } - } ], "related_apps": [ "os" ], "evaluator": { - "func": "exact_match", + "func": "infeasible", "result": { - "type": "vm_command_line", - "command": "gsettings get org.gnome.desktop.interface show-battery-percentage", - "shell": true }, "expected": { - "type": "rule", - "rules": { - "expected": "true\n" - } } } } \ No newline at end of file