add multi-apps 5 examples by ruisheng 2024-03-06

desktop_env/controllers/setup.py

@@ -450,6 +450,8 @@ class SetupController:
                     query(str): query pattern string to search files or folder in google drive to delete, please refer to
                         https://developers.google.com/drive/api/guides/search-files?hl=en about how to write query string.
                     trash(bool): whether to delete files permanently or move to trash. By default, trash=false, completely delete it.
+                for mkdirs:
+                    path(List[str]): the path in the google drive to create folder
                 for upload:
                     path(str): remote url to download file
                     dest(List[str]): the path in the google drive to store the downloaded file

desktop_env/evaluators/getters/__init__.py

@@ -23,9 +23,10 @@ from .chrome import (
     get_active_tab_url_parse,
     get_gotoRecreationPage_and_get_html_content,
     get_url_dashPart,
-    get_active_url_from_accessTree
+    get_active_url_from_accessTree,
+    get_info_from_website
 )
-from .file import get_cloud_file, get_vm_file, get_cache_file
+from .file import get_cloud_file, get_vm_file, get_cache_file, get_content_from_vm_file
 from .general import get_vm_command_line, get_vm_terminal_output
 from .gimp import get_gimp_config_file
 from .impress import get_audio_in_slide

desktop_env/evaluators/getters/chrome.py

@@ -11,7 +11,7 @@ import lxml.etree
 import requests
 from lxml.cssselect import CSSSelector
 from lxml.etree import _Element
-from playwright.sync_api import sync_playwright
+from playwright.sync_api import sync_playwright, expect
 from pydrive.auth import GoogleAuth
 from pydrive.drive import GoogleDrive, GoogleDriveFileList, GoogleDriveFile
 
@@ -35,6 +35,89 @@ WARNING:
 """
 
 
+def get_info_from_website(env, config: Dict[Any, Any]) -> Any:
+    """ Get information from a website. Especially useful when the information may be updated through time.
+    Args:
+        env (Any): The environment object.
+        config (Dict[Any, Any]): The configuration dictionary.
+            - url (str): The URL of the website to visit
+            - infos (List[Dict[str, str]]): The list of information to be extracted from the website. Each dictionary contains:
+                - action (str): chosen from 'inner_text', 'attribute', 'click_and_inner_text', 'click_and_attribute', etc., concretely,
+                    - inner_text: extract the inner text of the element specified by the selector
+                    - attribute: extract the attribute of the element specified by the selector
+                    - click_and_inner_text: click elements following the selector and then extract the inner text of the last element
+                    - click_and_attribute: click elements following the selector and then extract the attribute of the last element
+                - selector (Union[str, List[str]]): The CSS selector(s) of the element(s) to be extracted.
+                - attribute (str): optional for 'attribute' and 'click_and_attribute', the attribute to be extracted.
+            - backups (Any): The backup information to be returned if the extraction fails.
+    """
+    try:
+        host = env.vm_ip
+        port = 9222  # fixme: this port is hard-coded, need to be changed from config file
+        remote_debugging_url = f"http://{host}:{port}"
+        with sync_playwright() as p:
+            # connect to remote Chrome instance
+            try:
+                browser = p.chromium.connect_over_cdp(remote_debugging_url)
+            except Exception as e:
+                # If the connection fails (e.g., the agent close the browser instance), start a new browser instance
+                app = 'chromium' if 'arm' in platform.machine() else 'google-chrome'
+                payload = json.dumps({"command": [
+                    app,
+                    "--remote-debugging-port=1337"
+                ], "shell": False})
+                headers = {"Content-Type": "application/json"}
+                requests.post("http://" + host + ":5000/setup" + "/launch", headers=headers, data=payload)
+                time.sleep(5)
+                browser = p.chromium.connect_over_cdp(remote_debugging_url)
+
+            page = browser.contexts[0].new_page()
+            page.goto(config["url"])
+            page.wait_for_load_state('load')
+            infos = []
+            for info_dict in config.get('infos', []):
+                if page.url != config["url"]:
+                    page.goto(config["url"])
+                    page.wait_for_load_state('load')
+                action = info_dict.get('action', 'inner_text')
+                if action == "inner_text":
+                    ele = page.locator(info_dict['selector'])
+                    expect(ele).to_be_visible()
+                    infos.append(ele.inner_text())
+                elif action == "attribute":
+                    ele = page.locator(info_dict['selector'])
+                    expect(ele).to_be_visible()
+                    infos.append(ele.get_attribute(info_dict['attribute']))
+                elif action == 'click_and_inner_text':
+                    for idx, sel in enumerate(info_dict['selector']):
+                        if idx != len(info_dict['selector']) - 1:
+                            link = page.locator(sel)
+                            expect(link).to_be_visible()
+                            link.click()
+                            page.wait_for_load_state('load')
+                        else:
+                            ele = page.locator(sel)
+                            expect(ele).to_be_visible()
+                            infos.append(ele.inner_text())
+                elif action == 'click_and_attribute':
+                    for idx, sel in enumerate(info_dict['selector']):
+                        if idx != len(info_dict['selector']) - 1:
+                            link = page.locator(sel)
+                            expect(link).to_be_visible()
+                            link.click()
+                            page.wait_for_load_state('load')
+                        else:
+                            ele = page.locator(sel)
+                            expect(ele).to_be_visible()
+                            infos.append(ele.get_attribute(info_dict['attribute']))
+                else:
+                    raise NotImplementedError(f'The action {action} is not supported yet.')
+        return infos
+    except Exception as e:
+        logger.error(f'[ERROR]: failed to obtain information from the website: {config["url"]}. Use backup results instead.')
+        return config.get('backups', None)
+
+
 # The following ones just need to load info from the files of software, no need to connect to the software
 def get_default_search_engine(env, config: Dict[str, str]):
     os_type = env.vm_platform

desktop_env/evaluators/getters/file.py

@@ -1,8 +1,27 @@
 import os
 from typing import Dict, List, Set
 from typing import Optional, Any, Union
-
 import requests
+import pandas as pd
+
+
+def get_content_from_vm_file(env, config: Dict[str, Any]) -> Any:
+    """
+    Config:
+        path (str): absolute path on the VM to fetch
+    """
+
+    path = config["path"]
+    file_path = get_vm_file(env, {"path": path, "dest": os.path.basename(path)})
+    file_type, file_content = config['file_type'], config['file_content']
+    if file_type == 'xlsx':
+        if file_content == 'last_row':
+            df = pd.read_excel(file_path)
+            last_row = df.iloc[-1]
+            last_row_as_list = last_row.astype(str).tolist()
+            return last_row_as_list
+    else:
+        raise NotImplementedError(f"File type {file_type} not supported")
 
 
 def get_cloud_file(env, config: Dict[str, Any]) -> Union[str, List[str]]:

desktop_env/evaluators/getters/impress.py

@@ -40,20 +40,23 @@ def get_audio_in_slide(env, config: Dict[str, str]):
                             audio_file_path = audio_file_path.replace('\\', '/')
 
                             # Create a temporary directory to extract the audio file
-                            with tempfile.TemporaryDirectory() as tmpdirname:
+                            tmpdirname = os.path.dirname(ppt_file_localhost_path)
-                                # Extract the audio file
+                            myzip.extract(audio_file_path, tmpdirname)
-                                myzip.extract(audio_file_path, tmpdirname)
+                            audio_file_path = os.path.join(tmpdirname, audio_file_path)
-                                # Get the full path of the extracted audio file
+                            return audio_file_path
-                                extracted_audio_path = os.path.join(tmpdirname, audio_file_path)
+                            # with tempfile.TemporaryDirectory() as tmpdirname:
-                                # Return the extracted audio file path
+                            #     # Extract the audio file
-                                audio_file_path = extracted_audio_path
+                            #     myzip.extract(audio_file_path, tmpdirname)
-
+                            #     # Get the full path of the extracted audio file
+                            #     extracted_audio_path = os.path.join(tmpdirname, audio_file_path)
+                            #     # Return the extracted audio file path
+                            #     audio_file_path = extracted_audio_path
                         else:
                             # the audio file is external to the .pptx file
                             # Return the audio file path
                             assert target.startswith("file://"), target
                             audio_file_path = target[7:]
-
+                        break
     if audio_file_path is None:
         return None
 

desktop_env/evaluators/metrics/__init__.py

@@ -60,7 +60,8 @@ from .general import (
     fuzzy_match,
     check_include_exclude,
     check_direct_json_object,
-    diff_text_file
+    diff_text_file,
+    literal_match
 )
 from .gimp import (
     check_brightness_decrease_and_structure_sim,

desktop_env/evaluators/metrics/docs.py

@@ -58,6 +58,8 @@ def contains_page_break(docx_file):
 
 def compare_docx_files(file1, file2, **options):
     ignore_blanks = options.get('ignore_blanks', True)
+    ignore_case = options.get('ignore_case', False)
+    ignore_order = options.get('ignore_order', False)
     content_only = options.get('content_only', False)
 
     def get_paragraph_texts_odt(document):
@@ -82,11 +84,17 @@ def compare_docx_files(file1, file2, **options):
         doc2 = Document(file2)
         doc1_paragraphs = [p.text for p in doc1.paragraphs]
         doc2_paragraphs = [p.text for p in doc2.paragraphs]
+        if ignore_order:
+            doc1_paragraphs = sorted(doc1_paragraphs)
+            doc2_paragraphs = sorted(doc2_paragraphs)
     elif file1.endswith('.odt') and file2.endswith('.odt'):
         doc1 = load(file1)
         doc2 = load(file2)
         doc1_paragraphs = get_paragraph_texts_odt(doc1)
         doc2_paragraphs = get_paragraph_texts_odt(doc2)
+        if ignore_order:
+            doc1_paragraphs = sorted(doc1_paragraphs)
+            doc2_paragraphs = sorted(doc2_paragraphs)
     else:
         # Unsupported file types or mismatch
         print("Unsupported file types or mismatch between file types.")
@@ -96,6 +104,8 @@ def compare_docx_files(file1, file2, **options):
         # Compare the content of the documents
         text1 = re.sub(r'\s+', ' ', '\n'.join(doc1_paragraphs)).strip()
         text2 = re.sub(r'\s+', ' ', '\n'.join(doc2_paragraphs)).strip()
+        if ignore_case:
+            text1, text2 = text1.lower(), text2.lower()
         similarity = fuzz.ratio(text1, text2) / 100.0
         return similarity
 
@@ -103,6 +113,8 @@ def compare_docx_files(file1, file2, **options):
     if ignore_blanks:
         text1 = re.sub(r'\s+', ' ', '\n'.join(doc1_paragraphs)).strip()
         text2 = re.sub(r'\s+', ' ', '\n'.join(doc2_paragraphs)).strip()
+        if ignore_case:
+            text1, text2 = text1.lower(), text2.lower()
         if text1 != text2:
             return 0
     else:
@@ -111,6 +123,8 @@ def compare_docx_files(file1, file2, **options):
 
         # Compare each paragraph
         for p1, p2 in zip(doc1_paragraphs, doc2_paragraphs):
+            if ignore_case:
+                p1, p2 = p1.lower(), p2.lower()
             if p1 != p2:
                 return 0
 

desktop_env/evaluators/metrics/general.py

@@ -39,6 +39,24 @@ def exact_match(result, rules) -> float:
     else:
         return 0.
 
+
+def literal_match(result: Any, expected: Any, **options) -> float:
+    literal_type = options.get('type', 'str')
+    if literal_type == 'str':
+        ignore_case = options.get('ignore_case', False)
+        score = str(result) == str(expected) if not ignore_case else str(result).lower() == str(expected).lower()
+        return float(score)
+    elif literal_type == 'list':
+        if type(result) not in [list, tuple] or type(expected) not in [list, tuple] or len(result) != len(expected):
+            return .0
+        ignore_case = options.get('ignore_case', False)
+        result = [str(s) for s in result] if not ignore_case else [str(s).lower() for s in result]
+        expected = [str(s) for s in expected] if not ignore_case else [str(s).lower() for s in expected]
+        return float(result == expected)
+    else:
+        raise NotImplementedError(f"Type {type} not supported")
+
+
 def is_in_list(result, rules) -> float:
     expect = rules["expected"]
     if expect in result:

evaluation_examples/examples/multi_apps/22a4636f-8179-4357-8e87-d1743ece1f81.json

@@ -0,0 +1,108 @@
+{
+    "id": "22a4636f-8179-4357-8e87-d1743ece1f81",
+    "snapshot": "chrome",
+    "instruction": "Please help me convert the file \"Meeting-Agenda.docx\" to a pdf file and upload to the folder 'meetings' in my google drive.",
+    "source": "https://marketplace.uipath.com/listings/merge-pdfs-from-gmail-email-attachments-and-upload-to-gogle-drive",
+    "config": [
+        {
+            "type": "googledrive",
+            "parameters": {
+                "settings_file": "evaluation_examples/settings/googledrive/settings.yml",
+                "operation": [
+                    "delete",
+                    "mkdirs"
+                ],
+                "args": [
+                    {
+                        "query": "title = 'Meeting-Agenda.docx' or title = 'Meeting-Agenda.pdf'",
+                        "trash": false
+                    },
+                    {
+                        "path": [
+                            "meetings"
+                        ]
+                    }
+                ]
+            }
+        },
+        {
+            "type": "launch",
+            "parameters": {
+                "command": [
+                    "google-chrome",
+                    "--remote-debugging-port=1337"
+                ]
+            }
+        },
+        {
+            "type": "launch",
+            "parameters": {
+                "command": [
+                    "socat",
+                    "tcp-listen:9222,fork",
+                    "tcp:localhost:1337"
+                ]
+            }
+        },
+        {
+            "type": "chrome_open_tabs",
+            "parameters": {
+                "urls_to_open": [
+                    "https://news.google.com",
+                    "https://x.com",
+                    "https://www.deepl.com"
+                ]
+            }
+        },
+        {
+            "type": "login",
+            "parameters": {
+                "settings_file": "evaluation_examples/settings/google/settings.json",
+                "platform": "googledrive"
+            }
+        },
+        {
+            "type": "download",
+            "parameters": {
+                "files": [
+                    {
+                        "url": "https://drive.usercontent.google.com/download?id=17fkMG4guromDzAHNCzzJieZHa2nJDBpc&export=download&authuser=0&confirm=t",
+                        "path": "/home/user/Desktop/Meeting-Agenda.docx"
+                    }
+                ]
+            }
+        },
+        {
+            "type": "launch",
+            "parameters": {
+                "command": [
+                    "libreoffice",
+                    "--writer",
+                    "/home/user/Desktop/Meeting-Agenda.docx"
+                ]
+            }
+        }
+    ],
+    "trajectory": "trajectories/",
+    "related_apps": [
+        "libreoffice_writer",
+        "chrome"
+    ],
+    "evaluator": {
+        "func": "compare_pdfs",
+        "result": {
+            "type": "googledrive_file",
+            "settings_file": "evaluation_examples/settings/googledrive/settings.yml",
+            "path": [
+                "meetings",
+                "Meeting-Agenda.pdf"
+            ],
+            "dest": "Meeting-Agenda.pdf"
+        },
+        "expected": {
+            "type": "cloud_file",
+            "path": "https://drive.usercontent.google.com/download?id=1wHWQ6mTJcNLtrL83GrUPzxx2HeOC4L8T&export=download&authuser=0&confirm=t",
+            "dest": "gold-Meeting-Agenda.pdf"
+        }
+    }
+}

evaluation_examples/examples/multi_apps/236833a3-5704-47fc-888c-4f298f09f799.json

@@ -0,0 +1,67 @@
+{
+    "id": "236833a3-5704-47fc-888c-4f298f09f799",
+    "snapshot": "chrome",
+    "instruction": "Find daily papers on Huggingface and take down all the titles, authors and the abstracts of papers on 1st March, 2024 in the doc file 'paper_reading_2024_03_01.docx' on desktop. Each paragraph (split by empty lines) conforms to the following format:\nTitle: xxx\nAuthors: xxx, xxx, xxx\nAbstract: xxxxxxxx.\nArxiv PDF: https://xxxx.pdf",
+    "source": "authors",
+    "config": [
+        {
+            "type": "launch",
+            "parameters": {
+                "command": [
+                    "google-chrome",
+                    "--remote-debugging-port=1337"
+                ]
+            }
+        },
+        {
+            "type": "launch",
+            "parameters": {
+                "command": [
+                    "socat",
+                    "tcp-listen:9222,fork",
+                    "tcp:localhost:1337"
+                ]
+            }
+        },
+        {
+            "type": "chrome_open_tabs",
+            "parameters": {
+                "urls_to_open": [
+                    "https://huggingface.co/"
+                ]
+            }
+        },
+        {
+            "type": "launch",
+            "parameters": {
+                "command": [
+                    "libreoffice",
+                    "--writer"
+                ]
+            }
+        }
+    ],
+    "trajectory": "trajectories/",
+    "related_apps": [
+        "chrome",
+        "libreoffice_writer"
+    ],
+    "evaluator": {
+        "func": "compare_docx_files",
+        "result": {
+            "type": "vm_file",
+            "path": "/home/user/Desktop/paper_reading_2024_03_01.docx",
+            "dest": "paper_reading_2024_03_01.docx"
+        },
+        "expected": {
+            "type": "cloud_file",
+            "path": "https://drive.usercontent.google.com/download?id=1TUTihXD93bIlekuYy_44fmXAhI1KVol4&export=download&authuser=0&confirm=t",
+            "dest": "gold_paper_reading_2024_03_01.docx"
+        },
+        "options": {
+            "ignore_blanks": true,
+            "ignore_case": true,
+            "ignore_order": true
+        }
+    }
+}

evaluation_examples/examples/multi_apps/5990457f-2adb-467b-a4af-5c857c92d762.json

@@ -0,0 +1,116 @@
+{
+    "id": "5990457f-2adb-467b-a4af-5c857c92d762",
+    "snapshot": "chrome",
+    "instruction": "Append one entry of AI researcher Yann LeCun from Google Scholar into an existing table researchers.xlsx.",
+    "source": "authors",
+    "config": [
+        {
+            "type": "launch",
+            "parameters": {
+                "command": [
+                    "google-chrome",
+                    "--remote-debugging-port=1337"
+                ]
+            }
+        },
+        {
+            "type": "launch",
+            "parameters": {
+                "command": [
+                    "socat",
+                    "tcp-listen:9222,fork",
+                    "tcp:localhost:1337"
+                ]
+            }
+        },
+        {
+            "type": "chrome_open_tabs",
+            "parameters": {
+                "urls_to_open": [
+                    "https://arxiv.org/abs/2005.14165",
+                    "https://wallhaven.cc/"
+                ]
+            }
+        },
+        {
+            "type": "download",
+            "parameters": {
+                "files": [
+                    {
+                        "url": "https://drive.usercontent.google.com/download?id=1EbQ75SHLthiZCBqCJtO0fLXJZcKrNler&export=download&authuser=0&confirm=t",
+                        "path": "/home/user/Desktop/researchers.xlsx"
+                    }
+                ]
+            }
+        },
+        {
+            "type": "launch",
+            "parameters": {
+                "command": [
+                    "nautilus",
+                    "/home/user/Desktop"
+                ]
+            }
+        }
+    ],
+    "trajectory": "trajectories/",
+    "related_apps": [
+        "chrome",
+        "libreoffice_calc"
+    ],
+    "evaluator": {
+        "func": "literal_match",
+        "result": {
+            "type": "content_from_vm_file",
+            "path": "/home/user/Desktop/researchers.xlsx",
+            "file_type": "xlsx",
+            "file_content": "last_row"
+        },
+        "expected": {
+            "type": "info_from_website",
+            "url": "https://scholar.google.com/citations?user=WLN3QrAAAAAJ&hl=en",
+            "infos": [
+                {
+                    "action": "inner_text",
+                    "selector": "#gsc_prf_in"
+                },
+                {
+                    "action": "inner_text",
+                    "selector": "#gsc_rsb_st > tbody > tr:nth-child(1) > td:nth-child(2)"
+                },
+                {
+                    "action": "inner_text",
+                    "selector": "#gsc_rsb_st > tbody > tr:nth-child(2) > td:nth-child(2)"
+                },
+                {
+                    "action": "inner_text",
+                    "selector": "#gsc_rsb_st > tbody > tr:nth-child(3) > td:nth-child(2)"
+                },
+                {
+                    "action": "inner_text",
+                    "selector": "#gsc_a_b > tr:nth-child(1) > td.gsc_a_t > a"
+                },
+                {
+                    "action": "click_and_attribute",
+                    "selector": [
+                        "#gsc_a_b > tr:nth-child(1) > td.gsc_a_t > a",
+                        "#gsc_oci_title_gg > div:nth-child(1) > a"
+                    ],
+                    "attribute": "href"
+                }
+            ],
+            "backups": [
+                "Yann LeCun",
+                "345074",
+                "147",
+                "372",
+                "Deep learning",
+                "https://creativecoding.soe.ucsc.edu/courses/cs523/slides/week3/DeepLearning_LeCun.pdf"
+            ]
+        },
+        "options": {
+            "type": "list",
+            "ignore_case": true
+        }
+    }
+}

evaluation_examples/examples/multi_apps/67890eb6-6ce5-4c00-9e3d-fb4972699b06.json

@@ -0,0 +1,110 @@
+{
+    "id": "67890eb6-6ce5-4c00-9e3d-fb4972699b06",
+    "snapshot": "libreoffice_calc",
+    "instruction": "I am an NLP researcher. Check out the best long paper awards of ACL from 2019 to 2022 and record the 'title', 'year', 'author list' and 'PDF link' into table best_awards_acl.xslx on the desktop. Separate authors only by commas, use offical aclanthology urls not arxiv, and sort rows by years in descending orders.",
+    "source": "authors",
+    "config": [
+        {
+            "type": "download",
+            "parameters": {
+                "files": [
+                    {
+                        "url": "https://drive.google.com/uc?export=download&id=1v5DgViUgAL771kBWy-qBddVGyjGmgFhK",
+                        "path": "/home/user/Desktop/best_awards_acl.xlsx"
+                    }
+                ]
+            }
+        },
+        {
+            "type": "open",
+            "parameters": {
+                "path": "/home/user/Desktop/best_awards_acl.xlsx"
+            }
+        },
+        {
+            "type": "launch",
+            "parameters": {
+                "command": [
+                    "google-chrome",
+                    "--remote-debugging-port=1337"
+                ]
+            }
+        },
+        {
+            "type": "launch",
+            "parameters": {
+                "command": [
+                    "socat",
+                    "tcp-listen:9222,fork",
+                    "tcp:localhost:1337"
+                ]
+            }
+        },
+        {
+            "type": "chrome_open_tabs",
+            "parameters": {
+                "urls_to_open": [
+                    "https://aclanthology.org/"
+                ]
+            }
+        }
+    ],
+    "trajectory": "trajectories/",
+    "related_apps": [
+        "libreoffice_calc",
+        "chrome"
+    ],
+    "evaluator": {
+        "postconfig": [
+            {
+                "type": "activate_window",
+                "parameters": {
+                    "window_name": "best_awards_acl.xlsx - LibreOffice Calc",
+                    "strict": true
+                }
+            },
+            {
+                "type": "sleep",
+                "parameters": {
+                    "seconds": 0.5
+                }
+            },
+            {
+                "type": "execute",
+                "parameters": {
+                    "command": [
+                        "python",
+                        "-c",
+                        "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);"
+                    ]
+                }
+            },
+            {
+                "type": "sleep",
+                "parameters": {
+                    "seconds": 0.5
+                }
+            }
+        ],
+        "func": "compare_table",
+        "result": {
+            "type": "vm_file",
+            "path": "/home/user/Desktop/best_awards_acl.xlsx",
+            "dest": "best_awards_acl.xlsx"
+        },
+        "expected": {
+            "type": "cloud_file",
+            "path": "https://drive.usercontent.google.com/download?id=17ORdOPl3sZGk3s4Wm0vESgImKZjdZBqX&export=download&authuser=0&confirm=t",
+            "dest": "gold_best_awards_acl.xlsx"
+        },
+        "options": {
+            "rules": [
+              {
+                "type": "sheet_data",
+                "sheet_idx0": 0,
+                "sheet_idx1": "EI0"
+              }
+            ]
+          }
+    }
+}

evaluation_examples/examples/multi_apps/778efd0a-153f-4842-9214-f05fc176b877.json

@@ -0,0 +1,89 @@
+{
+    "id": "778efd0a-153f-4842-9214-f05fc176b877",
+    "snapshot": "vlc",
+    "instruction": "I'm using libreoffice impress to write slideshows. I found that the video being played by VLC media player had a good soundtrack. Please extract the audio to planet.wav and use it as background music for this slideshow.",
+    "source": "https://researchguides.case.edu/c.php?g=1286426",
+    "config": [
+        {
+            "type": "download",
+            "parameters": {
+            "files": [
+                    {
+                        "url": "https://drive.usercontent.google.com/download?id=1I-ArULOnZzlGkS9UyIuE8Dyuyus27iZt&export=download&authuser=0&confirm=t",
+                        "path": "/home/user/Desktop/Minimalist_Business_Slides.pptx"
+                    },
+                    {
+                        "url": "https://drive.usercontent.google.com/download?id=1HiW-PokIfWRyRvLwlkiVKKNCB2h2bcx7&export=download&authuser=0&confirm=t",
+                        "path": "/home/user/Desktop/planet.mp4"
+                    }
+                ]
+            }
+        },
+        {
+            "type": "open",
+            "parameters": {
+                "path": "/home/user/Desktop/Minimalist_Business_Slides.pptx"
+            }
+        },
+        {
+            "type": "launch",
+            "parameters": {
+                "command": [
+                    "vlc",
+                    "--repeat",
+                    "/home/user/Desktop/planet.mp4"
+                ]
+            }
+        }
+    ],
+    "trajectory": "trajectories/",
+    "related_apps": [
+        "libreoffice_impress",
+        "vlc"
+    ],
+    "evaluator": {
+        "postconfig": [
+            {
+                "type": "activate_window",
+                "parameters": {
+                    "window_name": "Minimalist_Business_Slides.pptx - LibreOffice Impress",
+                    "strict": true
+                }
+            },
+            {
+                "type": "sleep",
+                "parameters": {
+                    "seconds": 0.5
+                }
+            },
+            {
+                "type": "execute",
+                "parameters": {
+                    "command": [
+                        "python",
+                        "-c",
+                        "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);"
+                    ]
+                }
+            },
+            {
+                "type": "sleep",
+                "parameters": {
+                    "seconds": 0.5
+                }
+            }
+        ],
+        "func": "compare_audios",
+        "result": {
+            "type": "audio_in_slide",
+            "ppt_file_path": "/home/user/Desktop/Minimalist_Business_Slides.pptx",
+            "slide_index": 0,
+            "dest": "planet.wav"
+        },
+        "expected": {
+            "type": "cloud_file",
+            "path": "https://drive.usercontent.google.com/download?id=1F_iBPLgVg-2g1LQ5rxKyCrFa9UitZ9yn&export=download&authuser=0&confirm=t",
+            "dest": "gold_planet.wav"
+        }
+    }
+}

evaluation_examples/settings/googledrive/credentials.json

@@ -1 +0,0 @@
-{"access_token": "ya29.a0AfB_byCmR_-BUvipM02LYvRdhSIsjxhdK4a1tpDABrZOjTOWPkPhs4gy070hbrq0tZBeld813_HqZ4q5GTeLzddfVVTWfiHdDSLlT4Bf5_f4ZURSB53XJAdAyYeI7LcT4RwF1WoAQhtldbIBJ4rizVk7L6O3486u6e9OaCgYKARQSARISFQHGX2Mi1U-dePZ0efbg8nydUEsP9Q0171", "client_id": "786888752612-rgng5v9hcq4as7pn0b40gt9r5lekmht9.apps.googleusercontent.com", "client_secret": "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", "refresh_token": "1//0edHIvL2N4T8ICgYIARAAGA4SNwF-L9IrKhoX-pALW0nJ18niS1Gy3Lg9wF_G1joCoGHLM8v4-WJsibIB04KjWYCp_40Cs1WS7es", "token_expiry": "2024-03-02T13:59:28Z", "token_uri": "https://oauth2.googleapis.com/token", "user_agent": null, "revoke_uri": "https://oauth2.googleapis.com/revoke", "id_token": null, "id_token_jwt": null, "token_response": {"access_token": "ya29.a0AfB_byCmR_-BUvipM02LYvRdhSIsjxhdK4a1tpDABrZOjTOWPkPhs4gy070hbrq0tZBeld813_HqZ4q5GTeLzddfVVTWfiHdDSLlT4Bf5_f4ZURSB53XJAdAyYeI7LcT4RwF1WoAQhtldbIBJ4rizVk7L6O3486u6e9OaCgYKARQSARISFQHGX2Mi1U-dePZ0efbg8nydUEsP9Q0171", "expires_in": 3599, "refresh_token": "1//0edHIvL2N4T8ICgYIARAAGA4SNwF-L9IrKhoX-pALW0nJ18niS1Gy3Lg9wF_G1joCoGHLM8v4-WJsibIB04KjWYCp_40Cs1WS7es", "scope": "https://www.googleapis.com/auth/drive", "token_type": "Bearer"}, "scopes": ["https://www.googleapis.com/auth/drive"], "token_info_uri": "https://oauth2.googleapis.com/tokeninfo", "invalid": false, "_class": "OAuth2Credentials", "_module": "oauth2client.client"}