From 64594701aea742d9b7055e04528ba69eab250567 Mon Sep 17 00:00:00 2001 From: Timothyxxx <384084775@qq.com> Date: Fri, 26 Jan 2024 00:15:04 +0800 Subject: [PATCH 1/6] Load OS Ubuntu examples batch 2 --- desktop_env/evaluators/getters/__init__.py | 2 +- desktop_env/evaluators/getters/info.py | 4 + desktop_env/evaluators/metrics/__init__.py | 2 +- desktop_env/evaluators/metrics/os.py | 77 ++++++------ .../23393935-50c7-4a86-aeea-2b78fd089c5c.json | 112 +++++++++++++++--- .../3ce045a0-877b-42aa-8d2c-b4a863336ab8.json | 9 +- .../5c433d22-ed9a-4e31-91f5-54cf3e8acd63.json | 2 +- .../5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57.json | 40 +++++-- .../6ebbfb01-ea72-4226-a2a6-dc428e111ed2.json | 57 ++++++++- .../a4d98375-215b-4a4d-aee9-3d4370fccc41.json | 35 +++--- .../b6781586-6346-41cd-935a-a6b1487918fc.json | 10 +- .../ddc75b62-7311-4af8-bfb3-859558542b36.json | 45 +++++-- .../e0df059f-28a6-4169-924f-b9623e7184cc.json | 44 ++++--- .../fe41f596-a71b-4c2f-9b2f-9dcd40b568c3.json | 9 +- 14 files changed, 324 insertions(+), 124 deletions(-) diff --git a/desktop_env/evaluators/getters/__init__.py b/desktop_env/evaluators/getters/__init__.py index 9e2e636..0ce7ea7 100644 --- a/desktop_env/evaluators/getters/__init__.py +++ b/desktop_env/evaluators/getters/__init__.py @@ -4,7 +4,7 @@ from .chrome import get_default_search_engine, get_cookie_data, get_bookmarks, g from .file import get_cloud_file, get_vm_file, get_cache_file from .general import get_vm_command_line from .impress import get_audio_in_slide -from .info import get_vm_screen_size, get_vm_window_size, get_vm_wallpaper +from .info import get_vm_screen_size, get_vm_window_size, get_vm_wallpaper, get_list_directory from .misc import get_rule, get_accessibility_tree from .replay import get_replay from .vlc import get_vlc_playing_info, get_vlc_config diff --git a/desktop_env/evaluators/getters/info.py b/desktop_env/evaluators/getters/info.py index 3d41ee5..0c88fd2 100644 --- a/desktop_env/evaluators/getters/info.py +++ b/desktop_env/evaluators/getters/info.py @@ -18,3 +18,7 @@ def get_vm_wallpaper(env, config: dict) -> Union[str, bytes]: f.write(content) return _path + + +def get_list_directory(env, config: dict) -> dict: + return env.controller.get_vm_directory_tree(config["path"]) diff --git a/desktop_env/evaluators/metrics/__init__.py b/desktop_env/evaluators/metrics/__init__.py index b2214fd..d863dc7 100644 --- a/desktop_env/evaluators/metrics/__init__.py +++ b/desktop_env/evaluators/metrics/__init__.py @@ -25,4 +25,4 @@ from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, co check_qt_slider_colours, check_global_key_play_pause from .vscode import compare_text_file, compare_config, compare_answer, is_extension_installed, check_json_settings, \ check_json_keybindings -from .os import check_gnome_favorite_apps +from .os import check_gnome_favorite_apps, is_utc_0, check_text_enlarged, check_moved_jpgs diff --git a/desktop_env/evaluators/metrics/os.py b/desktop_env/evaluators/metrics/os.py index 3386466..55688fb 100644 --- a/desktop_env/evaluators/metrics/os.py +++ b/desktop_env/evaluators/metrics/os.py @@ -17,54 +17,43 @@ def check_gnome_favorite_apps(apps_str: str, rule): return 0 -# TODO: log in to the system before running this -def is_logout_successful(): - try: - subprocess.run(["whoami"]) - return 0 # Task not successful - except subprocess.CalledProcessError: - return 1 # Task successful +def is_utc_0(timedatectl_output): + """ + Format as: + Local time: Thu 2024-01-25 12:56:06 WET + Universal time: Thu 2024-01-25 12:56:06 UTC + RTC time: Thu 2024-01-25 12:56:05 + Time zone: Atlantic/Faroe (WET, +0000) +System clock synchronized: yes + NTP service: inactive + RTC in local TZ: no + """ + + utc_line = timedatectl_output.split("\n")[3] + + if utc_line.endswith("+0000)"): + return 1 + else: + return 0 -def is_battery_percentage_displayed(): - # GNOME schema and key for the setting - schema = "org.gnome.desktop.interface" - key = "show-battery-percentage" - - try: - # use gsettings to get the current setting - result = subprocess.run(['gsettings', 'get', schema, key], capture_output=True, text=True) - # examine the output is 'true' - if result.stdout.strip() == 'true': - return 1. - else: - return 0. - except Exception as e: - print(f"An error occurred: {e}") - return 0. +def check_text_enlarged(scaling_factor_str): + scaling_factor = float(scaling_factor_str) + if scaling_factor > 1.0: + return 1 + else: + return 0 -def check_auto_lock_settings(): - # The schema and keys for the GNOME desktop lock settings - schema = "org.gnome.desktop.screensaver" - lock_enabled_key = "lock-enabled" - lock_delay_key = "lock-delay" +def check_moved_jpgs(directory_list, rule): - try: - # Check if the screen lock is enabled - lock_enabled = subprocess.run(['gsettings', 'get', schema, lock_enabled_key], capture_output=True, text=True) - is_lock_enabled = lock_enabled.stdout.strip() == 'true' + expected_jpgs = rule["expected"] + moved_jpgs = [node['name'] for node in directory_list['children']] - # Check the delay before the screen is locked - lock_delay = subprocess.run(['gsettings', 'get', schema, lock_delay_key], capture_output=True, text=True) - # Extract the numerical value from the output - delay_seconds = lock_delay + if len(moved_jpgs) != len(expected_jpgs): + return 0 - # Print the results - if is_lock_enabled: - return 1. - else: - return 0. - - except Exception as e: - return 0. + if set(moved_jpgs) == set(expected_jpgs): + return 1 + else: + return 0 diff --git a/evaluation_examples/examples/os/23393935-50c7-4a86-aeea-2b78fd089c5c.json b/evaluation_examples/examples/os/23393935-50c7-4a86-aeea-2b78fd089c5c.json index ee41922..654a749 100644 --- a/evaluation_examples/examples/os/23393935-50c7-4a86-aeea-2b78fd089c5c.json +++ b/evaluation_examples/examples/os/23393935-50c7-4a86-aeea-2b78fd089c5c.json @@ -1,17 +1,101 @@ { - "id": "23393935-50c7-4a86-aeea-2b78fd089c5c", - "snapshot": "os", - "instruction": "Recursively go through the folders of the 'photos' directory and copy any .jpg files found into another directory named 'cpjpg'.", - "source": "https://superuser.com/questions/91307/copying-only-jpg-from-a-directory-structure-to-another-location-linux", - "trajectory": "trajectories/", - "related_apps": [ - "os" - ], - "evaluator": { - "func": "", - "result": { - }, - "expected": { + "id": "23393935-50c7-4a86-aeea-2b78fd089c5c", + "snapshot": "os", + "instruction": "Recursively go through the folders of the 'photos' directory and copy any .jpg files found into another directory named 'cpjpg'.", + "source": "https://superuser.com/questions/91307/copying-only-jpg-from-a-directory-structure-to-another-location-linux", + "trajectory": "trajectories/", + "config": [ + { + "type": "execute", + "parameters": { + "command": "mkdir -p ~/Desktop/photos && mkdir -p ~/Desktop/cpjpg", + "shell": true + } + }, + { + "type": "execute", + "parameters": { + "command": "mkdir -p ~/Desktop/photos/vacation && mkdir -p ~/Desktop/photos/family && mkdir -p ~/Desktop/photos/events", + "shell": true + } + }, + { + "type": "execute", + "parameters": { + "command": "mkdir -p ~/Desktop/photos/vacation/thailand && mkdir -p ~/Desktop/photos/vacation/hk", + "shell": true + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1H7R6flpv6vgyBOHurzR0CyvPRIobuIpI&export=download&authuser=0&confirm=t&uuid=1b41e87d-7a84-4aa6-af40-cf940e80d956&at=APZUnTWus2n1pP6l2wmIyXkCUHGs:1706196492307", + "path": "/home/user/Desktop/photos/vacation/thailand/monk252520thailand252520wat252520arun252520scaled25255B225255D.jpg" + } + ] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1Mh3kIViHaayxNC88TpgpjbmIEl17LA3d&export=download&authuser=0&confirm=t&uuid=a8355b84-aa2e-4502-8d0f-3723ffe7d442&at=APZUnTW4nzDFgDgNeERMRKEQ4bTU:1706196493503", + "path": "/home/user/Desktop/photos/vacation/hk/hong-kong-china.jpg" + } + ] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1EhmbZ3QP_mdvvppC5zKA3o--Ce2F-ocO&export=download&authuser=0&confirm=t&uuid=70e05133-dcac-4168-b555-82da15a10d58&at=APZUnTXJ371jG7qiPLHBLt2zy7ij:1706196494612", + "path": "/home/user/Desktop/photos/vacation/hk/hk_group_photo.jpg" + } + ] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1p_r_BMCLFG872uH9Qa-sLnqXppHvDx95&export=download&authuser=0&confirm=t&uuid=af59d107-8e46-4d2c-b72a-e4b42655959a&at=APZUnTU2hBidSnefJuUQTyJCEVFx:1706196491030", + "path": "/home/user/Desktop/photos/family/us_3.png" + } + ] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1YxryNHfLPcFODnUh31CYGHGj3ZFRhvbA&export=download&authuser=0&confirm=t&uuid=0925ae33-ab33-42b9-901c-d90f11e0afcd&at=APZUnTVey7VeC9zYOmTwl0x02Iwf:1706196495770", + "path": "/home/user/Desktop/photos/events/emnlp2023.jpg" + } + ] } } - } \ No newline at end of file + ], + "related_apps": [ + "os" + ], + "evaluator": { + "func": "check_moved_jpgs", + "result": { + "type": "list_directory", + "path": "/home/user/Desktop/cpjpg" + }, + "expected": { + "type": "rule", + "rules": { + "expected": ["emnlp2023.jpg", "hk_group_photo.jpg", "hong-kong-china.jpg", "monk252520thailand252520wat252520arun252520scaled25255B225255D.jpg"] + } + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/os/3ce045a0-877b-42aa-8d2c-b4a863336ab8.json b/evaluation_examples/examples/os/3ce045a0-877b-42aa-8d2c-b4a863336ab8.json index 9c67040..99df0f7 100644 --- a/evaluation_examples/examples/os/3ce045a0-877b-42aa-8d2c-b4a863336ab8.json +++ b/evaluation_examples/examples/os/3ce045a0-877b-42aa-8d2c-b4a863336ab8.json @@ -1,7 +1,7 @@ { "id": "3ce045a0-877b-42aa-8d2c-b4a863336ab8", "snapshot": "os", - "instruction": "I want to make the text on the screen larger", + "instruction": "My glasses are broken, and I'm having trouble seeing small things clearly. Could you help me enlarge the text on my screen so it's easier to read?", "source": "https://help.ubuntu.com/lts/ubuntu-help/a11y-font-size.html.en", "trajectory": "trajectories/", "config": [], @@ -9,10 +9,11 @@ "os" ], "evaluator": { - "func": "", + "func": "check_text_enlarged", "result": { - }, - "expected": { + "type": "vm_command_line", + "command": "gsettings get org.gnome.desktop.interface text-scaling-factor", + "shell": true } } } \ No newline at end of file diff --git a/evaluation_examples/examples/os/5c433d22-ed9a-4e31-91f5-54cf3e8acd63.json b/evaluation_examples/examples/os/5c433d22-ed9a-4e31-91f5-54cf3e8acd63.json index 5482ee2..ebbb114 100644 --- a/evaluation_examples/examples/os/5c433d22-ed9a-4e31-91f5-54cf3e8acd63.json +++ b/evaluation_examples/examples/os/5c433d22-ed9a-4e31-91f5-54cf3e8acd63.json @@ -1,7 +1,7 @@ { "id": "5c433d22-ed9a-4e31-91f5-54cf3e8acd63", "snapshot": "os", - "instruction": "I want to change my system language to Chinese(simplified). Can you help me?", + "instruction": "I want to change my system language to Chinese (simplified). Can you help me?", "source": "https://help.ubuntu.com/lts/ubuntu-help/session-language.html.zh-CN", "trajectory": "trajectories/", "config": [], diff --git a/evaluation_examples/examples/os/5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57.json b/evaluation_examples/examples/os/5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57.json index b6c00af..7ce6f21 100644 --- a/evaluation_examples/examples/os/5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57.json +++ b/evaluation_examples/examples/os/5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57.json @@ -1,18 +1,44 @@ { "id": "5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57", "snapshot": "os", - "instruction": "I am currently using an Ubuntu system, and I have wrongly deleted a file named \"test\". Could you help me recover it from the Trash?", + "instruction": "I am currently using an Ubuntu system, and I have wrongly deleted a poster of party night. Could you help me recover it from the Trash?", "source": "https://help.ubuntu.com/lts/ubuntu-help/files-recover.html.en", "trajectory": "trajectories/", - "config": [], + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1XaTnC_lLbR_tGTz8tcN2Tp6cNrMlNW3R&export=download&authuser=0&confirm=t&uuid=89e69a23-43cf-4316-833a-fb9d3e281460&at=APZUnTWn5zZTH4GlClO6lV1i4WwP:1706184669922", + "path": "poster_party_night.webp" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": "mv ~/poster_party_night.webp ~/.local/share/Trash/files/", + "shell": true + } + } + ], "related_apps": [ "os" ], "evaluator": { - "func": "", - "result": { - }, - "expected": { - } + "func": "exact_match", + "result": { + "type": "vm_command_line", + "command": "[ -f /home/user/Desktop/poster_party_night.webp ] && echo 'File exists.' || echo 'File does not exist.'", + "shell": true + }, + "expected": { + "type": "rule", + "rules":{ + "expected": "File exists.\n" + } + } } } \ No newline at end of file diff --git a/evaluation_examples/examples/os/6ebbfb01-ea72-4226-a2a6-dc428e111ed2.json b/evaluation_examples/examples/os/6ebbfb01-ea72-4226-a2a6-dc428e111ed2.json index 1d253f4..673163b 100644 --- a/evaluation_examples/examples/os/6ebbfb01-ea72-4226-a2a6-dc428e111ed2.json +++ b/evaluation_examples/examples/os/6ebbfb01-ea72-4226-a2a6-dc428e111ed2.json @@ -4,15 +4,60 @@ "instruction": "Could you please help me to set Bash as my default shell on the current Ubuntu system?", "source": "https://superuser.com/questions/46748/how-do-i-make-bash-my-default-shell-on-ubuntu", "trajectory": "trajectories/", - "config": [], + "config": [ + { + "type": "execute", + "parameters": { + "command": "echo password | sudo -S apt-get install zsh -y", + "shell": true + } + }, + { + "type": "execute", + "parameters": { + "command": "echo password | sudo -S chsh -s $(which zsh)", + "shell": true + } + }, + { + "type": "execute", + "parameters": { + "command": "echo password | sudo -S loginctl terminate-user user", + "shell": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 10 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; time.sleep(2); pyautogui.press(\"enter\"); pyautogui.type(\"password\"); time.sleep(2); pyautogui.press(\"enter\");" + ] + } + } + ], "related_apps": [ "os" ], "evaluator": { - "func": "", - "result": { - }, - "expected": { - } + "func": "exact_match", + "result": { + "type": "vm_command_line", + "command": "echo $SHELL", + "shell": true + }, + "expected": { + "type": "rule", + "rules":{ + "expected": "/bin/bash\n" + } + } } } \ No newline at end of file diff --git a/evaluation_examples/examples/os/a4d98375-215b-4a4d-aee9-3d4370fccc41.json b/evaluation_examples/examples/os/a4d98375-215b-4a4d-aee9-3d4370fccc41.json index 9b6cd0f..3675d05 100644 --- a/evaluation_examples/examples/os/a4d98375-215b-4a4d-aee9-3d4370fccc41.json +++ b/evaluation_examples/examples/os/a4d98375-215b-4a4d-aee9-3d4370fccc41.json @@ -1,17 +1,24 @@ { - "id": "a4d98375-215b-4a4d-aee9-3d4370fccc41", - "snapshot": "os", - "instruction": "I want to have my computer automatically locked after I leaved. Can you help me?", - "source": "https://help.ubuntu.com/lts/ubuntu-help/privacy-screen-lock.html.en", - "trajectory": "trajectories/", - "related_apps": [ - "os" - ], - "evaluator": { - "func": "", - "result": { - }, - "expected": { + "id": "a4d98375-215b-4a4d-aee9-3d4370fccc41", + "snapshot": "os", + "instruction": "I want to have my computer automatically locked after I leaved. Can you help me?", + "source": "https://help.ubuntu.com/lts/ubuntu-help/privacy-screen-lock.html.en", + "trajectory": "trajectories/", + "related_apps": [ + "os" + ], + "evaluator": { + "func": "exact_match", + "result": { + "type": "vm_command_line", + "command": "gsettings get org.gnome.desktop.screensaver lock-enabled", + "shell": true + }, + "expected": { + "type": "rule", + "rules": { + "expected": "true\n" } } - } \ No newline at end of file + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/os/b6781586-6346-41cd-935a-a6b1487918fc.json b/evaluation_examples/examples/os/b6781586-6346-41cd-935a-a6b1487918fc.json index 7b43485..46964d3 100644 --- a/evaluation_examples/examples/os/b6781586-6346-41cd-935a-a6b1487918fc.json +++ b/evaluation_examples/examples/os/b6781586-6346-41cd-935a-a6b1487918fc.json @@ -4,15 +4,17 @@ "instruction": "I want to set my current time zone to UTC+0. Can you help me?", "source": "https://help.ubuntu.com/lts/ubuntu-help/clock-timezone.html.en", "trajectory": "trajectories/", - "config": [], + "config": [ + ], "related_apps": [ "os" ], "evaluator": { - "func": "", + "func": "is_utc_0", "result": { - }, - "expected": { + "type": "vm_command_line", + "command": "timedatectl status", + "shell": true } } } \ No newline at end of file diff --git a/evaluation_examples/examples/os/ddc75b62-7311-4af8-bfb3-859558542b36.json b/evaluation_examples/examples/os/ddc75b62-7311-4af8-bfb3-859558542b36.json index b50c3a9..c54f368 100644 --- a/evaluation_examples/examples/os/ddc75b62-7311-4af8-bfb3-859558542b36.json +++ b/evaluation_examples/examples/os/ddc75b62-7311-4af8-bfb3-859558542b36.json @@ -1,17 +1,36 @@ { - "id": "ddc75b62-7311-4af8-bfb3-859558542b36", - "snapshot": "os", - "instruction": "I want to uninstall the Firefox on my system. Can you help me", - "source": "https://help.ubuntu.com/lts/ubuntu-help/addremove-remove.html.en", - "trajectory": "trajectories/", - "related_apps": [ - "os" + "id": "ddc75b62-7311-4af8-bfb3-859558542b36", + "snapshot": "os", + "instruction": "I want to uninstall the Mahjongg on my system. Can you help me?", + "source": "https://help.ubuntu.com/lts/ubuntu-help/addremove-remove.html.en", + "trajectory": "trajectories/", + "related_apps": [ + "os" + ], + "evaluator": { + "postconfig": [ + { + "type": "sleep", + "parameters": { + "seconds": 1 + } + } ], - "evaluator": { - "func": "", - "result": { - }, - "expected": { + "func": "check_include_exclude", + "result": { + "type": "vm_command_line", + "command": "dpkg -l | grep mahjongg", + "shell": true + }, + "expected": { + "type": "rule", + "rules": { + "include": [ + ], + "exclude": [ + "mahjongg" + ] } } - } \ No newline at end of file + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/os/e0df059f-28a6-4169-924f-b9623e7184cc.json b/evaluation_examples/examples/os/e0df059f-28a6-4169-924f-b9623e7184cc.json index 0563b9a..c9e41d8 100644 --- a/evaluation_examples/examples/os/e0df059f-28a6-4169-924f-b9623e7184cc.json +++ b/evaluation_examples/examples/os/e0df059f-28a6-4169-924f-b9623e7184cc.json @@ -1,17 +1,33 @@ { - "id": "e0df059f-28a6-4169-924f-b9623e7184cc", - "snapshot": "os", - "instruction": "I have a directory named \"test1\". Can you help me change its name into \"test2\"?", - "source": "https://help.ubuntu.com/lts/ubuntu-help/files-rename.html.en", - "trajectory": "trajectories/", - "related_apps": [ - "os" - ], - "evaluator": { - "func": "", - "result": { - }, - "expected": { + "id": "e0df059f-28a6-4169-924f-b9623e7184cc", + "snapshot": "os", + "instruction": "I have a directory named \"todo_list_Jan_1\". Can you help me change its name into \"todo_list_Jan_2\"?", + "source": "https://help.ubuntu.com/lts/ubuntu-help/files-rename.html.en", + "trajectory": "trajectories/", + "config": [ + { + "type": "execute", + "parameters": { + "command": "echo 'password' | sudo -S mkdir ~/Desktop/todo_list_Jan_1", + "shell": true } } - } \ No newline at end of file + ], + "related_apps": [ + "os" + ], + "evaluator": { + "func": "exact_match", + "result": { + "type": "vm_command_line", + "command": "[ -d ~/Desktop/todo_list_Jan_2 ] && echo 'Directory exists.' || echo 'Directory does not exist.'", + "shell": true + }, + "expected": { + "type": "rule", + "rules":{ + "expected": "Directory exists.\n" + } + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/os/fe41f596-a71b-4c2f-9b2f-9dcd40b568c3.json b/evaluation_examples/examples/os/fe41f596-a71b-4c2f-9b2f-9dcd40b568c3.json index b5ee6cd..c613775 100644 --- a/evaluation_examples/examples/os/fe41f596-a71b-4c2f-9b2f-9dcd40b568c3.json +++ b/evaluation_examples/examples/os/fe41f596-a71b-4c2f-9b2f-9dcd40b568c3.json @@ -8,10 +8,17 @@ "os" ], "evaluator": { - "func": "", + "func": "exact_match", "result": { + "type": "vm_command_line", + "command": "gsettings get org.gnome.desktop.interface show-battery-percentage", + "shell": true }, "expected": { + "type": "rule", + "rules":{ + "expected": "true\n" + } } } } \ No newline at end of file From 5ac80dc309d9a061461cbb7e9af5ff913bb2cfe3 Mon Sep 17 00:00:00 2001 From: rhythmcao Date: Fri, 26 Jan 2024 00:53:35 +0800 Subject: [PATCH 2/6] update examples --- desktop_env/controllers/setup.py | 2 +- desktop_env/evaluators/getters/chrome.py | 65 +++++---- .../0aa56709-3293-5849-ad47-e377f49fd3a0.json | 119 +++++++++++++++ .../492c2c87-b04a-544a-b5dd-eb808036bf85.json | 89 ++++++++++++ .../49f981ee-f793-5e27-9a53-083d66934ea1.json | 126 ++++++++++++++++ .../8aa9e870-b0c9-5417-be80-03154e83c7a3.json | 102 +++++++++++++ .../8ff98608-8e0e-526e-9413-d744554ba708.json | 86 +++++++++++ .../46407397-a7d5-4c6b-92c6-dbe038b1457b.json | 136 ++++++++++++++++++ .../b52b40a5-ad70-4c53-b5b0-5650a8387052.json | 111 ++++++++++++++ .../settings/google/settings.json | 2 +- .../settings/googledrive/credentials.json | 2 +- .../settings/thunderbird/settings.json | 5 + 12 files changed, 818 insertions(+), 27 deletions(-) create mode 100644 evaluation_examples/examples/dbt/0aa56709-3293-5849-ad47-e377f49fd3a0.json create mode 100644 evaluation_examples/examples/dbt/492c2c87-b04a-544a-b5dd-eb808036bf85.json create mode 100644 evaluation_examples/examples/dbt/49f981ee-f793-5e27-9a53-083d66934ea1.json create mode 100644 evaluation_examples/examples/dbt/8aa9e870-b0c9-5417-be80-03154e83c7a3.json create mode 100644 evaluation_examples/examples/dbt/8ff98608-8e0e-526e-9413-d744554ba708.json create mode 100644 evaluation_examples/examples/multi_apps/46407397-a7d5-4c6b-92c6-dbe038b1457b.json create mode 100644 evaluation_examples/examples/multi_apps/b52b40a5-ad70-4c53-b5b0-5650a8387052.json create mode 100644 evaluation_examples/settings/thunderbird/settings.json diff --git a/desktop_env/controllers/setup.py b/desktop_env/controllers/setup.py index b0f9b5b..c56832b 100644 --- a/desktop_env/controllers/setup.py +++ b/desktop_env/controllers/setup.py @@ -536,7 +536,7 @@ class SetupController: page.goto(url) logger.info(f"Opened new page: {url}") settings = json.load(open(config['settings_file'])) - email, password = settings['account'], settings['password'] + email, password = settings['email'], settings['password'] try: page.wait_for_selector('input[type="email"]', state="visible", timeout=3000) diff --git a/desktop_env/evaluators/getters/chrome.py b/desktop_env/evaluators/getters/chrome.py index 89145c1..bcfab09 100644 --- a/desktop_env/evaluators/getters/chrome.py +++ b/desktop_env/evaluators/getters/chrome.py @@ -469,34 +469,51 @@ def get_googledrive_file(env, config: Dict[str, Any]) -> str: To retrieve the file, we provide two options in config dict: 1. query: a list of queries to search the file, each query is a string that follows the format of Google Drive search query 2. path: a list of path to the file, 'folder/subfolder/filename' -> ['folder', 'subfolder', 'filename'] + 3. query_list: query extends to list to download multiple files + 4. path_list: path extends to list to download multiple files + dest: target file name or list. If *_list is used, dest should also be a list of the same length. Return the downloaded filepath locally. """ settings_file = config.get('settings_file', 'evaluation_examples/settings/googledrive/settings.json') auth = GoogleAuth(settings_file=settings_file) drive = GoogleDrive(auth) - _path = os.path.join(env.cache_dir, config['dest']) + + def get_single_file(_query, _path): + parent_id = 'root' + try: + for q in _query: + search = f'( {q} ) and "{parent_id}" in parents' + filelist: GoogleDriveFileList = drive.ListFile({'q': search}).GetList() + if len(filelist) == 0: # target file not found + return None + file: GoogleDriveFile = filelist[0] # HACK: if multiple candidates, just use the first one + parent_id = file['id'] + + file.GetContentFile(_path, mimetype=file['mimeType']) + except: + logger.info('[ERROR]: Failed to download the file from Google Drive') + return None + return _path if 'query' in config: - query = config['query'] - if type(query) != list: query = [query] - else: - paths = config['path'] - if type(paths) != list: paths = [paths] - query = [f"title = {fp} and mimeType = 'application/vnd.google-apps.folder' and trashed = false" if idx < len(paths) - 1 - else f'title = {fp} and trashed = false' for idx, fp in enumerate(paths)] - parent_id = 'root' - - try: - for q in query: - search = f'( {q} ) and "{parent_id}" in parents' - filelist: GoogleDriveFileList = drive.ListFile({'q': search}).GetList() - if len(filelist) == 0: # target file not found - return None - file: GoogleDriveFile = filelist[0] # HACK: if multiple candidates, just use the first one - parent_id = file['id'] - - file.GetContentFile(_path, mimetype=file['mimeType']) - except: - logger.info('[ERROR]: Failed to download the file from Google Drive') - return None - return _path \ No newline at end of file + return get_single_file(config['query'], os.path.join(env.cache_dir, config['dest'])) + elif 'path' in config: + query = [f"title = {fp} and mimeType = 'application/vnd.google-apps.folder' and trashed = false" if idx < len(config['path']) - 1 + else f'title = {fp} and trashed = false' for idx, fp in enumerate(config['path'])] + return get_single_file(query, os.path.join(env.cache_dir, config['dest'])) + elif 'query_list' in config: + _path_list = [] + assert len(config['query_list']) == len(config['dest']) + for idx, query in enumerate(config['query_list']): + dest = config['dest'][idx] + _path_list.append(get_single_file(query, os.path.join(env.cache_dir, dest))) + return _path_list + else: # path_list in config + _path_list = [] + assert len(config['path_list']) == len(config['dest']) + for idx, path in enumerate(config['path_list']): + query = [f"title = {fp} and mimeType = 'application/vnd.google-apps.folder' and trashed = false" if jdx < len(path) - 1 + else f'title = {fp} and trashed = false' for jdx, fp in enumerate(path)] + dest = config['dest'][idx] + _path_list.append(get_single_file(query, os.path.join(env.cache_dir, dest))) + return _path_list \ No newline at end of file diff --git a/evaluation_examples/examples/dbt/0aa56709-3293-5849-ad47-e377f49fd3a0.json b/evaluation_examples/examples/dbt/0aa56709-3293-5849-ad47-e377f49fd3a0.json new file mode 100644 index 0000000..4e74142 --- /dev/null +++ b/evaluation_examples/examples/dbt/0aa56709-3293-5849-ad47-e377f49fd3a0.json @@ -0,0 +1,119 @@ +{ + "id": "0aa56709-3293-5849-ad47-e377f49fd3a0", + "snapshot": "dbt", + "instruction": "Update models/schema.yml file to include some description fields. 1) The description fields include what each table is about. 2) For the primary key column of each table, add the description \"Primary key\". 3) Also insert one description for column customers.first_order_date that \"NULL when a customer has not yet placed an order.\". Then, use dbt docs command to generate the documentation for this dbt project and launch the documentation in a local website with port 8020.", + "source": [ + "https://docs.getdbt.com/guides/manual-install?step=13" + ], + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1TZmmW7wYnWUQVMCH5JOBCxxdgm_QN-vz&export=download&authuser=0&confirm=t&uuid=43113538-c0e4-4e23-8a18-12b727b9f890&at=APZUnTWgSRXjNNZZIt8ni7rsTxoy:1705910001969", + "path": "/home/user/projects/jaffle_shop.zip" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": ["/bin/bash", "-c", "unzip -oq /home/user/projects/jaffle_shop.zip -d /home/user/projects/ && rm /home/user/projects/jaffle_shop.zip && mkdir -p /home/user/.dbt"] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1xkXjiFhRNoiX_-0zikfdwzJ8UvdXEVAt&export=download&authuser=0&confirm=t&uuid=bc8d0eb8-99b3-4be7-b110-c0a56246d2d2&at=APZUnTUaPD_Z6ov6uMWDNf5rSy3-:1705978221799", + "path": "/home/user/.dbt/profiles.yml" + } + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "dbt", + "duckdb" + ], + "evaluator": { + "func": [ + "check_yaml_file", + "check_dbt_command", + "check_dbt_command", + "check_dbt_command", + "check_dbt_command" + ], + "conj": "and", + "result": [ + { + "type": "vm_file", + "path": "/home/user/projects/jaffle_shop/models/schema.yml", + "dest": "schema.yml" + }, + { + "type": "vm_command_line", + "command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; if [ -s target/catalog.json ] && [ -s target/index.html ] ; then echo \"dbt docs generate succeed\" ; else echo \"dbt docs generate failed\" ; fi"] + }, + { + "type": "vm_command_line", + "command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; curl -o index.html http://localhost:8020; if [ -s index.html ] ; then echo \"dbt docs serve succeed\" ; else echo \"dbt docs serve failed\" ; fi"] + }, + { + "type": "vm_command_line", + "command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; diff index.html target/index.html > diff.log; if [ -s diff.log ] ; then echo \"dbt docs serve failed\" ; else echo \"dbt docs serve succeed\" ; fi ; rm -rf diff.log index.html"] + }, + { + "type": "vm_command_line", + "command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; dbt docs generate"] + } + ], + "expected": [ + { + "type": "rule", + "rules": [ + ["not_null", ["models", ["name", "customers"], "description"], ""], + ["not_null", ["models", ["name", "stg_customers"], "description"], ""], + ["not_null", ["models", ["name", "stg_orders"], "description"], ""], + ["match", ["models", ["name", "customers"], "columns", ["name", "customer_id"], "description"], "Primary key"], + ["match", ["models", ["name", "stg_customers"], "columns", ["name", "customer_id"], "description"], "Primary key"], + ["match", ["models", ["name", "stg_orders"], "columns", ["name", "order_id"], "description"], "Primary key"], + ["match", ["models", ["name", "customers"], "columns", ["name", "first_order_date"], "description"], "NULL when a customer has not yet placed an order."] + ] + }, + { + "type": "rule", + "rules": [ + ["contains", "dbt docs generate succeed", ""], + ["excludes", "dbt docs generate failed", ""] + ] + }, + { + "type": "rule", + "rules": [ + ["contains", "dbt docs serve succeed", ""], + ["excludes", "dbt docs serve failed", ""] + ] + }, + { + "type": "rule", + "rules": [ + ["contains", "dbt docs serve succeed", ""], + ["excludes", "dbt docs serve failed", ""] + ] + }, + { + "type": "rule", + "rules": [ + ["excludes", "error", ""], + ["excludes", "Error", ""], + ["contains", "Catalog written to", ""] + ] + } + ] + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/dbt/492c2c87-b04a-544a-b5dd-eb808036bf85.json b/evaluation_examples/examples/dbt/492c2c87-b04a-544a-b5dd-eb808036bf85.json new file mode 100644 index 0000000..9dbfe00 --- /dev/null +++ b/evaluation_examples/examples/dbt/492c2c87-b04a-544a-b5dd-eb808036bf85.json @@ -0,0 +1,89 @@ +{ + "id": "492c2c87-b04a-544a-b5dd-eb808036bf85", + "snapshot": "dbt", + "instruction": "Separate the single dbt model customers into separate ones and use the ``ref`` function to build models on top of others. 1) Create a new SQL file, models/stg_customers.sql, with the SQL from the customers common table expressions (CTE) in the original query. 2) Create a second new SQL file, models/stg_orders.sql, with the SQL from the orders CTE in the original query. These two stg models are materialized as view. 3) Edit the SQL in models/customers.sql file to refer two these two staged models and execute dbt run. Please ensure the running succeeds.", + "source": [ + "https://docs.getdbt.com/guides/manual-install?step=11" + ], + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1-Nn0leqKdVnA1gQ6s50wkGSa2xZauEUe&export=download&authuser=0&confirm=t&uuid=d7596b15-ae39-43cb-a940-03bee8a0c961&at=APZUnTXwp5xMjHpi5AYVF3Z_bfw2:1705582080467", + "path": "/home/user/projects/jaffle_shop.zip" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": ["/bin/bash", "-c", "unzip -oq /home/user/projects/jaffle_shop.zip -d /home/user/projects/ && rm /home/user/projects/jaffle_shop.zip && mkdir -p /home/user/.dbt"] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1xkXjiFhRNoiX_-0zikfdwzJ8UvdXEVAt&export=download&authuser=0&confirm=t&uuid=bc8d0eb8-99b3-4be7-b110-c0a56246d2d2&at=APZUnTUaPD_Z6ov6uMWDNf5rSy3-:1705978221799", + "path": "/home/user/.dbt/profiles.yml" + } + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "dbt", + "duckdb" + ], + "evaluator": { + "func": [ + "check_database", + "check_dbt_command" + ], + "conj": "and", + "result": [ + { + "type": "vm_file", + "path": "/home/user/projects/jaffle_shop/jaffle_shop.duckdb", + "dest": "jaffle_shop.duckdb" + }, + { + "type": "vm_command_line", + "command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; dbt run"] + } + ], + "expected": [ + { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=1rtRXtgKiwyWSVPCbDVkjYXh8AMni7t1W&export=download&authuser=0&confirm=t&uuid=6e4331f7-4e46-4894-910f-9af77450534d&at=APZUnTXySypK2OcEdOb8FVHQnbd7:1705583620817", + "dest": "gold_jaffle_shop.duckdb" + }, + { + "type": "rule", + "rules": [ + ["contains", "of 3 OK created sql view model .*\\.stg_customers", ""], + ["contains", "of 3 OK created sql view model .*\\.stg_orders", ""], + ["contains", "3 of 3 OK created sql table model .*\\.customers", ""], + ["contains", "Completed successfully", ""], + ["contains", "PASS=3", ""], + ["contains", "TOTAL=3", ""] + ] + } + ], + "options": [ + { + "db_type": "duckdb", + "check_type": [ + "table-schema-content", + "view-schema-content" + ] + }, + {} + ] + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/dbt/49f981ee-f793-5e27-9a53-083d66934ea1.json b/evaluation_examples/examples/dbt/49f981ee-f793-5e27-9a53-083d66934ea1.json new file mode 100644 index 0000000..5b4a21e --- /dev/null +++ b/evaluation_examples/examples/dbt/49f981ee-f793-5e27-9a53-083d66934ea1.json @@ -0,0 +1,126 @@ +{ + "id": "49f981ee-f793-5e27-9a53-083d66934ea1", + "snapshot": "dbt", + "instruction": "Add dbt test for the three models in jaffle_shop project. 1) Create a YAML file in the models directory, named models/schema.yml. 2) Ensure that customer_id and order_id are unique and not empty in related tables or views. Column status must be value from placed, shipped, completed, return_pending, returned. And customer_id in stg_orders is not null and is a foreign key in the referenced stg_customers. 3) Run dbt test, and confirm that all tests passed.", + "source": [ + "https://docs.getdbt.com/guides/manual-install?step=12" + ], + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1QhPSdctnfYk0O5Tuo2NzldMDw0cKZi16&export=download&authuser=0&confirm=t&uuid=d68dcd34-304d-4e66-a215-99abd3954c24&at=APZUnTXihMvZPodIOVVxg3S0tUs4:1705587177507", + "path": "/home/user/projects/jaffle_shop.zip" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": ["/bin/bash", "-c", "unzip -oq /home/user/projects/jaffle_shop.zip -d /home/user/projects/ && rm /home/user/projects/jaffle_shop.zip && mkdir -p /home/user/.dbt"] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1xkXjiFhRNoiX_-0zikfdwzJ8UvdXEVAt&export=download&authuser=0&confirm=t&uuid=bc8d0eb8-99b3-4be7-b110-c0a56246d2d2&at=APZUnTUaPD_Z6ov6uMWDNf5rSy3-:1705978221799", + "path": "/home/user/.dbt/profiles.yml" + } + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "dbt", + "duckdb" + ], + "evaluator": { + "func": "check_dbt_command", + "result": { + "type": "dbt_test_result", + "pre-processing": ["mv", "/home/user/projects/jaffle_shop/jaffle_shop.duckdb", "/home/user/Desktop/jaffle_shop.duckdb"], + "path": [ + "https://drive.usercontent.google.com/download?id=1AZ6hCtbyN8Ypzf0e2nkkxivxdAmKNHZQ&export=download&authuser=0&confirm=t&uuid=dffccf1e-a42b-45e7-966d-72d289a1062e&at=APZUnTUsGPahiUmvgXxji9x9Ii7o:1705668446168", + "https://drive.usercontent.google.com/download?id=1z0hApNSqvs2oUwJiBQmFXrsomAxa1RhJ&export=download&authuser=0&confirm=t&uuid=0a2ba1be-1e15-4e8d-8458-0a661eaeef6f&at=APZUnTVZMe3y1OpU6ipsm0U6Ryb6:1705668535249", + "https://drive.usercontent.google.com/download?id=1OYsLQSYAdaAyu0sa6Y8IstmWz7wXneUN&export=download&authuser=0&confirm=t&uuid=33818f3c-a125-44d4-b9ba-7c5465976250&at=APZUnTXWFrNUtydf460ZWA-2jJrg:1705668651799", + "https://drive.usercontent.google.com/download?id=1KgGuJMeCXpIG2_TAKIJsT-YoIbSWswYp&export=download&authuser=0&confirm=t&uuid=4f2aadaa-2bc0-4ecf-b725-19c73375e370&at=APZUnTVt5cFNMzCsRxekJUaBPfhw:1705668748132", + "https://drive.usercontent.google.com/download?id=1OJ0xgAF1KqhovIvkimp3J5ZM-crfZU6g&export=download&authuser=0&confirm=t&uuid=48bcfac9-d40c-40c5-ac07-da9288a4ebea&at=APZUnTVUl1FaKtXy4oVpkYnGKIdt:1705669212860", + "https://drive.usercontent.google.com/download?id=10mr1jhdA52_bKOgoDgG0mAal1nzXkG21&export=download&authuser=0&confirm=t&uuid=5adde4ce-59d5-4da0-ada8-9c7df3e7434c&at=APZUnTVwJ4WDHSnfUv8yB08jr1b7:1705669280449", + "https://drive.usercontent.google.com/download?id=1_Dh9lwVDo8TfB0jg8QsdemRZ14r9O-uY&export=download&authuser=0&confirm=t&uuid=a8445889-8d13-4b66-88e7-4b675a943bec&at=APZUnTWrfFiAMJVLNFuteOctpjzs:1705669328524", + "https://drive.usercontent.google.com/download?id=1n4fLzFWj9dEqRdLOSwSB5SB8O3kHyEBZ&export=download&authuser=0&confirm=t&uuid=78cd4358-cc9e-4d2c-b74f-93b7760de4d3&at=APZUnTXbEj7FEQL5DEU5g4wsDFHs:1705669396969", + "https://drive.usercontent.google.com/download?id=1sjwB1aoi5UzbHQI-mb0hph2qd4qRKZR_&export=download&authuser=0&confirm=t&uuid=2fe93a52-d820-462e-9cce-36daa3a5ce30&at=APZUnTXPsOgN81L1RkenHg__oXUB:1705669474430", + "https://drive.usercontent.google.com/download?id=1ujsyfS7F3A6YCgZJ7N9ixcRlgNdAAdXJ&export=download&authuser=0&confirm=t&uuid=7e1b73c9-a14b-4d8b-800c-77f16ac5f897&at=APZUnTWBVbznnKO-zYj7UI6qk-qz:1705669501449", + "https://drive.usercontent.google.com/download?id=1tujswM-r4GKav5CpmciF-H3zYAdEq-uL&export=download&authuser=0&confirm=t&uuid=32253ad7-d343-4961-bc14-1e4cf4fee244&at=APZUnTWiRpT1pHF6Qp17y1VqcT7Z:1705676183100" + ], + "dest": "/home/user/projects/jaffle_shop/jaffle_shop.duckdb", + "command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; dbt test"], + "post-processing": ["/bin/bash", "-c", "mv /home/user/Desktop/jaffle_shop.duckdb /home/user/projects/jaffle_shop/jaffle_shop.duckdb"] + }, + "expected": { + "type": "rule", + "rules": [ + [ + ["excludes", "Nothing to do", ""], + ["contains", "Completed successfully", ""], + ["contains", "ERROR=0", ""] + ], + [ + ["excludes", "Nothing to do", ""], + ["excludes", "Completed successfully", ""], + ["excludes", "ERROR=0", ""] + ], + [ + ["excludes", "Nothing to do", ""], + ["excludes", "Completed successfully", ""], + ["excludes", "ERROR=0", ""] + ], + [ + ["excludes", "Nothing to do", ""], + ["excludes", "Completed successfully", ""], + ["excludes", "ERROR=0", ""] + ], + [ + ["excludes", "Nothing to do", ""], + ["excludes", "Completed successfully", ""], + ["excludes", "ERROR=0", ""] + ], + [ + ["excludes", "Nothing to do", ""], + ["excludes", "Completed successfully", ""], + ["excludes", "ERROR=0", ""] + ], + [ + ["excludes", "Nothing to do", ""], + ["excludes", "Completed successfully", ""], + ["excludes", "ERROR=0", ""] + ], + [ + ["excludes", "Nothing to do", ""], + ["excludes", "Completed successfully", ""], + ["excludes", "ERROR=0", ""] + ], + [ + ["excludes", "Nothing to do", ""], + ["excludes", "Completed successfully", ""], + ["excludes", "ERROR=0", ""] + ], + [ + ["excludes", "Nothing to do", ""], + ["excludes", "Completed successfully", ""], + ["excludes", "ERROR=0", ""] + ], + [ + ["excludes", "Nothing to do", ""], + ["contains", "Completed successfully", ""], + ["contains", "ERROR=0", ""] + ] + ] + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/dbt/8aa9e870-b0c9-5417-be80-03154e83c7a3.json b/evaluation_examples/examples/dbt/8aa9e870-b0c9-5417-be80-03154e83c7a3.json new file mode 100644 index 0000000..042c2c5 --- /dev/null +++ b/evaluation_examples/examples/dbt/8aa9e870-b0c9-5417-be80-03154e83c7a3.json @@ -0,0 +1,102 @@ +{ + "id": "8aa9e870-b0c9-5417-be80-03154e83c7a3", + "snapshot": "dbt", + "instruction": "Create a dbt project called jaffle_shop under ~/projects and configure the connection to duckdb with path jaffle_shop/jaffle_shop.duckdb for the target dev and prod. Load data from jaffle-shop-data.zip into that database using dbt.", + "source": [ + "https://docs.getdbt.com/docs/core/connect-data-platform/duckdb-setup#", + "https://docs.getdbt.com/guides/manual-install?step=3", + "https://docs.getdbt.com/reference/commands/seed" + ], + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=18pbzQOnAZEZ2psFLWJJQIyLlQBSNRxWs&export=download&authuser=0&confirm=t&uuid=d8045faa-0f74-4c34-ab98-55ab41bcc2c9&at=APZUnTUoi3u-jMpwO8osMEll4gDr:1705548224569", + "path": "/home/user/projects/jaffle-shop-data.zip" + } + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "dbt", + "duckdb" + ], + "evaluator": { + "func": [ + "check_yaml_file", + "check_dbt_command", + "check_database", + "check_dbt_command" + ], + "conj": "and", + "result": [ + { + "type": "dbt_profiles", + "paths": [ + "/home/user/.dbt/profiles.yml", + "/home/user/projects/jaffle_shop/profiles.yml", + "$DBT_PROFILES_DIR" + ], + "dest": "profiles.yml" + }, + { + "type": "vm_command_line", + "command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; dbt debug"] + }, + { + "type": "vm_file", + "path": "/home/user/projects/jaffle_shop/jaffle_shop.duckdb", + "dest": "jaffle_shop.duckdb" + }, + { + "type": "vm_command_line", + "command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; dbt seed"] + } + ], + "expected": [ + { + "type": "rule", + "rules": [ + ["match", ["jaffle_shop", "outputs", "dev", "type"], "duckdb"], + ["match", ["jaffle_shop", "outputs", "prod", "type"], "duckdb"], + ["in", ["jaffle_shop", "outputs", "dev", "path"], ["jaffle_shop.duckdb", "/home/user/projects/jaffle_shop/jaffle_shop.duckdb"]], + ["in", ["jaffle_shop", "outputs", "prod", "path"], ["jaffle_shop.duckdb", "/home/user/projects/jaffle_shop/jaffle_shop.duckdb"]] + ] + }, + { + "type": "rule", + "rules": [ + ["contains", "OK connection ok", ""], + ["contains", "All checks passed", ""] + ] + }, + { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=12xJuEcBxGqPHjJriUPnkuGRkw9HsZniJ&export=download&authuser=0&confirm=t&uuid=f2b7147f-5114-461d-a939-486750c19029&at=APZUnTVBf4QJMyDPk5BmfZzTecU5:1705548962476", + "dest": "gold_jaffle_shop.duckdb" + }, + { + "type": "rule", + "rules": [ + ["excludes", "Nothing to do", ""], + ["contains", "Completed successfully", ""], + ["contains", "PASS=3", ""], + ["contains", "TOTAL=3", ""] + ] + } + ], + "options": [ + {}, + {}, + { + "db_type": "duckdb", + "check_type": ["table-schema-content"] + }, + {} + ] + } + } \ No newline at end of file diff --git a/evaluation_examples/examples/dbt/8ff98608-8e0e-526e-9413-d744554ba708.json b/evaluation_examples/examples/dbt/8ff98608-8e0e-526e-9413-d744554ba708.json new file mode 100644 index 0000000..33117cb --- /dev/null +++ b/evaluation_examples/examples/dbt/8ff98608-8e0e-526e-9413-d744554ba708.json @@ -0,0 +1,86 @@ +{ + "id": "8ff98608-8e0e-526e-9413-d744554ba708", + "snapshot": "dbt", + "instruction": "Delete the dbt example models and write one dbt model \"customers\" from existing tables raw_customers and raw_orders. It contains columns customer_ids, first_name, last_name, first_order_date, most_recent_order_date and number_of_orders (default to 0). Set the materialization method as table and build it with dbt run to make it work.", + "source": [ + "https://docs.getdbt.com/guides/manual-install?step=10", + "https://docs.getdbt.com/guides/manual-install?step=8", + "https://docs.getdbt.com/guides/manual-install?step=9" + ], + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1iqimXoJUa2fd16I_qCZGHbVMHABJAh7D&export=download&authuser=0&confirm=t&uuid=65b73e29-0b30-415d-9928-d1ccf3be9880&at=APZUnTWCBNMro4gX8whSahDtx05S:1705563480661", + "path": "/home/user/projects/jaffle_shop.zip" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": ["/bin/bash", "-c", "unzip -oq /home/user/projects/jaffle_shop.zip -d /home/user/projects/ && rm /home/user/projects/jaffle_shop.zip && mkdir -p /home/user/.dbt"] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1xkXjiFhRNoiX_-0zikfdwzJ8UvdXEVAt&export=download&authuser=0&confirm=t&uuid=bc8d0eb8-99b3-4be7-b110-c0a56246d2d2&at=APZUnTUaPD_Z6ov6uMWDNf5rSy3-:1705978221799", + "path": "/home/user/.dbt/profiles.yml" + } + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "dbt", + "duckdb" + ], + "evaluator": { + "func": [ + "check_database", + "check_dbt_command" + ], + "conj": "and", + "result": [ + { + "type": "vm_file", + "path": "/home/user/projects/jaffle_shop/jaffle_shop.duckdb", + "dest": "jaffle_shop.duckdb" + }, + { + "type": "vm_command_line", + "command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; dbt run"] + } + ], + "expected": [ + { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=1BnTQOiDIKriWGhLRj7C8MPgfk4JH-uf9&export=download&authuser=0&confirm=t&uuid=cee74971-d66b-4da0-8cdd-cb9f1b9da8e7&at=APZUnTWI7GqSBLnQ_g-NNsS7kDxq:1705567576749", + "dest": "gold_jaffle_shop.duckdb" + }, + { + "type": "rule", + "rules": [ + ["contains", "1 of 1 OK created sql table model .*\\.customers", ""], + ["contains", "Completed successfully", ""], + ["contains", "PASS=1", ""], + ["contains", "TOTAL=1", ""] + ] + } + ], + "options": [ + { + "db_type": "duckdb", + "check_type": ["table-schema-content"] + }, + {} + ] + } + } \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/46407397-a7d5-4c6b-92c6-dbe038b1457b.json b/evaluation_examples/examples/multi_apps/46407397-a7d5-4c6b-92c6-dbe038b1457b.json new file mode 100644 index 0000000..5475c70 --- /dev/null +++ b/evaluation_examples/examples/multi_apps/46407397-a7d5-4c6b-92c6-dbe038b1457b.json @@ -0,0 +1,136 @@ +{ + "id": "46407397-a7d5-4c6b-92c6-dbe038b1457b", + "snapshot": "chrome", + "instruction": "Help me export charts, graph or other images from docx files received in email xxx in Thunderbird and upload them in the figures/ folder in Google Drive for later use (use pure numbers to name them).", + "source": "https://marketplace.uipath.com/listings/merge-pdfs-from-gmail-email-attachments-and-upload-to-gogle-drive", + "config": [ + { + "type": "googledrive", + "parameters": { + "settings_file": "evaluation_examples/settings/googledrive/settings.yml", + "operation": ["delete"], + "args": [ + { + "query": "title = 'figures' and 'root' in parents and mimeType = 'application/vnd.google-apps.folder'", + "trash": false + } + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "google-chrome", + "--remote-debugging-port=1337" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "socat", + "tcp-listen:9222,fork", + "tcp:localhost:1337" + ] + } + }, + { + "type": "chrome_open_tabs", + "parameters": { + "urls_to_open": [ + "https://www.freerice.com/", + "https://www.hku.hk/", + "https://about.meta.com/technologies/facebook-app/" + ] + } + }, + { + "type": "login", + "parameters": { + "settings_file": "evaluation_examples/settings/google/settings.json", + "platform": "googledrive" + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=11b93787-7076-47ba-b04b-b63a4e9aab02&at=APZUnTV_3ASC-R55FmBXgTLcC46e:1706187828620", + "path": "/home/user/thunderbird-profile.tar.gz" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "tar", + "-xzv", + "--recursive-unlink", + "-f", + "/home/user/thunderbird-profile.tar.gz", + "-C", + "/home/user/" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "/usr/bin/thunderbird" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "thunderbird", + "chrome" + ], + "evaluator": { + "func": "compare_figures", + "result": { + "type": "googledrive_file", + "settings_file": "evaluation_examples/settings/googledrive/settings.yml", + "query_list": [ + [ + "title = 'figures' and trashed = false and 'root' in parents and mimeType = 'application/vnd.google-apps.folder'", + "title = '1.png' and trashed = false" + ], + [ + "title = 'figures' and trashed = false and 'root' in parents and mimeType = 'application/vnd.google-apps.folder'", + "title = '2.png' and trashed = false" + ], + [ + "title = 'figures' and trashed = false and 'root' in parents and mimeType = 'application/vnd.google-apps.folder'", + "title = '3.png' and trashed = false" + ] + ], + "dest": [ + "1.png", + "2.png", + "3.png" + ] + }, + "expected": { + "type": "cloud_file", + "path": [ + "file1", + "file2", + "file3" + ], + "dest": [ + "1_gold.png", + "2_gold.png", + "3_gold.png" + ], + "multi": true, + "gives": [0, 1, 2] + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/b52b40a5-ad70-4c53-b5b0-5650a8387052.json b/evaluation_examples/examples/multi_apps/b52b40a5-ad70-4c53-b5b0-5650a8387052.json new file mode 100644 index 0000000..9440ddb --- /dev/null +++ b/evaluation_examples/examples/multi_apps/b52b40a5-ad70-4c53-b5b0-5650a8387052.json @@ -0,0 +1,111 @@ +{ + "id": "b52b40a5-ad70-4c53-b5b0-5650a8387052", + "snapshot": "chrome", + "instruction": "Could you help me merge all PDF files in the email attachment in Thunderbird into one file and upload it to attachment_full.pdf in Google Drive?", + "source": "https://marketplace.uipath.com/listings/merge-pdfs-from-gmail-email-attachments-and-upload-to-gogle-drive", + "config": [ + { + "type": "googledrive", + "parameters": { + "settings_file": "evaluation_examples/settings/googledrive/settings.yml", + "operation": ["delete"], + "args": [ + { + "query": "title = 'attachment_full.pdf' and 'root' in parents", + "trash": false + } + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "google-chrome", + "--remote-debugging-port=1337" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "socat", + "tcp-listen:9222,fork", + "tcp:localhost:1337" + ] + } + }, + { + "type": "chrome_open_tabs", + "parameters": { + "urls_to_open": [ + "https://www.freerice.com/", + "https://www.hku.hk/", + "https://about.meta.com/technologies/facebook-app/" + ] + } + }, + { + "type": "login", + "parameters": { + "settings_file": "evaluation_examples/settings/google/settings.json", + "platform": "googledrive" + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=11b93787-7076-47ba-b04b-b63a4e9aab02&at=APZUnTV_3ASC-R55FmBXgTLcC46e:1706187828620", + "path": "/home/user/thunderbird-profile.tar.gz" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "tar", + "-xzv", + "--recursive-unlink", + "-f", + "/home/user/thunderbird-profile.tar.gz", + "-C", + "/home/user/" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "/usr/bin/thunderbird" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "thunderbird", + "chrome" + ], + "evaluator": { + "func": "compare_pdfs", + "result": { + "type": "googledrive_file", + "settings_file": "evaluation_examples/settings/googledrive/settings.yml", + "query": [ + "title = 'attachment_full.pdf' and trashed = false and 'root' in parents" + ], + "dest": "attachment_full.pdf" + }, + "expected": { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=1-FaONI6f5g9XyJAlx8vPFgTP_SEZ-1vV&export=download&authuser=0&confirm=t&uuid=b0ad08b7-5b4e-4372-99fa-0952cd096144&at=APZUnTVrZK9_alT_gchTKE6ZYeod:1706194464805", + "dest": "attachment_full_gold.pdf" + } + } +} \ No newline at end of file diff --git a/evaluation_examples/settings/google/settings.json b/evaluation_examples/settings/google/settings.json index a72dd31..ae70605 100644 --- a/evaluation_examples/settings/google/settings.json +++ b/evaluation_examples/settings/google/settings.json @@ -1,4 +1,4 @@ { - "account": "xlang2024anonym@gmail.com", + "email": "xlang2024anonym@gmail.com", "password": "q]wN~0iD>H:6" } \ No newline at end of file diff --git a/evaluation_examples/settings/googledrive/credentials.json b/evaluation_examples/settings/googledrive/credentials.json index 491c002..1174a00 100644 --- a/evaluation_examples/settings/googledrive/credentials.json +++ b/evaluation_examples/settings/googledrive/credentials.json @@ -1 +1 @@ -{"access_token": "ya29.a0AfB_byAHYEB_El-ry3ChNYPLX14Tmq9Nq5j640ghesztCRfoMEPMW79ENOZT8Ab-5wdY7UFxL2ih5l5nhsBf3QENT8W7l7X3QmtQrhyUnDzCfWW2tzYxrNR34iisY0OFWAR9JSfyQPvkpkWQOq0T0QGkBZ8Rg60PVGraNgaCgYKAfQSARISFQHGX2MiTg-uFBVMQnVo8b2H7tuOqw0173", "client_id": "786888752612-6cv6lermep9n6704s4kv20h08lotias9.apps.googleusercontent.com", "client_secret": "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", "refresh_token": "1//0e0qXy4xW1Ud5CgYIARAAGA4SNwF-L9IrWfaomed_CK0R7zZffcpT-GIXf3y2ZjqqAD0UP6UkbaMV9F_OEC6pBVaaX4TYnBKx3os", "token_expiry": "2024-01-25T05:51:17Z", "token_uri": "https://oauth2.googleapis.com/token", "user_agent": null, "revoke_uri": "https://oauth2.googleapis.com/revoke", "id_token": null, "id_token_jwt": null, "token_response": {"access_token": "ya29.a0AfB_byAHYEB_El-ry3ChNYPLX14Tmq9Nq5j640ghesztCRfoMEPMW79ENOZT8Ab-5wdY7UFxL2ih5l5nhsBf3QENT8W7l7X3QmtQrhyUnDzCfWW2tzYxrNR34iisY0OFWAR9JSfyQPvkpkWQOq0T0QGkBZ8Rg60PVGraNgaCgYKAfQSARISFQHGX2MiTg-uFBVMQnVo8b2H7tuOqw0173", "expires_in": 3599, "scope": "https://www.googleapis.com/auth/drive", "token_type": "Bearer"}, "scopes": ["https://www.googleapis.com/auth/drive"], "token_info_uri": "https://oauth2.googleapis.com/tokeninfo", "invalid": false, "_class": "OAuth2Credentials", "_module": "oauth2client.client"} \ No newline at end of file +{"access_token": "ya29.a0AfB_byCwgV4hMbm-I9xpB2y3R5GGCfe63GrdDTE22Wv_upDecjO_Ey86lJtENTD_AWVgpB6iR4aidDsq44sJNOBVIMvZtPOfu1Hib3tiyXjS-P4ID0i0NHi21-ZGaewUe1RYBRp_1klpd5KAttaAZC8R-Tn-uSbhKXI0HwaCgYKAaASARISFQHGX2MiD2nlZt98Zb8FQIBcLc7n-A0173", "client_id": "786888752612-6cv6lermep9n6704s4kv20h08lotias9.apps.googleusercontent.com", "client_secret": "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", "refresh_token": "1//0e0qXy4xW1Ud5CgYIARAAGA4SNwF-L9IrWfaomed_CK0R7zZffcpT-GIXf3y2ZjqqAD0UP6UkbaMV9F_OEC6pBVaaX4TYnBKx3os", "token_expiry": "2024-01-25T16:47:34Z", "token_uri": "https://oauth2.googleapis.com/token", "user_agent": null, "revoke_uri": "https://oauth2.googleapis.com/revoke", "id_token": null, "id_token_jwt": null, "token_response": {"access_token": "ya29.a0AfB_byCwgV4hMbm-I9xpB2y3R5GGCfe63GrdDTE22Wv_upDecjO_Ey86lJtENTD_AWVgpB6iR4aidDsq44sJNOBVIMvZtPOfu1Hib3tiyXjS-P4ID0i0NHi21-ZGaewUe1RYBRp_1klpd5KAttaAZC8R-Tn-uSbhKXI0HwaCgYKAaASARISFQHGX2MiD2nlZt98Zb8FQIBcLc7n-A0173", "expires_in": 3599, "scope": "https://www.googleapis.com/auth/drive", "token_type": "Bearer"}, "scopes": ["https://www.googleapis.com/auth/drive"], "token_info_uri": "https://oauth2.googleapis.com/tokeninfo", "invalid": false, "_class": "OAuth2Credentials", "_module": "oauth2client.client"} \ No newline at end of file diff --git a/evaluation_examples/settings/thunderbird/settings.json b/evaluation_examples/settings/thunderbird/settings.json new file mode 100644 index 0000000..aab6d7d --- /dev/null +++ b/evaluation_examples/settings/thunderbird/settings.json @@ -0,0 +1,5 @@ +{ + "email": "anonym-x2024@outlook.com", + "password": "gTCI\";=@y7|QJ0nDa_kN3Sb&>", + "user": "Anonym Tester" +} \ No newline at end of file From 3e7cfa8699251fb797861d0be00822bd70dfb201 Mon Sep 17 00:00:00 2001 From: tsuky_chen <3107760494@qq.com> Date: Fri, 26 Jan 2024 02:07:26 +0800 Subject: [PATCH 3/6] load libreoffice writer eval -batch 2 --- desktop_env/controllers/setup.py | 1 - desktop_env/envs/desktop_env.py | 4 +- desktop_env/evaluators/getters/__init__.py | 2 +- desktop_env/evaluators/metrics/__init__.py | 6 +- desktop_env/evaluators/metrics/docs.py | 47 +++++++++-- .../2bd59342-0664-4ccb-ba87-79379096cc08.json | 7 +- .../39aa4e37-dc91-482e-99af-132a612d40f3.json | 79 +++++++++++++++++++ ...ce3-2793-461f-ab86-43680ccbae25.json.json} | 0 .../0a0faba3-5580-44df-965d-f562a99b291c.json | 27 ++++++- .../6a33f9b9-0a56-4844-9c3f-96ec3ffb3ba2.json | 30 ++++++- .../6f81754e-285d-4ce0-b59e-af7edb02d108.json | 25 ++++++ .../72b810ef-4156-4d09-8f08-a0cf57e7cefe.json | 30 ++++++- .../8472fece-c7dd-4241-8d65-9b3cd1a0b568.json | 28 ++++++- .../88fe4b2d-3040-4c70-9a70-546a47764b48.json | 27 ++++++- .../b21acd93-60fd-4127-8a43-2f5178f4a830.json | 30 ++++++- .../d53ff5ee-3b1a-431e-b2be-30ed2673079b.json | 27 ++++++- .../e246f6d8-78d7-44ac-b668-fcf47946cb50.json | 28 ++++++- main.py | 6 +- 18 files changed, 362 insertions(+), 42 deletions(-) create mode 100644 evaluation_examples/examples/libreoffice_calc/39aa4e37-dc91-482e-99af-132a612d40f3.json rename evaluation_examples/examples/libreoffice_calc/{a01fbce3-2793-461f-ab86-43680ccbae25.json.nosetup => a01fbce3-2793-461f-ab86-43680ccbae25.json.json} (100%) diff --git a/desktop_env/controllers/setup.py b/desktop_env/controllers/setup.py index 9c6b559..f400606 100644 --- a/desktop_env/controllers/setup.py +++ b/desktop_env/controllers/setup.py @@ -105,7 +105,6 @@ class SetupController: cache_path: str = os.path.join(self.cache_dir, "{:}_{:}".format( uuid.uuid5(uuid.NAMESPACE_URL, url), os.path.basename(path))) - if not url or not path: raise Exception(f"Setup Download - Invalid URL ({url}) or path ({path}).") diff --git a/desktop_env/envs/desktop_env.py b/desktop_env/envs/desktop_env.py index 2f4287e..92539fa 100644 --- a/desktop_env/envs/desktop_env.py +++ b/desktop_env/envs/desktop_env.py @@ -47,8 +47,8 @@ class DesktopEnv(gym.Env): path_to_vm: str, action_space: str = "computer_13", task_config: Dict[str, Any] = None, - tmp_dir: str = "tmp", - cache_dir: str = "cache", + tmp_dir: str = "D:\\NJU\\HKUNLP\\Desktop-Env\\tmp", + cache_dir: str = "D:\\NJU\\HKUNLP\\Desktop-Env\\cache", screen_size: Tuple[int] = (1920, 1080) ): """ diff --git a/desktop_env/evaluators/getters/__init__.py b/desktop_env/evaluators/getters/__init__.py index dd81060..83cd30e 100644 --- a/desktop_env/evaluators/getters/__init__.py +++ b/desktop_env/evaluators/getters/__init__.py @@ -7,4 +7,4 @@ from .misc import get_rule, get_accessibility_tree from .replay import get_replay from .vlc import get_vlc_playing_info, get_vlc_config from .vscode import get_vscode_config -from .impress import get_audio_in_slide +# from .impress import get_audio_in_slide diff --git a/desktop_env/evaluators/metrics/__init__.py b/desktop_env/evaluators/metrics/__init__.py index 3ab04c2..95a5834 100644 --- a/desktop_env/evaluators/metrics/__init__.py +++ b/desktop_env/evaluators/metrics/__init__.py @@ -5,7 +5,7 @@ from .docs import find_default_font, contains_page_break, compare_docx_files, co from .docs import is_first_line_centered, check_file_exists, compare_contains_image from .docs import evaluate_colored_words_in_tables, check_highlighted_words, evaluate_strike_through_last_paragraph, \ evaluate_conversion, evaluate_spacing, check_italic_font_size_14, evaluate_alignment, get_unique_train_ids, \ - check_no_duplicates + check_no_duplicates, compare_init_lines from .general import exact_match, fuzzy_match from .general import check_csv, check_accessibility_tree, run_sqlite3, check_json from .gimp import increase_saturation, decrease_brightness, check_file_exists, compare_triangle_positions @@ -16,7 +16,7 @@ from .pdf import check_pdf_pages #from .table import check_sheet_list, check_xlsx_freeze, check_xlsx_zoom, check_data_validations from .table import compare_table from .thunderbird import check_thunderbird_prefs, check_thunderbird_filter -from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, compare_images, compare_audios, \ - compare_videos +# from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, compare_images, compare_audios, \ + # compare_videos from .thunderbird import check_thunderbird_prefs, check_thunderbird_filter from .vscode import compare_text_file, compare_config, compare_answer, is_extension_installed, check_json_settings, check_json_keybindings diff --git a/desktop_env/evaluators/metrics/docs.py b/desktop_env/evaluators/metrics/docs.py index a3532d0..a45c300 100644 --- a/desktop_env/evaluators/metrics/docs.py +++ b/desktop_env/evaluators/metrics/docs.py @@ -56,15 +56,34 @@ def compare_docx_files(file1, file2): doc2_paragraphs = [p.text for p in doc2.paragraphs] if len(doc1_paragraphs) != len(doc2_paragraphs): + # print(len(doc1_paragraphs)) + # print(len(doc2_paragraphs)) return 0 # Compare each paragraph for p1, p2 in zip(doc1_paragraphs, doc2_paragraphs): if p1 != p2: + # print(p1) + # print(p2) return 0 return 1 +def compare_init_lines(file1, file2): + doc1 = Document(file1) + doc2 = Document(file2) + + doc1_paragraphs = [p.text for p in doc1.paragraphs] + doc2_paragraphs = [p.text for p in doc2.paragraphs] + + # Compare each paragraph + for p1, p2 in zip(doc1_paragraphs, doc2_paragraphs): + if p1 != p2: + # print(p1) + # print(p2) + return 0 + + return 1 def compare_docx_tables(docx_file1, docx_file2): doc1 = Document(docx_file1) @@ -93,6 +112,8 @@ def compare_docx_tables(docx_file1, docx_file2): def compare_line_spacing(docx_file1, docx_file2): + if not compare_docx_files(docx_file1, docx_file2): + return 0 doc1 = Document(docx_file1) doc2 = Document(docx_file2) @@ -200,8 +221,10 @@ def compare_contains_image(docx_file1, docx_file2): # print(find_default_font("Ani", config_path)) -def evaluate_colored_words_in_tables(file_path): - document = Document(file_path) +def evaluate_colored_words_in_tables(file_path1, file_path2): + if not compare_docx_files(file_path1, file_path2): + return 0 + document = Document(file_path1) for table in document.tables: # Iterate through rows and cells in the table @@ -221,8 +244,10 @@ def evaluate_colored_words_in_tables(file_path): return 1 # All words in tables are correctly colored -def check_highlighted_words(file_path): - document = Document(file_path) +def check_highlighted_words(file_path1, file_path2): + if not compare_docx_files(file_path1, file_path2): + return 0 + document = Document(file_path1) for paragraph in document.paragraphs: for run in paragraph.runs: @@ -232,8 +257,10 @@ def check_highlighted_words(file_path): return 1 # No highlighted words found -def evaluate_strike_through_last_paragraph(file_path): - document = Document(file_path) +def evaluate_strike_through_last_paragraph(file_path1, file_path2): + if not compare_docx_files(file_path1, file_path2): + return 0 + document = Document(file_path1) # Get the last paragraph last_paragraph = document.paragraphs[-1] @@ -278,8 +305,10 @@ def evaluate_spacing(file_path): return 0 -def check_italic_font_size_14(path): - document = Document(path) +def check_italic_font_size_14(path1, path2): + if not compare_docx_files(path1, path2): + return 0 + document = Document(path1) for paragraph in document.paragraphs: for run in paragraph.runs: if run.italic: @@ -365,6 +394,8 @@ def compare_docx_lines(file1, file2): doc2 = Document(file2) doc2_lines = [p.text.strip() for p in doc2.paragraphs if p.text.strip()] + # print(doc1_lines) + # print(doc2_lines) # Convert the list of lines to sets and compare return set(doc1_lines) == set(doc2_lines) diff --git a/evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json b/evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json index a37953d..fbf8b33 100644 --- a/evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json +++ b/evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json @@ -19,8 +19,9 @@ "type": "open", "parameters": { "path": "/home/user/OrderId_Month_Chart.xlsx" - ] - }, + } + } + ], "trajectory": "trajectories/2bd59342-0664-4ccb-ba87-79379096cc08", "related_apps": [ "libreoffice calc" @@ -77,4 +78,4 @@ ] } } -} +} \ No newline at end of file diff --git a/evaluation_examples/examples/libreoffice_calc/39aa4e37-dc91-482e-99af-132a612d40f3.json b/evaluation_examples/examples/libreoffice_calc/39aa4e37-dc91-482e-99af-132a612d40f3.json new file mode 100644 index 0000000..0658400 --- /dev/null +++ b/evaluation_examples/examples/libreoffice_calc/39aa4e37-dc91-482e-99af-132a612d40f3.json @@ -0,0 +1,79 @@ +{ + "id": "39aa4e37-dc91-482e-99af-132a612d40f3", + "snapshot": "libreoffice_calc", + "instruction": "Help me rename sheet1 \"LARS_Science_Assessment\"", + "source": "https://www.libreofficehelp.com/add-insert-delete-copy-move-rename-a-worksheet-in-libreoffice-calc/", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1Y9KUHsACajMU1kWz_XjW0VBO3RleDBXw&export=download&authuser=0&confirm=t&uuid=73fe859e-7fe1-449b-83cd-2ddaab067f90&at=APZUnTW-Gp9kn8VhJVCXi5VWxjMT:1706099595303", + "path": "/home/user/LARS_Science_Assessment_Resource_List_7-30-21.xlsx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/LARS_Science_Assessment_Resource_List_7-30-21.xlsx" + } + } + ], + "trajectory": "trajectories/0cecd4f3-74de-457b-ba94-29ad6b5dafb6", + "related_apps": [ + "libreoffice_calc" + ], + "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "LARS_Science_Assessment_Resource_List_7-30-21.xlsx - LibreOffice Calc", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + ] + } + } + ], + "func": "compare_table", + "result": { + "type": "vm_file", + "path": "/home/user/LARS_Science_Assessment_Resource_List_7-30-21.xlsx", + "dest": "LARS_Science_Assessment_Resource_List_7-30-21.xlsx" + }, + "expected": { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=1JC8iKt5_XjhksBDSePi0THwrfM5oF-5A&export=download&authuser=0&confirm=t&uuid=6124298b-1487-4a85-b7d9-d9ba133c26e0&at=APZUnTVfW7o4XruPUft-Dph7um6J:1706099593830", + "dest": "LARS_Science_Assessment_Resource_List_7-30-21_gold.xlsx" + }, + "options": { + "rules": [ + { + "type": "sheet_name" + }, + { + "type": "sheet_data", + "sheet_idx0": 0, + "sheet_idx1": "EI0" + } + ] + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json.nosetup b/evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json.json similarity index 100% rename from evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json.nosetup rename to evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json.json diff --git a/evaluation_examples/examples/libreoffice_writer/0a0faba3-5580-44df-965d-f562a99b291c.json b/evaluation_examples/examples/libreoffice_writer/0a0faba3-5580-44df-965d-f562a99b291c.json index 4a20b1e..3ce8e7f 100644 --- a/evaluation_examples/examples/libreoffice_writer/0a0faba3-5580-44df-965d-f562a99b291c.json +++ b/evaluation_examples/examples/libreoffice_writer/0a0faba3-5580-44df-965d-f562a99b291c.json @@ -27,7 +27,32 @@ "libreoffice_writer" ], "evaluator": { - "func": "compare_docx_files", + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "04 CHIN9505 EBook Purchasing info 2021 Jan.docx - LibreOffice Writer", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')" + ] + } + } + ], + "func": "compare_init_lines", "expected": { "type": "cloud_file", "path": "https://drive.google.com/uc?id=1yyHGj8KUHDMsZmc1QeJ1KkvSEGy83jMR&export=download", diff --git a/evaluation_examples/examples/libreoffice_writer/6a33f9b9-0a56-4844-9c3f-96ec3ffb3ba2.json b/evaluation_examples/examples/libreoffice_writer/6a33f9b9-0a56-4844-9c3f-96ec3ffb3ba2.json index a25712b..3280355 100644 --- a/evaluation_examples/examples/libreoffice_writer/6a33f9b9-0a56-4844-9c3f-96ec3ffb3ba2.json +++ b/evaluation_examples/examples/libreoffice_writer/6a33f9b9-0a56-4844-9c3f-96ec3ffb3ba2.json @@ -9,7 +9,7 @@ "parameters": { "files": [ { - "url": "https://drive.usercontent.google.com/download?id=10hgB73d_DoQXQVgUjvgXFUCP1Hd9YxDb&export=download&authuser=0&confirm=t&uuid=df2bb3c3-e75e-4cbc-a6bc-24120512a4e1&at=APZUnTUak54ZfgQDNxbt_PqBvNQu:1706017722797", + "url": "https://drive.usercontent.google.com/download?id=10hgB73d_DoQXQVgUjvgXFUCP1Hd9YxDb&export=download&authuser=0&confirm=t&uuid=845f9616-2fb7-476a-abab-8b620d482ac2&at=APZUnTXB71PxHF7Dq9TC2OL_cRLm:1706199789147", "path": "Desktop/sample-recruitment-phone-script.docx" } ] @@ -27,10 +27,32 @@ "libreoffice_writer" ], "evaluator": { - "func": [ - "compare_docx_files", - "check_highlighted_words" + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "sample-recruitment-phone-script.docx - LibreOffice Writer", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')" + ] + } + } ], + "func": "check_highlighted_words", "expected": { "type": "cloud_file", "path": "https://drive.usercontent.google.com/download?id=1s9Dsy66-zxbCAgeTyCh0P7AT7P4jF6o3&export=download&authuser=0&confirm=t&uuid=1239f2a1-8c86-45a4-8e7d-36388ac22a69&at=APZUnTVZQzXQAMNsKKQzOw5ppT8A:1706017721589", diff --git a/evaluation_examples/examples/libreoffice_writer/6f81754e-285d-4ce0-b59e-af7edb02d108.json b/evaluation_examples/examples/libreoffice_writer/6f81754e-285d-4ce0-b59e-af7edb02d108.json index 9bb14b7..aaa6dae 100644 --- a/evaluation_examples/examples/libreoffice_writer/6f81754e-285d-4ce0-b59e-af7edb02d108.json +++ b/evaluation_examples/examples/libreoffice_writer/6f81754e-285d-4ce0-b59e-af7edb02d108.json @@ -27,6 +27,31 @@ "libreoffice_writer" ], "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "HK_train_record.docx - LibreOffice Writer", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')" + ] + } + } + ], "func": "compare_docx_lines", "result": { "type": "vm_file", diff --git a/evaluation_examples/examples/libreoffice_writer/72b810ef-4156-4d09-8f08-a0cf57e7cefe.json b/evaluation_examples/examples/libreoffice_writer/72b810ef-4156-4d09-8f08-a0cf57e7cefe.json index c21d2ad..b0b0532 100644 --- a/evaluation_examples/examples/libreoffice_writer/72b810ef-4156-4d09-8f08-a0cf57e7cefe.json +++ b/evaluation_examples/examples/libreoffice_writer/72b810ef-4156-4d09-8f08-a0cf57e7cefe.json @@ -18,7 +18,7 @@ { "type": "open", "parameters": { - "path": "GEOG2169_Course_Outline_2022-23.docx" + "path": "Desktop/GEOG2169_Course_Outline_2022-23.docx" } } ], @@ -27,10 +27,32 @@ "libreoffice_writer" ], "evaluator": { - "func": [ - "evaluate_strike_through_last_paragraph", - "compare_docx_files" + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "GEOG2169_Course_Outline_2022-23.docx - LibreOffice Writer", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')" + ] + } + } ], + "func": "evaluate_strike_through_last_paragraph", "expected": { "type": "cloud_file", "path": "https://drive.google.com/uc?id=1IpAnQRYo1whrnzIGyo8UldZf4Tli-yVT&export=download", diff --git a/evaluation_examples/examples/libreoffice_writer/8472fece-c7dd-4241-8d65-9b3cd1a0b568.json b/evaluation_examples/examples/libreoffice_writer/8472fece-c7dd-4241-8d65-9b3cd1a0b568.json index 055f65e..b941cbc 100644 --- a/evaluation_examples/examples/libreoffice_writer/8472fece-c7dd-4241-8d65-9b3cd1a0b568.json +++ b/evaluation_examples/examples/libreoffice_writer/8472fece-c7dd-4241-8d65-9b3cd1a0b568.json @@ -27,10 +27,32 @@ "libreoffice_writer" ], "evaluator": { - "func": [ - "evaluate_colored_words_in_tables", - "compare_docx_files" + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "Dolch_Sight_Words_Primer.docx - LibreOffice Writer", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')" + ] + } + } ], + "func": "evaluate_colored_words_in_tables", "expected": { "type": "cloud_file", "path": "https://drive.google.com/uc?id=1ksn444K17lFOdm5pELrQYvuZHkOsKq69&export=download", diff --git a/evaluation_examples/examples/libreoffice_writer/88fe4b2d-3040-4c70-9a70-546a47764b48.json b/evaluation_examples/examples/libreoffice_writer/88fe4b2d-3040-4c70-9a70-546a47764b48.json index 9a75fcf..863fd45 100644 --- a/evaluation_examples/examples/libreoffice_writer/88fe4b2d-3040-4c70-9a70-546a47764b48.json +++ b/evaluation_examples/examples/libreoffice_writer/88fe4b2d-3040-4c70-9a70-546a47764b48.json @@ -18,7 +18,7 @@ { "type": "open", "parameters": { - "path": "CCCH9003_Tutorial_guidelines.docx" + "path": "Desktop/CCCH9003_Tutorial_guidelines.docx" } } ], @@ -27,6 +27,31 @@ "libreoffice_writer" ], "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "CCCH9003_Tutorial_guidelines.docx - LibreOffice Writer", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')" + ] + } + } + ], "func": "compare_docx_files", "expected": { "type": "cloud_file", diff --git a/evaluation_examples/examples/libreoffice_writer/b21acd93-60fd-4127-8a43-2f5178f4a830.json b/evaluation_examples/examples/libreoffice_writer/b21acd93-60fd-4127-8a43-2f5178f4a830.json index 6464762..50a3cfd 100644 --- a/evaluation_examples/examples/libreoffice_writer/b21acd93-60fd-4127-8a43-2f5178f4a830.json +++ b/evaluation_examples/examples/libreoffice_writer/b21acd93-60fd-4127-8a43-2f5178f4a830.json @@ -18,7 +18,7 @@ { "type": "open", "parameters": { - "path": "CCHU9045_Course_Outline_2019-20.docx" + "path": "Desktop/CCHU9045_Course_Outline_2019-20.docx" } } ], @@ -27,10 +27,32 @@ "libreoffice_writer" ], "evaluator": { - "func": [ - "compare_line_spacing", - "compare_docx_files" + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "CCHU9045_Course_Outline_2019-20.docx - LibreOffice Writer", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')" + ] + } + } ], + "func": "compare_line_spacing", "expected": { "type": "cloud_file", "path": "https://drive.google.com/uc?id=16LN7uYSSXk_xwgc4IZXnN2Z1nCmPJfLm&export=download", diff --git a/evaluation_examples/examples/libreoffice_writer/d53ff5ee-3b1a-431e-b2be-30ed2673079b.json b/evaluation_examples/examples/libreoffice_writer/d53ff5ee-3b1a-431e-b2be-30ed2673079b.json index 3200a06..de4479b 100644 --- a/evaluation_examples/examples/libreoffice_writer/d53ff5ee-3b1a-431e-b2be-30ed2673079b.json +++ b/evaluation_examples/examples/libreoffice_writer/d53ff5ee-3b1a-431e-b2be-30ed2673079b.json @@ -18,7 +18,7 @@ { "type": "open", "parameters": { - "path": "presentation_instruction_2023_Feb.docx" + "path": "Desktop/presentation_instruction_2023_Feb.docx" } } ], @@ -28,6 +28,31 @@ ], "evaluator": { "func": "compare_docx_files", + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "presentation_instruction_2023_Feb.docx - LibreOffice Writer", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')" + ] + } + } + ], "expected": { "type": "cloud_file", "path": "https://drive.google.com/uc?id=1bB1N2TWN0puZ6DwUFS_TDjvRWchaGP9T&export=download", diff --git a/evaluation_examples/examples/libreoffice_writer/e246f6d8-78d7-44ac-b668-fcf47946cb50.json b/evaluation_examples/examples/libreoffice_writer/e246f6d8-78d7-44ac-b668-fcf47946cb50.json index 462bee9..bce074c 100644 --- a/evaluation_examples/examples/libreoffice_writer/e246f6d8-78d7-44ac-b668-fcf47946cb50.json +++ b/evaluation_examples/examples/libreoffice_writer/e246f6d8-78d7-44ac-b668-fcf47946cb50.json @@ -27,10 +27,32 @@ "libreoffice_writer" ], "evaluator": { - "func": [ - "check_italic_font_size_14", - "compare_docx_files" + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "Y22-2119-assign4.docx - LibreOffice Writer", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')" + ] + } + } ], + "func": "check_italic_font_size_14", "expected": { "type": "cloud_file", "path": "https://drive.google.com/uc?id=1GTZ-DkMxpdYx38z_s0ab85Ejgxv3qfEp&export=download", diff --git a/main.py b/main.py index d4c3dc1..2c98071 100644 --- a/main.py +++ b/main.py @@ -46,12 +46,12 @@ def human_agent(): Runs the Gym environment with human input. """ - with open("evaluation_examples/examples/vs_code/59ed65c7-e9a6-43db-833f-76d6730c0004.json", "r") as f: + with open("evaluation_examples/examples/libreoffice_writer/72b810ef-4156-4d09-8f08-a0cf57e7cefe.json", "r") as f: example = json.load(f) - example["snapshot"] = "vscode_setup" + example["snapshot"] = "base18" env = DesktopEnv( - path_to_vm=r"C:\Users\tianbaox\Documents\Virtual Machines\Ubuntu\Ubuntu.vmx", + path_to_vm=r"D:\Ubuntu\Ubuntu\Ubuntu.vmx", action_space="computer_13", task_config=example ) From f0b073989a317e783018e28aa638596b9d04a8c8 Mon Sep 17 00:00:00 2001 From: tsuky_chen <3107760494@qq.com> Date: Fri, 26 Jan 2024 02:21:04 +0800 Subject: [PATCH 4/6] feat:fix --- desktop_env/evaluators/getters/__init__.py | 2 +- desktop_env/evaluators/metrics/__init__.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/desktop_env/evaluators/getters/__init__.py b/desktop_env/evaluators/getters/__init__.py index 59952c9..faa9f2e 100644 --- a/desktop_env/evaluators/getters/__init__.py +++ b/desktop_env/evaluators/getters/__init__.py @@ -9,5 +9,5 @@ from .misc import get_rule, get_accessibility_tree from .replay import get_replay from .vlc import get_vlc_playing_info, get_vlc_config from .vscode import get_vscode_config -# from .impress import get_audio_in_slide +from .impress import get_audio_in_slide diff --git a/desktop_env/evaluators/metrics/__init__.py b/desktop_env/evaluators/metrics/__init__.py index 40fdecd..f368699 100644 --- a/desktop_env/evaluators/metrics/__init__.py +++ b/desktop_env/evaluators/metrics/__init__.py @@ -22,8 +22,8 @@ from .slides import check_presenter_console_disable, check_image_stretch_and_cen from .table import compare_table from .thunderbird import check_thunderbird_prefs, check_thunderbird_filter -# from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, compare_images, compare_audios, \ - # compare_videos +from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, compare_images, compare_audios, \ + compare_videos from .thunderbird import check_thunderbird_prefs, check_thunderbird_filter from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, compare_images, compare_audios, \ compare_videos, check_qt_bgcone, check_one_instance_when_started_from_file, check_qt_minimal_view, \ From a30f07c5d12012b434507bbe7a82eec9f8796bcf Mon Sep 17 00:00:00 2001 From: Timothyxxx <384084775@qq.com> Date: Fri, 26 Jan 2024 12:39:15 +0800 Subject: [PATCH 5/6] Revert the path --- desktop_env/envs/desktop_env.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/desktop_env/envs/desktop_env.py b/desktop_env/envs/desktop_env.py index b008fd1..ef7a1ad 100644 --- a/desktop_env/envs/desktop_env.py +++ b/desktop_env/envs/desktop_env.py @@ -47,8 +47,8 @@ class DesktopEnv(gym.Env): path_to_vm: str, action_space: str = "computer_13", task_config: Dict[str, Any] = None, - tmp_dir: str = "D:\\NJU\\HKUNLP\\Desktop-Env\\tmp", - cache_dir: str = "D:\\NJU\\HKUNLP\\Desktop-Env\\cache", + tmp_dir: str = "tmp", + cache_dir: str = "cache", screen_size: Tuple[int] = (1920, 1080) ): """ From 0d05add4329dd2616cf5a527ba02825a0ee61267 Mon Sep 17 00:00:00 2001 From: David Chang Date: Fri, 26 Jan 2024 12:46:43 +0800 Subject: [PATCH 6/6] ver Jan26th fixed path of trajectory in cacl/39aa4e37 --- .../39aa4e37-dc91-482e-99af-132a612d40f3.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/evaluation_examples/examples/libreoffice_calc/39aa4e37-dc91-482e-99af-132a612d40f3.json b/evaluation_examples/examples/libreoffice_calc/39aa4e37-dc91-482e-99af-132a612d40f3.json index 0658400..ed98840 100644 --- a/evaluation_examples/examples/libreoffice_calc/39aa4e37-dc91-482e-99af-132a612d40f3.json +++ b/evaluation_examples/examples/libreoffice_calc/39aa4e37-dc91-482e-99af-132a612d40f3.json @@ -22,7 +22,7 @@ } } ], - "trajectory": "trajectories/0cecd4f3-74de-457b-ba94-29ad6b5dafb6", + "trajectory": "trajectories/39aa4e37-dc91-482e-99af-132a612d40f3", "related_apps": [ "libreoffice_calc" ], @@ -76,4 +76,4 @@ ] } } -} \ No newline at end of file +}