From ab71ebb2ba6e6e0165f399166da6129e381693dc Mon Sep 17 00:00:00 2001 From: Timothyxxx <384084775@qq.com> Date: Thu, 4 Jan 2024 17:05:17 +0800 Subject: [PATCH] Initialize VLC getters and metrics, fix some bugs in infra logic, needs to be refactored later on --- desktop_env/controllers/python.py | 19 +++++- desktop_env/controllers/setup.py | 66 ++++++++++++------- desktop_env/envs/desktop_env.py | 3 +- desktop_env/evaluators/getters/__init__.py | 1 + desktop_env/evaluators/getters/file.py | 5 +- desktop_env/evaluators/metrics/__init__.py | 3 +- .../59f21cfb-0120-4326-b255-a5b827b38967.json | 36 +++++++++- 7 files changed, 101 insertions(+), 32 deletions(-) diff --git a/desktop_env/controllers/python.py b/desktop_env/controllers/python.py index 2b0dfb7..5469800 100644 --- a/desktop_env/controllers/python.py +++ b/desktop_env/controllers/python.py @@ -80,7 +80,8 @@ class PythonController: y = parameters["y"] if "num_clicks" in parameters: num_clicks = parameters["num_clicks"] - self.execute_python_command(f"pyautogui.click(button='{button}', x={x}, y={y}, clicks={num_clicks})") + self.execute_python_command( + f"pyautogui.click(button='{button}', x={x}, y={y}, clicks={num_clicks})") else: self.execute_python_command(f"pyautogui.click(button='{button}', x={x}, y={y})") elif "button" in parameters and "x" not in parameters and "y" not in parameters: @@ -143,7 +144,8 @@ class PythonController: if "x" in parameters and "y" in parameters: x = parameters["x"] y = parameters["y"] - self.execute_python_command(f"pyautogui.dragTo({x}, {y}, duration=1.0, button='left', mouseDownUp=True)") + self.execute_python_command( + f"pyautogui.dragTo({x}, {y}, duration=1.0, button='left', mouseDownUp=True)") elif action_type == "SCROLL": # todo: check if it is related to the operating system, as https://github.com/TheDuckAI/DuckTrack/blob/main/ducktrack/playback.py pointed out @@ -206,3 +208,16 @@ class PythonController: else: raise Exception(f"Unknown action type: {action_type}") + + + def get_vlc_status(self, host='localhost', port=8080, password='password'): + url = f'http://{host}:{port}/requests/status.xml' + + response = requests.get(url, auth=('', password)) + + if response.status_code == 200: + print("File downloaded successfully") + return response.content + else: + print("Failed to get vlc status. Status code:", response.status_code) + return None diff --git a/desktop_env/controllers/setup.py b/desktop_env/controllers/setup.py index e28287b..2320e99 100644 --- a/desktop_env/controllers/setup.py +++ b/desktop_env/controllers/setup.py @@ -10,12 +10,11 @@ from typing import Any class SetupController: - def __init__( self - , http_server: str - , cache_dir: str - ): - self.http_server = http_server + "/setup" + def __init__(self, http_server: str, cache_dir: str): + self.http_server: str = http_server + self.http_server_setup_root = http_server + "/setup" self.cache_dir: str = cache_dir + def reset_cache_dir(self, cache_dir: str): self.cache_dir = cache_dir @@ -48,6 +47,32 @@ class SetupController: # self._open_setup(config) # can add other setup steps + def _command_setup(self, command: str): + """ + Directly send a command into the virtual machine os for setting up. + """ + payload = json.dumps({"command": command}) + headers = { + 'Content-Type': 'application/json' + } + timeout = 5 + timout_whitelist = ["vlc"] + + try: + + response = requests.post(self.http_server + "/execute", headers=headers, data=payload, timeout=timeout) + if response.status_code == 200: + print("Command executed successfully:", response.text) + else: + print("Failed to execute command. Status code:", response.status_code) + except requests.exceptions.Timeout as e: + if command in timout_whitelist: + print("Command executed successfully:", command) + else: + print("An error occurred while trying to execute the command:", e) + except requests.exceptions.RequestException as e: + print("An error occurred while trying to execute the command:", e) + def _download_setup(self, files: List[Dict[str, str]]): """ Args: @@ -66,12 +91,9 @@ class SetupController: for f in files: url: str = f["url"] path: str = f["path"] - cache_path: str = os.path.join( self.cache_dir - , "{:}_{:}".format( - uuid.uuid5(uuid.NAMESPACE_URL, url) - , os.path.basename(path) - ) - ) + cache_path: str = os.path.join(self.cache_dir, "{:}_{:}".format( + uuid.uuid5(uuid.NAMESPACE_URL, url), + os.path.basename(path))) if not url or not path: raise Exception(f"Setup Download - Invalid URL ({url}) or path ({path}).") @@ -97,21 +119,21 @@ class SetupController: if not downloaded: raise requests.RequestException(f"Failed to download {url}. No retries left. Error: {e}") - #payload = json.dumps({"url": url, "path": path}) - #headers = { - #'Content-Type': 'application/json' - #} + # payload = json.dumps({"url": url, "path": path}) + # headers = { + # 'Content-Type': 'application/json' + # } - form = MultipartEncoder( { "file_path": path - , "file_data": (os.path.basename(path), open(cache_path, "rb")) - } - ) + form = MultipartEncoder({ + "file_path": path, + "file_data": (os.path.basename(path), open(cache_path, "rb")) + }) headers = {"Content-Type": form.content_type} print(form.content_type) # send request to server to upload file try: - response = requests.post(self.http_server + "/upload", headers=headers, data=form) + response = requests.post(self.http_server_setup_root + "/upload", headers=headers, data=form) if response.status_code == 200: print("Command executed successfully:", response.text) else: @@ -136,7 +158,7 @@ class SetupController: # send request to server to change wallpaper try: - response = requests.post(self.http_server + "/change_wallpaper", headers=headers, data=payload) + response = requests.post(self.http_server_setup_root + "/change_wallpaper", headers=headers, data=payload) if response.status_code == 200: print("Command executed successfully:", response.text) else: @@ -163,7 +185,7 @@ class SetupController: # send request to server to open file try: - response = requests.post(self.http_server + "/open_file", headers=headers, data=payload) + response = requests.post(self.http_server_setup_root + "/open_file", headers=headers, data=payload) if response.status_code == 200: print("Command executed successfully:", response.text) else: diff --git a/desktop_env/envs/desktop_env.py b/desktop_env/envs/desktop_env.py index 03086ff..c953f3c 100644 --- a/desktop_env/envs/desktop_env.py +++ b/desktop_env/envs/desktop_env.py @@ -85,7 +85,8 @@ class DesktopEnv(gym.Env): # Initialize emulator and controller print("Initializing...") self._start_emulator() - self.host = f"http://{self._get_vm_ip()}:5000" + self.vm_ip = self._get_vm_ip() + self.host = f"http://{self.vm_ip}:5000" self.controller = PythonController(http_server=self.host) self.setup_controller = SetupController(http_server=self.host, cache_dir=self.cache_dir) diff --git a/desktop_env/evaluators/getters/__init__.py b/desktop_env/evaluators/getters/__init__.py index 81a23fd..770c30c 100644 --- a/desktop_env/evaluators/getters/__init__.py +++ b/desktop_env/evaluators/getters/__init__.py @@ -1,2 +1,3 @@ from .file import get_cloud_file, get_vm_file from .misc import get_rule +from .vlc import get_vlc_playing_info diff --git a/desktop_env/evaluators/getters/file.py b/desktop_env/evaluators/getters/file.py index a9be430..25fd081 100644 --- a/desktop_env/evaluators/getters/file.py +++ b/desktop_env/evaluators/getters/file.py @@ -3,6 +3,7 @@ from typing import Dict import os import requests + def get_cloud_file(env, config: Dict[str, str]) -> str: """ Config: @@ -25,6 +26,7 @@ def get_cloud_file(env, config: Dict[str, str]) -> str: return _path + def get_vm_file(env, config: Dict[str, str]) -> str: """ Config: @@ -33,12 +35,9 @@ def get_vm_file(env, config: Dict[str, str]) -> str: """ _path = os.path.join(env.cache_dir, config["dest"]) - if os.path.exists(_path): - return _path file = env.controller.get_file(config["path"]) with open(_path, "wb") as f: f.write(file) return _path - diff --git a/desktop_env/evaluators/metrics/__init__.py b/desktop_env/evaluators/metrics/__init__.py index 498df17..80a26cf 100644 --- a/desktop_env/evaluators/metrics/__init__.py +++ b/desktop_env/evaluators/metrics/__init__.py @@ -2,4 +2,5 @@ from .table import compare_table from .table import check_sheet_list, check_xlsx_freeze, check_zoom from .docs import find_default_font, contains_page_break, compare_docx_files, compare_docx_tables, compare_line_spacing, compare_insert_equation from .docs import compare_font_names, compare_subscript_contains, has_page_numbers_in_footers -from .docs import is_first_line_centered, check_file_exists, compare_contains_image \ No newline at end of file +from .docs import is_first_line_centered, check_file_exists, compare_contains_image +from .vlc import is_vlc_playing diff --git a/evaluation_examples/examples/vlc/59f21cfb-0120-4326-b255-a5b827b38967.json b/evaluation_examples/examples/vlc/59f21cfb-0120-4326-b255-a5b827b38967.json index 512427e..fabd42b 100644 --- a/evaluation_examples/examples/vlc/59f21cfb-0120-4326-b255-a5b827b38967.json +++ b/evaluation_examples/examples/vlc/59f21cfb-0120-4326-b255-a5b827b38967.json @@ -1,12 +1,42 @@ { "id": "59f21cfb-0120-4326-b255-a5b827b38967", "snapshot": "base_setup", - "instruction": "Could you help me play the file at FILE_PATH?", + "instruction": "Play the music video on my desktop", "source": "https://docs.videolan.me/vlc-user/desktop/3.0/en/basic/media.html#playing-a-file", - "config": [], + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=14-vhVMVw53e0l-MDVBFbngFAE1jMqvgm&export=download&authuser=0&confirm=t&uuid=d31607ed-0075-4fe5-b68c-b24b6eec356e&at=APZUnTV0Wy0672VFGrQChgHmd1Ba:1704337791613", + "path": "Desktop/Rick Astley - Never Gonna Give You Up (Official Music Video).mp4" + } + ] + } + }, + { + "type": "command", + "parameters": { + "command": "vlc" + } + } + ], "trajectory": "trajectories/", "related_apps": [ "vlc" ], - "evaluator": "evaluation_dir" + "evaluator": { + "func": "is_vlc_playing", + "expected": { + "type": "rule", + "rules": { + "file_path": "Desktop/Rick Astley - Never Gonna Give You Up (Official Music Video).mp4" + } + }, + "result": { + "type": "vlc_playing_info", + "dest": "status.xml" + } + } }