diff --git a/README.md b/README.md
index 815c35d..96f1da2 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,8 @@
-# DesktopEnv: A Learning Environment for Human-like Computer Task Mastery
+# DesktopEnv: An Environment towards Human-like Computer Task Mastery
## Setup guide
+### For members of the team
1. Download OS image
1. Download kubuntu from
2. Download ubuntu from
@@ -22,7 +23,8 @@
2. `rm -rf ~/screenshot.png`
7. Set up python and install [mouse](https://github.com/boppreh/mouse/) and [keyboard](https://github.com/jordansissel/xdotool)
-
+### For users of the environment
+todo
## Road map (Proposed)
diff --git a/SERVER_SETUP.md b/SERVER_SETUP.md
index f67c01e..231bd9e 100644
--- a/SERVER_SETUP.md
+++ b/SERVER_SETUP.md
@@ -1,23 +1,6 @@
# Server Setup Guide
-- [Linux](#linux)
-- [Windows](#windows)
-
-## Linux
-
-
-
-1. `sudo apt install openssh-server`
-2. `sudo systemctl enable ssh --now`
-3. `sudo ufw disable` (disable firewall - safe for local network, otherwise `sudo ufw allow ssh`)
-4. `ip a` - find ip address
-5. ssh username@
-6. On host, run `ssh-copy-id @`
-
-
-## Windows
-
-1. Copy and paste the file `windows_server/main.py` to the windows vm
-2. Make sure `mouse` and `keyboard` are installed
+1. Copy and paste the file `server/main.py` to the windows vm
+2. Install the requirements `pip install -r requirements.txt`
3. Run the file `python main.py`
4. `ipconfig /all` and find the ip address
\ No newline at end of file
diff --git a/desktop_env/assets/cursor.png b/desktop_env/assets/cursor.png
new file mode 100644
index 0000000..8d9182a
Binary files /dev/null and b/desktop_env/assets/cursor.png differ
diff --git a/desktop_env/controllers/keyboard.py b/desktop_env/controllers/keyboard.py
deleted file mode 100644
index 5178491..0000000
--- a/desktop_env/controllers/keyboard.py
+++ /dev/null
@@ -1,56 +0,0 @@
-from abc import ABC, abstractmethod
-from fabric import Connection
-
-from .xdotool import XDoToolController
-from .python import PythonController
-
-class AbstractKeyboardController(ABC):
- @abstractmethod
- def type(self, text: str):
- raise NotImplementedError
-
- @abstractmethod
- def key(self, key: str):
- raise NotImplementedError
-
- @abstractmethod
- def key_down(self, key: str):
- raise NotImplementedError
-
- @abstractmethod
- def key_up(self, key: str):
- raise NotImplementedError
-
-
-class XDoToolKeyboardController(AbstractKeyboardController, XDoToolController):
- def __init__(self, ssh_connection: Connection):
- super().__init__(ssh_connection=ssh_connection)
-
- def type(self, text: str):
- self._execute_xdotool_command(f"type {text}")
-
- def key(self, key: str):
- self._execute_xdotool_command(f"key {key}")
-
- def key_down(self, key: str):
- self._execute_xdotool_command(f"keydown {key}")
-
- def key_up(self, key: str):
- self._execute_xdotool_command(f"keyup {key}")
-
-class PythonKeyboardController(AbstractKeyboardController, PythonController):
- def __init__(self, http_server: str):
- super().__init__(http_server=http_server)
- self.command = "python -c \"import keyboard; {command}\""
-
- def type(self, text: str):
- self._execute_python_command(self.command.format(command=f"keyboard.write('{text}')"))
-
- def key(self, key: str):
- self._execute_python_command(self.command.format(command=f"keyboard.press_and_release('{key}')"))
-
- def key_down(self, key: str):
- self._execute_python_command(self.command.format(command=f"keyboard.press('{key}')"))
-
- def key_up(self, key: str):
- self._execute_python_command(self.command.format(command=f"keyboard.release('{key}')"))
diff --git a/desktop_env/controllers/mouse.py b/desktop_env/controllers/mouse.py
deleted file mode 100644
index 45961be..0000000
--- a/desktop_env/controllers/mouse.py
+++ /dev/null
@@ -1,144 +0,0 @@
-from enum import Enum
-
-from abc import ABC, abstractmethod
-from fabric import Connection
-
-from .xdotool import XDoToolController
-from .python import PythonController
-class MouseClick(Enum):
- LEFT = 1
- MIDDLE = 2
- RIGHT = 3
- WHEEL_UP = 4
- WHEEL_DOWN = 5
-
-class AbstractMouseController(ABC):
- @abstractmethod
- def mouse_move(self, x: int, y: int):
- raise NotImplementedError
-
- @abstractmethod
- def left_down(self):
- raise NotImplementedError
-
- @abstractmethod
- def left_up(self):
- raise NotImplementedError
-
- @abstractmethod
- def left_click(self):
- raise NotImplementedError
-
- @abstractmethod
- def middle_down(self):
- raise NotImplementedError
-
- @abstractmethod
- def middle_up(self):
- raise NotImplementedError
-
- @abstractmethod
- def middle_click(self):
- raise NotImplementedError
-
- @abstractmethod
- def right_down(self):
- raise NotImplementedError
-
- @abstractmethod
- def right_up(self):
- raise NotImplementedError
-
- @abstractmethod
- def right_click(self):
- raise NotImplementedError
-
- @abstractmethod
- def scroll_up(self):
- raise NotImplementedError
-
- @abstractmethod
- def scroll_down(self):
- raise NotImplementedError
-
-
-class XDoToolMouseController(AbstractMouseController, XDoToolController):
- def __init__(self, ssh_connection: Connection):
- super().__init__(ssh_connection=ssh_connection)
-
- def mouse_move(self, x: int, y: int):
- self._execute_xdotool_command(f"mousemove {x} {y}")
-
- def left_down(self):
- self._execute_xdotool_command(f"mousedown 1")
-
- def left_up(self):
- self._execute_xdotool_command(f"mouseup 1")
-
- def left_click(self):
- self._execute_xdotool_command(f"click 1")
-
- def middle_down(self):
- self._execute_xdotool_command(f"mousedown 2")
-
- def middle_up(self):
- self._execute_xdotool_command(f"mouseup 2")
-
- def middle_click(self):
- self._execute_xdotool_command(f"click 2")
-
- def right_down(self):
- self._execute_xdotool_command(f"mousedown 3")
-
- def right_up(self):
- self._execute_xdotool_command(f"mouseup 3")
-
- def right_click(self):
- self._execute_xdotool_command(f"click 3")
-
- def scroll_up(self):
- self._execute_xdotool_command(f"click 4")
-
- def scroll_down(self):
- self._execute_xdotool_command(f"click 5")
-
-class PythonMouseController(AbstractMouseController, PythonController):
- def __init__(self, http_server: str):
- super().__init__(http_server=http_server)
- self.command = "python -c \"import mouse; {command}\""
-
- def mouse_move(self, x: int, y: int):
- self._execute_python_command(self.command.format(command=f"mouse.move({x}, {y})"))
-
- def left_down(self):
- self._execute_python_command(self.command.format(command="mouse.press(button='left')"))
-
- def left_up(self):
- self._execute_python_command(self.command.format(command="mouse.release(button='left')"))
-
- def left_click(self):
- self._execute_python_command(self.command.format(command="mouse.click(button='left')"))
-
- def middle_down(self):
- self._execute_python_command(self.command.format(command="mouse.press(button='middle')"))
-
- def middle_up(self):
- self._execute_python_command(self.command.format(command="mouse.release(button='middle')"))
-
- def middle_click(self):
- self._execute_python_command(self.command.format(command="mouse.click(button='middle')"))
-
- def right_down(self):
- self._execute_python_command(self.command.format(command="mouse.press(button='right')"))
-
- def right_up(self):
- self._execute_python_command(self.command.format(command="mouse.release(button='right')"))
-
- def right_click(self):
- self._execute_python_command(self.command.format(command="mouse.click(button='right')"))
-
- def scroll_up(self):
- self._execute_python_command(self.command.format(command="mouse.wheel(10)"))
-
- def scroll_down(self):
- self._execute_python_command(self.command.format(command="mouse.wheel(-10)"))
\ No newline at end of file
diff --git a/desktop_env/controllers/python.py b/desktop_env/controllers/python.py
index 6b5e627..2b0dfb7 100644
--- a/desktop_env/controllers/python.py
+++ b/desktop_env/controllers/python.py
@@ -1,34 +1,208 @@
-import requests
import json
+from typing import Any, Dict
+import requests
+from desktop_env.envs.actions import KEYBOARD_KEYS
+
class PythonController:
- def __init__(self, http_server: str):
+ def __init__(self, http_server: str, pkgs_prefix: str = "python -c \"import pyautogui; {command}\""):
self.http_server = http_server
-
- def _execute_python_command(self, command: str) -> None:
- payload = json.dumps({
- "command": command
- })
+ self.pkgs_prefix = pkgs_prefix # fixme: this is a hacky way to execute python commands. fix it and combine it with installation of packages
+
+ def get_screenshot(self):
+ """
+ Gets a screenshot from the server. With the cursor.
+ """
+ response = requests.get(self.http_server + "/screenshot")
+ if response.status_code == 200:
+ return response.content
+ else:
+ print("Failed to get screenshot. Status code:", response.status_code)
+ return None
+
+ def get_file(self, file_path: str):
+ """
+ Gets a file from the server.
+ """
+ response = requests.post(self.http_server + "/file", data={"file_path": file_path})
+ if response.status_code == 200:
+ print("File downloaded successfully")
+ return response.content
+ else:
+ print("Failed to get file. Status code:", response.status_code)
+ return None
+
+ def execute_python_command(self, command: str) -> None:
+ """
+ Executes a python command on the server.
+ It can be used to execute the pyautogui commands, or... any other python command. who knows?
+ """
+ command = self.pkgs_prefix.format(command=command)
+ payload = json.dumps({"command": command})
headers = {
'Content-Type': 'application/json'
}
-
+
try:
response = requests.post(self.http_server + "/execute", headers=headers, data=payload)
if response.status_code == 200:
print("Command executed successfully:", response.text)
else:
print("Failed to execute command. Status code:", response.status_code)
+ return response.json()
except requests.exceptions.RequestException as e:
print("An error occurred while trying to execute the command:", e)
-# example usage
-if __name__ == '__main__':
- # replace with your actual server URL of the vm
- server_url = "http://192.168.7.129:5000"
- controller = PythonController(server_url)
+ def execute_action(self, action: Dict[str, Any]):
+ """
+ Executes an action on the server computer.
+ """
- # example commands
- python_command = "python -c \"import keyboard; keyboard.write('hello world')\""
- python_command = "python -c \"import mouse; mouse.move(100,100);mouse.right_click()\""
- controller._execute_python_command(python_command)
+ action_type = action["action_type"]
+ parameters = action["parameters"] if "parameters" in action else {}
+
+ if action_type == "MOVE_TO":
+ if parameters == {} or None:
+ self.execute_python_command(f"pyautogui.moveTo()")
+ elif "x" in parameters and "y" in parameters:
+ x = parameters["x"]
+ y = parameters["y"]
+ self.execute_python_command(f"pyautogui.moveTo({x}, {y})")
+ else:
+ raise Exception(f"Unknown parameters: {parameters}")
+
+ elif action_type == "CLICK":
+ if parameters == {} or None:
+ self.execute_python_command(f"pyautogui.click()")
+ elif "button" in parameters and "x" in parameters and "y" in parameters:
+ button = parameters["button"]
+ x = parameters["x"]
+ y = parameters["y"]
+ if "num_clicks" in parameters:
+ num_clicks = parameters["num_clicks"]
+ self.execute_python_command(f"pyautogui.click(button='{button}', x={x}, y={y}, clicks={num_clicks})")
+ else:
+ self.execute_python_command(f"pyautogui.click(button='{button}', x={x}, y={y})")
+ elif "button" in parameters and "x" not in parameters and "y" not in parameters:
+ button = parameters["button"]
+ if "num_clicks" in parameters:
+ num_clicks = parameters["num_clicks"]
+ self.execute_python_command(f"pyautogui.click(button='{button}', clicks={num_clicks})")
+ else:
+ self.execute_python_command(f"pyautogui.click(button='{button}')")
+ elif "button" not in parameters and "x" in parameters and "y" in parameters:
+ x = parameters["x"]
+ y = parameters["y"]
+ if "num_clicks" in parameters:
+ num_clicks = parameters["num_clicks"]
+ self.execute_python_command(f"pyautogui.click(x={x}, y={y}, clicks={num_clicks})")
+ else:
+ self.execute_python_command(f"pyautogui.click(x={x}, y={y})")
+ else:
+ raise Exception(f"Unknown parameters: {parameters}")
+
+ elif action_type == "MOUSE_DOWN":
+ if parameters == {} or None:
+ self.execute_python_command(f"pyautogui.mouseDown()")
+ elif "button" in parameters:
+ button = parameters["button"]
+ self.execute_python_command(f"pyautogui.mouseDown(button='{button}')")
+ else:
+ raise Exception(f"Unknown parameters: {parameters}")
+
+ elif action_type == "MOUSE_UP":
+ if parameters == {} or None:
+ self.execute_python_command(f"pyautogui.mouseUp()")
+ elif "button" in parameters:
+ button = parameters["button"]
+ self.execute_python_command(f"pyautogui.mouseUp(button='{button}')")
+ else:
+ raise Exception(f"Unknown parameters: {parameters}")
+
+ elif action_type == "RIGHT_CLICK":
+ if parameters == {} or None:
+ self.execute_python_command(f"pyautogui.rightClick()")
+ elif "x" in parameters and "y" in parameters:
+ x = parameters["x"]
+ y = parameters["y"]
+ self.execute_python_command(f"pyautogui.rightClick(x={x}, y={y})")
+ else:
+ raise Exception(f"Unknown parameters: {parameters}")
+
+ elif action_type == "DOUBLE_CLICK":
+ if parameters == {} or None:
+ self.execute_python_command(f"pyautogui.doubleClick()")
+ elif "x" in parameters and "y" in parameters:
+ x = parameters["x"]
+ y = parameters["y"]
+ self.execute_python_command(f"pyautogui.doubleClick(x={x}, y={y})")
+ else:
+ raise Exception(f"Unknown parameters: {parameters}")
+
+ elif action_type == "DRAG_TO":
+ if "x" in parameters and "y" in parameters:
+ x = parameters["x"]
+ y = parameters["y"]
+ self.execute_python_command(f"pyautogui.dragTo({x}, {y}, duration=1.0, button='left', mouseDownUp=True)")
+
+ elif action_type == "SCROLL":
+ # todo: check if it is related to the operating system, as https://github.com/TheDuckAI/DuckTrack/blob/main/ducktrack/playback.py pointed out
+ if "dx" in parameters and "dy" in parameters:
+ dx = parameters["dx"]
+ dy = parameters["dy"]
+ self.execute_python_command(f"pyautogui.hscroll({dx})")
+ self.execute_python_command(f"pyautogui.vscroll({dy})")
+ elif "dx" in parameters and "dy" not in parameters:
+ dx = parameters["dx"]
+ self.execute_python_command(f"pyautogui.hscroll({dx})")
+ elif "dx" not in parameters and "dy" in parameters:
+ dy = parameters["dy"]
+ self.execute_python_command(f"pyautogui.vscroll({dy})")
+ else:
+ raise Exception(f"Unknown parameters: {parameters}")
+
+ elif action_type == "TYPING":
+ if "text" not in parameters:
+ raise Exception(f"Unknown parameters: {parameters}")
+ text = parameters["text"]
+ self.execute_python_command(f"pyautogui.typewrite('{text}')")
+
+ elif action_type == "PRESS":
+ if "key" not in parameters:
+ raise Exception(f"Unknown parameters: {parameters}")
+ key = parameters["key"]
+ if key.lower() not in KEYBOARD_KEYS:
+ raise Exception(f"Key must be one of {KEYBOARD_KEYS}")
+ self.execute_python_command(f"pyautogui.press('{key}')")
+
+ elif action_type == "KEY_DOWN":
+ if "key" not in parameters:
+ raise Exception(f"Unknown parameters: {parameters}")
+ key = parameters["key"]
+ if key.lower() not in KEYBOARD_KEYS:
+ raise Exception(f"Key must be one of {KEYBOARD_KEYS}")
+ self.execute_python_command(f"pyautogui.keyDown('{key}')")
+
+ elif action_type == "KEY_UP":
+ if "key" not in parameters:
+ raise Exception(f"Unknown parameters: {parameters}")
+ key = parameters["key"]
+ if key.lower() not in KEYBOARD_KEYS:
+ raise Exception(f"Key must be one of {KEYBOARD_KEYS}")
+ self.execute_python_command(f"pyautogui.keyUp('{key}')")
+
+ elif action_type == "HOTKEY":
+ if "keys" not in parameters:
+ raise Exception(f"Unknown parameters: {parameters}")
+ keys = parameters["keys"]
+ if not isinstance(keys, list):
+ raise Exception(f"Keys must be a list of keys")
+ for key in keys:
+ if key.lower() not in KEYBOARD_KEYS:
+ raise Exception(f"Key must be one of {KEYBOARD_KEYS}")
+
+ keys_para_rep = "', '".join(keys)
+ self.execute_python_command(f"pyautogui.hotkey('{keys_para_rep}')")
+
+ else:
+ raise Exception(f"Unknown action type: {action_type}")
diff --git a/desktop_env/controllers/setup.py b/desktop_env/controllers/setup.py
new file mode 100644
index 0000000..1b3e581
--- /dev/null
+++ b/desktop_env/controllers/setup.py
@@ -0,0 +1,96 @@
+import requests
+import json
+
+
+class SetupController:
+ def __init__(self, http_server: str):
+ self.http_server = http_server + "/setup"
+
+ def setup(self, config):
+ """
+ Setup Config:
+ {
+ download: list[tuple[string]], # a list of tuples of url of file to download and the save path
+ ...
+ }
+ """
+ self._download_setup(config)
+ self._change_wallpaper(config)
+ # self._tidy_desktop(config) todo: implement this
+ self._open_setup(config)
+ # can add other setup steps
+
+ def _download_setup(self, config):
+ if not config:
+ return
+ if not 'download' in config:
+ return
+ for url, path in config['download']:
+ if not url or not path:
+ raise Exception(f"Setup Download - Invalid URL ({url}) or path ({path}).")
+
+ payload = json.dumps({"url": url, "path": path})
+ headers = {
+ 'Content-Type': 'application/json'
+ }
+
+ # send request to server to download file
+ try:
+ response = requests.post(self.http_server + "/download_file", headers=headers, data=payload)
+ if response.status_code == 200:
+ print("Command executed successfully:", response.text)
+ else:
+ print("Failed to download file. Status code:", response.text)
+ except requests.exceptions.RequestException as e:
+ print("An error occurred while trying to send the request:", e)
+
+ def _change_wallpaper(self, config):
+ if not config:
+ return
+ if not 'wallpaper' in config:
+ return
+ path = config['wallpaper']
+ if not path:
+ raise Exception(f"Setup Wallpaper - Invalid path ({path}).")
+
+ payload = json.dumps({"path": path})
+ headers = {
+ 'Content-Type': 'application/json'
+ }
+
+ # send request to server to change wallpaper
+ try:
+ response = requests.post(self.http_server + "/change_wallpaper", headers=headers, data=payload)
+ if response.status_code == 200:
+ print("Command executed successfully:", response.text)
+ else:
+ print("Failed to change wallpaper. Status code:", response.text)
+ except requests.exceptions.RequestException as e:
+ print("An error occurred while trying to send the request:", e)
+
+ def _tidy_desktop(self, config):
+ raise NotImplementedError
+
+ def _open_setup(self, config):
+ if not config:
+ return
+ if not 'open' in config:
+ return
+ for path in config['open']:
+ if not path:
+ raise Exception(f"Setup Open - Invalid path ({path}).")
+
+ payload = json.dumps({"path": path})
+ headers = {
+ 'Content-Type': 'application/json'
+ }
+
+ # send request to server to open file
+ try:
+ response = requests.post(self.http_server + "/open_file", headers=headers, data=payload)
+ if response.status_code == 200:
+ print("Command executed successfully:", response.text)
+ else:
+ print("Failed to open file. Status code:", response.text)
+ except requests.exceptions.RequestException as e:
+ print("An error occurred while trying to send the request:", e)
diff --git a/desktop_env/controllers/xdotool.py b/desktop_env/controllers/xdotool.py
deleted file mode 100644
index abb268f..0000000
--- a/desktop_env/controllers/xdotool.py
+++ /dev/null
@@ -1,11 +0,0 @@
-from fabric import Connection
-from typing import List
-
-
-class XDoToolController:
- def __init__(self, ssh_connection: Connection):
- self.ssh_connection = ssh_connection
-
- def _execute_xdotool_command(self, command: List[str]) -> None:
- result = self.ssh_connection.run(f"DISPLAY=:0 xdotool {command}", hide=True)
- return result.stdout.strip()
diff --git a/desktop_env/envs/actions.py b/desktop_env/envs/actions.py
new file mode 100644
index 0000000..e03ccf0
--- /dev/null
+++ b/desktop_env/envs/actions.py
@@ -0,0 +1,190 @@
+X_MAX = 1920 # TODO: get the screen resolution
+Y_MAX = 1080
+
+KEYBOARD_KEYS = ['\t', '\n', '\r', ' ', '!', '"', '#', '$', '%', '&', "'", '(', ')', '*', '+', ',', '-', '.', '/', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ':', ';', '<', '=', '>', '?', '@', '[', '\\', ']', '^', '_', '`', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '{', '|', '}', '~', 'accept', 'add', 'alt', 'altleft', 'altright', 'apps', 'backspace', 'browserback', 'browserfavorites', 'browserforward', 'browserhome', 'browserrefresh', 'browsersearch', 'browserstop', 'capslock', 'clear', 'convert', 'ctrl', 'ctrlleft', 'ctrlright', 'decimal', 'del', 'delete', 'divide', 'down', 'end', 'enter', 'esc', 'escape', 'execute', 'f1', 'f10', 'f11', 'f12', 'f13', 'f14', 'f15', 'f16', 'f17', 'f18', 'f19', 'f2', 'f20', 'f21', 'f22', 'f23', 'f24', 'f3', 'f4', 'f5', 'f6', 'f7', 'f8', 'f9', 'final', 'fn', 'hanguel', 'hangul', 'hanja', 'help', 'home', 'insert', 'junja', 'kana', 'kanji', 'launchapp1', 'launchapp2', 'launchmail', 'launchmediaselect', 'left', 'modechange', 'multiply', 'nexttrack', 'nonconvert', 'num0', 'num1', 'num2', 'num3', 'num4', 'num5', 'num6', 'num7', 'num8', 'num9', 'numlock', 'pagedown', 'pageup', 'pause', 'pgdn', 'pgup', 'playpause', 'prevtrack', 'print', 'printscreen', 'prntscrn', 'prtsc', 'prtscr', 'return', 'right', 'scrolllock', 'select', 'separator', 'shift', 'shiftleft', 'shiftright', 'sleep', 'stop', 'subtract', 'tab', 'up', 'volumedown', 'volumemute', 'volumeup', 'win', 'winleft', 'winright', 'yen', 'command', 'option', 'optionleft', 'optionright']
+
+ACTION_SPACE = [
+ {
+ "action_type": "MOVE_TO",
+ "note": "move the cursor to the specified position",
+ "parameters": {
+ "x": {
+ "type": float,
+ "range": [0, X_MAX],
+ "optional": False,
+ },
+ "y": {
+ "type": float,
+ "range": [0, Y_MAX],
+ "optional": False,
+ }
+ }
+ },
+ {
+ "action_type": "CLICK",
+ "note": "click the left button if the button not specified, otherwise click the specified button; click at the current position if x and y are not specified, otherwise click at the specified position",
+ "parameters": {
+ "button": {
+ "type": str,
+ "range": ["left", "right", "middle"],
+ "optional": True,
+ },
+ "x": {
+ "type": float,
+ "range": [0, X_MAX],
+ "optional": True,
+ },
+ "y": {
+ "type": float,
+ "range": [0, Y_MAX],
+ "optional": True,
+ },
+ "num_clicks": {
+ "type": int,
+ "range": [1, 2, 3],
+ "optional": True,
+ },
+ }
+ },
+ {
+ "action_type": "MOUSE_DOWN",
+ "note": "press the left button if the button not specified, otherwise press the specified button",
+ "parameters": {
+ "button": {
+ "type": str,
+ "range": ["left", "right", "middle"],
+ "optional": True,
+ }
+ }
+ },
+ {
+ "action_type": "MOUSE_UP",
+ "note": "release the left button if the button not specified, otherwise release the specified button",
+ "parameters": {
+ "button": {
+ "type": str,
+ "range": ["left", "right", "middle"],
+ "optional": True,
+ }
+ }
+ },
+ {
+ "action_type": "RIGHT_CLICK",
+ "note": "right click at the current position if x and y are not specified, otherwise right click at the specified position",
+ "parameters": {
+ "x": {
+ "type": float,
+ "range": [0, X_MAX],
+ "optional": True,
+ },
+ "y": {
+ "type": float,
+ "range": [0, Y_MAX],
+ "optional": True,
+ }
+ }
+ },
+ {
+ "action_type": "DOUBLE_CLICK",
+ "note": "double click at the current position if x and y are not specified, otherwise double click at the specified position",
+ "parameters": {
+ "x": {
+ "type": float,
+ "range": [0, X_MAX],
+ "optional": True,
+ },
+ "y": {
+ "type": float,
+ "range": [0, Y_MAX],
+ "optional": True,
+ }
+ }
+ },
+ {
+ "action_type": "DRAG_TO",
+ "note": "drag the cursor to the specified position with the left button pressed",
+ "parameters": {
+ "x": {
+ "type": float,
+ "range": [0, X_MAX],
+ "optional": False,
+ },
+ "y": {
+ "type": float,
+ "range": [0, Y_MAX],
+ "optional": False,
+ }
+ }
+ },
+ {
+ "action_type": "SCROLL",
+ "note": "scroll the mouse wheel up or down",
+ "parameters": {
+ "dx": {
+ "type": int,
+ "range": None,
+ "optional": False,
+ },
+ "dy": {
+ "type": int,
+ "range": None,
+ "optional": False,
+ }
+ }
+ },
+ {
+ "action_type": "TYPING",
+ "note": "type the specified text",
+ "parameters": {
+ "text": {
+ "type": str,
+ "range": None,
+ "optional": False,
+ }
+ }
+ },
+ {
+ "action_type": "PRESS",
+ "note": "press the specified key and release it",
+ "parameters": {
+ "key": {
+ "type": str,
+ "range": KEYBOARD_KEYS,
+ "optional": False,
+ }
+ }
+ },
+ {
+ "action_type": "KEY_DOWN",
+ "note": "press the specified key",
+ "parameters": {
+ "key": {
+ "type": str,
+ "range": KEYBOARD_KEYS,
+ "optional": False,
+ }
+ }
+ },
+ {
+ "action_type": "KEY_UP",
+ "note": "release the specified key",
+ "parameters": {
+ "key": {
+ "type": str,
+ "range": KEYBOARD_KEYS,
+ "optional": False,
+ }
+ }
+ },
+ {
+ "action_type": "HOTKEY",
+ "note": "press the specified key combination",
+ "parameters": {
+ "keys": {
+ "type": list,
+ "range": [KEYBOARD_KEYS],
+ "optional": False,
+ }
+ }
+ }
+]
diff --git a/desktop_env/envs/desktop_env.py b/desktop_env/envs/desktop_env.py
index 962d176..183411d 100644
--- a/desktop_env/envs/desktop_env.py
+++ b/desktop_env/envs/desktop_env.py
@@ -1,78 +1,61 @@
-from enum import Enum
-from typing import Literal, List, Tuple
+from __future__ import annotations
+
+import os
import subprocess
-from fabric import Connection
import time
+import uuid
+import platform
+from typing import List
import gymnasium as gym
-from gymnasium import spaces
-import numpy as np
-from PIL import Image
+import requests
-from desktop_env.controllers.mouse import MouseClick, AbstractMouseController, XDoToolMouseController, PythonMouseController
-from desktop_env.controllers.keyboard import AbstractKeyboardController, XDoToolKeyboardController, PythonKeyboardController
-
-class Action(Enum):
- CLICK = 0
- MOUSE_DOWN = 1
- MOUSE_UP = 2
- MOUSE_MOVE = 3
- KEY = 4
- KEY_DOWN = 5
- KEY_UP = 6
- TYPE = 7
+from desktop_env.controllers.python import PythonController
+from desktop_env.controllers.setup import SetupController
+from desktop_env.evaluators import eval_funcs
-VM_TYPE = Literal['ubuntu', 'windows']
+def _execute_command(command: List[str]) -> None:
+ if command[:4] == ["vmrun", "-T", "ws", "start"]:
+ p = subprocess.Popen(command)
+ p.wait()
+ else:
+ result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60, text=True)
+ if result.returncode != 0:
+ raise Exception("\033[91m" + result.stdout + result.stderr + "\033[0m")
+ return result.stdout
class DesktopEnv(gym.Env):
"""DesktopEnv with OpenAI Gym interface."""
- def __init__(self, path_to_vm: str, username: str, password: str,
- host: str, snapshot_path: str = "some_point_browser", vm_os: VM_TYPE = "ubuntu"):
+ def __init__(
+ self,
+ path_to_vm: str,
+ snapshot_path: str = "base",
+ instruction: str = None,
+ config: dict = None,
+ evaluator: dict = None,
+ action_space: str = "computer_13",
+ ):
+ # Initialize environment variables
self.path_to_vm = path_to_vm
- self.username = username
- self.password = password
- self.host = host
self.snapshot_path = snapshot_path # todo: handling the logic of snapshot directory
- self.screen_width = 800
- self.screen_height = 800
- # Define the action and observation space
- self.action_space = spaces.Dict({
- "action_type": spaces.Discrete(len(Action)),
- "click_type": spaces.Discrete(len(MouseClick)),
- "x": spaces.Discrete(self.screen_width),
- "y": spaces.Discrete(self.screen_height),
- "key": spaces.MultiDiscrete([128] * 10), # max 10 characters, ASCII
- "text": spaces.MultiDiscrete([128] * 10) # max 10 characters, ASCII
- })
-
- self.observation_space = spaces.Box(low=0, high=255, shape=(self.screen_width, self.screen_height, 3), dtype=np.uint8)
-
- # Additional setup
- self.metadata = {'render.modes': ['rgb_array']}
-
- # Initialize emulator
+ # Initialize emulator and controller
print("Initializing...")
self._start_emulator()
+ self.host = f"http://{self._get_vm_ip()}:5000"
+ self.controller = PythonController(http_server=self.host)
+ self.setup_controller = SetupController(http_server=self.host)
+ self.instruction = instruction
+ self.config = config
+ self.evaluator = evaluator
- # set up controllers
- self.mouse_controller, self.keyboard_controller = self._create_controllers(vm_os)
-
- def _create_controllers(self, vm_os: VM_TYPE) -> Tuple[AbstractMouseController, AbstractKeyboardController]:
- if vm_os == "ubuntu":
- ssh_connection = Connection(host=self.host, user=self.username, connect_kwargs={"password": self.password})
- mouse_controller = XDoToolMouseController(ssh_connection)
- keyboard_controller = XDoToolKeyboardController(ssh_connection)
- elif vm_os == "windows":
- mouse_controller = PythonMouseController(http_server=self.host)
- keyboard_controller = PythonKeyboardController(http_server=self.host)
- else:
- raise NotImplementedError(vm_os)
-
- return mouse_controller, keyboard_controller
+ # mode: human or machine
+ assert action_space in ["computer_13", "pyautogui"]
+ self.action_space = action_space
+ # todo: define the action space and the observation space as gym did, or extend theirs
def _start_emulator(self):
while True:
@@ -84,108 +67,120 @@ class DesktopEnv(gym.Env):
break
else:
print("Starting VM...")
- self._execute_command(["vmrun", "-T", "ws", "start", self.path_to_vm])
- time.sleep(5)
+ _execute_command(["vmrun", "-T", "ws", "start", self.path_to_vm])
+ time.sleep(3)
except subprocess.CalledProcessError as e:
print(f"Error executing command: {e.output.decode().strip()}")
- def _execute_command(self, command: List[str]) -> None:
- process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
- stdout, stderr = process.communicate()
- if process.returncode != 0:
- print(f"Error executing command: {command}")
- return None
- else:
- return stdout.decode()
+ def _get_vm_ip(self):
+ max_retries = 10
+ print("Getting IP Address...")
+ for _ in range(max_retries):
+ try:
+ output = _execute_command(["vmrun", "-T", "ws", "getGuestIPAddress", self.path_to_vm]).strip()
+ print(f"IP address: {output}")
+ return output
+ except:
+ time.sleep(5)
+ print("Retrying...")
+ raise Exception("Failed to get VM IP address!")
def _save_state(self):
- self._execute_command(["vmrun", "-T", "ws" "snapshot", self.path_to_vm, self.snapshot_path])
+ _execute_command(["vmrun", "-T", "ws" "snapshot", self.path_to_vm, self.snapshot_path])
def _get_screenshot(self):
- image_path = "./screenshot.png"
- self._execute_command(
- ["vmrun", "-T", "ws", "-gu", self.username, "-gp", self.password, "captureScreen", self.path_to_vm,
- image_path])
+ random_uuid = str(uuid.uuid4())
+ os.makedirs(os.path.join("tmp", random_uuid), exist_ok=True)
+ image_path = os.path.join("tmp", random_uuid, "screenshot.png")
+
+ # Get the screenshot and save to the image_path
+ screenshot = self.controller.get_screenshot()
+ with open(image_path, "wb") as f:
+ f.write(screenshot)
+
return image_path
def _get_obs(self):
screenshot_image_path = self._get_screenshot()
- with Image.open(screenshot_image_path) as img:
- return np.array(img)
+ return screenshot_image_path
- def reset(self):
+ def reset(self, seed=None, options=None):
print("Resetting environment...")
print("Reverting to snapshot to {}...".format(self.snapshot_path))
- self._execute_command(["vmrun", "-T", "ws", "revertToSnapshot", self.path_to_vm, self.snapshot_path])
+ _execute_command(["vmrun", "-T", "ws", "revertToSnapshot", self.path_to_vm, self.snapshot_path])
+ time.sleep(5)
print("Starting emulator...")
self._start_emulator()
print("Emulator started.")
+ print("Setting up environment...")
+ self.setup_controller.setup(self.config)
+
+ time.sleep(5)
+ print("Environment setup complete.")
+
observation = self._get_obs()
return observation
- def step(self, action):
- action_type = Action(action['action_type'])
- if action_type == Action.CLICK:
- click = MouseClick(action['click_type'])
- if click == MouseClick.LEFT:
- self.mouse_controller.left_click()
- elif click == MouseClick.MIDDLE:
- self.mouse_controller.middle_click()
- elif click == MouseClick.RIGHT:
- self.mouse_controller.right_click()
- elif click == MouseClick.WHEEL_UP:
- self.mouse_controller.scroll_up()
- elif click == MouseClick.WHEEL_DOWN:
- self.mouse_controller.scroll_down()
- elif action_type == Action.MOUSE_DOWN:
- click = MouseClick(action['click_type'])
- if click == MouseClick.LEFT:
- self.mouse_controller.left_down()
- elif click == MouseClick.MIDDLE:
- self.mouse_controller.middle_down()
- elif click == MouseClick.RIGHT:
- self.mouse_controller.right_down()
- elif click == MouseClick.WHEEL_UP:
- self.mouse_controller.scroll_up()
- elif click == MouseClick.WHEEL_DOWN:
- self.mouse_controller.scroll_down()
- elif action_type == Action.MOUSE_UP:
- click = MouseClick(action['click_type'])
- if click == MouseClick.LEFT:
- self.mouse_controller.left_up()
- elif click == MouseClick.MIDDLE:
- self.mouse_controller.middle_up()
- elif click == MouseClick.RIGHT:
- self.mouse_controller.right_up()
- elif click == MouseClick.WHEEL_UP:
- self.mouse_controller.scroll_up()
- elif click == MouseClick.WHEEL_DOWN:
- self.mouse_controller.scroll_down()
- elif action_type == Action.MOUSE_MOVE:
- self.mouse_controller.mouse_move(x = action['x'], y = action['y'])
- elif action_type == Action.KEY:
- key_sequence = ''.join(map(chr, action['key'])) # Convert integer array to string
- self.keyboard_controller.key(key_sequence)
- elif action_type == Action.KEY_DOWN:
- key_sequence = ''.join(map(chr, action['key'])) # Convert integer array to string
- self.keyboard_controller.key_down(key_sequence)
- elif action_type == Action.KEY_UP:
- key_sequence = ''.join(map(chr, action['key'])) # Convert integer array to string
- self.keyboard_controller.key_up(key_sequence)
- elif action_type == Action.TYPE:
- text = ''.join(map(chr, action['text'])) # Convert integer array to string
- self.keyboard_controller.type(text)
+ def step(self, action, pause=0.5):
+ # fixme: add reminding logic here, decide if the action is valid for the current action_space
+ if self.action_space == "computer_13":
+ # the set of all possible actions defined in the action representation
+ self.controller.execute_action(action)
+ elif self.action_space == "pyautogui":
+ # the set of all possible python commands insides `pyautogui`
+ self.controller.execute_python_command(action)
- # Capture new state
- observation = self._get_obs()
- reward = 0 # Define reward calculation
- done = False # Define episode termination condition
+ # todo: maybe for the better here we need to add a logic to wait until the rendering is done
+ time.sleep(pause)
+ observation = {
+ "screenshot": self._get_obs(),
+ "instruction": self.instruction
+ }
+ reward = 0 # todo: Define reward calculation for each example
+ done = False # todo: Define episode termination condition for each example
info = {}
return observation, reward, done, info
+ def evaluate(self):
+ """
+ Evaluate whether the task is successfully completed.
+ """
+ def copy_file_to_local(_file_info):
+ random_uuid = str(uuid.uuid4())
+ os.makedirs(os.path.join("tmp", random_uuid), exist_ok=True)
+ _path = os.path.join("tmp", random_uuid, "tmp.xlsx")
+ if _file_info["type"] == "cloud_file":
+ url = _file_info["path"]
+ response = requests.get(url, stream=True)
+ response.raise_for_status()
+
+ with open(_path, 'wb') as f:
+ for chunk in response.iter_content(chunk_size=8192):
+ if chunk:
+ f.write(chunk)
+ elif _file_info["type"] == "vm_file":
+ # fixme: stream this part maybe as well
+ file = self.controller.get_file(_file_info["path"])
+ with open(_path, "wb") as f:
+ f.write(file)
+ else:
+ raise NotImplementedError
+
+ return _path
+
+ # todo: make this more flexible by refactoring
+ eval_func = eval_funcs[self.evaluator["func"]]
+ eval_func_vars = {}
+
+ for var_name, file_info in self.evaluator["paths"].items():
+ path = copy_file_to_local(file_info)
+ eval_func_vars[var_name] = path
+
+ return eval_func(**eval_func_vars)
+
def render(self, mode='rgb_array'):
if mode == 'rgb_array':
return self._get_obs()
@@ -193,4 +188,4 @@ class DesktopEnv(gym.Env):
raise ValueError('Unsupported render mode: {}'.format(mode))
def close(self):
- self._execute_command(["vmrun", "stop", self.path_to_vm])
+ _execute_command(["vmrun", "stop", self.path_to_vm])
diff --git a/desktop_env/evaluators/__init__.py b/desktop_env/evaluators/__init__.py
new file mode 100644
index 0000000..32a19e7
--- /dev/null
+++ b/desktop_env/evaluators/__init__.py
@@ -0,0 +1,5 @@
+from .table import compare_table
+
+eval_funcs = {
+ "compare_table(expected, actual)": compare_table
+}
diff --git a/desktop_env/evaluators/replay.py b/desktop_env/evaluators/replay.py
new file mode 100644
index 0000000..e69de29
diff --git a/desktop_env/evaluators/table.py b/desktop_env/evaluators/table.py
new file mode 100644
index 0000000..bd49297
--- /dev/null
+++ b/desktop_env/evaluators/table.py
@@ -0,0 +1,14 @@
+def compare_table(expected, actual):
+ import pandas as pd
+ df1 = pd.read_excel(expected)
+ df2 = pd.read_excel(actual)
+
+ # Compare the DataFrames
+ return 1 if df1.equals(df2) else 0
+
+
+if __name__ == '__main__':
+ path1 = ""
+ path2 = ""
+
+ print(compare_table(path1, path2))
diff --git a/desktop_env/server/main.py b/desktop_env/server/main.py
new file mode 100644
index 0000000..8bc1323
--- /dev/null
+++ b/desktop_env/server/main.py
@@ -0,0 +1,184 @@
+import os
+from pathlib import Path
+import platform
+import subprocess
+import requests
+
+import Xlib.display
+import pyautogui
+from PIL import ImageGrab, Image
+from flask import Flask, request, jsonify, send_file
+
+app = Flask(__name__)
+
+pyautogui.PAUSE = 0
+pyautogui.DARWIN_CATCH_UP_TIME = 0
+
+
+@app.route('/execute', methods=['POST'])
+def execute_command():
+ data = request.json
+ # The 'command' key in the JSON request should contain the command to be executed.
+ command = data.get('command', '')
+
+ # Execute the command without any safety checks.
+ try:
+ result = subprocess.run(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+ return jsonify({
+ 'status': 'success',
+ 'output': result.stdout,
+ 'error': result.stderr
+ })
+ except Exception as e:
+ return jsonify({
+ 'status': 'error',
+ 'message': str(e)
+ }), 500
+
+
+@app.route('/screenshot', methods=['GET'])
+def capture_screen_with_cursor():
+ file_path = os.path.join("screenshots", "screenshot.png")
+ user_platform = platform.system()
+
+ # Ensure the screenshots directory exists
+ os.makedirs(os.path.dirname(file_path), exist_ok=True)
+
+ if user_platform == "Windows":
+ def _download_image(url, path):
+ response = requests.get(url)
+ with open(path, 'wb') as file:
+ file.write(response.content)
+
+ cursor_path = os.path.join("screenshots", "cursor.png")
+ if not os.path.exists(cursor_path):
+ cursor_url = "https://vip.helloimg.com/images/2023/12/02/oQPzmt.png"
+ _download_image(cursor_url, cursor_path)
+ screenshot = pyautogui.screenshot()
+ cursor_x, cursor_y = pyautogui.position()
+ cursor = Image.open(cursor_path)
+ screenshot.paste(cursor, (cursor_x, cursor_y), cursor)
+ screenshot.save(file_path)
+ elif user_platform == "Linux":
+ # Use xlib to prevent scrot dependency for Linux
+ screen = Xlib.display.Display().screen()
+ size = screen.width_in_pixels, screen.height_in_pixels
+ screenshot = ImageGrab.grab(bbox=(0, 0, size[0], size[1]))
+ screenshot.save(file_path)
+ elif user_platform == "Darwin": # (Mac OS)
+ # Use the screencapture utility to capture the screen with the cursor
+ subprocess.run(["screencapture", "-C", file_path])
+ else:
+ print(f"The platform you're using ({user_platform}) is not currently supported")
+
+ return send_file(file_path, mimetype='image/png')
+
+
+@app.route('/file', methods=['POST'])
+def get_file():
+ # Retrieve filename from the POST request
+ if 'file_path' in request.form:
+ file_path = request.form['file_path']
+ else:
+ return jsonify({"error": "file_path is required"}), 400
+
+ try:
+ # Check if the file exists and send it to the user
+ return send_file(file_path, as_attachment=True)
+ except FileNotFoundError:
+ # If the file is not found, return a 404 error
+ return jsonify({"error": "File not found"}), 404
+
+
+@app.route('/platform', methods=['GET'])
+def get_platform():
+ return platform.system()
+
+
+@app.route('/cursor_position', methods=['GET'])
+def get_cursor_position():
+ return pyautogui.position().x, pyautogui.position().y
+
+
+@app.route("/setup/change_wallpaper", methods=['POST'])
+def change_wallpaper():
+ data = request.json
+ path = data.get('path', None)
+
+ if not path:
+ return "Path not supplied!", 400
+
+ path = Path(path)
+
+ if not path.exists():
+ return f"File not found: {path}", 404
+
+ try:
+ user_platform = platform.system()
+ if user_platform == "Windows":
+ import ctypes
+ ctypes.windll.user32.SystemParametersInfoW(20, 0, str(path), 3)
+ elif user_platform == "Linux":
+ import subprocess
+ subprocess.run(["gsettings", "set", "org.gnome.desktop.background", "picture-uri", f"file://{path}"])
+ elif user_platform == "Darwin": # (Mac OS)
+ import subprocess
+ subprocess.run(
+ ["osascript", "-e", f'tell application "Finder" to set desktop picture to POSIX file "{path}"'])
+ return "Wallpaper changed successfully"
+ except Exception as e:
+ return f"Failed to change wallpaper. Error: {e}", 500
+
+
+@app.route("/setup/download_file", methods=['POST'])
+def download_file():
+ data = request.json
+ url = data.get('url', None)
+ path = data.get('path', None)
+
+ if not url or not path:
+ return "Path or URL not supplied!", 400
+
+ path = Path(path)
+ path.parent.mkdir(parents=True, exist_ok=True)
+
+ max_retries = 3
+ for i in range(max_retries):
+ try:
+ response = requests.get(url, stream=True)
+ response.raise_for_status()
+
+ with open(path, 'wb') as f:
+ for chunk in response.iter_content(chunk_size=8192):
+ if chunk:
+ f.write(chunk)
+ return "File downloaded successfully"
+
+ except requests.RequestException as e:
+ print(f"Failed to download {url}. Retrying... ({max_retries - i - 1} attempts left)")
+
+ return f"Failed to download {url}. No retries left. Error: {e}", 500
+
+
+@app.route("/setup/open_file", methods=['POST'])
+def open_file():
+ data = request.json
+ path = data.get('path', None)
+
+ if not path:
+ return "Path not supplied!", 400
+
+ path = Path(path)
+
+ if not path.exists():
+ return f"File not found: {path}", 404
+
+ try:
+ os.startfile(path)
+ return "File opened successfully"
+ except Exception as e:
+ return f"Failed to open {path}. Error: {e}", 500
+
+
+if __name__ == '__main__':
+ app.run(debug=True, host="0.0.0.0")
diff --git a/desktop_env/server/requirements.txt b/desktop_env/server/requirements.txt
new file mode 100644
index 0000000..f4cb1ab
--- /dev/null
+++ b/desktop_env/server/requirements.txt
@@ -0,0 +1,5 @@
+python3-xlib==0.15
+PyAutoGUI==0.9.54
+Pillow==10.1.0
+git+https://github.com/moses-palmer/pynput.git@refs/pull/541/head # to make sure that it works on Apple Silicon
+requests
diff --git a/desktop_env/windows_server/main.py b/desktop_env/windows_server/main.py
deleted file mode 100644
index 56b7fc2..0000000
--- a/desktop_env/windows_server/main.py
+++ /dev/null
@@ -1,29 +0,0 @@
-from flask import Flask, request, jsonify
-import subprocess
-
-app = Flask(__name__)
-
-@app.route('/execute', methods=['POST'])
-def execute_command():
- data = request.json
- # The 'command' key in the JSON request should contain the command to be executed.
- command = data.get('command', '')
-
- # Execute the command without any safety checks.
- try:
- process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- stdout, stderr = process.communicate()
-
- return jsonify({
- 'status': 'success',
- 'output': stdout.decode(),
- 'error': stderr.decode()
- })
- except Exception as e:
- return jsonify({
- 'status': 'error',
- 'message': str(e)
- }), 500
-
-if __name__ == '__main__':
- app.run(debug=True, host="0.0.0.0")
diff --git a/evaluation_examples/README.md b/evaluation_examples/README.md
new file mode 100644
index 0000000..a5bf2db
--- /dev/null
+++ b/evaluation_examples/README.md
@@ -0,0 +1,24 @@
+# Evaluation examples
+
+Here we put the data examples to benchmark the ability of agents when interacting with GUI.
+The examples are stored in `./examples` where each data item formatted as:
+
+```
+{
+ "id": "uid", # unique id
+ "snapshot": "snapshot_id", # the snapshot id of the environment, with some data already there and apps already opened, or just desktop
+ "instruction": "natural_language_instruction", # the natural language instruction of the task, what we want the agent to do
+ "source": "website_url", # where we know this example, some forum, or some website, or some paper
+ "config": {xxx}, # the scripts to setup the donwload and open files actions, as the initial state of a task
+ "trajectory": "trajectory_directory", # the trajectory directory, which contains the action sequence file, the screenshots and the recording video
+ "related_apps": ["app1", "app2", ...], # the related apps, which are opened during the task
+ "evaluator": "evaluation_dir", # the directory of the evaluator, which contains the evaluation script for this example
+…
+}
+```
+
+The `./trajectories` file contains the annotated trajectories for each data item in `./examples` for finishing the task.
+
+For now, it is under construction, and only tested on Windows 10. Please:
+- Modify the path accordingly to run the evaluation;
+- Remind us if some parts are overfit to our environment.
diff --git a/evaluation_examples/examples/0bf05a7d-b28b-44d2-955a-50b41e24012a.json b/evaluation_examples/examples/0bf05a7d-b28b-44d2-955a-50b41e24012a.json
new file mode 100644
index 0000000..6f92ac5
--- /dev/null
+++ b/evaluation_examples/examples/0bf05a7d-b28b-44d2-955a-50b41e24012a.json
@@ -0,0 +1,22 @@
+{
+ "id": "0bf05a7d-b28b-44d2-955a-50b41e24012a",
+ "snapshot": "libreoffice_calc",
+ "instruction": "I would like to pad all the numbers in the 'Old ID' column with zeros in front, to fill them up to seven digits in the 'New 7 Digit ID' column.",
+ "source": "https://www.youtube.com/shorts/FPAQaDTS8VY",
+ "config": {
+ "download": [
+ [
+ "",
+ "C:\\Users\\tianbaox\\Desktop\\Customers_New_7digit_Id.xlsx"
+ ]
+ ],
+ "open": [
+ "C:\\Users\\tianbaox\\Desktop\\Customers_New_7digit_Id.xlsx"
+ ]
+ },
+ "trajectory": "trajectories/0bf05a7d-b28b-44d2-955a-50b41e24012a",
+ "related_apps": [
+ "libreoffice calc"
+ ],
+ "evaluator": "evaluation_dir"
+}
diff --git a/evaluation_examples/examples/2bd59342-0664-4ccb-ba87-79379096cc08.json b/evaluation_examples/examples/2bd59342-0664-4ccb-ba87-79379096cc08.json
new file mode 100644
index 0000000..98d773a
--- /dev/null
+++ b/evaluation_examples/examples/2bd59342-0664-4ccb-ba87-79379096cc08.json
@@ -0,0 +1,22 @@
+{
+ "id": "2bd59342-0664-4ccb-ba87-79379096cc08",
+ "snapshot": "libreoffice_calc",
+ "instruction": "Make sparkline chart line by line",
+ "source": "https://www.youtube.com/shorts/L3Z-F1QTQFY",
+ "config": {
+ "download": [
+ [
+ "",
+ "C:\\Users\\tianbaox\\Desktop\\OrderId_Month_Chart.xlsx"
+ ]
+ ],
+ "open": [
+ "C:\\Users\\tianbaox\\Desktop\\OrderId_Month_Chart.xlsx"
+ ]
+ },
+ "trajectory": "trajectories/2bd59342-0664-4ccb-ba87-79379096cc08",
+ "related_apps": [
+ "libreoffice calc"
+ ],
+ "evaluator": "evaluation_dir"
+}
diff --git a/evaluation_examples/examples/37608790-6147-45d0-9f20-1137bb35703d.json b/evaluation_examples/examples/37608790-6147-45d0-9f20-1137bb35703d.json
new file mode 100644
index 0000000..e25e698
--- /dev/null
+++ b/evaluation_examples/examples/37608790-6147-45d0-9f20-1137bb35703d.json
@@ -0,0 +1,34 @@
+{
+ "id": "37608790-6147-45d0-9f20-1137bb35703d",
+ "snapshot": "libreoffice_calc",
+ "instruction": "Help me fill the columns of First Name, Last Name and Rank",
+ "source": "https://www.youtube.com/shorts/uzPo_CPCHH8",
+ "config": {
+ "download": [
+ [
+ "https://drive.usercontent.google.com/download?id=1wDqap5cBfxnlqTNrZG61k_wDWTujl6AU&export=download&authuser=0&confirm=t&uuid=fd183b89-76b7-4dc5-880e-1045ed769562&at=APZUnTWp9RMafMg0xohhBWazN3YD:1701785710674",
+ "C:\\Users\\tianbaox\\Desktop\\Employee_Roles_and_Ranks.xlsx"
+ ]
+ ],
+ "open": [
+ "C:\\Users\\tianbaox\\Desktop\\Employee_Roles_and_Ranks.xlsx"
+ ]
+ },
+ "trajectory": "trajectories/37608790-6147-45d0-9f20-1137bb35703d",
+ "related_apps": [
+ "libreoffice calc"
+ ],
+ "evaluator": {
+ "func": "compare_table(expected, actual)",
+ "paths": {
+ "expected": {
+ "type": "cloud_file",
+ "path": "https://drive.usercontent.google.com/download?id=1dxpiUqP_CVvQp5tddxlwO3Cp1BqJ-ZDE&export=download&authuser=0&confirm=t&uuid=ccd204c7-07ce-4fdf-a5d4-a7e4f37b9ce6&at=APZUnTVBs7TgrVrDXpkiU8S7WbQo:1702360836747"
+ },
+ "actual": {
+ "type": "vm_file",
+ "path": "C:\\Users\\tianbaox\\Desktop\\Employee_Roles_and_Ranks.xlsx"
+ }
+ }
+ }
+}
diff --git a/evaluation_examples/examples/7a4e4bc8-922c-4c84-865c-25ba34136be1.json b/evaluation_examples/examples/7a4e4bc8-922c-4c84-865c-25ba34136be1.json
new file mode 100644
index 0000000..dfe1da0
--- /dev/null
+++ b/evaluation_examples/examples/7a4e4bc8-922c-4c84-865c-25ba34136be1.json
@@ -0,0 +1,22 @@
+{
+ "id": "7a4e4bc8-922c-4c84-865c-25ba34136be1",
+ "snapshot": "libreoffice_calc",
+ "instruction": "Reorder the columns to be \"Data\", \"First Name\", \"Last Name\", \"Order ID\", \"Sales\"",
+ "source": "https://www.youtube.com/shorts/bvUhr1AHs44",
+ "config": {
+ "download": [
+ [
+ "",
+ "C:\\Users\\tianbaox\\Desktop\\Name_Order_Id_move_column.xlsx"
+ ]
+ ],
+ "open": [
+ "C:\\Users\\tianbaox\\Desktop\\Name_Order_Id_move_column.xlsx"
+ ]
+ },
+ "trajectory": "trajectories/7a4e4bc8-922c-4c84-865c-25ba34136be1",
+ "related_apps": [
+ "libreoffice calc"
+ ],
+ "evaluator": "evaluation_dir"
+}
diff --git a/evaluation_examples/examples/7b802dad-6e0f-4204-9815-d4e3f57627d8.json b/evaluation_examples/examples/7b802dad-6e0f-4204-9815-d4e3f57627d8.json
new file mode 100644
index 0000000..d26f5ce
--- /dev/null
+++ b/evaluation_examples/examples/7b802dad-6e0f-4204-9815-d4e3f57627d8.json
@@ -0,0 +1,22 @@
+{
+ "id": "7b802dad-6e0f-4204-9815-d4e3f57627d8",
+ "snapshot": "libreoffice_calc",
+ "instruction": "I would like to sort this table based on cell color, placing all the rows marked with pink at the beginning, while keeping their order among themselves unchanged.",
+ "source": "https://www.youtube.com/shorts/Of-lzeP1usE",
+ "config": {
+ "download": [
+ [
+ "",
+ "C:\\Users\\tianbaox\\Desktop\\Customer_Sort_by_cell_color.xlsx"
+ ]
+ ],
+ "open": [
+ "C:\\Users\\tianbaox\\Desktop\\Customer_Sort_by_cell_color.xlsx"
+ ]
+ },
+ "trajectory": "trajectories/7b802dad-6e0f-4204-9815-d4e3f57627d8",
+ "related_apps": [
+ "libreoffice calc"
+ ],
+ "evaluator": "evaluation_dir"
+}
diff --git a/evaluation_examples/examples/7efeb4b1-3d19-4762-b163-63328d66303b.json b/evaluation_examples/examples/7efeb4b1-3d19-4762-b163-63328d66303b.json
new file mode 100644
index 0000000..dcb0eb6
--- /dev/null
+++ b/evaluation_examples/examples/7efeb4b1-3d19-4762-b163-63328d66303b.json
@@ -0,0 +1,22 @@
+{
+ "id": "7efeb4b1-3d19-4762-b163-63328d66303b",
+ "snapshot": "libreoffice_calc",
+ "instruction": "Fill in the Serieal Numbers in \"Serial #\" column",
+ "source": "https://www.youtube.com/shorts/4jzXfZNhfmk",
+ "config": {
+ "download": [
+ [
+ "",
+ "C:\\Users\\tianbaox\\Desktop\\Order_Sales_Serial#.xlsx"
+ ]
+ ],
+ "open": [
+ "C:\\Users\\tianbaox\\Desktop\\Order_Sales_Serial#.xlsx"
+ ]
+ },
+ "trajectory": "trajectories/",
+ "related_apps": [
+ "libreoffice calc"
+ ],
+ "evaluator": "evaluation_dir"
+}
diff --git a/evaluation_examples/examples/a9f325aa-8c05-4e4f-8341-9e4358565f4f.json b/evaluation_examples/examples/a9f325aa-8c05-4e4f-8341-9e4358565f4f.json
new file mode 100644
index 0000000..057ed2f
--- /dev/null
+++ b/evaluation_examples/examples/a9f325aa-8c05-4e4f-8341-9e4358565f4f.json
@@ -0,0 +1,22 @@
+{
+ "id": "a9f325aa-8c05-4e4f-8341-9e4358565f4f",
+ "snapshot": "libreoffice_calc",
+ "instruction": "Clean the messy movie titles and put them in the cleaned column",
+ "source": "https://www.youtube.com/shorts/A0gmEBRKXWs",
+ "config": {
+ "download": [
+ [
+ "",
+ "C:\\Users\\tianbaox\\Desktop\\"
+ ]
+ ],
+ "open": [
+ "C:\\Users\\tianbaox\\Desktop\\"
+ ]
+ },
+ "trajectory": "trajectories/a9f325aa-8c05-4e4f-8341-9e4358565f4f",
+ "related_apps": [
+ "libreoffice calc"
+ ],
+ "evaluator": "evaluation_dir"
+}
diff --git a/evaluation_examples/examples/d681960f-7bc3-4286-9913-a8812ba3261a.json b/evaluation_examples/examples/d681960f-7bc3-4286-9913-a8812ba3261a.json
new file mode 100644
index 0000000..509fd77
--- /dev/null
+++ b/evaluation_examples/examples/d681960f-7bc3-4286-9913-a8812ba3261a.json
@@ -0,0 +1,34 @@
+{
+ "id": "d681960f-7bc3-4286-9913-a8812ba3261a",
+ "snapshot": "libreoffice_calc",
+ "instruction": "According to the green table shown above, calculate and give each student a grade",
+ "source": "https://www.youtube.com/shorts/d7U1S_IsTVM",
+ "config": {
+ "download": [
+ [
+ "https://drive.usercontent.google.com/download?id=1wodZjx1KjThUsrtF6ZJaCTy1fQX4E9vA&export=download&authuser=0&confirm=t&uuid=d07ca312-1abc-40f2-81cd-d06e27119854&at=APZUnTWwjnxsHQYapSvpLR8NmlfV:1701785087048",
+ "C:\\Users\\tianbaox\\Desktop\\Student_Grades_and_Remarks.xlsx"
+ ]
+ ],
+ "open": [
+ "C:\\Users\\tianbaox\\Desktop\\Student_Grades_and_Remarks.xlsx"
+ ]
+ },
+ "trajectory": "trajectories/d681960f-7bc3-4286-9913-a8812ba3261a",
+ "related_apps": [
+ "libreoffice calc"
+ ],
+ "evaluator": {
+ "func": "compare_table(expected, actual)",
+ "paths": {
+ "expected": {
+ "type": "cloud_file",
+ "path": "https://drive.usercontent.google.com/download?id=1kfEHJH1n0yCsQp443IIFvdD9uWv0DWMr&export=download&authuser=0&confirm=t&uuid=d9907f65-8d39-4ecc-8747-b4ed7e6011f5&at=APZUnTXpPAnlh5sD6q-R8oQtqL6g:1702362952170"
+ },
+ "actual": {
+ "type": "vm_file",
+ "path": "C:\\Users\\tianbaox\\Desktop\\Student_Grades_and_Remarks.xlsx"
+ }
+ }
+ }
+}
diff --git a/evaluation_examples/examples/eb03d19a-b88d-4de4-8a64-ca0ac66f426b.json b/evaluation_examples/examples/eb03d19a-b88d-4de4-8a64-ca0ac66f426b.json
new file mode 100644
index 0000000..02e0da2
--- /dev/null
+++ b/evaluation_examples/examples/eb03d19a-b88d-4de4-8a64-ca0ac66f426b.json
@@ -0,0 +1,22 @@
+{
+ "id": "eb03d19a-b88d-4de4-8a64-ca0ac66f426b",
+ "snapshot": "libreoffice_calc",
+ "instruction": "Traverse the table and paste it below",
+ "source": "https://www.youtube.com/shorts/t9JLUaT55UQ",
+ "config": {
+ "download": [
+ [
+ "",
+ "C:\\Users\\tianbaox\\Desktop\\"
+ ]
+ ],
+ "open": [
+ "C:\\Users\\tianbaox\\Desktop\\"
+ ]
+ },
+ "trajectory": "trajectories/eb03d19a-b88d-4de4-8a64-ca0ac66f426b",
+ "related_apps": [
+ "libreoffice calc"
+ ],
+ "evaluator": "evaluation_dir"
+}
diff --git a/evaluation_examples/examples/ecb0df7a-4e8d-4a03-b162-053391d3afaf.json b/evaluation_examples/examples/ecb0df7a-4e8d-4a03-b162-053391d3afaf.json
new file mode 100644
index 0000000..6c5142a
--- /dev/null
+++ b/evaluation_examples/examples/ecb0df7a-4e8d-4a03-b162-053391d3afaf.json
@@ -0,0 +1,22 @@
+{
+ "id": "ecb0df7a-4e8d-4a03-b162-053391d3afaf",
+ "snapshot": "libreoffice_calc",
+ "instruction": "Enable each cell in the column\"Pass/Fail/Held\" is a drop down list",
+ "source": "https://www.youtube.com/shorts/tXOovKn0H68",
+ "config": {
+ "download": [
+ [
+ "",
+ "C:\\Users\\tianbaox\\Desktop\\"
+ ]
+ ],
+ "open": [
+ "C:\\Users\\tianbaox\\Desktop\\"
+ ]
+ },
+ "trajectory": "trajectories/ecb0df7a-4e8d-4a03-b162-053391d3afaf",
+ "related_apps": [
+ "libreoffice calc"
+ ],
+ "evaluator": "evaluation_dir"
+}
diff --git a/evaluation_examples/examples/f9584479-3d0d-4c79-affa-9ad7afdd8850.json b/evaluation_examples/examples/f9584479-3d0d-4c79-affa-9ad7afdd8850.json
new file mode 100644
index 0000000..5a01d56
--- /dev/null
+++ b/evaluation_examples/examples/f9584479-3d0d-4c79-affa-9ad7afdd8850.json
@@ -0,0 +1,34 @@
+{
+ "id": "f9584479-3d0d-4c79-affa-9ad7afdd8850",
+ "snapshot": "libreoffice_calc",
+ "instruction": "Fill the missing row and column which show the total value",
+ "source": "https://youtube.com/shorts/feldd-Pn48c?si=9xJiem2uAHm6Jshb",
+ "config": {
+ "download": [
+ [
+ "https://drive.usercontent.google.com/download?id=1rwhniaClEkF8XFzdfaNUA6GmAiy4syMZ&export=download&authuser=0&confirm=t&uuid=6fdd5b04-85f4-45e1-ad74-368f8f2a82ab&at=APZUnTUP-JxPxLfNls6jXWghblQ5:1701766091851",
+ "C:\\Users\\tianbaox\\Desktop\\Quarterly_Product_Sales_by_Zone.xlsx"
+ ]
+ ],
+ "open": [
+ "C:\\Users\\tianbaox\\Desktop\\Quarterly_Product_Sales_by_Zone.xlsx"
+ ]
+ },
+ "trajectory": "trajectories/f9584479-3d0d-4c79-affa-9ad7afdd8850",
+ "related_apps": [
+ "libreoffice calc"
+ ],
+ "evaluator": {
+ "func": "compare_table(expected, actual)",
+ "paths": {
+ "expected": {
+ "type": "cloud_file",
+ "path": "https://drive.usercontent.google.com/download?id=17f1wZuJPvUEc5at_Fy3c18VFdOk0x7xz&export=download&authuser=0&confirm=t&uuid=6d2edffd-0ce0-426e-9820-8af25b4667f3&at=APZUnTVh7JS85dwZBaV2hytWQgDK:1702361510956"
+ },
+ "actual": {
+ "type": "vm_file",
+ "path": "C:\\Users\\tianbaox\\Desktop\\Quarterly_Product_Sales_by_Zone.xlsx"
+ }
+ }
+ }
+}
diff --git a/evaluation_examples/examples/template.json b/evaluation_examples/examples/template.json
new file mode 100644
index 0000000..f8efe3b
--- /dev/null
+++ b/evaluation_examples/examples/template.json
@@ -0,0 +1,13 @@
+{
+ "id": "",
+ "snapshot": "libreoffice_calc",
+ "instruction": "",
+ "source": "",
+ "config": {
+ },
+ "trajectory": "trajectories/",
+ "related_apps": [
+ "libreoffice calc"
+ ],
+ "evaluator": "evaluation_dir"
+}
diff --git a/main.py b/main.py
index 2cbcd18..d7c3073 100644
--- a/main.py
+++ b/main.py
@@ -1,56 +1,51 @@
-from pprint import pprint
-from desktop_env.envs.desktop_env import DesktopEnv, Action, MouseClick
-
-def get_human_action():
- """
- Prompts the human player for an action and returns a structured action.
- """
- print("\nAvailable actions:", [action.name for action in Action])
- action_type = None
- while action_type not in [action.value for action in Action]:
- action_type = Action[input("Enter the type of action: ".strip())].value
-
- action = {"action_type": action_type}
-
- if action_type == Action.CLICK.value or action_type == Action.MOUSE_DOWN.value or action_type == Action.MOUSE_UP.value:
- print("\n Available clicks:", [action.name for action in MouseClick])
- click_type = input("Enter click type: ")
- action["click_type"] = MouseClick[click_type].value
-
- if action_type == Action.MOUSE_MOVE.value:
- x = int(input("Enter x-coordinate for mouse move: "))
- y = int(input("Enter y-coordinate for mouse move: "))
- action["x"] = x
- action["y"] = y
-
- if action_type == Action.KEY.value:
- key = input("Enter the key to press: ")
- action["key"] = [ord(c) for c in key]
-
- if action_type == Action.TYPE.value:
- text = input("Enter the text to type: ")
- action["text"] = [ord(c) for c in text]
-
- return action
+import json
+from desktop_env.envs.desktop_env import DesktopEnv
def human_agent():
"""
Runs the Gym environment with human input.
"""
- env = DesktopEnv(path_to_vm="/home/yuri/vmware/Windows 10 x64/Windows 10 x64.vmx",
- # path_to_vm="/home/yuri/vmware/Ubuntu 64-bit/Ubuntu 64-bit.vmx",
- username="user",
- password="password",
- # host="192.168.7.128",
- host="http://192.168.7.129:5000",
- vm_os="windows")
+
+ with open("evaluation_examples/examples/37608790-6147-45d0-9f20-1137bb35703d.json", "r") as f:
+ example = json.load(f)
+
+ env = DesktopEnv(
+ # path_to_vm=r"""C:\Users\tianbaox\Downloads\Windows 10 x64\Windows 10 x64.vmx""",
+ path_to_vm=r"""C:\Users\tianbaox\Documents\Virtual Machines\Win10\Win10.vmx""",
+ # path_to_vm="/home/yuri/vmware/Ubuntu 64-bit/Ubuntu 64-bit.vmx",
+ action_space="computer_13",
+ snapshot_path="base_setup3",
+ instruction=example["instruction"],
+ config=example["config"],
+ evaluator=example["evaluator"]
+ )
+
+ # reset the environment to certain snapshot
observation = env.reset()
done = False
- while not done:
- action = get_human_action()
- observation, reward, done, info = env.step(action)
+ trajectory = [
+ {
+ "action_type": "MOVE_TO",
+ "parameters": {
+ "x": 754,
+ "y": 1057
+ }
+ },
+ {"action_type": "CLICK", "parameters": {"button": "right", "num_clicks": 1}}
+ ]
+
+ for i in range(len(trajectory)):
+ # action = get_human_action()
+
+ # action = {
+ # "action_type": 0,
+ # "click_type": 3,
+ # }
+ print(trajectory[i])
+
+ observation, reward, done, info = env.step(trajectory[i], pause=5)
print("Observation:", observation)
print("Reward:", reward)
print("Info:", info)
@@ -61,8 +56,12 @@ def human_agent():
print("The episode is done.")
break
+ result = env.evaluate()
+ print("Result:", result)
+
env.close()
print("Environment closed.")
+
if __name__ == "__main__":
human_agent()
diff --git a/mm_agents/gpt_4v_agent.py b/mm_agents/gpt_4v_agent.py
index 663bf1e..d0288e1 100644
--- a/mm_agents/gpt_4v_agent.py
+++ b/mm_agents/gpt_4v_agent.py
@@ -1,8 +1,12 @@
+# fixme: Need to be rewrite on new action space
+
import os
+import re
import base64
from desktop_env.envs.desktop_env import Action, MouseClick
-import json5
+import json
import requests
+from mm_agents.gpt_4v_prompt import SYS_PROMPT
# Function to encode the image
@@ -11,6 +15,38 @@ def encode_image(image_path):
return base64.b64encode(image_file.read()).decode('utf-8')
+def parse_actions_from_string(input_string):
+ # Search for a JSON string within the input string
+ actions = []
+ matches = re.findall(r'```json\s+(.*?)\s+```', input_string, re.DOTALL)
+ if matches:
+ # Assuming there's only one match, parse the JSON string into a dictionary
+ try:
+ for match in matches:
+ action_dict = json.loads(match)
+ actions.append(action_dict)
+ return actions
+ except json.JSONDecodeError as e:
+ return f"Failed to parse JSON: {e}"
+ else:
+ matches = re.findall(r'```\s+(.*?)\s+```', input_string, re.DOTALL)
+ if matches:
+ # Assuming there's only one match, parse the JSON string into a dictionary
+ try:
+ for match in matches:
+ action_dict = json.loads(match)
+ actions.append(action_dict)
+ return actions
+ except json.JSONDecodeError as e:
+ return f"Failed to parse JSON: {e}"
+ else:
+ try:
+ action_dict = json.loads(input_string)
+ return [action_dict]
+ except json.JSONDecodeError as e:
+ raise ValueError("Invalid response format: " + input_string)
+
+
class GPT4v_Agent:
def __init__(self, api_key, instruction, model="gpt-4-vision-preview", max_tokens=300):
self.instruction = instruction
@@ -22,18 +58,13 @@ class GPT4v_Agent:
"Authorization": f"Bearer {api_key}"
}
- # load prompt from file
- self.prompt = ""
- with open("gpt_4v_prompt.txt", "r") as f:
- self.prompt = f.read()
-
self.trajectory = [
{
"role": "system",
"content": [
{
"type": "text",
- "text": self.prompt
+ "text": SYS_PROMPT
},
]
}
@@ -56,6 +87,12 @@ class GPT4v_Agent:
}
]
})
+ traj_to_show = []
+ for i in range(len(self.trajectory)):
+ traj_to_show.append(self.trajectory[i]["content"][0]["text"])
+ if len(self.trajectory[i]["content"]) > 1:
+ traj_to_show.append("screenshot_obs")
+ print("Trajectory:", traj_to_show)
payload = {
"model": self.model,
"messages": self.trajectory,
@@ -63,11 +100,15 @@ class GPT4v_Agent:
}
response = requests.post("https://api.openai.com/v1/chat/completions", headers=self.headers, json=payload)
- action = self.parse_action(response.json()['choices'][0]['message']['content'])
+ try:
+ actions = self.parse_actions(response.json()['choices'][0]['message']['content'])
+ except:
+ print("Failed to parse action from response:", response.json()['choices'][0]['message']['content'])
+ actions = None
- return action
+ return actions
- def parse_action(self, response: str):
+ def parse_actions(self, response: str):
# response example
"""
```json
@@ -79,12 +120,7 @@ class GPT4v_Agent:
"""
# parse from the response
- if response.startswith("```json"):
- action = json5.loads(response[7:-3])
- elif response.startswith("```"):
- action = json5.loads(response[3:-3])
- else:
- action = json5.loads(response)
+ actions = parse_actions_from_string(response)
# add action into the trajectory
self.trajectory.append({
@@ -98,25 +134,28 @@ class GPT4v_Agent:
})
# parse action
- parsed_action = {}
- action_type = Action[action['action_type']].value
- parsed_action["action_type"] = action_type
+ parsed_actions = []
+ for action in actions:
+ parsed_action = {}
+ action_type = Action[action['action_type']].value
+ parsed_action["action_type"] = action_type
- if action_type == Action.CLICK.value or action_type == Action.MOUSE_DOWN.value or action_type == Action.MOUSE_UP.value:
- parsed_action["click_type"] = MouseClick[action['click_type']].value
+ if action_type == Action.CLICK.value or action_type == Action.MOUSE_DOWN.value or action_type == Action.MOUSE_UP.value:
+ parsed_action["click_type"] = MouseClick[action['click_type']].value
- if action_type == Action.MOUSE_MOVE.value:
- parsed_action["x"] = action["x"]
- parsed_action["y"] = action["y"]
+ if action_type == Action.MOUSE_MOVE.value:
+ parsed_action["x"] = action["x"]
+ parsed_action["y"] = action["y"]
- # fixme: could these two actions be merged??
- if action_type == Action.KEY.value:
- parsed_action["key"] = [ord(c) for c in action["key"]]
+ if action_type == Action.KEY.value:
+ parsed_action["key"] = action["key"] # handle the condition of single key and multiple keys
- if action_type == Action.TYPE.value:
- parsed_action["text"] = [ord(c) for c in action["text"]]
+ if action_type == Action.TYPE.value:
+ parsed_action["text"] = action["text"]
- return parsed_action
+ parsed_actions.append(parsed_action)
+
+ return parsed_actions
if __name__ == '__main__':
@@ -125,4 +164,3 @@ if __name__ == '__main__':
agent = GPT4v_Agent(api_key=api_key, instruction="Open Google Sheet")
print(agent.predict(obs="stackoverflow.png"))
-
diff --git a/mm_agents/gpt_4v_prompt_action.py b/mm_agents/gpt_4v_prompt_action.py
new file mode 100644
index 0000000..11705e3
--- /dev/null
+++ b/mm_agents/gpt_4v_prompt_action.py
@@ -0,0 +1,54 @@
+SYS_PROMPT = """
+You will act as an agent which follow my instruction and perform desktop computer tasks as instructed. You must have good knowledge of computer and good internet connection.
+For each step, you will get an observation of an image, which is the screenshot of the computer screen. And you will predict the action of the computer based on the image.
+Here is the description of the action space:
+
+Firstly you need to predict the class of your action, select from one below:
+- **MOUSE_MOVE**: move the mouse to a specific position
+- **CLICK**: click on the screen
+- **MOUSE_DOWN**: press the mouse button
+- **MOUSE_UP**: release the mouse button
+- **KEY**: press a key on the keyboard
+- **KEY_DOWN**: press a key on the keyboard
+- **KEY_UP**: release a key on the keyboard
+- **TYPE**: type a string on the keyboard
+
+Then you need to predict the parameters of your action:
+- For MOUSE_MOVE, you need to predict the x and y coordinate of the mouse cursor, the left top corner of the screen is (0, 0), the right bottom corner of the screen is (1920, 1080)
+for example, format as:
+```
+{
+ "action_type": "MOUSE_MOVE",
+ "x": 1319.11,
+ "y": 65.06
+}
+```
+- For [CLICK, MOUSE_DOWN, MOUSE_UP], you need to specify the click_type as well, select from [LEFT, MIDDLE, RIGHT, WHEEL_UP, WHEEL_DOWN], which means you click the left button, middle button, right button, wheel up or wheel down of your mouse:
+for example, format as:
+```
+{
+ "action_type": "CLICK",
+ "click_type": "LEFT"
+}
+```
+- For [KEY, KEY_DOWN, KEY_UP], you need to choose a(multiple) key(s) from the keyboard
+for example, format as:
+```
+{
+ "action_type": "KEY",
+ "key": "ctrl+c"
+}
+```
+- For TYPE, you need to specify the text you want to type
+for example, format as:
+```
+{
+ "action_type": "TYPE",
+ "text": "hello world"
+}
+```
+
+For every step, you should only return the action_type and the parameters of your action as a dict, without any other things. You MUST wrap the dict with backticks (\`).
+You can predict multiple actions at one step, but you should only return one action for each step.
+You MUST choose and ONLY CHOOSE from the action space above, otherwise your action will be considered as invalid and you will get a penalty.
+"""
\ No newline at end of file
diff --git a/mm_agents/gpt_4v_prompt_code.py b/mm_agents/gpt_4v_prompt_code.py
new file mode 100644
index 0000000..f04602c
--- /dev/null
+++ b/mm_agents/gpt_4v_prompt_code.py
@@ -0,0 +1,8 @@
+SYS_PROMPT = """
+You will act as an agent which follow my instruction and perform desktop computer tasks as instructed. You must have good knowledge of computer and good internet connection.
+For each step, you will get an observation of an image, which is the screenshot of the computer screen. And you will predict the action of the computer based on the image.
+
+You are required to use `pyautogui` to perform the action.
+Return one line or multiple lines of python code to perform the action each time, be time efficient.
+Return `None` if you cannot perform the action.
+"""
\ No newline at end of file
diff --git a/mm_agents/sam_test.py b/mm_agents/sam_test.py
new file mode 100644
index 0000000..9d4ce44
--- /dev/null
+++ b/mm_agents/sam_test.py
@@ -0,0 +1,124 @@
+import torch
+from PIL import Image
+import requests
+from transformers import SamModel, SamProcessor
+import numpy as np
+import matplotlib.pyplot as plt
+import os
+os.environ["KMP_DUPLICATE_LIB_OK"]="TRUE"
+
+def show_mask(mask, ax, random_color=False):
+ if random_color:
+ color = np.concatenate([np.random.random(3), np.array([0.6])], axis=0)
+ else:
+ color = np.array([30 / 255, 144 / 255, 255 / 255, 0.6])
+ h, w = mask.shape[-2:]
+ mask_image = mask.reshape(h, w, 1) * color.reshape(1, 1, -1)
+ ax.imshow(mask_image)
+
+
+def show_box(box, ax):
+ x0, y0 = box[0], box[1]
+ w, h = box[2] - box[0], box[3] - box[1]
+ ax.add_patch(plt.Rectangle((x0, y0), w, h, edgecolor='green', facecolor=(0, 0, 0, 0), lw=2))
+
+
+def show_boxes_on_image(raw_image, boxes):
+ plt.figure(figsize=(10, 10))
+ plt.imshow(raw_image)
+ for box in boxes:
+ show_box(box, plt.gca())
+ plt.axis('on')
+ plt.show()
+
+
+def show_points_on_image(raw_image, input_points, input_labels=None):
+ plt.figure(figsize=(10, 10))
+ plt.imshow(raw_image)
+ input_points = np.array(input_points)
+ if input_labels is None:
+ labels = np.ones_like(input_points[:, 0])
+ else:
+ labels = np.array(input_labels)
+ show_points(input_points, labels, plt.gca())
+ plt.axis('on')
+ plt.show()
+
+
+def show_points_and_boxes_on_image(raw_image, boxes, input_points, input_labels=None):
+ plt.figure(figsize=(10, 10))
+ plt.imshow(raw_image)
+ input_points = np.array(input_points)
+ if input_labels is None:
+ labels = np.ones_like(input_points[:, 0])
+ else:
+ labels = np.array(input_labels)
+ show_points(input_points, labels, plt.gca())
+ for box in boxes:
+ show_box(box, plt.gca())
+ plt.axis('on')
+ plt.show()
+
+
+def show_points_and_boxes_on_image(raw_image, boxes, input_points, input_labels=None):
+ plt.figure(figsize=(10, 10))
+ plt.imshow(raw_image)
+ input_points = np.array(input_points)
+ if input_labels is None:
+ labels = np.ones_like(input_points[:, 0])
+ else:
+ labels = np.array(input_labels)
+ show_points(input_points, labels, plt.gca())
+ for box in boxes:
+ show_box(box, plt.gca())
+ plt.axis('on')
+ plt.show()
+
+
+def show_points(coords, labels, ax, marker_size=375):
+ pos_points = coords[labels == 1]
+ neg_points = coords[labels == 0]
+ ax.scatter(pos_points[:, 0], pos_points[:, 1], color='green', marker='*', s=marker_size, edgecolor='white',
+ linewidth=1.25)
+ ax.scatter(neg_points[:, 0], neg_points[:, 1], color='red', marker='*', s=marker_size, edgecolor='white',
+ linewidth=1.25)
+
+
+def show_masks_on_image(raw_image, masks, scores):
+ if len(masks.shape) == 4:
+ masks = masks.squeeze()
+ if scores.shape[0] == 1:
+ scores = scores.squeeze()
+
+ nb_predictions = scores.shape[-1]
+ fig, axes = plt.subplots(1, nb_predictions, figsize=(15, 15))
+
+ for i, (mask, score) in enumerate(zip(masks, scores)):
+ mask = mask.cpu().detach()
+ axes[i].imshow(np.array(raw_image))
+ show_mask(mask, axes[i])
+ axes[i].title.set_text(f"Mask {i + 1}, Score: {score.item():.3f}")
+ axes[i].axis("off")
+ plt.show()
+
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+model = SamModel.from_pretrained("facebook/sam-vit-huge").to(device)
+processor = SamProcessor.from_pretrained("facebook/sam-vit-huge")
+
+img_url = "https://huggingface.co/ybelkada/segment-anything/resolve/main/assets/car.png"
+raw_image = Image.open(requests.get(img_url, stream=True).raw).convert("RGB")
+
+plt.imshow(raw_image)
+
+inputs = processor(raw_image, return_tensors="pt").to(device)
+with torch.no_grad():
+ outputs = model(**inputs)
+
+masks = processor.image_processor.post_process_masks(
+ outputs.pred_masks.cpu(), inputs["original_sizes"].cpu(), inputs["reshaped_input_sizes"].cpu()
+)
+
+
+scores = outputs.iou_scores
+show_masks_on_image(raw_image, masks[0], scores)
diff --git a/requirements.txt b/requirements.txt
index cb56195..0a27eff 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,8 +1,16 @@
-numpy
-Pillow
+numpy~=1.24.3
+Pillow~=10.1.0
fabric
-gymnasium
-requests
-transformers
-torch
+gymnasium~=0.28.1
+requests~=2.31.0
+transformers~=4.35.2
+torch~=2.1.1+cu118
accelerate
+opencv-python~=4.8.1.78
+matplotlib~=3.7.4
+pynput~=1.7.6
+pyautogui~=0.9.54
+psutil~=5.9.6
+tqdm~=4.65.0
+pandas~=2.0.3
+flask~=3.0.0
\ No newline at end of file
diff --git a/screenshot.png b/screenshot.png
index 0ea0c0f..a0b20d0 100644
Binary files a/screenshot.png and b/screenshot.png differ
diff --git a/utils/complex_clicking.json b/utils/complex_clicking.json
new file mode 100644
index 0000000..fa67adf
--- /dev/null
+++ b/utils/complex_clicking.json
@@ -0,0 +1 @@
+[{"action_type": "MOVE_TO", "parameters": {"x": 501, "y": 382}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 501, "y": 382}}, {"action_type": "MOVE_TO", "parameters": {"x": 501, "y": 382}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 560, "y": 385}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 560, "y": 385}}, {"action_type": "MOVE_TO", "parameters": {"x": 560, "y": 385}}, {"action_type": "MOVE_TO", "parameters": {"x": 560, "y": 385}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 623, "y": 386}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 623, "y": 386}}, {"action_type": "MOVE_TO", "parameters": {"x": 623, "y": 386}}, {"action_type": "MOVE_TO", "parameters": {"x": 623, "y": 386}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 716, "y": 389}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 716, "y": 389}}, {"action_type": "MOVE_TO", "parameters": {"x": 716, "y": 389}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 639, "y": 409}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 639, "y": 409}}, {"action_type": "MOVE_TO", "parameters": {"x": 639, "y": 409}}, {"action_type": "MOVE_TO", "parameters": {"x": 639, "y": 409}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 711, "y": 409}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 711, "y": 409}}, {"action_type": "MOVE_TO", "parameters": {"x": 711, "y": 409}}, {"action_type": "MOVE_TO", "parameters": {"x": 711, "y": 409}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 644, "y": 448}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 644, "y": 448}}, {"action_type": "MOVE_TO", "parameters": {"x": 644, "y": 449}}, {"action_type": "MOVE_TO", "parameters": {"x": 644, "y": 449}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 720, "y": 446}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 720, "y": 446}}, {"action_type": "MOVE_TO", "parameters": {"x": 720, "y": 446}}, {"action_type": "MOVE_TO", "parameters": {"x": 720, "y": 446}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 660, "y": 458}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 660, "y": 458}}, {"action_type": "MOVE_TO", "parameters": {"x": 660, "y": 458}}, {"action_type": "MOVE_TO", "parameters": {"x": 660, "y": 458}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 692, "y": 459}}, {"action_type": "KEY_DOWN", "parameters": {"key": "s"}}, {"action_type": "MOVE_TO", "parameters": {"x": 690, "y": 459}}, {"action_type": "KEY_DOWN", "parameters": {"key": "f"}}, {"action_type": "MOVE_TO", "parameters": {"x": 675, "y": 460}}, {"action_type": "KEY_UP", "parameters": {"key": "s"}}, {"action_type": "KEY_DOWN", "parameters": {"key": "g"}}, {"action_type": "MOVE_TO", "parameters": {"x": 674, "y": 460}}, {"action_type": "KEY_UP", "parameters": {"key": "f"}}, {"action_type": "KEY_UP", "parameters": {"key": "g"}}, {"action_type": "MOVE_TO", "parameters": {"x": 637, "y": 482}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 637, "y": 482}}, {"action_type": "MOVE_TO", "parameters": {"x": 637, "y": 483}}, {"action_type": "MOVE_TO", "parameters": {"x": 637, "y": 483}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 601, "y": 501}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 601, "y": 501}}, {"action_type": "MOVE_TO", "parameters": {"x": 601, "y": 501}}, {"action_type": "MOVE_TO", "parameters": {"x": 601, "y": 501}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 557, "y": 501}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 557, "y": 501}}, {"action_type": "MOVE_TO", "parameters": {"x": 557, "y": 501}}, {"action_type": "MOVE_TO", "parameters": {"x": 557, "y": 501}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 568, "y": 509}}, {"action_type": "KEY_DOWN", "parameters": {"key": "g"}}, {"action_type": "KEY_DOWN", "parameters": {"key": "s"}}, {"action_type": "KEY_UP", "parameters": {"key": "g"}}, {"action_type": "KEY_DOWN", "parameters": {"key": "d"}}, {"action_type": "KEY_DOWN", "parameters": {"key": "g"}}, {"action_type": "MOVE_TO", "parameters": {"x": 570, "y": 511}}, {"action_type": "KEY_UP", "parameters": {"key": "d"}}, {"action_type": "MOVE_TO", "parameters": {"x": 613, "y": 516}}, {"action_type": "KEY_UP", "parameters": {"key": "g"}}, {"action_type": "MOVE_TO", "parameters": {"x": 616, "y": 513}}, {"action_type": "KEY_UP", "parameters": {"key": "s"}}, {"action_type": "MOVE_TO", "parameters": {"x": 600, "y": 337}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 600, "y": 337}}, {"action_type": "MOVE_TO", "parameters": {"x": 600, "y": 337}}, {"action_type": "MOVE_TO", "parameters": {"x": 600, "y": 337}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "KEY_DOWN", "parameters": {"key": "g"}}, {"action_type": "KEY_DOWN", "parameters": {"key": "s"}}, {"action_type": "MOVE_TO", "parameters": {"x": 600, "y": 341}}, {"action_type": "KEY_DOWN", "parameters": {"key": "d"}}, {"action_type": "MOVE_TO", "parameters": {"x": 600, "y": 342}}, {"action_type": "KEY_UP", "parameters": {"key": "g"}}, {"action_type": "KEY_DOWN", "parameters": {"key": "f"}}, {"action_type": "MOVE_TO", "parameters": {"x": 600, "y": 350}}, {"action_type": "KEY_DOWN", "parameters": {"key": "g"}}, {"action_type": "MOVE_TO", "parameters": {"x": 601, "y": 352}}, {"action_type": "KEY_UP", "parameters": {"key": "f"}}, {"action_type": "MOVE_TO", "parameters": {"x": 602, "y": 355}}, {"action_type": "KEY_UP", "parameters": {"key": "d"}}, {"action_type": "MOVE_TO", "parameters": {"x": 617, "y": 453}}, {"action_type": "KEY_UP", "parameters": {"key": "g"}}, {"action_type": "MOVE_TO", "parameters": {"x": 610, "y": 489}}, {"action_type": "KEY_UP", "parameters": {"key": "s"}}, {"action_type": "MOVE_TO", "parameters": {"x": 533, "y": 586}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 533, "y": 586}}, {"action_type": "MOVE_TO", "parameters": {"x": 533, "y": 586}}, {"action_type": "MOVE_TO", "parameters": {"x": 533, "y": 586}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 538, "y": 588}}, {"action_type": "KEY_DOWN", "parameters": {"key": "s"}}, {"action_type": "KEY_DOWN", "parameters": {"key": "g"}}, {"action_type": "MOVE_TO", "parameters": {"x": 541, "y": 590}}, {"action_type": "KEY_DOWN", "parameters": {"key": "f"}}, {"action_type": "MOVE_TO", "parameters": {"x": 541, "y": 591}}, {"action_type": "KEY_UP", "parameters": {"key": "g"}}, {"action_type": "MOVE_TO", "parameters": {"x": 544, "y": 595}}, {"action_type": "KEY_DOWN", "parameters": {"key": "g"}}, {"action_type": "MOVE_TO", "parameters": {"x": 545, "y": 597}}, {"action_type": "KEY_UP", "parameters": {"key": "f"}}, {"action_type": "MOVE_TO", "parameters": {"x": 536, "y": 605}}, {"action_type": "KEY_DOWN", "parameters": {"key": "f"}}, {"action_type": "MOVE_TO", "parameters": {"x": 526, "y": 606}}, {"action_type": "KEY_UP", "parameters": {"key": "g"}}, {"action_type": "MOVE_TO", "parameters": {"x": 513, "y": 606}}, {"action_type": "KEY_UP", "parameters": {"key": "f"}}, {"action_type": "MOVE_TO", "parameters": {"x": 490, "y": 604}}, {"action_type": "KEY_UP", "parameters": {"key": "s"}}, {"action_type": "MOVE_TO", "parameters": {"x": 489, "y": 604}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 451, "y": 604}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "right"}}, {"action_type": "MOVE_TO", "parameters": {"x": 451, "y": 604}}, {"action_type": "MOUSE_UP", "parameters": {"button": "right"}}, {"action_type": "MOVE_TO", "parameters": {"x": 563, "y": 590}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 563, "y": 590}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 570, "y": 579}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "right"}}, {"action_type": "MOVE_TO", "parameters": {"x": 570, "y": 579}}, {"action_type": "MOUSE_UP", "parameters": {"button": "right"}}, {"action_type": "MOVE_TO", "parameters": {"x": 615, "y": 612}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 615, "y": 612}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 623, "y": 458}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 623, "y": 458}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 626, "y": 455}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "right"}}, {"action_type": "MOVE_TO", "parameters": {"x": 626, "y": 455}}, {"action_type": "MOUSE_UP", "parameters": {"button": "right"}}, {"action_type": "MOVE_TO", "parameters": {"x": 647, "y": 456}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "right"}}, {"action_type": "MOVE_TO", "parameters": {"x": 647, "y": 456}}, {"action_type": "MOUSE_UP", "parameters": {"button": "right"}}, {"action_type": "MOVE_TO", "parameters": {"x": 647, "y": 458}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 559, "y": 458}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 557, "y": 459}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 557, "y": 459}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 570, "y": 427}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 570, "y": 427}}, {"action_type": "MOVE_TO", "parameters": {"x": 570, "y": 427}}, {"action_type": "MOVE_TO", "parameters": {"x": 570, "y": 427}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 628, "y": 423}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 628, "y": 423}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 553, "y": 375}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 647, "y": 504}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 649, "y": 504}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 649, "y": 504}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 573, "y": 371}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 573, "y": 371}}, {"action_type": "MOVE_TO", "parameters": {"x": 573, "y": 371}}, {"action_type": "MOVE_TO", "parameters": {"x": 573, "y": 371}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 564, "y": 335}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 3}}, {"action_type": "MOVE_TO", "parameters": {"x": 564, "y": 335}}, {"action_type": "MOVE_TO", "parameters": {"x": 567, "y": 333}}, {"action_type": "MOVE_TO", "parameters": {"x": 567, "y": 333}}, {"action_type": "MOVE_TO", "parameters": {"x": 567, "y": 333}}, {"action_type": "MOVE_TO", "parameters": {"x": 567, "y": 333}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 573, "y": 332}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 573, "y": 332}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 618, "y": 411}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 618, "y": 411}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 755, "y": 1057}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 754, "y": 1057}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 701, "y": 295}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 701, "y": 295}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}]
\ No newline at end of file
diff --git a/utils/complex_clicking.jsonl b/utils/complex_clicking.jsonl
new file mode 100644
index 0000000..1c5e212
--- /dev/null
+++ b/utils/complex_clicking.jsonl
@@ -0,0 +1,1788 @@
+{"time_stamp": 20885.5093285, "action": "move", "x": 727, "y": 298}
+{"time_stamp": 20885.5486486, "action": "move", "x": 727, "y": 298}
+{"time_stamp": 20885.5726277, "action": "move", "x": 728, "y": 299}
+{"time_stamp": 20885.5812855, "action": "move", "x": 728, "y": 300}
+{"time_stamp": 20885.5891249, "action": "move", "x": 728, "y": 300}
+{"time_stamp": 20885.5965132, "action": "move", "x": 728, "y": 300}
+{"time_stamp": 20885.6055747, "action": "move", "x": 728, "y": 301}
+{"time_stamp": 20885.6125424, "action": "move", "x": 728, "y": 302}
+{"time_stamp": 20885.6209172, "action": "move", "x": 727, "y": 303}
+{"time_stamp": 20885.6286479, "action": "move", "x": 727, "y": 305}
+{"time_stamp": 20885.6370033, "action": "move", "x": 726, "y": 305}
+{"time_stamp": 20885.6447023, "action": "move", "x": 725, "y": 306}
+{"time_stamp": 20885.6524801, "action": "move", "x": 724, "y": 308}
+{"time_stamp": 20885.6606371, "action": "move", "x": 723, "y": 309}
+{"time_stamp": 20885.6687156, "action": "move", "x": 720, "y": 311}
+{"time_stamp": 20885.6770625, "action": "move", "x": 719, "y": 312}
+{"time_stamp": 20885.6844749, "action": "move", "x": 717, "y": 314}
+{"time_stamp": 20885.6923004, "action": "move", "x": 715, "y": 316}
+{"time_stamp": 20885.7016409, "action": "move", "x": 712, "y": 317}
+{"time_stamp": 20885.7081761, "action": "move", "x": 710, "y": 318}
+{"time_stamp": 20885.7164073, "action": "move", "x": 705, "y": 320}
+{"time_stamp": 20885.7245609, "action": "move", "x": 703, "y": 322}
+{"time_stamp": 20885.7324495, "action": "move", "x": 697, "y": 324}
+{"time_stamp": 20885.7407729, "action": "move", "x": 692, "y": 325}
+{"time_stamp": 20885.7485978, "action": "move", "x": 685, "y": 328}
+{"time_stamp": 20885.7568341, "action": "move", "x": 679, "y": 329}
+{"time_stamp": 20885.7647292, "action": "move", "x": 672, "y": 332}
+{"time_stamp": 20885.7728367, "action": "move", "x": 666, "y": 333}
+{"time_stamp": 20885.7803453, "action": "move", "x": 659, "y": 335}
+{"time_stamp": 20885.7887313, "action": "move", "x": 653, "y": 336}
+{"time_stamp": 20885.7967785, "action": "move", "x": 645, "y": 339}
+{"time_stamp": 20885.8046519, "action": "move", "x": 638, "y": 340}
+{"time_stamp": 20885.8125112, "action": "move", "x": 630, "y": 343}
+{"time_stamp": 20885.8206433, "action": "move", "x": 621, "y": 346}
+{"time_stamp": 20885.8285957, "action": "move", "x": 613, "y": 349}
+{"time_stamp": 20885.8363896, "action": "move", "x": 605, "y": 351}
+{"time_stamp": 20885.8442534, "action": "move", "x": 597, "y": 354}
+{"time_stamp": 20885.8524552, "action": "move", "x": 590, "y": 358}
+{"time_stamp": 20885.8608368, "action": "move", "x": 586, "y": 360}
+{"time_stamp": 20885.8688078, "action": "move", "x": 580, "y": 364}
+{"time_stamp": 20885.8765587, "action": "move", "x": 575, "y": 368}
+{"time_stamp": 20885.8845853, "action": "move", "x": 569, "y": 371}
+{"time_stamp": 20885.8924835, "action": "move", "x": 565, "y": 374}
+{"time_stamp": 20885.9006509, "action": "move", "x": 561, "y": 377}
+{"time_stamp": 20885.9091894, "action": "move", "x": 557, "y": 380}
+{"time_stamp": 20885.9167535, "action": "move", "x": 554, "y": 382}
+{"time_stamp": 20885.925305, "action": "move", "x": 550, "y": 385}
+{"time_stamp": 20885.9326388, "action": "move", "x": 547, "y": 387}
+{"time_stamp": 20885.94096, "action": "move", "x": 543, "y": 389}
+{"time_stamp": 20885.9485579, "action": "move", "x": 540, "y": 389}
+{"time_stamp": 20885.9566463, "action": "move", "x": 538, "y": 390}
+{"time_stamp": 20885.9645797, "action": "move", "x": 536, "y": 391}
+{"time_stamp": 20885.9745125, "action": "move", "x": 535, "y": 391}
+{"time_stamp": 20885.9805225, "action": "move", "x": 533, "y": 391}
+{"time_stamp": 20885.9890978, "action": "move", "x": 532, "y": 391}
+{"time_stamp": 20886.0056633, "action": "move", "x": 529, "y": 391}
+{"time_stamp": 20886.0125014, "action": "move", "x": 528, "y": 391}
+{"time_stamp": 20886.021609, "action": "move", "x": 527, "y": 391}
+{"time_stamp": 20886.0366457, "action": "move", "x": 527, "y": 391}
+{"time_stamp": 20886.1166137, "action": "move", "x": 526, "y": 391}
+{"time_stamp": 20886.1245303, "action": "move", "x": 526, "y": 391}
+{"time_stamp": 20886.1325257, "action": "move", "x": 525, "y": 391}
+{"time_stamp": 20886.1404761, "action": "move", "x": 525, "y": 390}
+{"time_stamp": 20886.1484418, "action": "move", "x": 525, "y": 389}
+{"time_stamp": 20886.1565568, "action": "move", "x": 524, "y": 389}
+{"time_stamp": 20886.1649685, "action": "move", "x": 524, "y": 388}
+{"time_stamp": 20886.1727309, "action": "move", "x": 524, "y": 387}
+{"time_stamp": 20886.1805062, "action": "move", "x": 523, "y": 386}
+{"time_stamp": 20886.1885398, "action": "move", "x": 523, "y": 385}
+{"time_stamp": 20886.1971866, "action": "move", "x": 523, "y": 384}
+{"time_stamp": 20886.2045313, "action": "move", "x": 522, "y": 383}
+{"time_stamp": 20886.2124502, "action": "move", "x": 521, "y": 383}
+{"time_stamp": 20886.2206215, "action": "move", "x": 521, "y": 382}
+{"time_stamp": 20886.228563, "action": "move", "x": 521, "y": 382}
+{"time_stamp": 20886.4130623, "action": "move", "x": 520, "y": 382}
+{"time_stamp": 20886.4210048, "action": "move", "x": 520, "y": 382}
+{"time_stamp": 20886.4286417, "action": "move", "x": 518, "y": 382}
+{"time_stamp": 20886.4363667, "action": "move", "x": 516, "y": 382}
+{"time_stamp": 20886.4446138, "action": "move", "x": 513, "y": 382}
+{"time_stamp": 20886.4523317, "action": "move", "x": 510, "y": 382}
+{"time_stamp": 20886.4604978, "action": "move", "x": 507, "y": 382}
+{"time_stamp": 20886.4683192, "action": "move", "x": 505, "y": 382}
+{"time_stamp": 20886.4767501, "action": "move", "x": 503, "y": 382}
+{"time_stamp": 20886.4847723, "action": "move", "x": 502, "y": 382}
+{"time_stamp": 20886.4925822, "action": "move", "x": 501, "y": 382}
+{"time_stamp": 20886.5413184, "action": "move", "x": 501, "y": 382}
+{"time_stamp": 20886.5416084, "action": "click", "x": 501, "y": 382, "button": "left", "pressed": true}
+{"time_stamp": 20886.6066953, "action": "move", "x": 501, "y": 382}
+{"time_stamp": 20886.6068837, "action": "click", "x": 501, "y": 382, "button": "left", "pressed": false}
+{"time_stamp": 20886.6769357, "action": "click", "x": 501, "y": 382, "button": "left", "pressed": true}
+{"time_stamp": 20886.67733, "action": "move", "x": 501, "y": 382}
+{"time_stamp": 20886.7654135, "action": "click", "x": 501, "y": 382, "button": "left", "pressed": false}
+{"time_stamp": 20886.765583, "action": "move", "x": 501, "y": 382}
+{"time_stamp": 20886.8606369, "action": "move", "x": 501, "y": 382}
+{"time_stamp": 20886.8768414, "action": "move", "x": 502, "y": 382}
+{"time_stamp": 20886.8854348, "action": "move", "x": 502, "y": 382}
+{"time_stamp": 20886.8928413, "action": "move", "x": 503, "y": 382}
+{"time_stamp": 20886.9020618, "action": "move", "x": 504, "y": 382}
+{"time_stamp": 20886.9085807, "action": "move", "x": 504, "y": 382}
+{"time_stamp": 20886.9172158, "action": "move", "x": 505, "y": 382}
+{"time_stamp": 20886.9245427, "action": "move", "x": 506, "y": 382}
+{"time_stamp": 20886.9329781, "action": "move", "x": 507, "y": 382}
+{"time_stamp": 20886.9409988, "action": "move", "x": 508, "y": 382}
+{"time_stamp": 20886.9514492, "action": "move", "x": 509, "y": 382}
+{"time_stamp": 20886.956446, "action": "move", "x": 511, "y": 382}
+{"time_stamp": 20886.9651331, "action": "move", "x": 512, "y": 382}
+{"time_stamp": 20886.9723365, "action": "move", "x": 513, "y": 382}
+{"time_stamp": 20886.9804385, "action": "move", "x": 515, "y": 382}
+{"time_stamp": 20886.988291, "action": "move", "x": 518, "y": 382}
+{"time_stamp": 20886.9969054, "action": "move", "x": 521, "y": 383}
+{"time_stamp": 20887.0047228, "action": "move", "x": 525, "y": 383}
+{"time_stamp": 20887.0131814, "action": "move", "x": 531, "y": 383}
+{"time_stamp": 20887.0207604, "action": "move", "x": 536, "y": 383}
+{"time_stamp": 20887.0290548, "action": "move", "x": 542, "y": 383}
+{"time_stamp": 20887.0367786, "action": "move", "x": 546, "y": 383}
+{"time_stamp": 20887.0446786, "action": "move", "x": 550, "y": 383}
+{"time_stamp": 20887.0527477, "action": "move", "x": 553, "y": 384}
+{"time_stamp": 20887.0614582, "action": "move", "x": 555, "y": 384}
+{"time_stamp": 20887.0688024, "action": "move", "x": 558, "y": 384}
+{"time_stamp": 20887.0767121, "action": "move", "x": 559, "y": 384}
+{"time_stamp": 20887.084849, "action": "move", "x": 559, "y": 384}
+{"time_stamp": 20887.0928521, "action": "move", "x": 560, "y": 384}
+{"time_stamp": 20887.1407264, "action": "move", "x": 560, "y": 385}
+{"time_stamp": 20887.1419092, "action": "move", "x": 560, "y": 385}
+{"time_stamp": 20887.142089, "action": "click", "x": 560, "y": 385, "button": "left", "pressed": true}
+{"time_stamp": 20887.2210254, "action": "move", "x": 560, "y": 385}
+{"time_stamp": 20887.2210899, "action": "click", "x": 560, "y": 385, "button": "left", "pressed": false}
+{"time_stamp": 20887.2773118, "action": "move", "x": 560, "y": 385}
+{"time_stamp": 20887.2773927, "action": "click", "x": 560, "y": 385, "button": "left", "pressed": true}
+{"time_stamp": 20887.3731484, "action": "move", "x": 560, "y": 385}
+{"time_stamp": 20887.373358, "action": "click", "x": 560, "y": 385, "button": "left", "pressed": false}
+{"time_stamp": 20887.4849951, "action": "move", "x": 561, "y": 385}
+{"time_stamp": 20887.5008889, "action": "move", "x": 561, "y": 385}
+{"time_stamp": 20887.5091234, "action": "move", "x": 562, "y": 385}
+{"time_stamp": 20887.5166536, "action": "move", "x": 563, "y": 385}
+{"time_stamp": 20887.5245447, "action": "move", "x": 566, "y": 385}
+{"time_stamp": 20887.5326788, "action": "move", "x": 568, "y": 385}
+{"time_stamp": 20887.5406298, "action": "move", "x": 571, "y": 385}
+{"time_stamp": 20887.5490931, "action": "move", "x": 575, "y": 385}
+{"time_stamp": 20887.5572623, "action": "move", "x": 578, "y": 385}
+{"time_stamp": 20887.5646613, "action": "move", "x": 582, "y": 385}
+{"time_stamp": 20887.5725499, "action": "move", "x": 586, "y": 386}
+{"time_stamp": 20887.5805645, "action": "move", "x": 590, "y": 386}
+{"time_stamp": 20887.5884829, "action": "move", "x": 596, "y": 386}
+{"time_stamp": 20887.5966087, "action": "move", "x": 601, "y": 386}
+{"time_stamp": 20887.6048227, "action": "move", "x": 606, "y": 386}
+{"time_stamp": 20887.6126744, "action": "move", "x": 610, "y": 386}
+{"time_stamp": 20887.6224145, "action": "move", "x": 613, "y": 386}
+{"time_stamp": 20887.6288013, "action": "move", "x": 615, "y": 386}
+{"time_stamp": 20887.6367294, "action": "move", "x": 617, "y": 386}
+{"time_stamp": 20887.6446103, "action": "move", "x": 619, "y": 386}
+{"time_stamp": 20887.6525411, "action": "move", "x": 620, "y": 386}
+{"time_stamp": 20887.6607461, "action": "move", "x": 620, "y": 386}
+{"time_stamp": 20887.6692536, "action": "move", "x": 621, "y": 386}
+{"time_stamp": 20887.6768085, "action": "move", "x": 622, "y": 386}
+{"time_stamp": 20887.6929049, "action": "move", "x": 623, "y": 386}
+{"time_stamp": 20887.733524, "action": "move", "x": 623, "y": 386}
+{"time_stamp": 20887.733667, "action": "click", "x": 623, "y": 386, "button": "left", "pressed": true}
+{"time_stamp": 20887.8132072, "action": "move", "x": 623, "y": 386}
+{"time_stamp": 20887.813383, "action": "click", "x": 623, "y": 386, "button": "left", "pressed": false}
+{"time_stamp": 20887.8777183, "action": "move", "x": 623, "y": 386}
+{"time_stamp": 20887.878104, "action": "click", "x": 623, "y": 386, "button": "left", "pressed": true}
+{"time_stamp": 20887.9732206, "action": "move", "x": 623, "y": 386}
+{"time_stamp": 20887.9734133, "action": "click", "x": 623, "y": 386, "button": "left", "pressed": false}
+{"time_stamp": 20888.0129113, "action": "move", "x": 624, "y": 386}
+{"time_stamp": 20888.0217306, "action": "move", "x": 626, "y": 386}
+{"time_stamp": 20888.0287836, "action": "move", "x": 628, "y": 386}
+{"time_stamp": 20888.0368828, "action": "move", "x": 632, "y": 386}
+{"time_stamp": 20888.0447276, "action": "move", "x": 638, "y": 387}
+{"time_stamp": 20888.0528328, "action": "move", "x": 644, "y": 387}
+{"time_stamp": 20888.06039, "action": "move", "x": 653, "y": 387}
+{"time_stamp": 20888.0693848, "action": "move", "x": 662, "y": 387}
+{"time_stamp": 20888.0763292, "action": "move", "x": 672, "y": 387}
+{"time_stamp": 20888.084808, "action": "move", "x": 681, "y": 387}
+{"time_stamp": 20888.0925201, "action": "move", "x": 688, "y": 387}
+{"time_stamp": 20888.1005118, "action": "move", "x": 694, "y": 389}
+{"time_stamp": 20888.1086534, "action": "move", "x": 700, "y": 389}
+{"time_stamp": 20888.1165696, "action": "move", "x": 704, "y": 389}
+{"time_stamp": 20888.1247116, "action": "move", "x": 707, "y": 389}
+{"time_stamp": 20888.1327588, "action": "move", "x": 709, "y": 389}
+{"time_stamp": 20888.1405444, "action": "move", "x": 711, "y": 389}
+{"time_stamp": 20888.1484073, "action": "move", "x": 712, "y": 389}
+{"time_stamp": 20888.1566511, "action": "move", "x": 712, "y": 389}
+{"time_stamp": 20888.1646907, "action": "move", "x": 713, "y": 389}
+{"time_stamp": 20888.172634, "action": "move", "x": 715, "y": 389}
+{"time_stamp": 20888.181099, "action": "move", "x": 715, "y": 389}
+{"time_stamp": 20888.1888003, "action": "move", "x": 716, "y": 389}
+{"time_stamp": 20888.2045134, "action": "move", "x": 716, "y": 389}
+{"time_stamp": 20888.2451223, "action": "move", "x": 716, "y": 389}
+{"time_stamp": 20888.2453113, "action": "click", "x": 716, "y": 389, "button": "left", "pressed": true}
+{"time_stamp": 20888.3249476, "action": "move", "x": 716, "y": 389}
+{"time_stamp": 20888.3250101, "action": "click", "x": 716, "y": 389, "button": "left", "pressed": false}
+{"time_stamp": 20888.3910788, "action": "click", "x": 716, "y": 389, "button": "left", "pressed": true}
+{"time_stamp": 20888.391679, "action": "move", "x": 716, "y": 389}
+{"time_stamp": 20888.48527, "action": "move", "x": 716, "y": 389}
+{"time_stamp": 20888.4855224, "action": "click", "x": 716, "y": 389, "button": "left", "pressed": false}
+{"time_stamp": 20888.5086267, "action": "move", "x": 716, "y": 390}
+{"time_stamp": 20888.5166186, "action": "move", "x": 715, "y": 390}
+{"time_stamp": 20888.5247422, "action": "move", "x": 712, "y": 391}
+{"time_stamp": 20888.5326865, "action": "move", "x": 709, "y": 393}
+{"time_stamp": 20888.5404878, "action": "move", "x": 706, "y": 394}
+{"time_stamp": 20888.5485758, "action": "move", "x": 701, "y": 396}
+{"time_stamp": 20888.5566001, "action": "move", "x": 697, "y": 398}
+{"time_stamp": 20888.5644666, "action": "move", "x": 691, "y": 400}
+{"time_stamp": 20888.5725089, "action": "move", "x": 685, "y": 402}
+{"time_stamp": 20888.5815362, "action": "move", "x": 680, "y": 402}
+{"time_stamp": 20888.5884777, "action": "move", "x": 673, "y": 405}
+{"time_stamp": 20888.5965067, "action": "move", "x": 666, "y": 406}
+{"time_stamp": 20888.6046281, "action": "move", "x": 660, "y": 406}
+{"time_stamp": 20888.6126925, "action": "move", "x": 653, "y": 408}
+{"time_stamp": 20888.6209347, "action": "move", "x": 649, "y": 408}
+{"time_stamp": 20888.6289274, "action": "move", "x": 646, "y": 409}
+{"time_stamp": 20888.6365993, "action": "move", "x": 643, "y": 409}
+{"time_stamp": 20888.6451626, "action": "move", "x": 640, "y": 409}
+{"time_stamp": 20888.6527268, "action": "move", "x": 640, "y": 409}
+{"time_stamp": 20888.6607615, "action": "move", "x": 639, "y": 409}
+{"time_stamp": 20888.7131175, "action": "move", "x": 639, "y": 409}
+{"time_stamp": 20888.7132072, "action": "click", "x": 639, "y": 409, "button": "left", "pressed": true}
+{"time_stamp": 20888.7970238, "action": "move", "x": 639, "y": 409}
+{"time_stamp": 20888.797195, "action": "click", "x": 639, "y": 409, "button": "left", "pressed": false}
+{"time_stamp": 20888.8768802, "action": "move", "x": 639, "y": 409}
+{"time_stamp": 20888.8771549, "action": "click", "x": 639, "y": 409, "button": "left", "pressed": true}
+{"time_stamp": 20888.9813895, "action": "move", "x": 639, "y": 409}
+{"time_stamp": 20888.9814519, "action": "click", "x": 639, "y": 409, "button": "left", "pressed": false}
+{"time_stamp": 20889.0045288, "action": "move", "x": 640, "y": 409}
+{"time_stamp": 20889.0123232, "action": "move", "x": 642, "y": 409}
+{"time_stamp": 20889.0205029, "action": "move", "x": 645, "y": 409}
+{"time_stamp": 20889.0286024, "action": "move", "x": 649, "y": 409}
+{"time_stamp": 20889.0369849, "action": "move", "x": 653, "y": 409}
+{"time_stamp": 20889.0451786, "action": "move", "x": 657, "y": 409}
+{"time_stamp": 20889.0525366, "action": "move", "x": 664, "y": 409}
+{"time_stamp": 20889.0607887, "action": "move", "x": 672, "y": 409}
+{"time_stamp": 20889.0684144, "action": "move", "x": 680, "y": 409}
+{"time_stamp": 20889.0765469, "action": "move", "x": 687, "y": 409}
+{"time_stamp": 20889.0844633, "action": "move", "x": 694, "y": 409}
+{"time_stamp": 20889.0933272, "action": "move", "x": 698, "y": 409}
+{"time_stamp": 20889.1009552, "action": "move", "x": 704, "y": 409}
+{"time_stamp": 20889.1088402, "action": "move", "x": 707, "y": 409}
+{"time_stamp": 20889.1165285, "action": "move", "x": 709, "y": 409}
+{"time_stamp": 20889.1249949, "action": "move", "x": 710, "y": 409}
+{"time_stamp": 20889.1328139, "action": "move", "x": 711, "y": 409}
+{"time_stamp": 20889.1405815, "action": "move", "x": 711, "y": 409}
+{"time_stamp": 20889.1809298, "action": "move", "x": 711, "y": 409}
+{"time_stamp": 20889.1812352, "action": "click", "x": 711, "y": 409, "button": "left", "pressed": true}
+{"time_stamp": 20889.2689349, "action": "move", "x": 711, "y": 409}
+{"time_stamp": 20889.2691323, "action": "click", "x": 711, "y": 409, "button": "left", "pressed": false}
+{"time_stamp": 20889.3327678, "action": "move", "x": 711, "y": 409}
+{"time_stamp": 20889.3329603, "action": "click", "x": 711, "y": 409, "button": "left", "pressed": true}
+{"time_stamp": 20889.4292096, "action": "move", "x": 711, "y": 409}
+{"time_stamp": 20889.4294004, "action": "click", "x": 711, "y": 409, "button": "left", "pressed": false}
+{"time_stamp": 20889.4945422, "action": "move", "x": 711, "y": 410}
+{"time_stamp": 20889.5009056, "action": "move", "x": 709, "y": 411}
+{"time_stamp": 20889.5085702, "action": "move", "x": 707, "y": 413}
+{"time_stamp": 20889.5163096, "action": "move", "x": 705, "y": 414}
+{"time_stamp": 20889.5246812, "action": "move", "x": 703, "y": 416}
+{"time_stamp": 20889.5325334, "action": "move", "x": 699, "y": 419}
+{"time_stamp": 20889.5411918, "action": "move", "x": 695, "y": 421}
+{"time_stamp": 20889.5485218, "action": "move", "x": 689, "y": 425}
+{"time_stamp": 20889.557168, "action": "move", "x": 684, "y": 428}
+{"time_stamp": 20889.5647331, "action": "move", "x": 678, "y": 431}
+{"time_stamp": 20889.5726347, "action": "move", "x": 675, "y": 433}
+{"time_stamp": 20889.5805106, "action": "move", "x": 670, "y": 436}
+{"time_stamp": 20889.5885605, "action": "move", "x": 666, "y": 438}
+{"time_stamp": 20889.5965673, "action": "move", "x": 661, "y": 441}
+{"time_stamp": 20889.6052303, "action": "move", "x": 658, "y": 443}
+{"time_stamp": 20889.6125782, "action": "move", "x": 654, "y": 444}
+{"time_stamp": 20889.6208992, "action": "move", "x": 652, "y": 446}
+{"time_stamp": 20889.6288901, "action": "move", "x": 650, "y": 446}
+{"time_stamp": 20889.6366668, "action": "move", "x": 647, "y": 447}
+{"time_stamp": 20889.6443109, "action": "move", "x": 646, "y": 448}
+{"time_stamp": 20889.6523586, "action": "move", "x": 645, "y": 448}
+{"time_stamp": 20889.6606285, "action": "move", "x": 644, "y": 448}
+{"time_stamp": 20889.6696248, "action": "move", "x": 644, "y": 448}
+{"time_stamp": 20889.7571965, "action": "move", "x": 644, "y": 448}
+{"time_stamp": 20889.7574841, "action": "click", "x": 644, "y": 448, "button": "left", "pressed": true}
+{"time_stamp": 20889.8449651, "action": "move", "x": 644, "y": 448}
+{"time_stamp": 20889.8451911, "action": "click", "x": 644, "y": 448, "button": "left", "pressed": false}
+{"time_stamp": 20889.9248543, "action": "move", "x": 644, "y": 449}
+{"time_stamp": 20889.933104, "action": "move", "x": 644, "y": 449}
+{"time_stamp": 20889.9333437, "action": "click", "x": 644, "y": 449, "button": "left", "pressed": true}
+{"time_stamp": 20890.0368817, "action": "move", "x": 644, "y": 449}
+{"time_stamp": 20890.0370434, "action": "click", "x": 644, "y": 449, "button": "left", "pressed": false}
+{"time_stamp": 20890.1249922, "action": "move", "x": 644, "y": 449}
+{"time_stamp": 20890.1337683, "action": "move", "x": 644, "y": 449}
+{"time_stamp": 20890.1409876, "action": "move", "x": 645, "y": 449}
+{"time_stamp": 20890.1491922, "action": "move", "x": 646, "y": 449}
+{"time_stamp": 20890.1568611, "action": "move", "x": 647, "y": 449}
+{"time_stamp": 20890.164678, "action": "move", "x": 649, "y": 449}
+{"time_stamp": 20890.1724802, "action": "move", "x": 651, "y": 449}
+{"time_stamp": 20890.1807197, "action": "move", "x": 654, "y": 449}
+{"time_stamp": 20890.188609, "action": "move", "x": 657, "y": 449}
+{"time_stamp": 20890.1971871, "action": "move", "x": 660, "y": 449}
+{"time_stamp": 20890.2046771, "action": "move", "x": 663, "y": 449}
+{"time_stamp": 20890.2123347, "action": "move", "x": 666, "y": 449}
+{"time_stamp": 20890.2205577, "action": "move", "x": 670, "y": 449}
+{"time_stamp": 20890.2289758, "action": "move", "x": 673, "y": 449}
+{"time_stamp": 20890.2367732, "action": "move", "x": 676, "y": 449}
+{"time_stamp": 20890.2457101, "action": "move", "x": 679, "y": 449}
+{"time_stamp": 20890.2527043, "action": "move", "x": 683, "y": 449}
+{"time_stamp": 20890.2613182, "action": "move", "x": 686, "y": 449}
+{"time_stamp": 20890.268621, "action": "move", "x": 690, "y": 449}
+{"time_stamp": 20890.2767753, "action": "move", "x": 693, "y": 449}
+{"time_stamp": 20890.2850479, "action": "move", "x": 697, "y": 449}
+{"time_stamp": 20890.2927533, "action": "move", "x": 700, "y": 449}
+{"time_stamp": 20890.3008228, "action": "move", "x": 704, "y": 449}
+{"time_stamp": 20890.309706, "action": "move", "x": 707, "y": 448}
+{"time_stamp": 20890.3165572, "action": "move", "x": 712, "y": 447}
+{"time_stamp": 20890.3247849, "action": "move", "x": 714, "y": 447}
+{"time_stamp": 20890.3326016, "action": "move", "x": 716, "y": 447}
+{"time_stamp": 20890.3408413, "action": "move", "x": 717, "y": 447}
+{"time_stamp": 20890.348607, "action": "move", "x": 718, "y": 446}
+{"time_stamp": 20890.357255, "action": "move", "x": 719, "y": 446}
+{"time_stamp": 20890.4048309, "action": "move", "x": 720, "y": 446}
+{"time_stamp": 20890.4212384, "action": "move", "x": 720, "y": 446}
+{"time_stamp": 20890.4212982, "action": "click", "x": 720, "y": 446, "button": "left", "pressed": true}
+{"time_stamp": 20890.4930247, "action": "move", "x": 720, "y": 446}
+{"time_stamp": 20890.4932062, "action": "click", "x": 720, "y": 446, "button": "left", "pressed": false}
+{"time_stamp": 20890.5649939, "action": "move", "x": 720, "y": 446}
+{"time_stamp": 20890.5651866, "action": "click", "x": 720, "y": 446, "button": "left", "pressed": true}
+{"time_stamp": 20890.6613704, "action": "move", "x": 720, "y": 446}
+{"time_stamp": 20890.6615445, "action": "click", "x": 720, "y": 446, "button": "left", "pressed": false}
+{"time_stamp": 20890.700915, "action": "move", "x": 720, "y": 446}
+{"time_stamp": 20890.7087118, "action": "move", "x": 719, "y": 446}
+{"time_stamp": 20890.716447, "action": "move", "x": 718, "y": 447}
+{"time_stamp": 20890.7249567, "action": "move", "x": 716, "y": 447}
+{"time_stamp": 20890.7329681, "action": "move", "x": 715, "y": 447}
+{"time_stamp": 20890.7413977, "action": "move", "x": 713, "y": 448}
+{"time_stamp": 20890.748687, "action": "move", "x": 711, "y": 448}
+{"time_stamp": 20890.7563417, "action": "move", "x": 709, "y": 448}
+{"time_stamp": 20890.7652709, "action": "move", "x": 707, "y": 448}
+{"time_stamp": 20890.7736886, "action": "move", "x": 704, "y": 448}
+{"time_stamp": 20890.7806604, "action": "move", "x": 701, "y": 448}
+{"time_stamp": 20890.7885603, "action": "move", "x": 699, "y": 448}
+{"time_stamp": 20890.7964335, "action": "move", "x": 696, "y": 448}
+{"time_stamp": 20890.8045634, "action": "move", "x": 693, "y": 450}
+{"time_stamp": 20890.8138164, "action": "move", "x": 692, "y": 450}
+{"time_stamp": 20890.8203062, "action": "move", "x": 690, "y": 450}
+{"time_stamp": 20890.8286962, "action": "move", "x": 688, "y": 450}
+{"time_stamp": 20890.8370902, "action": "move", "x": 685, "y": 450}
+{"time_stamp": 20890.8446082, "action": "move", "x": 684, "y": 451}
+{"time_stamp": 20890.8540814, "action": "move", "x": 681, "y": 451}
+{"time_stamp": 20890.8607729, "action": "move", "x": 679, "y": 452}
+{"time_stamp": 20890.8693105, "action": "move", "x": 677, "y": 452}
+{"time_stamp": 20890.8769606, "action": "move", "x": 674, "y": 453}
+{"time_stamp": 20890.8845649, "action": "move", "x": 672, "y": 454}
+{"time_stamp": 20890.8928874, "action": "move", "x": 670, "y": 455}
+{"time_stamp": 20890.9013758, "action": "move", "x": 669, "y": 455}
+{"time_stamp": 20890.9087683, "action": "move", "x": 667, "y": 455}
+{"time_stamp": 20890.9166522, "action": "move", "x": 666, "y": 456}
+{"time_stamp": 20890.9248176, "action": "move", "x": 664, "y": 456}
+{"time_stamp": 20890.9333449, "action": "move", "x": 663, "y": 456}
+{"time_stamp": 20890.9403141, "action": "move", "x": 662, "y": 456}
+{"time_stamp": 20890.9491818, "action": "move", "x": 661, "y": 457}
+{"time_stamp": 20890.9573498, "action": "move", "x": 661, "y": 457}
+{"time_stamp": 20890.9643001, "action": "move", "x": 660, "y": 457}
+{"time_stamp": 20890.9806352, "action": "move", "x": 660, "y": 458}
+{"time_stamp": 20890.9982276, "action": "move", "x": 660, "y": 458}
+{"time_stamp": 20890.9982983, "action": "click", "x": 660, "y": 458, "button": "left", "pressed": true}
+{"time_stamp": 20891.0849092, "action": "move", "x": 660, "y": 458}
+{"time_stamp": 20891.0851035, "action": "click", "x": 660, "y": 458, "button": "left", "pressed": false}
+{"time_stamp": 20891.1492936, "action": "move", "x": 660, "y": 458}
+{"time_stamp": 20891.1494984, "action": "click", "x": 660, "y": 458, "button": "left", "pressed": true}
+{"time_stamp": 20891.245245, "action": "move", "x": 660, "y": 458}
+{"time_stamp": 20891.2455736, "action": "click", "x": 660, "y": 458, "button": "left", "pressed": false}
+{"time_stamp": 20891.2928605, "action": "move", "x": 660, "y": 458}
+{"time_stamp": 20891.300785, "action": "move", "x": 662, "y": 458}
+{"time_stamp": 20891.308618, "action": "move", "x": 663, "y": 458}
+{"time_stamp": 20891.3167525, "action": "move", "x": 665, "y": 458}
+{"time_stamp": 20891.3247433, "action": "move", "x": 667, "y": 458}
+{"time_stamp": 20891.3327507, "action": "move", "x": 670, "y": 458}
+{"time_stamp": 20891.3405584, "action": "move", "x": 674, "y": 458}
+{"time_stamp": 20891.348594, "action": "move", "x": 678, "y": 458}
+{"time_stamp": 20891.3568297, "action": "move", "x": 682, "y": 458}
+{"time_stamp": 20891.3648252, "action": "move", "x": 689, "y": 458}
+{"time_stamp": 20891.3726139, "action": "move", "x": 694, "y": 458}
+{"time_stamp": 20891.3803799, "action": "move", "x": 700, "y": 458}
+{"time_stamp": 20891.3886755, "action": "move", "x": 706, "y": 458}
+{"time_stamp": 20891.3968848, "action": "move", "x": 710, "y": 458}
+{"time_stamp": 20891.4046814, "action": "move", "x": 713, "y": 458}
+{"time_stamp": 20891.412626, "action": "move", "x": 715, "y": 458}
+{"time_stamp": 20891.4207024, "action": "move", "x": 716, "y": 458}
+{"time_stamp": 20891.4287184, "action": "move", "x": 717, "y": 458}
+{"time_stamp": 20891.4445718, "action": "move", "x": 718, "y": 458}
+{"time_stamp": 20891.6689423, "action": "move", "x": 717, "y": 458}
+{"time_stamp": 20891.6766172, "action": "move", "x": 716, "y": 458}
+{"time_stamp": 20891.6845902, "action": "move", "x": 714, "y": 458}
+{"time_stamp": 20891.6931544, "action": "move", "x": 712, "y": 458}
+{"time_stamp": 20891.7015555, "action": "move", "x": 709, "y": 458}
+{"time_stamp": 20891.7084966, "action": "move", "x": 706, "y": 458}
+{"time_stamp": 20891.7166826, "action": "move", "x": 703, "y": 458}
+{"time_stamp": 20891.7248871, "action": "move", "x": 700, "y": 458}
+{"time_stamp": 20891.7326873, "action": "move", "x": 699, "y": 458}
+{"time_stamp": 20891.7408329, "action": "move", "x": 697, "y": 458}
+{"time_stamp": 20891.7487206, "action": "move", "x": 696, "y": 458}
+{"time_stamp": 20892.3970057, "action": "move", "x": 696, "y": 458}
+{"time_stamp": 20892.4204299, "action": "move", "x": 695, "y": 458}
+{"time_stamp": 20892.4284802, "action": "move", "x": 694, "y": 458}
+{"time_stamp": 20892.4363669, "action": "move", "x": 694, "y": 458}
+{"time_stamp": 20892.4444876, "action": "move", "x": 693, "y": 459}
+{"time_stamp": 20892.4536681, "action": "move", "x": 692, "y": 459}
+{"time_stamp": 20892.4546344, "action": "press", "name": "s"}
+{"time_stamp": 20892.4614135, "action": "move", "x": 690, "y": 459}
+{"time_stamp": 20892.4617429, "action": "press", "name": "f"}
+{"time_stamp": 20892.4690644, "action": "move", "x": 689, "y": 459}
+{"time_stamp": 20892.4767087, "action": "move", "x": 688, "y": 459}
+{"time_stamp": 20892.4857089, "action": "move", "x": 687, "y": 459}
+{"time_stamp": 20892.4928085, "action": "move", "x": 685, "y": 459}
+{"time_stamp": 20892.5007438, "action": "move", "x": 684, "y": 459}
+{"time_stamp": 20892.5088411, "action": "move", "x": 684, "y": 459}
+{"time_stamp": 20892.5168065, "action": "move", "x": 683, "y": 459}
+{"time_stamp": 20892.5245842, "action": "move", "x": 682, "y": 459}
+{"time_stamp": 20892.5330854, "action": "move", "x": 682, "y": 459}
+{"time_stamp": 20892.5407152, "action": "move", "x": 681, "y": 459}
+{"time_stamp": 20892.5489345, "action": "move", "x": 681, "y": 459}
+{"time_stamp": 20892.5567583, "action": "move", "x": 680, "y": 459}
+{"time_stamp": 20892.565298, "action": "move", "x": 680, "y": 459}
+{"time_stamp": 20892.5727452, "action": "move", "x": 679, "y": 459}
+{"time_stamp": 20892.5806405, "action": "move", "x": 678, "y": 459}
+{"time_stamp": 20892.596976, "action": "move", "x": 678, "y": 459}
+{"time_stamp": 20892.6125852, "action": "move", "x": 677, "y": 459}
+{"time_stamp": 20892.6209153, "action": "move", "x": 677, "y": 459}
+{"time_stamp": 20892.636834, "action": "move", "x": 676, "y": 460}
+{"time_stamp": 20892.6446376, "action": "move", "x": 676, "y": 460}
+{"time_stamp": 20892.6525858, "action": "move", "x": 675, "y": 460}
+{"time_stamp": 20892.6612937, "action": "release", "name": "s"}
+{"time_stamp": 20892.6688063, "action": "press", "name": "g"}
+{"time_stamp": 20892.669381, "action": "move", "x": 674, "y": 460}
+{"time_stamp": 20892.6849101, "action": "move", "x": 674, "y": 460}
+{"time_stamp": 20892.7330967, "action": "release", "name": "f"}
+{"time_stamp": 20892.8128564, "action": "release", "name": "g"}
+{"time_stamp": 20892.9013366, "action": "move", "x": 673, "y": 460}
+{"time_stamp": 20892.9087461, "action": "move", "x": 673, "y": 460}
+{"time_stamp": 20892.916699, "action": "move", "x": 672, "y": 460}
+{"time_stamp": 20892.9249539, "action": "move", "x": 672, "y": 461}
+{"time_stamp": 20892.9330541, "action": "move", "x": 670, "y": 462}
+{"time_stamp": 20892.9407661, "action": "move", "x": 669, "y": 463}
+{"time_stamp": 20892.9486746, "action": "move", "x": 667, "y": 464}
+{"time_stamp": 20892.9574437, "action": "move", "x": 663, "y": 466}
+{"time_stamp": 20892.9648382, "action": "move", "x": 661, "y": 467}
+{"time_stamp": 20892.9725577, "action": "move", "x": 657, "y": 470}
+{"time_stamp": 20892.9812922, "action": "move", "x": 654, "y": 472}
+{"time_stamp": 20892.9887796, "action": "move", "x": 650, "y": 474}
+{"time_stamp": 20892.9968372, "action": "move", "x": 647, "y": 476}
+{"time_stamp": 20893.0047422, "action": "move", "x": 643, "y": 478}
+{"time_stamp": 20893.0127746, "action": "move", "x": 641, "y": 480}
+{"time_stamp": 20893.0206556, "action": "move", "x": 640, "y": 481}
+{"time_stamp": 20893.0289989, "action": "move", "x": 638, "y": 482}
+{"time_stamp": 20893.0367595, "action": "move", "x": 637, "y": 482}
+{"time_stamp": 20893.1013149, "action": "move", "x": 637, "y": 482}
+{"time_stamp": 20893.101488, "action": "click", "x": 637, "y": 482, "button": "left", "pressed": true}
+{"time_stamp": 20893.1889859, "action": "move", "x": 637, "y": 482}
+{"time_stamp": 20893.189078, "action": "click", "x": 637, "y": 482, "button": "left", "pressed": false}
+{"time_stamp": 20893.2451512, "action": "move", "x": 637, "y": 483}
+{"time_stamp": 20893.2538637, "action": "move", "x": 637, "y": 483}
+{"time_stamp": 20893.2539744, "action": "click", "x": 637, "y": 483, "button": "left", "pressed": true}
+{"time_stamp": 20893.3570077, "action": "move", "x": 637, "y": 483}
+{"time_stamp": 20893.3572079, "action": "click", "x": 637, "y": 483, "button": "left", "pressed": false}
+{"time_stamp": 20893.6286203, "action": "move", "x": 637, "y": 483}
+{"time_stamp": 20893.6370858, "action": "move", "x": 637, "y": 484}
+{"time_stamp": 20893.6446825, "action": "move", "x": 637, "y": 485}
+{"time_stamp": 20893.6610517, "action": "move", "x": 636, "y": 485}
+{"time_stamp": 20893.6695956, "action": "move", "x": 636, "y": 486}
+{"time_stamp": 20893.6765598, "action": "move", "x": 636, "y": 486}
+{"time_stamp": 20893.6844952, "action": "move", "x": 636, "y": 487}
+{"time_stamp": 20893.692857, "action": "move", "x": 635, "y": 487}
+{"time_stamp": 20893.7015009, "action": "move", "x": 635, "y": 488}
+{"time_stamp": 20893.7083389, "action": "move", "x": 634, "y": 489}
+{"time_stamp": 20893.7167498, "action": "move", "x": 633, "y": 489}
+{"time_stamp": 20893.7244764, "action": "move", "x": 631, "y": 490}
+{"time_stamp": 20893.7326701, "action": "move", "x": 631, "y": 491}
+{"time_stamp": 20893.740882, "action": "move", "x": 629, "y": 492}
+{"time_stamp": 20893.7487729, "action": "move", "x": 628, "y": 493}
+{"time_stamp": 20893.7568486, "action": "move", "x": 626, "y": 493}
+{"time_stamp": 20893.7644792, "action": "move", "x": 624, "y": 494}
+{"time_stamp": 20893.7727393, "action": "move", "x": 621, "y": 494}
+{"time_stamp": 20893.7807102, "action": "move", "x": 620, "y": 496}
+{"time_stamp": 20893.7886298, "action": "move", "x": 619, "y": 496}
+{"time_stamp": 20893.7966263, "action": "move", "x": 617, "y": 496}
+{"time_stamp": 20893.804543, "action": "move", "x": 616, "y": 496}
+{"time_stamp": 20893.8126195, "action": "move", "x": 614, "y": 497}
+{"time_stamp": 20893.8206312, "action": "move", "x": 613, "y": 497}
+{"time_stamp": 20893.8289317, "action": "move", "x": 612, "y": 497}
+{"time_stamp": 20893.8363868, "action": "move", "x": 610, "y": 497}
+{"time_stamp": 20893.8446788, "action": "move", "x": 609, "y": 498}
+{"time_stamp": 20893.8526979, "action": "move", "x": 609, "y": 498}
+{"time_stamp": 20893.8606439, "action": "move", "x": 608, "y": 498}
+{"time_stamp": 20893.8684564, "action": "move", "x": 607, "y": 498}
+{"time_stamp": 20893.8764223, "action": "move", "x": 606, "y": 498}
+{"time_stamp": 20893.8844535, "action": "move", "x": 605, "y": 498}
+{"time_stamp": 20893.893019, "action": "move", "x": 604, "y": 498}
+{"time_stamp": 20893.9014853, "action": "move", "x": 602, "y": 500}
+{"time_stamp": 20893.9085319, "action": "move", "x": 601, "y": 501}
+{"time_stamp": 20893.9413667, "action": "move", "x": 601, "y": 501}
+{"time_stamp": 20893.9415573, "action": "click", "x": 601, "y": 501, "button": "left", "pressed": true}
+{"time_stamp": 20894.0050118, "action": "move", "x": 601, "y": 501}
+{"time_stamp": 20894.005183, "action": "click", "x": 601, "y": 501, "button": "left", "pressed": false}
+{"time_stamp": 20894.1019599, "action": "move", "x": 601, "y": 501}
+{"time_stamp": 20894.1021359, "action": "click", "x": 601, "y": 501, "button": "left", "pressed": true}
+{"time_stamp": 20894.2133482, "action": "move", "x": 601, "y": 501}
+{"time_stamp": 20894.2135946, "action": "click", "x": 601, "y": 501, "button": "left", "pressed": false}
+{"time_stamp": 20894.2365269, "action": "move", "x": 601, "y": 501}
+{"time_stamp": 20894.2444302, "action": "move", "x": 600, "y": 501}
+{"time_stamp": 20894.2526341, "action": "move", "x": 599, "y": 501}
+{"time_stamp": 20894.2606308, "action": "move", "x": 598, "y": 501}
+{"time_stamp": 20894.2684493, "action": "move", "x": 596, "y": 501}
+{"time_stamp": 20894.2768659, "action": "move", "x": 594, "y": 501}
+{"time_stamp": 20894.2843968, "action": "move", "x": 592, "y": 501}
+{"time_stamp": 20894.2926081, "action": "move", "x": 589, "y": 501}
+{"time_stamp": 20894.3009174, "action": "move", "x": 585, "y": 501}
+{"time_stamp": 20894.3088379, "action": "move", "x": 582, "y": 501}
+{"time_stamp": 20894.3167216, "action": "move", "x": 578, "y": 501}
+{"time_stamp": 20894.3246355, "action": "move", "x": 574, "y": 501}
+{"time_stamp": 20894.3325733, "action": "move", "x": 570, "y": 501}
+{"time_stamp": 20894.3407124, "action": "move", "x": 567, "y": 501}
+{"time_stamp": 20894.3487423, "action": "move", "x": 564, "y": 501}
+{"time_stamp": 20894.3572054, "action": "move", "x": 561, "y": 501}
+{"time_stamp": 20894.364766, "action": "move", "x": 559, "y": 501}
+{"time_stamp": 20894.3724545, "action": "move", "x": 558, "y": 501}
+{"time_stamp": 20894.3804239, "action": "move", "x": 557, "y": 501}
+{"time_stamp": 20894.3885445, "action": "move", "x": 557, "y": 501}
+{"time_stamp": 20894.4209401, "action": "move", "x": 557, "y": 501}
+{"time_stamp": 20894.4210435, "action": "click", "x": 557, "y": 501, "button": "left", "pressed": true}
+{"time_stamp": 20894.5090442, "action": "move", "x": 557, "y": 501}
+{"time_stamp": 20894.5092702, "action": "click", "x": 557, "y": 501, "button": "left", "pressed": false}
+{"time_stamp": 20894.5814724, "action": "move", "x": 557, "y": 501}
+{"time_stamp": 20894.5816759, "action": "click", "x": 557, "y": 501, "button": "left", "pressed": true}
+{"time_stamp": 20894.6852405, "action": "move", "x": 557, "y": 501}
+{"time_stamp": 20894.6853152, "action": "click", "x": 557, "y": 501, "button": "left", "pressed": false}
+{"time_stamp": 20894.72529, "action": "move", "x": 557, "y": 500}
+{"time_stamp": 20894.7334281, "action": "move", "x": 557, "y": 500}
+{"time_stamp": 20894.7413806, "action": "move", "x": 557, "y": 498}
+{"time_stamp": 20894.7491397, "action": "move", "x": 558, "y": 497}
+{"time_stamp": 20894.7571967, "action": "move", "x": 558, "y": 497}
+{"time_stamp": 20894.7648075, "action": "move", "x": 559, "y": 496}
+{"time_stamp": 20894.7737153, "action": "move", "x": 560, "y": 494}
+{"time_stamp": 20894.7809335, "action": "move", "x": 561, "y": 493}
+{"time_stamp": 20894.7885676, "action": "move", "x": 562, "y": 491}
+{"time_stamp": 20894.7969193, "action": "move", "x": 562, "y": 490}
+{"time_stamp": 20894.8050925, "action": "move", "x": 564, "y": 489}
+{"time_stamp": 20894.8126294, "action": "move", "x": 565, "y": 487}
+{"time_stamp": 20894.8219121, "action": "move", "x": 565, "y": 486}
+{"time_stamp": 20894.8288081, "action": "move", "x": 567, "y": 485}
+{"time_stamp": 20894.8364699, "action": "move", "x": 567, "y": 485}
+{"time_stamp": 20894.8448329, "action": "move", "x": 568, "y": 484}
+{"time_stamp": 20895.0287889, "action": "move", "x": 568, "y": 484}
+{"time_stamp": 20895.0365856, "action": "move", "x": 568, "y": 485}
+{"time_stamp": 20895.0447459, "action": "move", "x": 568, "y": 485}
+{"time_stamp": 20895.0526688, "action": "move", "x": 568, "y": 486}
+{"time_stamp": 20895.0609373, "action": "move", "x": 568, "y": 487}
+{"time_stamp": 20895.0686669, "action": "move", "x": 568, "y": 488}
+{"time_stamp": 20895.0772702, "action": "move", "x": 568, "y": 489}
+{"time_stamp": 20895.085079, "action": "move", "x": 568, "y": 490}
+{"time_stamp": 20895.0927144, "action": "move", "x": 568, "y": 492}
+{"time_stamp": 20895.1006349, "action": "move", "x": 568, "y": 493}
+{"time_stamp": 20895.1116483, "action": "move", "x": 568, "y": 496}
+{"time_stamp": 20895.1186343, "action": "move", "x": 568, "y": 497}
+{"time_stamp": 20895.1269084, "action": "move", "x": 568, "y": 499}
+{"time_stamp": 20895.1325007, "action": "move", "x": 569, "y": 500}
+{"time_stamp": 20895.141051, "action": "move", "x": 569, "y": 501}
+{"time_stamp": 20895.1487169, "action": "move", "x": 569, "y": 502}
+{"time_stamp": 20895.1569922, "action": "move", "x": 569, "y": 505}
+{"time_stamp": 20895.1645112, "action": "move", "x": 569, "y": 506}
+{"time_stamp": 20895.1728555, "action": "move", "x": 569, "y": 508}
+{"time_stamp": 20895.1807891, "action": "move", "x": 568, "y": 508}
+{"time_stamp": 20895.1885483, "action": "move", "x": 568, "y": 509}
+{"time_stamp": 20895.1968378, "action": "move", "x": 568, "y": 509}
+{"time_stamp": 20895.4530369, "action": "press", "name": "g"}
+{"time_stamp": 20895.4608213, "action": "press", "name": "s"}
+{"time_stamp": 20895.5888185, "action": "release", "name": "g"}
+{"time_stamp": 20895.5971481, "action": "press", "name": "d"}
+{"time_stamp": 20895.7090891, "action": "press", "name": "g"}
+{"time_stamp": 20895.7329549, "action": "move", "x": 569, "y": 510}
+{"time_stamp": 20895.7408323, "action": "move", "x": 569, "y": 510}
+{"time_stamp": 20895.7489785, "action": "move", "x": 570, "y": 511}
+{"time_stamp": 20895.7495082, "action": "release", "name": "d"}
+{"time_stamp": 20895.7567307, "action": "move", "x": 571, "y": 511}
+{"time_stamp": 20895.7651018, "action": "move", "x": 574, "y": 512}
+{"time_stamp": 20895.7724507, "action": "move", "x": 576, "y": 513}
+{"time_stamp": 20895.780972, "action": "move", "x": 578, "y": 513}
+{"time_stamp": 20895.7883566, "action": "move", "x": 581, "y": 514}
+{"time_stamp": 20895.7970967, "action": "move", "x": 584, "y": 514}
+{"time_stamp": 20895.804844, "action": "move", "x": 586, "y": 514}
+{"time_stamp": 20895.8126859, "action": "move", "x": 589, "y": 516}
+{"time_stamp": 20895.8211345, "action": "move", "x": 592, "y": 516}
+{"time_stamp": 20895.8291578, "action": "move", "x": 595, "y": 516}
+{"time_stamp": 20895.8367481, "action": "move", "x": 598, "y": 516}
+{"time_stamp": 20895.8452571, "action": "move", "x": 601, "y": 516}
+{"time_stamp": 20895.8526732, "action": "move", "x": 604, "y": 516}
+{"time_stamp": 20895.8607219, "action": "move", "x": 606, "y": 516}
+{"time_stamp": 20895.8688309, "action": "move", "x": 608, "y": 516}
+{"time_stamp": 20895.8768682, "action": "move", "x": 611, "y": 516}
+{"time_stamp": 20895.8846081, "action": "move", "x": 611, "y": 516}
+{"time_stamp": 20895.8928316, "action": "move", "x": 612, "y": 516}
+{"time_stamp": 20895.9010238, "action": "move", "x": 613, "y": 516}
+{"time_stamp": 20895.9088821, "action": "release", "name": "g"}
+{"time_stamp": 20895.9091836, "action": "move", "x": 613, "y": 516}
+{"time_stamp": 20895.9163124, "action": "move", "x": 614, "y": 516}
+{"time_stamp": 20895.9256445, "action": "move", "x": 615, "y": 515}
+{"time_stamp": 20895.9327879, "action": "move", "x": 615, "y": 514}
+{"time_stamp": 20895.94085, "action": "move", "x": 616, "y": 513}
+{"time_stamp": 20895.9413621, "action": "release", "name": "s"}
+{"time_stamp": 20895.9485558, "action": "move", "x": 616, "y": 510}
+{"time_stamp": 20895.9573828, "action": "move", "x": 616, "y": 508}
+{"time_stamp": 20895.9648262, "action": "move", "x": 616, "y": 505}
+{"time_stamp": 20895.9729922, "action": "move", "x": 616, "y": 501}
+{"time_stamp": 20895.980594, "action": "move", "x": 616, "y": 496}
+{"time_stamp": 20895.9889191, "action": "move", "x": 616, "y": 490}
+{"time_stamp": 20895.9968327, "action": "move", "x": 615, "y": 483}
+{"time_stamp": 20896.0050667, "action": "move", "x": 615, "y": 475}
+{"time_stamp": 20896.0127859, "action": "move", "x": 613, "y": 466}
+{"time_stamp": 20896.0205709, "action": "move", "x": 611, "y": 456}
+{"time_stamp": 20896.0288443, "action": "move", "x": 609, "y": 448}
+{"time_stamp": 20896.0365843, "action": "move", "x": 607, "y": 438}
+{"time_stamp": 20896.0451688, "action": "move", "x": 605, "y": 428}
+{"time_stamp": 20896.0526357, "action": "move", "x": 603, "y": 419}
+{"time_stamp": 20896.0606531, "action": "move", "x": 601, "y": 410}
+{"time_stamp": 20896.0686481, "action": "move", "x": 600, "y": 402}
+{"time_stamp": 20896.0769381, "action": "move", "x": 600, "y": 393}
+{"time_stamp": 20896.0848505, "action": "move", "x": 598, "y": 385}
+{"time_stamp": 20896.0927354, "action": "move", "x": 598, "y": 378}
+{"time_stamp": 20896.1015438, "action": "move", "x": 598, "y": 371}
+{"time_stamp": 20896.1089939, "action": "move", "x": 598, "y": 364}
+{"time_stamp": 20896.1170905, "action": "move", "x": 598, "y": 359}
+{"time_stamp": 20896.1249644, "action": "move", "x": 598, "y": 355}
+{"time_stamp": 20896.1327329, "action": "move", "x": 598, "y": 351}
+{"time_stamp": 20896.1407712, "action": "move", "x": 598, "y": 348}
+{"time_stamp": 20896.1487951, "action": "move", "x": 598, "y": 346}
+{"time_stamp": 20896.1566564, "action": "move", "x": 598, "y": 343}
+{"time_stamp": 20896.165396, "action": "move", "x": 600, "y": 341}
+{"time_stamp": 20896.1729828, "action": "move", "x": 600, "y": 339}
+{"time_stamp": 20896.1809805, "action": "move", "x": 600, "y": 339}
+{"time_stamp": 20896.1885536, "action": "move", "x": 600, "y": 338}
+{"time_stamp": 20896.1969559, "action": "move", "x": 600, "y": 337}
+{"time_stamp": 20896.20478, "action": "move", "x": 600, "y": 337}
+{"time_stamp": 20896.2540541, "action": "move", "x": 600, "y": 337}
+{"time_stamp": 20896.2541297, "action": "click", "x": 600, "y": 337, "button": "left", "pressed": true}
+{"time_stamp": 20896.3331842, "action": "move", "x": 600, "y": 337}
+{"time_stamp": 20896.3333561, "action": "click", "x": 600, "y": 337, "button": "left", "pressed": false}
+{"time_stamp": 20896.4130811, "action": "move", "x": 600, "y": 337}
+{"time_stamp": 20896.4131751, "action": "click", "x": 600, "y": 337, "button": "left", "pressed": true}
+{"time_stamp": 20896.5171154, "action": "move", "x": 600, "y": 337}
+{"time_stamp": 20896.5173101, "action": "click", "x": 600, "y": 337, "button": "left", "pressed": false}
+{"time_stamp": 20896.9405954, "action": "press", "name": "g"}
+{"time_stamp": 20896.9899643, "action": "press", "name": "s"}
+{"time_stamp": 20896.9900284, "action": "move", "x": 600, "y": 337}
+{"time_stamp": 20897.0049804, "action": "move", "x": 600, "y": 337}
+{"time_stamp": 20897.0208878, "action": "move", "x": 600, "y": 337}
+{"time_stamp": 20897.0295527, "action": "move", "x": 600, "y": 338}
+{"time_stamp": 20897.0447319, "action": "move", "x": 600, "y": 339}
+{"time_stamp": 20897.0688472, "action": "move", "x": 600, "y": 339}
+{"time_stamp": 20897.0932512, "action": "move", "x": 600, "y": 340}
+{"time_stamp": 20897.1089638, "action": "move", "x": 600, "y": 340}
+{"time_stamp": 20897.1247167, "action": "move", "x": 600, "y": 341}
+{"time_stamp": 20897.1325686, "action": "move", "x": 600, "y": 341}
+{"time_stamp": 20897.1404315, "action": "move", "x": 600, "y": 341}
+{"time_stamp": 20897.1486001, "action": "press", "name": "d"}
+{"time_stamp": 20897.1567908, "action": "move", "x": 600, "y": 342}
+{"time_stamp": 20897.1571129, "action": "release", "name": "g"}
+{"time_stamp": 20897.1651369, "action": "press", "name": "f"}
+{"time_stamp": 20897.1651863, "action": "move", "x": 600, "y": 343}
+{"time_stamp": 20897.1753103, "action": "move", "x": 600, "y": 343}
+{"time_stamp": 20897.1807753, "action": "move", "x": 600, "y": 344}
+{"time_stamp": 20897.1885241, "action": "move", "x": 600, "y": 344}
+{"time_stamp": 20897.1967893, "action": "move", "x": 600, "y": 345}
+{"time_stamp": 20897.2043134, "action": "move", "x": 600, "y": 346}
+{"time_stamp": 20897.2124458, "action": "move", "x": 600, "y": 347}
+{"time_stamp": 20897.2211833, "action": "move", "x": 600, "y": 348}
+{"time_stamp": 20897.2294716, "action": "move", "x": 600, "y": 349}
+{"time_stamp": 20897.2375236, "action": "move", "x": 600, "y": 350}
+{"time_stamp": 20897.238652, "action": "press", "name": "g"}
+{"time_stamp": 20897.2459899, "action": "move", "x": 601, "y": 351}
+{"time_stamp": 20897.2528845, "action": "move", "x": 601, "y": 352}
+{"time_stamp": 20897.2533701, "action": "release", "name": "f"}
+{"time_stamp": 20897.2611989, "action": "move", "x": 601, "y": 353}
+{"time_stamp": 20897.2690938, "action": "move", "x": 602, "y": 355}
+{"time_stamp": 20897.2692582, "action": "release", "name": "d"}
+{"time_stamp": 20897.2765957, "action": "move", "x": 603, "y": 357}
+{"time_stamp": 20897.2847244, "action": "move", "x": 604, "y": 359}
+{"time_stamp": 20897.2925376, "action": "move", "x": 604, "y": 362}
+{"time_stamp": 20897.3005819, "action": "move", "x": 605, "y": 364}
+{"time_stamp": 20897.3086446, "action": "move", "x": 607, "y": 367}
+{"time_stamp": 20897.3166721, "action": "move", "x": 607, "y": 371}
+{"time_stamp": 20897.3247337, "action": "move", "x": 608, "y": 375}
+{"time_stamp": 20897.3327107, "action": "move", "x": 610, "y": 379}
+{"time_stamp": 20897.3407473, "action": "move", "x": 612, "y": 384}
+{"time_stamp": 20897.349125, "action": "move", "x": 613, "y": 390}
+{"time_stamp": 20897.3566944, "action": "move", "x": 613, "y": 394}
+{"time_stamp": 20897.3646482, "action": "move", "x": 615, "y": 400}
+{"time_stamp": 20897.3724976, "action": "move", "x": 616, "y": 406}
+{"time_stamp": 20897.3804689, "action": "move", "x": 616, "y": 412}
+{"time_stamp": 20897.3887257, "action": "move", "x": 616, "y": 419}
+{"time_stamp": 20897.3968946, "action": "move", "x": 617, "y": 426}
+{"time_stamp": 20897.404716, "action": "move", "x": 617, "y": 434}
+{"time_stamp": 20897.4127188, "action": "move", "x": 617, "y": 440}
+{"time_stamp": 20897.4205885, "action": "move", "x": 617, "y": 448}
+{"time_stamp": 20897.429361, "action": "move", "x": 617, "y": 453}
+{"time_stamp": 20897.4294988, "action": "release", "name": "g"}
+{"time_stamp": 20897.437084, "action": "move", "x": 617, "y": 460}
+{"time_stamp": 20897.4455231, "action": "move", "x": 617, "y": 465}
+{"time_stamp": 20897.4525771, "action": "move", "x": 617, "y": 470}
+{"time_stamp": 20897.4618174, "action": "move", "x": 617, "y": 475}
+{"time_stamp": 20897.4688916, "action": "move", "x": 616, "y": 478}
+{"time_stamp": 20897.4768471, "action": "move", "x": 615, "y": 482}
+{"time_stamp": 20897.4847687, "action": "move", "x": 613, "y": 485}
+{"time_stamp": 20897.4928286, "action": "move", "x": 612, "y": 486}
+{"time_stamp": 20897.5014608, "action": "move", "x": 610, "y": 489}
+{"time_stamp": 20897.501929, "action": "release", "name": "s"}
+{"time_stamp": 20897.5089432, "action": "move", "x": 608, "y": 490}
+{"time_stamp": 20897.5168199, "action": "move", "x": 607, "y": 491}
+{"time_stamp": 20897.5244068, "action": "move", "x": 606, "y": 492}
+{"time_stamp": 20897.5327922, "action": "move", "x": 604, "y": 493}
+{"time_stamp": 20897.5406936, "action": "move", "x": 603, "y": 495}
+{"time_stamp": 20897.5487703, "action": "move", "x": 601, "y": 496}
+{"time_stamp": 20897.5568244, "action": "move", "x": 600, "y": 498}
+{"time_stamp": 20897.5646813, "action": "move", "x": 598, "y": 501}
+{"time_stamp": 20897.5728571, "action": "move", "x": 597, "y": 504}
+{"time_stamp": 20897.5804318, "action": "move", "x": 594, "y": 506}
+{"time_stamp": 20897.588472, "action": "move", "x": 593, "y": 509}
+{"time_stamp": 20897.5968274, "action": "move", "x": 592, "y": 512}
+{"time_stamp": 20897.6048263, "action": "move", "x": 589, "y": 514}
+{"time_stamp": 20897.6129937, "action": "move", "x": 588, "y": 517}
+{"time_stamp": 20897.6212666, "action": "move", "x": 585, "y": 520}
+{"time_stamp": 20897.6289889, "action": "move", "x": 584, "y": 522}
+{"time_stamp": 20897.6366934, "action": "move", "x": 582, "y": 524}
+{"time_stamp": 20897.6446991, "action": "move", "x": 581, "y": 527}
+{"time_stamp": 20897.6527226, "action": "move", "x": 578, "y": 529}
+{"time_stamp": 20897.6608101, "action": "move", "x": 577, "y": 532}
+{"time_stamp": 20897.6696679, "action": "move", "x": 575, "y": 535}
+{"time_stamp": 20897.6766434, "action": "move", "x": 574, "y": 537}
+{"time_stamp": 20897.6853238, "action": "move", "x": 572, "y": 539}
+{"time_stamp": 20897.6925349, "action": "move", "x": 570, "y": 542}
+{"time_stamp": 20897.7012391, "action": "move", "x": 568, "y": 544}
+{"time_stamp": 20897.7086353, "action": "move", "x": 565, "y": 548}
+{"time_stamp": 20897.7169557, "action": "move", "x": 557, "y": 555}
+{"time_stamp": 20897.7251504, "action": "move", "x": 552, "y": 562}
+{"time_stamp": 20897.732686, "action": "move", "x": 546, "y": 570}
+{"time_stamp": 20897.7406708, "action": "move", "x": 542, "y": 577}
+{"time_stamp": 20897.7490054, "action": "move", "x": 539, "y": 581}
+{"time_stamp": 20897.7566354, "action": "move", "x": 536, "y": 583}
+{"time_stamp": 20897.7649678, "action": "move", "x": 534, "y": 585}
+{"time_stamp": 20897.772443, "action": "move", "x": 534, "y": 586}
+{"time_stamp": 20897.7810297, "action": "move", "x": 533, "y": 586}
+{"time_stamp": 20897.9492474, "action": "click", "x": 533, "y": 586, "button": "left", "pressed": true}
+{"time_stamp": 20897.9495623, "action": "move", "x": 533, "y": 586}
+{"time_stamp": 20898.0213384, "action": "click", "x": 533, "y": 586, "button": "left", "pressed": false}
+{"time_stamp": 20898.0216603, "action": "move", "x": 533, "y": 586}
+{"time_stamp": 20898.1018141, "action": "click", "x": 533, "y": 586, "button": "left", "pressed": true}
+{"time_stamp": 20898.102122, "action": "move", "x": 533, "y": 586}
+{"time_stamp": 20898.1976224, "action": "move", "x": 533, "y": 586}
+{"time_stamp": 20898.1978148, "action": "click", "x": 533, "y": 586, "button": "left", "pressed": false}
+{"time_stamp": 20898.3968409, "action": "move", "x": 533, "y": 586}
+{"time_stamp": 20898.4208372, "action": "move", "x": 533, "y": 586}
+{"time_stamp": 20898.428981, "action": "move", "x": 534, "y": 586}
+{"time_stamp": 20898.4527847, "action": "move", "x": 534, "y": 586}
+{"time_stamp": 20898.4610052, "action": "move", "x": 535, "y": 587}
+{"time_stamp": 20898.4768302, "action": "move", "x": 535, "y": 587}
+{"time_stamp": 20898.4851324, "action": "move", "x": 535, "y": 587}
+{"time_stamp": 20898.4924613, "action": "move", "x": 536, "y": 587}
+{"time_stamp": 20898.5011145, "action": "move", "x": 536, "y": 587}
+{"time_stamp": 20898.5171904, "action": "move", "x": 537, "y": 588}
+{"time_stamp": 20898.5334448, "action": "move", "x": 538, "y": 588}
+{"time_stamp": 20898.5409515, "action": "press", "name": "s"}
+{"time_stamp": 20898.5568672, "action": "press", "name": "g"}
+{"time_stamp": 20898.6689175, "action": "move", "x": 538, "y": 589}
+{"time_stamp": 20898.67725, "action": "move", "x": 539, "y": 589}
+{"time_stamp": 20898.6932681, "action": "move", "x": 539, "y": 589}
+{"time_stamp": 20898.7011419, "action": "move", "x": 539, "y": 589}
+{"time_stamp": 20898.7172393, "action": "move", "x": 540, "y": 590}
+{"time_stamp": 20898.7328772, "action": "move", "x": 540, "y": 590}
+{"time_stamp": 20898.7488405, "action": "move", "x": 540, "y": 590}
+{"time_stamp": 20898.7566429, "action": "move", "x": 541, "y": 590}
+{"time_stamp": 20898.7570806, "action": "press", "name": "f"}
+{"time_stamp": 20898.7655701, "action": "move", "x": 541, "y": 591}
+{"time_stamp": 20898.7661165, "action": "release", "name": "g"}
+{"time_stamp": 20898.7732461, "action": "move", "x": 542, "y": 591}
+{"time_stamp": 20898.7903493, "action": "move", "x": 542, "y": 592}
+{"time_stamp": 20898.7968452, "action": "move", "x": 543, "y": 592}
+{"time_stamp": 20898.8049081, "action": "move", "x": 543, "y": 592}
+{"time_stamp": 20898.8126414, "action": "move", "x": 543, "y": 593}
+{"time_stamp": 20898.8220168, "action": "move", "x": 544, "y": 593}
+{"time_stamp": 20898.8376648, "action": "move", "x": 544, "y": 594}
+{"time_stamp": 20898.8448776, "action": "move", "x": 544, "y": 595}
+{"time_stamp": 20898.8611272, "action": "press", "name": "g"}
+{"time_stamp": 20898.8612065, "action": "move", "x": 545, "y": 596}
+{"time_stamp": 20898.8687914, "action": "move", "x": 545, "y": 597}
+{"time_stamp": 20898.8855468, "action": "release", "name": "f"}
+{"time_stamp": 20898.8855758, "action": "move", "x": 545, "y": 597}
+{"time_stamp": 20898.8927039, "action": "move", "x": 545, "y": 598}
+{"time_stamp": 20898.9089199, "action": "move", "x": 545, "y": 598}
+{"time_stamp": 20898.9171943, "action": "move", "x": 545, "y": 599}
+{"time_stamp": 20898.933975, "action": "move", "x": 545, "y": 600}
+{"time_stamp": 20898.9409269, "action": "move", "x": 544, "y": 600}
+{"time_stamp": 20898.9488334, "action": "move", "x": 544, "y": 600}
+{"time_stamp": 20898.9569021, "action": "move", "x": 544, "y": 601}
+{"time_stamp": 20898.9649247, "action": "move", "x": 543, "y": 601}
+{"time_stamp": 20898.9727697, "action": "move", "x": 542, "y": 601}
+{"time_stamp": 20898.9806174, "action": "move", "x": 542, "y": 602}
+{"time_stamp": 20898.9884256, "action": "move", "x": 540, "y": 602}
+{"time_stamp": 20898.9966963, "action": "move", "x": 540, "y": 603}
+{"time_stamp": 20899.0045278, "action": "move", "x": 539, "y": 603}
+{"time_stamp": 20899.0124709, "action": "move", "x": 539, "y": 604}
+{"time_stamp": 20899.0204732, "action": "move", "x": 538, "y": 604}
+{"time_stamp": 20899.029026, "action": "move", "x": 536, "y": 605}
+{"time_stamp": 20899.0295155, "action": "press", "name": "f"}
+{"time_stamp": 20899.0367389, "action": "move", "x": 536, "y": 605}
+{"time_stamp": 20899.0448425, "action": "move", "x": 535, "y": 605}
+{"time_stamp": 20899.05265, "action": "move", "x": 533, "y": 605}
+{"time_stamp": 20899.0608828, "action": "move", "x": 531, "y": 606}
+{"time_stamp": 20899.0688469, "action": "move", "x": 530, "y": 606}
+{"time_stamp": 20899.0768381, "action": "move", "x": 528, "y": 606}
+{"time_stamp": 20899.0850529, "action": "move", "x": 526, "y": 606}
+{"time_stamp": 20899.0857579, "action": "release", "name": "g"}
+{"time_stamp": 20899.0930081, "action": "move", "x": 525, "y": 606}
+{"time_stamp": 20899.1008339, "action": "move", "x": 523, "y": 606}
+{"time_stamp": 20899.1095025, "action": "move", "x": 520, "y": 606}
+{"time_stamp": 20899.1166598, "action": "move", "x": 517, "y": 606}
+{"time_stamp": 20899.1245681, "action": "move", "x": 513, "y": 606}
+{"time_stamp": 20899.1251393, "action": "release", "name": "f"}
+{"time_stamp": 20899.1325041, "action": "move", "x": 509, "y": 605}
+{"time_stamp": 20899.1407724, "action": "move", "x": 505, "y": 605}
+{"time_stamp": 20899.1484747, "action": "move", "x": 501, "y": 604}
+{"time_stamp": 20899.1576801, "action": "move", "x": 498, "y": 604}
+{"time_stamp": 20899.1648432, "action": "move", "x": 496, "y": 604}
+{"time_stamp": 20899.1728463, "action": "move", "x": 494, "y": 604}
+{"time_stamp": 20899.1805951, "action": "move", "x": 492, "y": 604}
+{"time_stamp": 20899.1888087, "action": "move", "x": 490, "y": 604}
+{"time_stamp": 20899.1892202, "action": "release", "name": "s"}
+{"time_stamp": 20899.1964966, "action": "move", "x": 490, "y": 604}
+{"time_stamp": 20899.2051025, "action": "move", "x": 489, "y": 604}
+{"time_stamp": 20899.2128797, "action": "move", "x": 489, "y": 604}
+{"time_stamp": 20899.2932387, "action": "move", "x": 489, "y": 604}
+{"time_stamp": 20899.2933216, "action": "click", "x": 489, "y": 604, "button": "left", "pressed": true}
+{"time_stamp": 20899.357334, "action": "click", "x": 489, "y": 604, "button": "left", "pressed": false}
+{"time_stamp": 20899.35741, "action": "move", "x": 489, "y": 604}
+{"time_stamp": 20899.4047508, "action": "move", "x": 488, "y": 604}
+{"time_stamp": 20899.413393, "action": "move", "x": 487, "y": 604}
+{"time_stamp": 20899.4211033, "action": "move", "x": 485, "y": 604}
+{"time_stamp": 20899.428782, "action": "move", "x": 482, "y": 604}
+{"time_stamp": 20899.4365141, "action": "move", "x": 478, "y": 604}
+{"time_stamp": 20899.4451451, "action": "move", "x": 474, "y": 604}
+{"time_stamp": 20899.4529168, "action": "move", "x": 470, "y": 604}
+{"time_stamp": 20899.4610206, "action": "move", "x": 466, "y": 604}
+{"time_stamp": 20899.4687884, "action": "move", "x": 462, "y": 604}
+{"time_stamp": 20899.4773181, "action": "move", "x": 459, "y": 604}
+{"time_stamp": 20899.4849895, "action": "move", "x": 456, "y": 604}
+{"time_stamp": 20899.4926383, "action": "move", "x": 454, "y": 604}
+{"time_stamp": 20899.5011268, "action": "move", "x": 452, "y": 604}
+{"time_stamp": 20899.509338, "action": "move", "x": 451, "y": 604}
+{"time_stamp": 20899.5254322, "action": "move", "x": 451, "y": 604}
+{"time_stamp": 20899.6689377, "action": "move", "x": 451, "y": 604}
+{"time_stamp": 20899.6690538, "action": "click", "x": 451, "y": 604, "button": "right", "pressed": true}
+{"time_stamp": 20899.7894017, "action": "move", "x": 451, "y": 604}
+{"time_stamp": 20899.7894678, "action": "click", "x": 451, "y": 604, "button": "right", "pressed": false}
+{"time_stamp": 20899.8688655, "action": "move", "x": 451, "y": 604}
+{"time_stamp": 20899.8768756, "action": "move", "x": 451, "y": 603}
+{"time_stamp": 20899.901308, "action": "move", "x": 451, "y": 603}
+{"time_stamp": 20899.9168511, "action": "move", "x": 451, "y": 602}
+{"time_stamp": 20899.9490845, "action": "move", "x": 452, "y": 602}
+{"time_stamp": 20899.9735701, "action": "move", "x": 452, "y": 602}
+{"time_stamp": 20899.9810005, "action": "move", "x": 452, "y": 602}
+{"time_stamp": 20899.997036, "action": "move", "x": 453, "y": 602}
+{"time_stamp": 20900.0125471, "action": "move", "x": 454, "y": 601}
+{"time_stamp": 20900.0206053, "action": "move", "x": 454, "y": 601}
+{"time_stamp": 20900.0291704, "action": "move", "x": 455, "y": 601}
+{"time_stamp": 20900.0365503, "action": "move", "x": 456, "y": 601}
+{"time_stamp": 20900.0447876, "action": "move", "x": 457, "y": 601}
+{"time_stamp": 20900.0688537, "action": "move", "x": 457, "y": 601}
+{"time_stamp": 20900.1014405, "action": "move", "x": 457, "y": 600}
+{"time_stamp": 20900.1087084, "action": "move", "x": 457, "y": 600}
+{"time_stamp": 20900.1172566, "action": "move", "x": 457, "y": 600}
+{"time_stamp": 20900.1248056, "action": "move", "x": 455, "y": 599}
+{"time_stamp": 20900.1339202, "action": "move", "x": 454, "y": 598}
+{"time_stamp": 20900.1406654, "action": "move", "x": 452, "y": 598}
+{"time_stamp": 20900.148739, "action": "move", "x": 448, "y": 598}
+{"time_stamp": 20900.1567952, "action": "move", "x": 446, "y": 597}
+{"time_stamp": 20900.164782, "action": "move", "x": 443, "y": 597}
+{"time_stamp": 20900.1728803, "action": "move", "x": 439, "y": 596}
+{"time_stamp": 20900.1808106, "action": "move", "x": 435, "y": 595}
+{"time_stamp": 20900.1885365, "action": "move", "x": 432, "y": 595}
+{"time_stamp": 20900.1975774, "action": "move", "x": 429, "y": 594}
+{"time_stamp": 20900.2048903, "action": "move", "x": 426, "y": 594}
+{"time_stamp": 20900.2128035, "action": "move", "x": 422, "y": 594}
+{"time_stamp": 20900.2214436, "action": "move", "x": 420, "y": 593}
+{"time_stamp": 20900.2298458, "action": "move", "x": 417, "y": 593}
+{"time_stamp": 20900.2365394, "action": "move", "x": 417, "y": 592}
+{"time_stamp": 20900.3490146, "action": "move", "x": 417, "y": 592}
+{"time_stamp": 20900.3565645, "action": "move", "x": 417, "y": 592}
+{"time_stamp": 20900.3650853, "action": "move", "x": 418, "y": 592}
+{"time_stamp": 20900.3725616, "action": "move", "x": 419, "y": 592}
+{"time_stamp": 20900.3806073, "action": "move", "x": 421, "y": 592}
+{"time_stamp": 20900.3885327, "action": "move", "x": 423, "y": 594}
+{"time_stamp": 20900.3970652, "action": "move", "x": 426, "y": 594}
+{"time_stamp": 20900.4049898, "action": "move", "x": 428, "y": 596}
+{"time_stamp": 20900.4129694, "action": "move", "x": 432, "y": 598}
+{"time_stamp": 20900.4208664, "action": "move", "x": 436, "y": 600}
+{"time_stamp": 20900.4289081, "action": "move", "x": 439, "y": 601}
+{"time_stamp": 20900.436759, "action": "move", "x": 443, "y": 603}
+{"time_stamp": 20900.4452925, "action": "move", "x": 446, "y": 605}
+{"time_stamp": 20900.4528455, "action": "move", "x": 449, "y": 606}
+{"time_stamp": 20900.4609552, "action": "move", "x": 451, "y": 609}
+{"time_stamp": 20900.4688287, "action": "move", "x": 454, "y": 609}
+{"time_stamp": 20900.4768249, "action": "move", "x": 455, "y": 610}
+{"time_stamp": 20900.4846859, "action": "move", "x": 457, "y": 611}
+{"time_stamp": 20900.4926668, "action": "move", "x": 458, "y": 613}
+{"time_stamp": 20900.5015629, "action": "move", "x": 459, "y": 613}
+{"time_stamp": 20900.5088113, "action": "move", "x": 460, "y": 614}
+{"time_stamp": 20900.5169789, "action": "move", "x": 461, "y": 615}
+{"time_stamp": 20900.5248581, "action": "move", "x": 463, "y": 617}
+{"time_stamp": 20900.5331803, "action": "move", "x": 464, "y": 618}
+{"time_stamp": 20900.5407634, "action": "move", "x": 466, "y": 619}
+{"time_stamp": 20900.5492806, "action": "move", "x": 467, "y": 621}
+{"time_stamp": 20900.5567272, "action": "move", "x": 469, "y": 623}
+{"time_stamp": 20900.5645438, "action": "move", "x": 470, "y": 623}
+{"time_stamp": 20900.5726761, "action": "move", "x": 473, "y": 625}
+{"time_stamp": 20900.5807711, "action": "move", "x": 474, "y": 625}
+{"time_stamp": 20900.5886926, "action": "move", "x": 475, "y": 626}
+{"time_stamp": 20900.5966472, "action": "move", "x": 477, "y": 627}
+{"time_stamp": 20900.6048138, "action": "move", "x": 478, "y": 627}
+{"time_stamp": 20900.6133154, "action": "move", "x": 479, "y": 628}
+{"time_stamp": 20900.6207569, "action": "move", "x": 480, "y": 628}
+{"time_stamp": 20900.6290941, "action": "move", "x": 481, "y": 628}
+{"time_stamp": 20900.6369918, "action": "move", "x": 481, "y": 628}
+{"time_stamp": 20900.6445749, "action": "move", "x": 482, "y": 629}
+{"time_stamp": 20900.6610371, "action": "move", "x": 482, "y": 629}
+{"time_stamp": 20900.6687919, "action": "move", "x": 483, "y": 629}
+{"time_stamp": 20900.676888, "action": "move", "x": 483, "y": 629}
+{"time_stamp": 20900.6848441, "action": "move", "x": 484, "y": 629}
+{"time_stamp": 20900.6930848, "action": "move", "x": 484, "y": 629}
+{"time_stamp": 20900.7011483, "action": "move", "x": 485, "y": 629}
+{"time_stamp": 20900.7730054, "action": "move", "x": 485, "y": 629}
+{"time_stamp": 20900.7809148, "action": "move", "x": 486, "y": 630}
+{"time_stamp": 20900.7887239, "action": "move", "x": 486, "y": 630}
+{"time_stamp": 20900.7970894, "action": "move", "x": 487, "y": 631}
+{"time_stamp": 20900.8050801, "action": "move", "x": 488, "y": 631}
+{"time_stamp": 20900.8129503, "action": "move", "x": 489, "y": 631}
+{"time_stamp": 20900.8207206, "action": "move", "x": 490, "y": 631}
+{"time_stamp": 20900.8287425, "action": "move", "x": 490, "y": 632}
+{"time_stamp": 20900.8366932, "action": "move", "x": 491, "y": 632}
+{"time_stamp": 20900.844556, "action": "move", "x": 492, "y": 632}
+{"time_stamp": 20900.8524867, "action": "move", "x": 492, "y": 633}
+{"time_stamp": 20900.8607337, "action": "move", "x": 493, "y": 633}
+{"time_stamp": 20900.8686741, "action": "move", "x": 494, "y": 633}
+{"time_stamp": 20900.8845067, "action": "move", "x": 494, "y": 634}
+{"time_stamp": 20900.8930174, "action": "move", "x": 495, "y": 634}
+{"time_stamp": 20900.9009817, "action": "move", "x": 496, "y": 634}
+{"time_stamp": 20900.9087741, "action": "move", "x": 496, "y": 635}
+{"time_stamp": 20900.924878, "action": "move", "x": 496, "y": 635}
+{"time_stamp": 20900.9412006, "action": "move", "x": 497, "y": 635}
+{"time_stamp": 20900.9649302, "action": "move", "x": 497, "y": 635}
+{"time_stamp": 20901.2051008, "action": "move", "x": 497, "y": 636}
+{"time_stamp": 20901.2528748, "action": "move", "x": 498, "y": 636}
+{"time_stamp": 20901.269966, "action": "move", "x": 498, "y": 636}
+{"time_stamp": 20901.2767534, "action": "move", "x": 499, "y": 636}
+{"time_stamp": 20901.2849057, "action": "move", "x": 501, "y": 634}
+{"time_stamp": 20901.2925479, "action": "move", "x": 502, "y": 633}
+{"time_stamp": 20901.3007555, "action": "move", "x": 504, "y": 631}
+{"time_stamp": 20901.3091741, "action": "move", "x": 507, "y": 628}
+{"time_stamp": 20901.3169104, "action": "move", "x": 509, "y": 626}
+{"time_stamp": 20901.3248928, "action": "move", "x": 512, "y": 623}
+{"time_stamp": 20901.3347843, "action": "move", "x": 515, "y": 619}
+{"time_stamp": 20901.3404907, "action": "move", "x": 519, "y": 616}
+{"time_stamp": 20901.3486625, "action": "move", "x": 523, "y": 613}
+{"time_stamp": 20901.3566329, "action": "move", "x": 530, "y": 609}
+{"time_stamp": 20901.3648629, "action": "move", "x": 534, "y": 605}
+{"time_stamp": 20901.3726594, "action": "move", "x": 540, "y": 602}
+{"time_stamp": 20901.3805335, "action": "move", "x": 545, "y": 600}
+{"time_stamp": 20901.3888518, "action": "move", "x": 550, "y": 597}
+{"time_stamp": 20901.3976185, "action": "move", "x": 554, "y": 595}
+{"time_stamp": 20901.4050503, "action": "move", "x": 557, "y": 593}
+{"time_stamp": 20901.4128986, "action": "move", "x": 559, "y": 593}
+{"time_stamp": 20901.4207486, "action": "move", "x": 560, "y": 592}
+{"time_stamp": 20901.4288321, "action": "move", "x": 561, "y": 592}
+{"time_stamp": 20901.4365618, "action": "move", "x": 562, "y": 591}
+{"time_stamp": 20901.4449998, "action": "move", "x": 562, "y": 591}
+{"time_stamp": 20901.4526752, "action": "move", "x": 563, "y": 591}
+{"time_stamp": 20901.4766617, "action": "move", "x": 563, "y": 590}
+{"time_stamp": 20901.5174645, "action": "move", "x": 563, "y": 590}
+{"time_stamp": 20901.5176765, "action": "click", "x": 563, "y": 590, "button": "left", "pressed": true}
+{"time_stamp": 20901.6057084, "action": "move", "x": 563, "y": 590}
+{"time_stamp": 20901.6059026, "action": "click", "x": 563, "y": 590, "button": "left", "pressed": false}
+{"time_stamp": 20901.6610496, "action": "move", "x": 563, "y": 590}
+{"time_stamp": 20901.6685742, "action": "move", "x": 563, "y": 590}
+{"time_stamp": 20901.6765931, "action": "move", "x": 563, "y": 589}
+{"time_stamp": 20901.6847297, "action": "move", "x": 564, "y": 589}
+{"time_stamp": 20901.7021417, "action": "move", "x": 565, "y": 588}
+{"time_stamp": 20901.7091577, "action": "move", "x": 565, "y": 587}
+{"time_stamp": 20901.7173053, "action": "move", "x": 566, "y": 585}
+{"time_stamp": 20901.7248688, "action": "move", "x": 566, "y": 585}
+{"time_stamp": 20901.7333101, "action": "move", "x": 567, "y": 584}
+{"time_stamp": 20901.7407301, "action": "move", "x": 567, "y": 583}
+{"time_stamp": 20901.7495808, "action": "move", "x": 567, "y": 583}
+{"time_stamp": 20901.7569349, "action": "move", "x": 568, "y": 582}
+{"time_stamp": 20901.7655372, "action": "move", "x": 569, "y": 582}
+{"time_stamp": 20901.7726439, "action": "move", "x": 569, "y": 581}
+{"time_stamp": 20901.7815934, "action": "move", "x": 569, "y": 580}
+{"time_stamp": 20901.7891204, "action": "move", "x": 569, "y": 580}
+{"time_stamp": 20901.7970255, "action": "move", "x": 570, "y": 579}
+{"time_stamp": 20901.8049346, "action": "move", "x": 570, "y": 579}
+{"time_stamp": 20901.8693852, "action": "click", "x": 570, "y": 579, "button": "right", "pressed": true}
+{"time_stamp": 20901.8694495, "action": "move", "x": 570, "y": 579}
+{"time_stamp": 20901.973499, "action": "move", "x": 570, "y": 579}
+{"time_stamp": 20901.9737069, "action": "click", "x": 570, "y": 579, "button": "right", "pressed": false}
+{"time_stamp": 20902.1328903, "action": "move", "x": 570, "y": 579}
+{"time_stamp": 20902.148925, "action": "move", "x": 570, "y": 580}
+{"time_stamp": 20902.1567769, "action": "move", "x": 570, "y": 581}
+{"time_stamp": 20902.1648877, "action": "move", "x": 570, "y": 582}
+{"time_stamp": 20902.1730706, "action": "move", "x": 571, "y": 583}
+{"time_stamp": 20902.1806709, "action": "move", "x": 572, "y": 585}
+{"time_stamp": 20902.1887332, "action": "move", "x": 574, "y": 586}
+{"time_stamp": 20902.1969065, "action": "move", "x": 577, "y": 589}
+{"time_stamp": 20902.2047584, "action": "move", "x": 578, "y": 590}
+{"time_stamp": 20902.2125785, "action": "move", "x": 581, "y": 592}
+{"time_stamp": 20902.2210181, "action": "move", "x": 584, "y": 593}
+{"time_stamp": 20902.2295276, "action": "move", "x": 586, "y": 595}
+{"time_stamp": 20902.2367552, "action": "move", "x": 589, "y": 596}
+{"time_stamp": 20902.2445903, "action": "move", "x": 592, "y": 598}
+{"time_stamp": 20902.2527143, "action": "move", "x": 594, "y": 598}
+{"time_stamp": 20902.2609025, "action": "move", "x": 596, "y": 600}
+{"time_stamp": 20902.2686708, "action": "move", "x": 597, "y": 601}
+{"time_stamp": 20902.2773601, "action": "move", "x": 599, "y": 602}
+{"time_stamp": 20902.2845503, "action": "move", "x": 600, "y": 603}
+{"time_stamp": 20902.293289, "action": "move", "x": 601, "y": 604}
+{"time_stamp": 20902.3009463, "action": "move", "x": 603, "y": 604}
+{"time_stamp": 20902.3092534, "action": "move", "x": 604, "y": 605}
+{"time_stamp": 20902.3169421, "action": "move", "x": 606, "y": 606}
+{"time_stamp": 20902.3247474, "action": "move", "x": 607, "y": 607}
+{"time_stamp": 20902.3326989, "action": "move", "x": 608, "y": 608}
+{"time_stamp": 20902.3414815, "action": "move", "x": 610, "y": 608}
+{"time_stamp": 20902.349085, "action": "move", "x": 611, "y": 610}
+{"time_stamp": 20902.3567567, "action": "move", "x": 612, "y": 610}
+{"time_stamp": 20902.3650359, "action": "move", "x": 612, "y": 610}
+{"time_stamp": 20902.3733836, "action": "move", "x": 613, "y": 611}
+{"time_stamp": 20902.3805226, "action": "move", "x": 613, "y": 612}
+{"time_stamp": 20902.3965957, "action": "move", "x": 614, "y": 612}
+{"time_stamp": 20902.4127349, "action": "move", "x": 615, "y": 612}
+{"time_stamp": 20902.4449086, "action": "move", "x": 615, "y": 612}
+{"time_stamp": 20902.7573756, "action": "move", "x": 615, "y": 612}
+{"time_stamp": 20902.7574353, "action": "click", "x": 615, "y": 612, "button": "left", "pressed": true}
+{"time_stamp": 20902.8371472, "action": "move", "x": 615, "y": 612}
+{"time_stamp": 20902.8373418, "action": "click", "x": 615, "y": 612, "button": "left", "pressed": false}
+{"time_stamp": 20902.8927749, "action": "move", "x": 615, "y": 612}
+{"time_stamp": 20902.9008522, "action": "move", "x": 615, "y": 609}
+{"time_stamp": 20902.9086919, "action": "move", "x": 615, "y": 607}
+{"time_stamp": 20902.9166991, "action": "move", "x": 615, "y": 605}
+{"time_stamp": 20902.9248996, "action": "move", "x": 615, "y": 602}
+{"time_stamp": 20902.9327824, "action": "move", "x": 613, "y": 598}
+{"time_stamp": 20902.9408915, "action": "move", "x": 613, "y": 593}
+{"time_stamp": 20902.9487593, "action": "move", "x": 613, "y": 589}
+{"time_stamp": 20902.9568104, "action": "move", "x": 613, "y": 583}
+{"time_stamp": 20902.9649619, "action": "move", "x": 613, "y": 578}
+{"time_stamp": 20902.9730329, "action": "move", "x": 613, "y": 571}
+{"time_stamp": 20902.9809696, "action": "move", "x": 613, "y": 566}
+{"time_stamp": 20902.9887028, "action": "move", "x": 614, "y": 560}
+{"time_stamp": 20902.9978931, "action": "move", "x": 614, "y": 555}
+{"time_stamp": 20903.0047735, "action": "move", "x": 615, "y": 548}
+{"time_stamp": 20903.0130782, "action": "move", "x": 615, "y": 541}
+{"time_stamp": 20903.0206539, "action": "move", "x": 615, "y": 534}
+{"time_stamp": 20903.0290465, "action": "move", "x": 616, "y": 529}
+{"time_stamp": 20903.0365904, "action": "move", "x": 616, "y": 521}
+{"time_stamp": 20903.0448781, "action": "move", "x": 616, "y": 514}
+{"time_stamp": 20903.0529212, "action": "move", "x": 616, "y": 507}
+{"time_stamp": 20903.0618288, "action": "move", "x": 616, "y": 502}
+{"time_stamp": 20903.0688895, "action": "move", "x": 616, "y": 496}
+{"time_stamp": 20903.0776968, "action": "move", "x": 616, "y": 492}
+{"time_stamp": 20903.0845372, "action": "move", "x": 616, "y": 488}
+{"time_stamp": 20903.0939629, "action": "move", "x": 616, "y": 483}
+{"time_stamp": 20903.1009437, "action": "move", "x": 616, "y": 481}
+{"time_stamp": 20903.1086878, "action": "move", "x": 616, "y": 478}
+{"time_stamp": 20903.1168128, "action": "move", "x": 616, "y": 477}
+{"time_stamp": 20903.1253411, "action": "move", "x": 616, "y": 474}
+{"time_stamp": 20903.1329869, "action": "move", "x": 616, "y": 473}
+{"time_stamp": 20903.1409398, "action": "move", "x": 616, "y": 471}
+{"time_stamp": 20903.1486387, "action": "move", "x": 616, "y": 470}
+{"time_stamp": 20903.1568731, "action": "move", "x": 616, "y": 470}
+{"time_stamp": 20903.165031, "action": "move", "x": 616, "y": 469}
+{"time_stamp": 20903.1725484, "action": "move", "x": 616, "y": 469}
+{"time_stamp": 20903.1807188, "action": "move", "x": 616, "y": 468}
+{"time_stamp": 20903.1888779, "action": "move", "x": 616, "y": 467}
+{"time_stamp": 20903.197386, "action": "move", "x": 617, "y": 466}
+{"time_stamp": 20903.2048049, "action": "move", "x": 617, "y": 466}
+{"time_stamp": 20903.2128183, "action": "move", "x": 617, "y": 465}
+{"time_stamp": 20903.2207641, "action": "move", "x": 618, "y": 464}
+{"time_stamp": 20903.2288693, "action": "move", "x": 619, "y": 464}
+{"time_stamp": 20903.2369101, "action": "move", "x": 619, "y": 463}
+{"time_stamp": 20903.2447845, "action": "move", "x": 619, "y": 463}
+{"time_stamp": 20903.252746, "action": "move", "x": 619, "y": 462}
+{"time_stamp": 20903.2608172, "action": "move", "x": 620, "y": 462}
+{"time_stamp": 20903.2692625, "action": "move", "x": 620, "y": 462}
+{"time_stamp": 20903.2849104, "action": "move", "x": 620, "y": 462}
+{"time_stamp": 20903.3007822, "action": "move", "x": 620, "y": 461}
+{"time_stamp": 20903.3170094, "action": "move", "x": 620, "y": 460}
+{"time_stamp": 20903.3250649, "action": "move", "x": 621, "y": 460}
+{"time_stamp": 20903.341077, "action": "move", "x": 621, "y": 460}
+{"time_stamp": 20903.3488418, "action": "move", "x": 622, "y": 459}
+{"time_stamp": 20903.3657115, "action": "move", "x": 623, "y": 459}
+{"time_stamp": 20903.4048669, "action": "move", "x": 623, "y": 458}
+{"time_stamp": 20903.4215683, "action": "move", "x": 623, "y": 458}
+{"time_stamp": 20903.4217388, "action": "click", "x": 623, "y": 458, "button": "left", "pressed": true}
+{"time_stamp": 20903.517215, "action": "move", "x": 623, "y": 458}
+{"time_stamp": 20903.5173176, "action": "click", "x": 623, "y": 458, "button": "left", "pressed": false}
+{"time_stamp": 20903.5809427, "action": "move", "x": 623, "y": 458}
+{"time_stamp": 20903.5887693, "action": "move", "x": 621, "y": 458}
+{"time_stamp": 20903.5969091, "action": "move", "x": 619, "y": 458}
+{"time_stamp": 20903.6049275, "action": "move", "x": 616, "y": 458}
+{"time_stamp": 20903.6128935, "action": "move", "x": 611, "y": 456}
+{"time_stamp": 20903.6206287, "action": "move", "x": 607, "y": 456}
+{"time_stamp": 20903.6286577, "action": "move", "x": 603, "y": 456}
+{"time_stamp": 20903.6365295, "action": "move", "x": 597, "y": 456}
+{"time_stamp": 20903.6449877, "action": "move", "x": 592, "y": 455}
+{"time_stamp": 20903.6526372, "action": "move", "x": 587, "y": 454}
+{"time_stamp": 20903.6610373, "action": "move", "x": 583, "y": 453}
+{"time_stamp": 20903.6694081, "action": "move", "x": 580, "y": 452}
+{"time_stamp": 20903.6771488, "action": "move", "x": 577, "y": 451}
+{"time_stamp": 20903.687229, "action": "move", "x": 574, "y": 451}
+{"time_stamp": 20903.6928974, "action": "move", "x": 572, "y": 450}
+{"time_stamp": 20903.7015873, "action": "move", "x": 571, "y": 450}
+{"time_stamp": 20903.8138373, "action": "move", "x": 571, "y": 450}
+{"time_stamp": 20903.8209979, "action": "move", "x": 572, "y": 450}
+{"time_stamp": 20903.8290731, "action": "move", "x": 573, "y": 450}
+{"time_stamp": 20903.85309, "action": "move", "x": 573, "y": 450}
+{"time_stamp": 20903.869115, "action": "move", "x": 574, "y": 450}
+{"time_stamp": 20903.87683, "action": "move", "x": 574, "y": 450}
+{"time_stamp": 20903.884761, "action": "move", "x": 576, "y": 450}
+{"time_stamp": 20903.892816, "action": "move", "x": 577, "y": 450}
+{"time_stamp": 20903.9009909, "action": "move", "x": 578, "y": 450}
+{"time_stamp": 20903.9090068, "action": "move", "x": 581, "y": 450}
+{"time_stamp": 20903.9167878, "action": "move", "x": 582, "y": 451}
+{"time_stamp": 20903.9247452, "action": "move", "x": 584, "y": 451}
+{"time_stamp": 20903.9329348, "action": "move", "x": 587, "y": 452}
+{"time_stamp": 20903.9411388, "action": "move", "x": 590, "y": 452}
+{"time_stamp": 20903.9489712, "action": "move", "x": 593, "y": 452}
+{"time_stamp": 20903.956519, "action": "move", "x": 596, "y": 452}
+{"time_stamp": 20903.9648777, "action": "move", "x": 600, "y": 453}
+{"time_stamp": 20903.9729454, "action": "move", "x": 603, "y": 453}
+{"time_stamp": 20903.9807449, "action": "move", "x": 607, "y": 453}
+{"time_stamp": 20903.9887615, "action": "move", "x": 609, "y": 453}
+{"time_stamp": 20903.9968865, "action": "move", "x": 612, "y": 454}
+{"time_stamp": 20904.0051229, "action": "move", "x": 615, "y": 454}
+{"time_stamp": 20904.0124996, "action": "move", "x": 617, "y": 454}
+{"time_stamp": 20904.0205993, "action": "move", "x": 620, "y": 454}
+{"time_stamp": 20904.0287838, "action": "move", "x": 623, "y": 454}
+{"time_stamp": 20904.041454, "action": "move", "x": 624, "y": 455}
+{"time_stamp": 20904.044792, "action": "move", "x": 625, "y": 455}
+{"time_stamp": 20904.0528514, "action": "move", "x": 626, "y": 455}
+{"time_stamp": 20904.1015353, "action": "move", "x": 626, "y": 455}
+{"time_stamp": 20904.1017341, "action": "click", "x": 626, "y": 455, "button": "right", "pressed": true}
+{"time_stamp": 20904.1974785, "action": "move", "x": 626, "y": 455}
+{"time_stamp": 20904.1976601, "action": "click", "x": 626, "y": 455, "button": "right", "pressed": false}
+{"time_stamp": 20904.3487735, "action": "move", "x": 626, "y": 455}
+{"time_stamp": 20904.3574017, "action": "move", "x": 626, "y": 456}
+{"time_stamp": 20904.3647981, "action": "move", "x": 626, "y": 459}
+{"time_stamp": 20904.3729217, "action": "move", "x": 626, "y": 460}
+{"time_stamp": 20904.3811012, "action": "move", "x": 626, "y": 462}
+{"time_stamp": 20904.3888477, "action": "move", "x": 628, "y": 464}
+{"time_stamp": 20904.396823, "action": "move", "x": 630, "y": 467}
+{"time_stamp": 20904.4056814, "action": "move", "x": 631, "y": 471}
+{"time_stamp": 20904.4125825, "action": "move", "x": 634, "y": 474}
+{"time_stamp": 20904.4204439, "action": "move", "x": 635, "y": 478}
+{"time_stamp": 20904.4290132, "action": "move", "x": 638, "y": 481}
+{"time_stamp": 20904.437161, "action": "move", "x": 640, "y": 483}
+{"time_stamp": 20904.4445543, "action": "move", "x": 644, "y": 487}
+{"time_stamp": 20904.4527602, "action": "move", "x": 647, "y": 492}
+{"time_stamp": 20904.4608077, "action": "move", "x": 650, "y": 494}
+{"time_stamp": 20904.4687643, "action": "move", "x": 654, "y": 497}
+{"time_stamp": 20904.4768431, "action": "move", "x": 658, "y": 501}
+{"time_stamp": 20904.4848179, "action": "move", "x": 662, "y": 504}
+{"time_stamp": 20904.4930456, "action": "move", "x": 665, "y": 505}
+{"time_stamp": 20904.5014081, "action": "move", "x": 667, "y": 506}
+{"time_stamp": 20904.5090166, "action": "move", "x": 669, "y": 508}
+{"time_stamp": 20904.5169428, "action": "move", "x": 669, "y": 508}
+{"time_stamp": 20904.5247595, "action": "move", "x": 669, "y": 508}
+{"time_stamp": 20904.6131723, "action": "move", "x": 669, "y": 508}
+{"time_stamp": 20904.6208739, "action": "move", "x": 669, "y": 508}
+{"time_stamp": 20904.6287357, "action": "move", "x": 669, "y": 506}
+{"time_stamp": 20904.6367283, "action": "move", "x": 669, "y": 505}
+{"time_stamp": 20904.6445169, "action": "move", "x": 667, "y": 504}
+{"time_stamp": 20904.6529893, "action": "move", "x": 666, "y": 501}
+{"time_stamp": 20904.6610396, "action": "move", "x": 664, "y": 498}
+{"time_stamp": 20904.6685725, "action": "move", "x": 663, "y": 496}
+{"time_stamp": 20904.6766821, "action": "move", "x": 661, "y": 492}
+{"time_stamp": 20904.6848569, "action": "move", "x": 659, "y": 490}
+{"time_stamp": 20904.6936349, "action": "move", "x": 656, "y": 485}
+{"time_stamp": 20904.701148, "action": "move", "x": 653, "y": 482}
+{"time_stamp": 20904.7089867, "action": "move", "x": 652, "y": 479}
+{"time_stamp": 20904.7171869, "action": "move", "x": 650, "y": 477}
+{"time_stamp": 20904.725438, "action": "move", "x": 649, "y": 473}
+{"time_stamp": 20904.7331133, "action": "move", "x": 647, "y": 471}
+{"time_stamp": 20904.7419294, "action": "move", "x": 646, "y": 469}
+{"time_stamp": 20904.7490213, "action": "move", "x": 645, "y": 466}
+{"time_stamp": 20904.7569795, "action": "move", "x": 644, "y": 464}
+{"time_stamp": 20904.7651285, "action": "move", "x": 644, "y": 463}
+{"time_stamp": 20904.7754758, "action": "move", "x": 643, "y": 462}
+{"time_stamp": 20904.7818648, "action": "move", "x": 643, "y": 462}
+{"time_stamp": 20904.7971013, "action": "move", "x": 643, "y": 461}
+{"time_stamp": 20904.9246533, "action": "move", "x": 643, "y": 460}
+{"time_stamp": 20904.9409977, "action": "move", "x": 643, "y": 460}
+{"time_stamp": 20904.9488488, "action": "move", "x": 643, "y": 459}
+{"time_stamp": 20904.9568397, "action": "move", "x": 644, "y": 459}
+{"time_stamp": 20904.9655409, "action": "move", "x": 644, "y": 459}
+{"time_stamp": 20904.9728529, "action": "move", "x": 645, "y": 458}
+{"time_stamp": 20904.9811637, "action": "move", "x": 645, "y": 458}
+{"time_stamp": 20904.9886765, "action": "move", "x": 646, "y": 458}
+{"time_stamp": 20904.9968743, "action": "move", "x": 646, "y": 457}
+{"time_stamp": 20905.0048128, "action": "move", "x": 647, "y": 456}
+{"time_stamp": 20905.0211041, "action": "move", "x": 647, "y": 456}
+{"time_stamp": 20905.1420584, "action": "move", "x": 647, "y": 456}
+{"time_stamp": 20905.1423107, "action": "click", "x": 647, "y": 456, "button": "right", "pressed": true}
+{"time_stamp": 20905.2451675, "action": "move", "x": 647, "y": 456}
+{"time_stamp": 20905.2453541, "action": "click", "x": 647, "y": 456, "button": "right", "pressed": false}
+{"time_stamp": 20905.5728227, "action": "move", "x": 647, "y": 456}
+{"time_stamp": 20905.5968156, "action": "move", "x": 647, "y": 456}
+{"time_stamp": 20905.6208009, "action": "move", "x": 647, "y": 457}
+{"time_stamp": 20905.6374996, "action": "move", "x": 647, "y": 458}
+{"time_stamp": 20905.6610489, "action": "move", "x": 647, "y": 458}
+{"time_stamp": 20906.0374202, "action": "move", "x": 647, "y": 458}
+{"time_stamp": 20906.0376232, "action": "click", "x": 647, "y": 458, "button": "left", "pressed": true}
+{"time_stamp": 20906.1087085, "action": "move", "x": 647, "y": 458}
+{"time_stamp": 20906.1169458, "action": "move", "x": 646, "y": 458}
+{"time_stamp": 20906.1250641, "action": "move", "x": 643, "y": 458}
+{"time_stamp": 20906.1333901, "action": "move", "x": 640, "y": 458}
+{"time_stamp": 20906.1409726, "action": "move", "x": 635, "y": 458}
+{"time_stamp": 20906.1489911, "action": "move", "x": 630, "y": 458}
+{"time_stamp": 20906.1567427, "action": "move", "x": 621, "y": 458}
+{"time_stamp": 20906.1651691, "action": "move", "x": 613, "y": 458}
+{"time_stamp": 20906.1729339, "action": "move", "x": 604, "y": 458}
+{"time_stamp": 20906.1821839, "action": "move", "x": 596, "y": 458}
+{"time_stamp": 20906.188685, "action": "move", "x": 587, "y": 458}
+{"time_stamp": 20906.1968051, "action": "move", "x": 578, "y": 458}
+{"time_stamp": 20906.2046645, "action": "move", "x": 573, "y": 458}
+{"time_stamp": 20906.212676, "action": "move", "x": 567, "y": 458}
+{"time_stamp": 20906.220874, "action": "move", "x": 565, "y": 458}
+{"time_stamp": 20906.2291344, "action": "move", "x": 562, "y": 458}
+{"time_stamp": 20906.2368927, "action": "move", "x": 561, "y": 458}
+{"time_stamp": 20906.2450354, "action": "move", "x": 560, "y": 458}
+{"time_stamp": 20906.2526308, "action": "move", "x": 559, "y": 458}
+{"time_stamp": 20906.389629, "action": "move", "x": 559, "y": 458}
+{"time_stamp": 20906.3896958, "action": "click", "x": 559, "y": 458, "button": "left", "pressed": false}
+{"time_stamp": 20906.4690405, "action": "move", "x": 559, "y": 458}
+{"time_stamp": 20906.4858845, "action": "move", "x": 560, "y": 458}
+{"time_stamp": 20906.5015721, "action": "move", "x": 561, "y": 458}
+{"time_stamp": 20906.5168131, "action": "move", "x": 561, "y": 458}
+{"time_stamp": 20906.5248614, "action": "move", "x": 562, "y": 458}
+{"time_stamp": 20906.5343155, "action": "move", "x": 563, "y": 458}
+{"time_stamp": 20906.5409562, "action": "move", "x": 566, "y": 458}
+{"time_stamp": 20906.5499087, "action": "move", "x": 567, "y": 458}
+{"time_stamp": 20906.5565177, "action": "move", "x": 569, "y": 459}
+{"time_stamp": 20906.5660774, "action": "move", "x": 571, "y": 459}
+{"time_stamp": 20906.5729523, "action": "move", "x": 574, "y": 459}
+{"time_stamp": 20906.5808837, "action": "move", "x": 577, "y": 460}
+{"time_stamp": 20906.5889095, "action": "move", "x": 582, "y": 461}
+{"time_stamp": 20906.5972368, "action": "move", "x": 585, "y": 461}
+{"time_stamp": 20906.6051975, "action": "move", "x": 588, "y": 462}
+{"time_stamp": 20906.6133521, "action": "move", "x": 591, "y": 462}
+{"time_stamp": 20906.6209863, "action": "move", "x": 593, "y": 463}
+{"time_stamp": 20906.6301822, "action": "move", "x": 595, "y": 463}
+{"time_stamp": 20906.6369125, "action": "move", "x": 597, "y": 463}
+{"time_stamp": 20906.6455795, "action": "move", "x": 598, "y": 463}
+{"time_stamp": 20906.6530063, "action": "move", "x": 599, "y": 463}
+{"time_stamp": 20906.6616393, "action": "move", "x": 600, "y": 463}
+{"time_stamp": 20906.6689704, "action": "move", "x": 600, "y": 463}
+{"time_stamp": 20906.8129431, "action": "move", "x": 600, "y": 463}
+{"time_stamp": 20906.8207995, "action": "move", "x": 600, "y": 463}
+{"time_stamp": 20906.8290663, "action": "move", "x": 598, "y": 463}
+{"time_stamp": 20906.8365692, "action": "move", "x": 597, "y": 463}
+{"time_stamp": 20906.8448471, "action": "move", "x": 596, "y": 463}
+{"time_stamp": 20906.8537618, "action": "move", "x": 594, "y": 463}
+{"time_stamp": 20906.8611989, "action": "move", "x": 592, "y": 463}
+{"time_stamp": 20906.8688715, "action": "move", "x": 588, "y": 463}
+{"time_stamp": 20906.8774325, "action": "move", "x": 584, "y": 462}
+{"time_stamp": 20906.8849445, "action": "move", "x": 579, "y": 462}
+{"time_stamp": 20906.8930184, "action": "move", "x": 574, "y": 460}
+{"time_stamp": 20906.901433, "action": "move", "x": 569, "y": 460}
+{"time_stamp": 20906.9088106, "action": "move", "x": 565, "y": 460}
+{"time_stamp": 20906.9169808, "action": "move", "x": 562, "y": 460}
+{"time_stamp": 20906.9248278, "action": "move", "x": 560, "y": 460}
+{"time_stamp": 20906.9334665, "action": "move", "x": 558, "y": 459}
+{"time_stamp": 20906.9408751, "action": "move", "x": 557, "y": 459}
+{"time_stamp": 20906.9491563, "action": "move", "x": 557, "y": 459}
+{"time_stamp": 20907.0293486, "action": "move", "x": 557, "y": 459}
+{"time_stamp": 20907.0295761, "action": "click", "x": 557, "y": 459, "button": "left", "pressed": true}
+{"time_stamp": 20907.1173399, "action": "move", "x": 557, "y": 459}
+{"time_stamp": 20907.1175071, "action": "click", "x": 557, "y": 459, "button": "left", "pressed": false}
+{"time_stamp": 20907.1971296, "action": "move", "x": 557, "y": 459}
+{"time_stamp": 20907.229007, "action": "move", "x": 557, "y": 458}
+{"time_stamp": 20907.2540716, "action": "move", "x": 557, "y": 458}
+{"time_stamp": 20907.2609685, "action": "move", "x": 557, "y": 457}
+{"time_stamp": 20907.2690786, "action": "move", "x": 558, "y": 457}
+{"time_stamp": 20907.2770335, "action": "move", "x": 558, "y": 456}
+{"time_stamp": 20907.2849045, "action": "move", "x": 558, "y": 455}
+{"time_stamp": 20907.292967, "action": "move", "x": 559, "y": 454}
+{"time_stamp": 20907.3019802, "action": "move", "x": 559, "y": 454}
+{"time_stamp": 20907.3089059, "action": "move", "x": 560, "y": 452}
+{"time_stamp": 20907.3165008, "action": "move", "x": 561, "y": 452}
+{"time_stamp": 20907.3248671, "action": "move", "x": 561, "y": 451}
+{"time_stamp": 20907.3330785, "action": "move", "x": 561, "y": 450}
+{"time_stamp": 20907.3410061, "action": "move", "x": 562, "y": 450}
+{"time_stamp": 20907.3491342, "action": "move", "x": 562, "y": 448}
+{"time_stamp": 20907.3569326, "action": "move", "x": 562, "y": 448}
+{"time_stamp": 20907.3650057, "action": "move", "x": 563, "y": 447}
+{"time_stamp": 20907.3726492, "action": "move", "x": 563, "y": 447}
+{"time_stamp": 20907.3813764, "action": "move", "x": 565, "y": 446}
+{"time_stamp": 20907.3895778, "action": "move", "x": 566, "y": 446}
+{"time_stamp": 20907.3973079, "action": "move", "x": 566, "y": 445}
+{"time_stamp": 20907.4049564, "action": "move", "x": 567, "y": 444}
+{"time_stamp": 20907.4129559, "action": "move", "x": 567, "y": 443}
+{"time_stamp": 20907.4205452, "action": "move", "x": 567, "y": 443}
+{"time_stamp": 20907.4288683, "action": "move", "x": 568, "y": 441}
+{"time_stamp": 20907.4365089, "action": "move", "x": 569, "y": 440}
+{"time_stamp": 20907.4449773, "action": "move", "x": 569, "y": 440}
+{"time_stamp": 20907.4527753, "action": "move", "x": 569, "y": 439}
+{"time_stamp": 20907.4605882, "action": "move", "x": 569, "y": 438}
+{"time_stamp": 20907.4690173, "action": "move", "x": 569, "y": 437}
+{"time_stamp": 20907.4776541, "action": "move", "x": 569, "y": 437}
+{"time_stamp": 20907.4849328, "action": "move", "x": 569, "y": 436}
+{"time_stamp": 20907.493594, "action": "move", "x": 569, "y": 435}
+{"time_stamp": 20907.5012365, "action": "move", "x": 569, "y": 434}
+{"time_stamp": 20907.5092137, "action": "move", "x": 570, "y": 433}
+{"time_stamp": 20907.5171008, "action": "move", "x": 570, "y": 432}
+{"time_stamp": 20907.52504, "action": "move", "x": 570, "y": 432}
+{"time_stamp": 20907.5328956, "action": "move", "x": 570, "y": 431}
+{"time_stamp": 20907.5408923, "action": "move", "x": 570, "y": 430}
+{"time_stamp": 20907.548728, "action": "move", "x": 570, "y": 429}
+{"time_stamp": 20907.5568182, "action": "move", "x": 570, "y": 429}
+{"time_stamp": 20907.5646886, "action": "move", "x": 570, "y": 428}
+{"time_stamp": 20907.5727879, "action": "move", "x": 570, "y": 428}
+{"time_stamp": 20907.580833, "action": "move", "x": 570, "y": 427}
+{"time_stamp": 20907.6459433, "action": "move", "x": 570, "y": 427}
+{"time_stamp": 20907.6461423, "action": "click", "x": 570, "y": 427, "button": "left", "pressed": true}
+{"time_stamp": 20907.7335124, "action": "move", "x": 570, "y": 427}
+{"time_stamp": 20907.7337527, "action": "click", "x": 570, "y": 427, "button": "left", "pressed": false}
+{"time_stamp": 20907.7815048, "action": "move", "x": 570, "y": 427}
+{"time_stamp": 20907.7817219, "action": "click", "x": 570, "y": 427, "button": "left", "pressed": true}
+{"time_stamp": 20907.87723, "action": "move", "x": 570, "y": 427}
+{"time_stamp": 20907.8775527, "action": "click", "x": 570, "y": 427, "button": "left", "pressed": false}
+{"time_stamp": 20908.0130113, "action": "move", "x": 570, "y": 427}
+{"time_stamp": 20908.0209147, "action": "move", "x": 570, "y": 427}
+{"time_stamp": 20908.0291035, "action": "move", "x": 571, "y": 427}
+{"time_stamp": 20908.044969, "action": "move", "x": 571, "y": 427}
+{"time_stamp": 20908.0527925, "action": "move", "x": 572, "y": 426}
+{"time_stamp": 20908.0610206, "action": "move", "x": 573, "y": 426}
+{"time_stamp": 20908.0691259, "action": "move", "x": 573, "y": 426}
+{"time_stamp": 20908.0767466, "action": "move", "x": 574, "y": 426}
+{"time_stamp": 20908.085181, "action": "move", "x": 574, "y": 425}
+{"time_stamp": 20908.0942297, "action": "move", "x": 575, "y": 425}
+{"time_stamp": 20908.1015232, "action": "move", "x": 576, "y": 425}
+{"time_stamp": 20908.109019, "action": "move", "x": 577, "y": 425}
+{"time_stamp": 20908.1168605, "action": "move", "x": 578, "y": 425}
+{"time_stamp": 20908.1250619, "action": "move", "x": 578, "y": 424}
+{"time_stamp": 20908.132763, "action": "move", "x": 579, "y": 424}
+{"time_stamp": 20908.1410655, "action": "move", "x": 580, "y": 424}
+{"time_stamp": 20908.1489293, "action": "move", "x": 580, "y": 424}
+{"time_stamp": 20908.1568685, "action": "move", "x": 581, "y": 424}
+{"time_stamp": 20908.1729893, "action": "move", "x": 581, "y": 424}
+{"time_stamp": 20908.2065538, "action": "move", "x": 582, "y": 424}
+{"time_stamp": 20908.2291259, "action": "move", "x": 582, "y": 423}
+{"time_stamp": 20908.6530966, "action": "move", "x": 583, "y": 423}
+{"time_stamp": 20908.6609559, "action": "move", "x": 584, "y": 423}
+{"time_stamp": 20908.6688077, "action": "move", "x": 587, "y": 423}
+{"time_stamp": 20908.6767405, "action": "move", "x": 590, "y": 423}
+{"time_stamp": 20908.6853282, "action": "move", "x": 594, "y": 423}
+{"time_stamp": 20908.6932469, "action": "move", "x": 600, "y": 423}
+{"time_stamp": 20908.7012446, "action": "move", "x": 604, "y": 423}
+{"time_stamp": 20908.7088607, "action": "move", "x": 609, "y": 423}
+{"time_stamp": 20908.7171192, "action": "move", "x": 615, "y": 423}
+{"time_stamp": 20908.7250348, "action": "move", "x": 620, "y": 423}
+{"time_stamp": 20908.7332986, "action": "move", "x": 623, "y": 423}
+{"time_stamp": 20908.7408305, "action": "move", "x": 626, "y": 423}
+{"time_stamp": 20908.7490316, "action": "move", "x": 627, "y": 423}
+{"time_stamp": 20908.7573569, "action": "move", "x": 627, "y": 423}
+{"time_stamp": 20908.7650299, "action": "move", "x": 628, "y": 423}
+{"time_stamp": 20908.8049364, "action": "move", "x": 628, "y": 423}
+{"time_stamp": 20908.8050094, "action": "click", "x": 628, "y": 423, "button": "left", "pressed": true}
+{"time_stamp": 20908.901774, "action": "move", "x": 628, "y": 423}
+{"time_stamp": 20908.9019889, "action": "click", "x": 628, "y": 423, "button": "left", "pressed": false}
+{"time_stamp": 20908.9891403, "action": "move", "x": 628, "y": 423}
+{"time_stamp": 20909.0130955, "action": "move", "x": 628, "y": 422}
+{"time_stamp": 20909.0210957, "action": "move", "x": 628, "y": 421}
+{"time_stamp": 20909.0298411, "action": "move", "x": 628, "y": 420}
+{"time_stamp": 20909.0370002, "action": "move", "x": 628, "y": 419}
+{"time_stamp": 20909.0450816, "action": "move", "x": 626, "y": 417}
+{"time_stamp": 20909.0528292, "action": "move", "x": 625, "y": 416}
+{"time_stamp": 20909.0617216, "action": "move", "x": 623, "y": 414}
+{"time_stamp": 20909.0688677, "action": "move", "x": 619, "y": 412}
+{"time_stamp": 20909.0767931, "action": "move", "x": 616, "y": 410}
+{"time_stamp": 20909.0853448, "action": "move", "x": 610, "y": 408}
+{"time_stamp": 20909.0930971, "action": "move", "x": 605, "y": 405}
+{"time_stamp": 20909.1011977, "action": "move", "x": 600, "y": 402}
+{"time_stamp": 20909.1091184, "action": "move", "x": 592, "y": 400}
+{"time_stamp": 20909.1168785, "action": "move", "x": 588, "y": 398}
+{"time_stamp": 20909.1253749, "action": "move", "x": 581, "y": 394}
+{"time_stamp": 20909.1330455, "action": "move", "x": 574, "y": 390}
+{"time_stamp": 20909.1410557, "action": "move", "x": 567, "y": 387}
+{"time_stamp": 20909.1491006, "action": "move", "x": 563, "y": 385}
+{"time_stamp": 20909.1568445, "action": "move", "x": 561, "y": 383}
+{"time_stamp": 20909.1649292, "action": "move", "x": 558, "y": 382}
+{"time_stamp": 20909.1729257, "action": "move", "x": 557, "y": 380}
+{"time_stamp": 20909.1807513, "action": "move", "x": 555, "y": 379}
+{"time_stamp": 20909.1887517, "action": "move", "x": 555, "y": 377}
+{"time_stamp": 20909.1969686, "action": "move", "x": 554, "y": 377}
+{"time_stamp": 20909.2060217, "action": "move", "x": 554, "y": 376}
+{"time_stamp": 20909.2126215, "action": "move", "x": 554, "y": 375}
+{"time_stamp": 20909.2698959, "action": "move", "x": 554, "y": 375}
+{"time_stamp": 20909.2770244, "action": "move", "x": 553, "y": 375}
+{"time_stamp": 20909.3496445, "action": "move", "x": 553, "y": 375}
+{"time_stamp": 20909.34993, "action": "click", "x": 553, "y": 375, "button": "left", "pressed": true}
+{"time_stamp": 20909.4209289, "action": "move", "x": 553, "y": 375}
+{"time_stamp": 20909.4291513, "action": "move", "x": 555, "y": 375}
+{"time_stamp": 20909.4368646, "action": "move", "x": 560, "y": 376}
+{"time_stamp": 20909.4449959, "action": "move", "x": 566, "y": 377}
+{"time_stamp": 20909.4527323, "action": "move", "x": 574, "y": 378}
+{"time_stamp": 20909.4623916, "action": "move", "x": 582, "y": 379}
+{"time_stamp": 20909.4689748, "action": "move", "x": 592, "y": 381}
+{"time_stamp": 20909.4768033, "action": "move", "x": 603, "y": 383}
+{"time_stamp": 20909.4846773, "action": "move", "x": 612, "y": 386}
+{"time_stamp": 20909.4930331, "action": "move", "x": 621, "y": 390}
+{"time_stamp": 20909.501255, "action": "move", "x": 629, "y": 393}
+{"time_stamp": 20909.5088933, "action": "move", "x": 635, "y": 397}
+{"time_stamp": 20909.5171043, "action": "move", "x": 640, "y": 401}
+{"time_stamp": 20909.5254182, "action": "move", "x": 643, "y": 405}
+{"time_stamp": 20909.5328418, "action": "move", "x": 646, "y": 408}
+{"time_stamp": 20909.5415916, "action": "move", "x": 647, "y": 412}
+{"time_stamp": 20909.5488226, "action": "move", "x": 648, "y": 417}
+{"time_stamp": 20909.5586638, "action": "move", "x": 648, "y": 421}
+{"time_stamp": 20909.5644839, "action": "move", "x": 648, "y": 427}
+{"time_stamp": 20909.5764305, "action": "move", "x": 648, "y": 431}
+{"time_stamp": 20909.5807324, "action": "move", "x": 648, "y": 436}
+{"time_stamp": 20909.5888791, "action": "move", "x": 648, "y": 442}
+{"time_stamp": 20909.5969952, "action": "move", "x": 647, "y": 447}
+{"time_stamp": 20909.6051702, "action": "move", "x": 647, "y": 454}
+{"time_stamp": 20909.613182, "action": "move", "x": 647, "y": 459}
+{"time_stamp": 20909.6221534, "action": "move", "x": 647, "y": 465}
+{"time_stamp": 20909.6288777, "action": "move", "x": 647, "y": 471}
+{"time_stamp": 20909.6379005, "action": "move", "x": 647, "y": 477}
+{"time_stamp": 20909.6450558, "action": "move", "x": 647, "y": 481}
+{"time_stamp": 20909.6534421, "action": "move", "x": 647, "y": 487}
+{"time_stamp": 20909.6608373, "action": "move", "x": 647, "y": 490}
+{"time_stamp": 20909.673098, "action": "move", "x": 647, "y": 494}
+{"time_stamp": 20909.6774157, "action": "move", "x": 647, "y": 497}
+{"time_stamp": 20909.6849938, "action": "move", "x": 647, "y": 499}
+{"time_stamp": 20909.6929131, "action": "move", "x": 647, "y": 501}
+{"time_stamp": 20909.7010057, "action": "move", "x": 647, "y": 502}
+{"time_stamp": 20909.7092304, "action": "move", "x": 647, "y": 503}
+{"time_stamp": 20909.7176735, "action": "move", "x": 647, "y": 504}
+{"time_stamp": 20909.8051763, "action": "move", "x": 647, "y": 504}
+{"time_stamp": 20909.8052405, "action": "click", "x": 647, "y": 504, "button": "left", "pressed": false}
+{"time_stamp": 20909.852911, "action": "move", "x": 647, "y": 504}
+{"time_stamp": 20909.8610037, "action": "move", "x": 647, "y": 505}
+{"time_stamp": 20909.8689777, "action": "move", "x": 647, "y": 505}
+{"time_stamp": 20909.8768065, "action": "move", "x": 647, "y": 506}
+{"time_stamp": 20909.8929638, "action": "move", "x": 647, "y": 506}
+{"time_stamp": 20909.9651205, "action": "move", "x": 647, "y": 506}
+{"time_stamp": 20909.9735798, "action": "move", "x": 647, "y": 506}
+{"time_stamp": 20909.9810372, "action": "move", "x": 647, "y": 505}
+{"time_stamp": 20909.9969856, "action": "move", "x": 648, "y": 505}
+{"time_stamp": 20910.0052334, "action": "move", "x": 648, "y": 504}
+{"time_stamp": 20910.0770109, "action": "move", "x": 649, "y": 504}
+{"time_stamp": 20910.1093585, "action": "move", "x": 649, "y": 504}
+{"time_stamp": 20910.1094196, "action": "click", "x": 649, "y": 504, "button": "left", "pressed": true}
+{"time_stamp": 20910.1978081, "action": "move", "x": 649, "y": 504}
+{"time_stamp": 20910.197881, "action": "click", "x": 649, "y": 504, "button": "left", "pressed": false}
+{"time_stamp": 20910.3809367, "action": "move", "x": 649, "y": 504}
+{"time_stamp": 20910.4289799, "action": "move", "x": 649, "y": 504}
+{"time_stamp": 20910.4611147, "action": "move", "x": 649, "y": 504}
+{"time_stamp": 20910.4695803, "action": "move", "x": 648, "y": 501}
+{"time_stamp": 20910.4768942, "action": "move", "x": 646, "y": 499}
+{"time_stamp": 20910.486076, "action": "move", "x": 644, "y": 497}
+{"time_stamp": 20910.4929244, "action": "move", "x": 643, "y": 493}
+{"time_stamp": 20910.5017329, "action": "move", "x": 642, "y": 492}
+{"time_stamp": 20910.5088159, "action": "move", "x": 639, "y": 488}
+{"time_stamp": 20910.5169781, "action": "move", "x": 636, "y": 485}
+{"time_stamp": 20910.5248625, "action": "move", "x": 634, "y": 480}
+{"time_stamp": 20910.5332216, "action": "move", "x": 630, "y": 475}
+{"time_stamp": 20910.5405812, "action": "move", "x": 626, "y": 469}
+{"time_stamp": 20910.5485548, "action": "move", "x": 620, "y": 462}
+{"time_stamp": 20910.5570139, "action": "move", "x": 615, "y": 454}
+{"time_stamp": 20910.5653158, "action": "move", "x": 609, "y": 446}
+{"time_stamp": 20910.5731145, "action": "move", "x": 604, "y": 438}
+{"time_stamp": 20910.5812951, "action": "move", "x": 599, "y": 429}
+{"time_stamp": 20910.5887473, "action": "move", "x": 594, "y": 422}
+{"time_stamp": 20910.5969663, "action": "move", "x": 589, "y": 413}
+{"time_stamp": 20910.6052146, "action": "move", "x": 585, "y": 406}
+{"time_stamp": 20910.6129937, "action": "move", "x": 582, "y": 399}
+{"time_stamp": 20910.621082, "action": "move", "x": 580, "y": 393}
+{"time_stamp": 20910.6303279, "action": "move", "x": 576, "y": 387}
+{"time_stamp": 20910.6366685, "action": "move", "x": 576, "y": 383}
+{"time_stamp": 20910.6446208, "action": "move", "x": 574, "y": 379}
+{"time_stamp": 20910.6525582, "action": "move", "x": 574, "y": 377}
+{"time_stamp": 20910.6618885, "action": "move", "x": 573, "y": 375}
+{"time_stamp": 20910.6685216, "action": "move", "x": 573, "y": 374}
+{"time_stamp": 20910.6773103, "action": "move", "x": 573, "y": 373}
+{"time_stamp": 20910.9729471, "action": "move", "x": 573, "y": 372}
+{"time_stamp": 20910.9970497, "action": "move", "x": 573, "y": 371}
+{"time_stamp": 20911.0049526, "action": "move", "x": 573, "y": 371}
+{"time_stamp": 20911.0543747, "action": "move", "x": 573, "y": 371}
+{"time_stamp": 20911.0544342, "action": "click", "x": 573, "y": 371, "button": "left", "pressed": true}
+{"time_stamp": 20911.1253175, "action": "move", "x": 573, "y": 371}
+{"time_stamp": 20911.1255146, "action": "click", "x": 573, "y": 371, "button": "left", "pressed": false}
+{"time_stamp": 20911.1893962, "action": "move", "x": 573, "y": 371}
+{"time_stamp": 20911.1894581, "action": "click", "x": 573, "y": 371, "button": "left", "pressed": true}
+{"time_stamp": 20911.2772173, "action": "move", "x": 573, "y": 371}
+{"time_stamp": 20911.2774168, "action": "click", "x": 573, "y": 371, "button": "left", "pressed": false}
+{"time_stamp": 20911.3735954, "action": "move", "x": 574, "y": 371}
+{"time_stamp": 20911.3970172, "action": "move", "x": 574, "y": 370}
+{"time_stamp": 20911.4130016, "action": "move", "x": 574, "y": 370}
+{"time_stamp": 20911.4210189, "action": "move", "x": 574, "y": 369}
+{"time_stamp": 20911.4290232, "action": "move", "x": 574, "y": 368}
+{"time_stamp": 20911.437391, "action": "move", "x": 574, "y": 367}
+{"time_stamp": 20911.4450491, "action": "move", "x": 574, "y": 365}
+{"time_stamp": 20911.4524841, "action": "move", "x": 574, "y": 364}
+{"time_stamp": 20911.4610051, "action": "move", "x": 573, "y": 363}
+{"time_stamp": 20911.4689653, "action": "move", "x": 573, "y": 360}
+{"time_stamp": 20911.4776057, "action": "move", "x": 572, "y": 358}
+{"time_stamp": 20911.4849593, "action": "move", "x": 571, "y": 357}
+{"time_stamp": 20911.4928413, "action": "move", "x": 570, "y": 355}
+{"time_stamp": 20911.5014124, "action": "move", "x": 570, "y": 353}
+{"time_stamp": 20911.5085918, "action": "move", "x": 569, "y": 351}
+{"time_stamp": 20911.5168119, "action": "move", "x": 569, "y": 350}
+{"time_stamp": 20911.5254012, "action": "move", "x": 567, "y": 348}
+{"time_stamp": 20911.5329073, "action": "move", "x": 566, "y": 347}
+{"time_stamp": 20911.5413472, "action": "move", "x": 566, "y": 346}
+{"time_stamp": 20911.5490057, "action": "move", "x": 565, "y": 344}
+{"time_stamp": 20911.5573987, "action": "move", "x": 565, "y": 343}
+{"time_stamp": 20911.5649727, "action": "move", "x": 564, "y": 342}
+{"time_stamp": 20911.5728592, "action": "move", "x": 564, "y": 341}
+{"time_stamp": 20911.5811959, "action": "move", "x": 563, "y": 341}
+{"time_stamp": 20911.5894587, "action": "move", "x": 563, "y": 341}
+{"time_stamp": 20911.6050234, "action": "move", "x": 563, "y": 340}
+{"time_stamp": 20911.6214642, "action": "move", "x": 563, "y": 340}
+{"time_stamp": 20911.6288865, "action": "move", "x": 563, "y": 340}
+{"time_stamp": 20911.6535106, "action": "move", "x": 563, "y": 339}
+{"time_stamp": 20911.6609509, "action": "move", "x": 563, "y": 339}
+{"time_stamp": 20911.6770742, "action": "move", "x": 563, "y": 338}
+{"time_stamp": 20911.6932968, "action": "move", "x": 563, "y": 337}
+{"time_stamp": 20911.7090873, "action": "move", "x": 563, "y": 337}
+{"time_stamp": 20911.7169515, "action": "move", "x": 563, "y": 337}
+{"time_stamp": 20911.7251378, "action": "move", "x": 563, "y": 336}
+{"time_stamp": 20911.7335977, "action": "move", "x": 563, "y": 336}
+{"time_stamp": 20911.7569874, "action": "move", "x": 564, "y": 335}
+{"time_stamp": 20911.7982343, "action": "move", "x": 564, "y": 335}
+{"time_stamp": 20911.798291, "action": "click", "x": 564, "y": 335, "button": "left", "pressed": true}
+{"time_stamp": 20912.0213764, "action": "move", "x": 564, "y": 335}
+{"time_stamp": 20912.0214364, "action": "click", "x": 564, "y": 335, "button": "left", "pressed": false}
+{"time_stamp": 20912.1169877, "action": "move", "x": 565, "y": 335}
+{"time_stamp": 20912.1403572, "action": "move", "x": 565, "y": 335}
+{"time_stamp": 20912.1502752, "action": "move", "x": 566, "y": 334}
+{"time_stamp": 20912.1652795, "action": "move", "x": 566, "y": 333}
+{"time_stamp": 20912.1892869, "action": "move", "x": 567, "y": 333}
+{"time_stamp": 20912.2453705, "action": "move", "x": 567, "y": 333}
+{"time_stamp": 20912.2456314, "action": "click", "x": 567, "y": 333, "button": "left", "pressed": true}
+{"time_stamp": 20912.3255209, "action": "move", "x": 567, "y": 333}
+{"time_stamp": 20912.3257194, "action": "click", "x": 567, "y": 333, "button": "left", "pressed": false}
+{"time_stamp": 20912.3975396, "action": "move", "x": 567, "y": 333}
+{"time_stamp": 20912.3977428, "action": "click", "x": 567, "y": 333, "button": "left", "pressed": true}
+{"time_stamp": 20912.4772747, "action": "move", "x": 567, "y": 333}
+{"time_stamp": 20912.4774559, "action": "click", "x": 567, "y": 333, "button": "left", "pressed": false}
+{"time_stamp": 20912.5170331, "action": "move", "x": 567, "y": 333}
+{"time_stamp": 20912.5250006, "action": "move", "x": 568, "y": 333}
+{"time_stamp": 20912.5330206, "action": "move", "x": 569, "y": 333}
+{"time_stamp": 20912.5494971, "action": "move", "x": 569, "y": 333}
+{"time_stamp": 20912.5570801, "action": "move", "x": 570, "y": 333}
+{"time_stamp": 20912.5651997, "action": "move", "x": 570, "y": 333}
+{"time_stamp": 20912.5730284, "action": "move", "x": 571, "y": 333}
+{"time_stamp": 20912.581419, "action": "move", "x": 571, "y": 333}
+{"time_stamp": 20912.5890085, "action": "move", "x": 572, "y": 333}
+{"time_stamp": 20912.5970459, "action": "move", "x": 572, "y": 332}
+{"time_stamp": 20912.6054759, "action": "move", "x": 573, "y": 332}
+{"time_stamp": 20912.6125877, "action": "move", "x": 573, "y": 332}
+{"time_stamp": 20913.0615122, "action": "move", "x": 573, "y": 332}
+{"time_stamp": 20913.0615766, "action": "click", "x": 573, "y": 332, "button": "left", "pressed": true}
+{"time_stamp": 20913.1253534, "action": "move", "x": 573, "y": 332}
+{"time_stamp": 20913.1255383, "action": "click", "x": 573, "y": 332, "button": "left", "pressed": false}
+{"time_stamp": 20913.1649149, "action": "move", "x": 574, "y": 332}
+{"time_stamp": 20913.1726433, "action": "move", "x": 574, "y": 332}
+{"time_stamp": 20913.1807086, "action": "move", "x": 574, "y": 333}
+{"time_stamp": 20913.1887133, "action": "move", "x": 574, "y": 335}
+{"time_stamp": 20913.1970498, "action": "move", "x": 576, "y": 336}
+{"time_stamp": 20913.2049206, "action": "move", "x": 576, "y": 337}
+{"time_stamp": 20913.2130203, "action": "move", "x": 577, "y": 339}
+{"time_stamp": 20913.2218437, "action": "move", "x": 578, "y": 341}
+{"time_stamp": 20913.2288568, "action": "move", "x": 579, "y": 343}
+{"time_stamp": 20913.2369158, "action": "move", "x": 581, "y": 345}
+{"time_stamp": 20913.2448247, "action": "move", "x": 582, "y": 348}
+{"time_stamp": 20913.2528064, "action": "move", "x": 585, "y": 352}
+{"time_stamp": 20913.2609073, "action": "move", "x": 586, "y": 355}
+{"time_stamp": 20913.2691133, "action": "move", "x": 590, "y": 360}
+{"time_stamp": 20913.2773732, "action": "move", "x": 593, "y": 364}
+{"time_stamp": 20913.2849304, "action": "move", "x": 596, "y": 371}
+{"time_stamp": 20913.2930878, "action": "move", "x": 600, "y": 377}
+{"time_stamp": 20913.3011352, "action": "move", "x": 603, "y": 383}
+{"time_stamp": 20913.3090124, "action": "move", "x": 606, "y": 389}
+{"time_stamp": 20913.3169507, "action": "move", "x": 609, "y": 394}
+{"time_stamp": 20913.3250383, "action": "move", "x": 612, "y": 401}
+{"time_stamp": 20913.3341516, "action": "move", "x": 613, "y": 405}
+{"time_stamp": 20913.3409443, "action": "move", "x": 615, "y": 408}
+{"time_stamp": 20913.3491641, "action": "move", "x": 617, "y": 410}
+{"time_stamp": 20913.3569501, "action": "move", "x": 617, "y": 410}
+{"time_stamp": 20913.3653744, "action": "move", "x": 618, "y": 411}
+{"time_stamp": 20913.3983456, "action": "move", "x": 618, "y": 411}
+{"time_stamp": 20913.3985633, "action": "click", "x": 618, "y": 411, "button": "left", "pressed": true}
+{"time_stamp": 20913.461381, "action": "move", "x": 618, "y": 411}
+{"time_stamp": 20913.4617251, "action": "move", "x": 618, "y": 411}
+{"time_stamp": 20913.4618623, "action": "click", "x": 618, "y": 411, "button": "left", "pressed": false}
+{"time_stamp": 20913.5571078, "action": "move", "x": 618, "y": 411}
+{"time_stamp": 20913.5809046, "action": "move", "x": 618, "y": 412}
+{"time_stamp": 20913.6053422, "action": "move", "x": 618, "y": 412}
+{"time_stamp": 20913.629084, "action": "move", "x": 618, "y": 413}
+{"time_stamp": 20913.661287, "action": "move", "x": 618, "y": 413}
+{"time_stamp": 20913.7572581, "action": "move", "x": 619, "y": 414}
+{"time_stamp": 20913.7733636, "action": "move", "x": 619, "y": 414}
+{"time_stamp": 20913.7811919, "action": "move", "x": 619, "y": 415}
+{"time_stamp": 20913.797434, "action": "move", "x": 620, "y": 416}
+{"time_stamp": 20913.8054243, "action": "move", "x": 620, "y": 417}
+{"time_stamp": 20913.8128871, "action": "move", "x": 621, "y": 417}
+{"time_stamp": 20913.8209318, "action": "move", "x": 621, "y": 419}
+{"time_stamp": 20913.8295056, "action": "move", "x": 623, "y": 420}
+{"time_stamp": 20913.8368152, "action": "move", "x": 624, "y": 421}
+{"time_stamp": 20913.8454679, "action": "move", "x": 625, "y": 423}
+{"time_stamp": 20913.8528129, "action": "move", "x": 626, "y": 423}
+{"time_stamp": 20913.8609935, "action": "move", "x": 626, "y": 425}
+{"time_stamp": 20913.8690673, "action": "move", "x": 627, "y": 426}
+{"time_stamp": 20913.8769586, "action": "move", "x": 628, "y": 427}
+{"time_stamp": 20913.8851888, "action": "move", "x": 629, "y": 428}
+{"time_stamp": 20913.893344, "action": "move", "x": 630, "y": 429}
+{"time_stamp": 20913.9011714, "action": "move", "x": 630, "y": 429}
+{"time_stamp": 20913.9088846, "action": "move", "x": 630, "y": 430}
+{"time_stamp": 20913.917037, "action": "move", "x": 631, "y": 430}
+{"time_stamp": 20913.9255765, "action": "move", "x": 631, "y": 431}
+{"time_stamp": 20913.9331618, "action": "move", "x": 631, "y": 431}
+{"time_stamp": 20914.2531646, "action": "move", "x": 632, "y": 432}
+{"time_stamp": 20914.2610357, "action": "move", "x": 632, "y": 432}
+{"time_stamp": 20914.2688307, "action": "move", "x": 632, "y": 432}
+{"time_stamp": 20914.2771917, "action": "move", "x": 632, "y": 433}
+{"time_stamp": 20914.2850601, "action": "move", "x": 633, "y": 434}
+{"time_stamp": 20914.2934044, "action": "move", "x": 634, "y": 436}
+{"time_stamp": 20914.3011978, "action": "move", "x": 635, "y": 439}
+{"time_stamp": 20914.3094538, "action": "move", "x": 638, "y": 446}
+{"time_stamp": 20914.3171653, "action": "move", "x": 643, "y": 454}
+{"time_stamp": 20914.3251099, "action": "move", "x": 649, "y": 466}
+{"time_stamp": 20914.3327128, "action": "move", "x": 655, "y": 479}
+{"time_stamp": 20914.3407581, "action": "move", "x": 666, "y": 502}
+{"time_stamp": 20914.3489255, "action": "move", "x": 676, "y": 530}
+{"time_stamp": 20914.3567984, "action": "move", "x": 686, "y": 558}
+{"time_stamp": 20914.365267, "action": "move", "x": 693, "y": 581}
+{"time_stamp": 20914.3729078, "action": "move", "x": 696, "y": 598}
+{"time_stamp": 20914.3810446, "action": "move", "x": 700, "y": 617}
+{"time_stamp": 20914.3893176, "action": "move", "x": 703, "y": 635}
+{"time_stamp": 20914.3972033, "action": "move", "x": 708, "y": 657}
+{"time_stamp": 20914.4054019, "action": "move", "x": 716, "y": 682}
+{"time_stamp": 20914.4131757, "action": "move", "x": 724, "y": 709}
+{"time_stamp": 20914.4209714, "action": "move", "x": 734, "y": 735}
+{"time_stamp": 20914.4290624, "action": "move", "x": 743, "y": 762}
+{"time_stamp": 20914.4371673, "action": "move", "x": 751, "y": 785}
+{"time_stamp": 20914.444998, "action": "move", "x": 755, "y": 799}
+{"time_stamp": 20914.453295, "action": "move", "x": 758, "y": 813}
+{"time_stamp": 20914.4609701, "action": "move", "x": 760, "y": 822}
+{"time_stamp": 20914.4688875, "action": "move", "x": 761, "y": 831}
+{"time_stamp": 20914.476972, "action": "move", "x": 761, "y": 836}
+{"time_stamp": 20914.4852847, "action": "move", "x": 762, "y": 840}
+{"time_stamp": 20914.493183, "action": "move", "x": 762, "y": 843}
+{"time_stamp": 20914.5011318, "action": "move", "x": 763, "y": 846}
+{"time_stamp": 20914.5090552, "action": "move", "x": 763, "y": 848}
+{"time_stamp": 20914.5169995, "action": "move", "x": 763, "y": 850}
+{"time_stamp": 20914.5249194, "action": "move", "x": 763, "y": 852}
+{"time_stamp": 20914.5329086, "action": "move", "x": 763, "y": 852}
+{"time_stamp": 20914.5412108, "action": "move", "x": 763, "y": 854}
+{"time_stamp": 20914.5497755, "action": "move", "x": 763, "y": 855}
+{"time_stamp": 20914.5568077, "action": "move", "x": 763, "y": 855}
+{"time_stamp": 20914.5651505, "action": "move", "x": 763, "y": 858}
+{"time_stamp": 20914.5729076, "action": "move", "x": 763, "y": 859}
+{"time_stamp": 20914.5815302, "action": "move", "x": 763, "y": 863}
+{"time_stamp": 20914.5889204, "action": "move", "x": 763, "y": 866}
+{"time_stamp": 20914.5993101, "action": "move", "x": 763, "y": 870}
+{"time_stamp": 20914.604969, "action": "move", "x": 763, "y": 875}
+{"time_stamp": 20914.6142152, "action": "move", "x": 762, "y": 881}
+{"time_stamp": 20914.6210746, "action": "move", "x": 761, "y": 887}
+{"time_stamp": 20914.629826, "action": "move", "x": 761, "y": 893}
+{"time_stamp": 20914.6371388, "action": "move", "x": 761, "y": 900}
+{"time_stamp": 20914.6451425, "action": "move", "x": 760, "y": 907}
+{"time_stamp": 20914.6526885, "action": "move", "x": 759, "y": 914}
+{"time_stamp": 20914.6612302, "action": "move", "x": 758, "y": 921}
+{"time_stamp": 20914.6691206, "action": "move", "x": 757, "y": 929}
+{"time_stamp": 20914.6775645, "action": "move", "x": 757, "y": 938}
+{"time_stamp": 20914.6849375, "action": "move", "x": 757, "y": 946}
+{"time_stamp": 20914.6927991, "action": "move", "x": 756, "y": 952}
+{"time_stamp": 20914.7013277, "action": "move", "x": 756, "y": 959}
+{"time_stamp": 20914.7089762, "action": "move", "x": 756, "y": 967}
+{"time_stamp": 20914.7169168, "action": "move", "x": 756, "y": 975}
+{"time_stamp": 20914.7248412, "action": "move", "x": 756, "y": 984}
+{"time_stamp": 20914.7332847, "action": "move", "x": 756, "y": 992}
+{"time_stamp": 20914.7409045, "action": "move", "x": 756, "y": 998}
+{"time_stamp": 20914.7492432, "action": "move", "x": 757, "y": 1007}
+{"time_stamp": 20914.756811, "action": "move", "x": 757, "y": 1015}
+{"time_stamp": 20914.7650875, "action": "move", "x": 757, "y": 1021}
+{"time_stamp": 20914.7732405, "action": "move", "x": 757, "y": 1028}
+{"time_stamp": 20914.781208, "action": "move", "x": 757, "y": 1034}
+{"time_stamp": 20914.7891933, "action": "move", "x": 757, "y": 1038}
+{"time_stamp": 20914.7970335, "action": "move", "x": 757, "y": 1040}
+{"time_stamp": 20914.8054292, "action": "move", "x": 757, "y": 1043}
+{"time_stamp": 20914.8131588, "action": "move", "x": 757, "y": 1045}
+{"time_stamp": 20914.8207724, "action": "move", "x": 757, "y": 1046}
+{"time_stamp": 20914.8292599, "action": "move", "x": 757, "y": 1046}
+{"time_stamp": 20914.8377593, "action": "move", "x": 757, "y": 1047}
+{"time_stamp": 20914.8450543, "action": "move", "x": 757, "y": 1047}
+{"time_stamp": 20914.8529288, "action": "move", "x": 756, "y": 1047}
+{"time_stamp": 20914.8619212, "action": "move", "x": 756, "y": 1048}
+{"time_stamp": 20914.8773142, "action": "move", "x": 756, "y": 1048}
+{"time_stamp": 20914.8929039, "action": "move", "x": 756, "y": 1049}
+{"time_stamp": 20914.9091474, "action": "move", "x": 756, "y": 1050}
+{"time_stamp": 20914.9258662, "action": "move", "x": 755, "y": 1050}
+{"time_stamp": 20914.9328988, "action": "move", "x": 755, "y": 1051}
+{"time_stamp": 20914.9412733, "action": "move", "x": 755, "y": 1052}
+{"time_stamp": 20914.9493841, "action": "move", "x": 755, "y": 1052}
+{"time_stamp": 20914.9570102, "action": "move", "x": 755, "y": 1053}
+{"time_stamp": 20914.9650747, "action": "move", "x": 755, "y": 1054}
+{"time_stamp": 20914.9726868, "action": "move", "x": 755, "y": 1055}
+{"time_stamp": 20914.9810211, "action": "move", "x": 755, "y": 1055}
+{"time_stamp": 20914.9969404, "action": "move", "x": 755, "y": 1056}
+{"time_stamp": 20915.004993, "action": "move", "x": 755, "y": 1057}
+{"time_stamp": 20915.0370925, "action": "move", "x": 755, "y": 1057}
+{"time_stamp": 20915.0373434, "action": "click", "x": 755, "y": 1057, "button": "left", "pressed": true}
+{"time_stamp": 20915.1171158, "action": "move", "x": 754, "y": 1057}
+{"time_stamp": 20915.125679, "action": "move", "x": 754, "y": 1057}
+{"time_stamp": 20915.125939, "action": "click", "x": 754, "y": 1057, "button": "left", "pressed": false}
+{"time_stamp": 20915.1651919, "action": "move", "x": 754, "y": 1057}
+{"time_stamp": 20915.1729477, "action": "move", "x": 754, "y": 1057}
+{"time_stamp": 20915.1809724, "action": "move", "x": 754, "y": 1056}
+{"time_stamp": 20915.1889093, "action": "move", "x": 753, "y": 1055}
+{"time_stamp": 20915.1996899, "action": "move", "x": 753, "y": 1054}
+{"time_stamp": 20915.2051687, "action": "move", "x": 753, "y": 1052}
+{"time_stamp": 20915.2128354, "action": "move", "x": 753, "y": 1051}
+{"time_stamp": 20915.2211657, "action": "move", "x": 753, "y": 1048}
+{"time_stamp": 20915.228887, "action": "move", "x": 753, "y": 1044}
+{"time_stamp": 20915.2368719, "action": "move", "x": 753, "y": 1038}
+{"time_stamp": 20915.2450598, "action": "move", "x": 754, "y": 1029}
+{"time_stamp": 20915.2529132, "action": "move", "x": 755, "y": 1019}
+{"time_stamp": 20915.2609538, "action": "move", "x": 756, "y": 1004}
+{"time_stamp": 20915.2689694, "action": "move", "x": 756, "y": 989}
+{"time_stamp": 20915.2766563, "action": "move", "x": 756, "y": 969}
+{"time_stamp": 20915.2885596, "action": "move", "x": 756, "y": 944}
+{"time_stamp": 20915.2929735, "action": "move", "x": 756, "y": 917}
+{"time_stamp": 20915.3040882, "action": "move", "x": 758, "y": 886}
+{"time_stamp": 20915.3089656, "action": "move", "x": 761, "y": 852}
+{"time_stamp": 20915.3171334, "action": "move", "x": 764, "y": 819}
+{"time_stamp": 20915.325026, "action": "move", "x": 768, "y": 785}
+{"time_stamp": 20915.3325308, "action": "move", "x": 769, "y": 749}
+{"time_stamp": 20915.3406827, "action": "move", "x": 771, "y": 718}
+{"time_stamp": 20915.3488368, "action": "move", "x": 771, "y": 685}
+{"time_stamp": 20915.3609822, "action": "move", "x": 773, "y": 655}
+{"time_stamp": 20915.3654554, "action": "move", "x": 773, "y": 616}
+{"time_stamp": 20915.3731016, "action": "move", "x": 770, "y": 580}
+{"time_stamp": 20915.3808956, "action": "move", "x": 768, "y": 547}
+{"time_stamp": 20915.3887073, "action": "move", "x": 766, "y": 516}
+{"time_stamp": 20915.397101, "action": "move", "x": 763, "y": 487}
+{"time_stamp": 20915.4049082, "action": "move", "x": 759, "y": 466}
+{"time_stamp": 20915.4132309, "action": "move", "x": 754, "y": 447}
+{"time_stamp": 20915.421175, "action": "move", "x": 746, "y": 425}
+{"time_stamp": 20915.4290594, "action": "move", "x": 738, "y": 407}
+{"time_stamp": 20915.4369077, "action": "move", "x": 726, "y": 388}
+{"time_stamp": 20915.4449497, "action": "move", "x": 717, "y": 372}
+{"time_stamp": 20915.4528974, "action": "move", "x": 710, "y": 359}
+{"time_stamp": 20915.4609101, "action": "move", "x": 703, "y": 348}
+{"time_stamp": 20915.468797, "action": "move", "x": 697, "y": 338}
+{"time_stamp": 20915.4766521, "action": "move", "x": 692, "y": 328}
+{"time_stamp": 20915.4847983, "action": "move", "x": 687, "y": 320}
+{"time_stamp": 20915.4929905, "action": "move", "x": 681, "y": 313}
+{"time_stamp": 20915.501352, "action": "move", "x": 678, "y": 308}
+{"time_stamp": 20915.5087319, "action": "move", "x": 676, "y": 306}
+{"time_stamp": 20915.5171601, "action": "move", "x": 674, "y": 304}
+{"time_stamp": 20915.5258167, "action": "move", "x": 674, "y": 303}
+{"time_stamp": 20915.5333029, "action": "move", "x": 673, "y": 302}
+{"time_stamp": 20915.6376355, "action": "move", "x": 673, "y": 302}
+{"time_stamp": 20915.6451407, "action": "move", "x": 675, "y": 302}
+{"time_stamp": 20915.6526475, "action": "move", "x": 678, "y": 302}
+{"time_stamp": 20915.6612996, "action": "move", "x": 681, "y": 302}
+{"time_stamp": 20915.6688079, "action": "move", "x": 684, "y": 302}
+{"time_stamp": 20915.6777765, "action": "move", "x": 687, "y": 300}
+{"time_stamp": 20915.6859168, "action": "move", "x": 690, "y": 300}
+{"time_stamp": 20915.693198, "action": "move", "x": 692, "y": 298}
+{"time_stamp": 20915.7010803, "action": "move", "x": 694, "y": 296}
+{"time_stamp": 20915.7089464, "action": "move", "x": 697, "y": 296}
+{"time_stamp": 20915.7165747, "action": "move", "x": 698, "y": 296}
+{"time_stamp": 20915.7246256, "action": "move", "x": 700, "y": 296}
+{"time_stamp": 20915.7332713, "action": "move", "x": 700, "y": 295}
+{"time_stamp": 20915.7409074, "action": "move", "x": 701, "y": 295}
+{"time_stamp": 20915.8458505, "action": "move", "x": 701, "y": 295}
+{"time_stamp": 20915.8461508, "action": "click", "x": 701, "y": 295, "button": "left", "pressed": true}
+{"time_stamp": 20915.9253627, "action": "move", "x": 701, "y": 295}
+{"time_stamp": 20915.9255724, "action": "click", "x": 701, "y": 295, "button": "left", "pressed": false}
diff --git a/utils/ducktrack.py b/utils/ducktrack.py
index 3fe7e1c..d42f85b 100644
--- a/utils/ducktrack.py
+++ b/utils/ducktrack.py
@@ -3,76 +3,97 @@ import sys, pathlib;
sys.path.append(str(pathlib.Path(__file__).parents[1]))
import os
+import math
import json
+import numpy as np
from typing import List
-from desktop_env.envs.desktop_env import Action, MouseClick
+from copy import deepcopy
+pynput2pyautogui_key = {
+ "alt_l": "altleft",
+ "alt_r": "altright",
+}
+COMMAND_KEYS = ['accept', 'add', 'alt', 'altleft', 'altright', 'apps', 'backspace', 'browserback', 'browserfavorites', 'browserforward', 'browserhome', 'browserrefresh', 'browsersearch', 'browserstop', 'capslock', 'clear', 'convert', 'ctrl', 'ctrlleft', 'ctrlright', 'decimal', 'del', 'delete', 'divide', 'down', 'end', 'enter', 'esc', 'escape', 'execute', 'f1', 'f10', 'f11', 'f12', 'f13', 'f14', 'f15', 'f16', 'f17', 'f18', 'f19', 'f2', 'f20', 'f21', 'f22', 'f23', 'f24', 'f3', 'f4', 'f5', 'f6', 'f7', 'f8', 'f9', 'final', 'fn', 'hanguel', 'hangul', 'hanja', 'help', 'home', 'insert', 'junja', 'kana', 'kanji', 'launchapp1', 'launchapp2', 'launchmail', 'launchmediaselect', 'left', 'modechange', 'multiply', 'nexttrack', 'nonconvert', 'num0', 'num1', 'num2', 'num3', 'num4', 'num5', 'num6', 'num7', 'num8', 'num9', 'numlock', 'pagedown', 'pageup', 'pause', 'pgdn', 'pgup', 'playpause', 'prevtrack', 'print', 'printscreen', 'prntscrn', 'prtsc', 'prtscr', 'return', 'right', 'scrolllock', 'select', 'separator', 'shift', 'shiftleft', 'shiftright', 'sleep', 'stop', 'subtract', 'tab', 'up', 'volumedown', 'volumemute', 'volumeup', 'win', 'winleft', 'winright', 'yen', 'command', 'option', 'optionleft', 'optionright', 'alt_l', 'alt_r']
+typingkey2str = {
+ "space" : " ",
+}
class DuckTrackEventActionConverter:
- def __init__(self, human_readable: str, compress_move: bool = True):
- self.human_readable = human_readable
- self.compress_move = compress_move
+ def __init__(self, ):
+ """"""
- def enum_to_str(self, enum):
- """Converts an enum to its string representation if HUMAN_READABLE is True, otherwise returns its value."""
- return enum.name if self.human_readable else enum.value
+ ### Enumerations ###
+ def move_event_to_action(self, event: dict, action_space: str = "computer_13"):
+ """Converts a mouse move event to its corresponding action."""
+ if action_space == "computer_13":
+ return {
+ "action_type": "MOVE_TO",
+ "parameters": {
+ "x": event["x"],
+ "y": event["y"]
+ }
+ }
+ elif action_space == "pyautogui":
+ return "pyautogui.moveTo({}, {})".format(event["x"], event["y"])
- def compress_mouse_move(self, data: List[dict], index: int):
- """Compresses consecutive mouse move events into first and last move events."""
- first_move, last_move = data[index], data[index]
- while index < len(data) and data[index]["action"] == "move":
- last_move = data[index]
- index += 1
- return first_move, last_move, index
+ def click_event_to_action(self, event: dict, action_space: str = "computer_13"):
+ """Converts a mouse click event to its corresponding action."""
+ action = {
+ "action_type": None,
+ "parameters": {
+ "button": None
+ }
+ }
- def move_event_to_action(self, event: dict):
- return {"action_type": self.enum_to_str(Action.MOUSE_MOVE),
- "x": event["x"],
- "y": event["y"]}
-
- def click_event_to_action(self, event: dict):
- action = {}
mouse_button = event["button"]
mouse_pressed = event["pressed"]
if mouse_pressed:
- action["action_type"] = self.enum_to_str(Action.MOUSE_DOWN)
+ action["action_type"] = "MOUSE_DOWN"
elif not mouse_pressed:
- action["action_type"] = self.enum_to_str(Action.MOUSE_UP)
+ action["action_type"] = "MOUSE_UP"
else:
raise NotImplementedError(mouse_pressed)
- if mouse_button == "left":
- action["click_type"] = self.enum_to_str(MouseClick.LEFT)
- elif mouse_button == "right":
- action["click_type"] = self.enum_to_str(MouseClick.RIGHT)
- elif mouse_button == "middle":
- action["click_type"] = self.enum_to_str(MouseClick.MIDDLE)
+ if mouse_button in ["left", "right", "middle"]:
+ action["parameters"]["button"] = mouse_button
else:
raise NotImplementedError(mouse_button)
return action
- def press_event_to_action(self, event: dict):
- return {"action_type": self.enum_to_str(Action.KEY_DOWN),
- "key": [ord(c) for c in event["name"]]}
+ def press_event_to_action(self, event: dict, action_space: str = "computer_13"):
+ """Converts a key down event to its corresponding action."""
+ # NOTE: the `key down`, `press` have the same meaning here, while different in pyautogui
+ return {
+ "action_type": "KEY_DOWN",
+ "parameters": {
+ "key": event["name"] if event["name"] not in pynput2pyautogui_key else pynput2pyautogui_key[
+ event["name"]]
+ }
+ }
- def release_event_to_action(self, event: dict):
- return {"action_type": self.enum_to_str(Action.KEY_UP),
- "key": [ord(c) for c in event["name"]]}
+ def release_event_to_action(self, event: dict, action_space: str = "computer_13"):
+ """Converts a key release event to its corresponding action."""
+ return {
+ "action_type": "KEY_UP",
+ "parameters": {
+ "key": event["name"] if event["name"] not in pynput2pyautogui_key else pynput2pyautogui_key[
+ event["name"]]
+ }
+ }
- def scroll_event_to_action(self, event: dict):
- # TODO: need to confirm if df < 0 means scroll up or down
- if event["dy"] < 0:
- down = False
- else:
- down = True
+ def scroll_event_to_action(self, event: dict, action_space: str = "computer_13"):
+ """Converts a scroll event to its corresponding action."""
+ return {
+ "action_type": "SCROLL",
+ "parameters": {
+ "dx": event["dx"],
+ "dy": event["dy"]
+ }
+ }
- return {"action_type": self.enum_to_str(Action.CLICK),
- "click_type": self.enum_to_str(MouseClick.WHEEL_DOWN) if down else self.enum_to_str(
- MouseClick.WHEEL_UP)}
-
- def event_to_action(self, event: dict):
+ def event_to_action(self, event: dict, action_space: str = "computer_13"):
"""Converts an event to its corresponding action based on the event type."""
if event["action"] == "move":
return self.move_event_to_action(event)
@@ -87,114 +108,243 @@ class DuckTrackEventActionConverter:
else:
raise NotImplementedError(event["action"])
- def ducktrack_event_file_to_action(self, ducktrack_event_file: str, out_file: str, compress_move: bool = None):
+ ### Compressing ###
+ def compress_mouse_move(self, data: List[dict], index: int):
+ """Compresses consecutive mouse move events into the last move events."""
+ last_move = data[index]
+ while index < len(data) and data[index]["action"] == "move":
+ last_move = data[index]
+ index += 1
+ return last_move, index
+
+ def compress_scroll(self, data: List[dict], index: int):
+ """Compresses consecutive scroll events into a single scroll event."""
+ last_scroll = data[index]
+ consecutive_dx, consecutive_dy = data[index]["dx"], data[index]["dy"]
+ while index < len(data) and data[index]["action"] == "scroll" and np.sign(data[index]["dx"]) == np.sign(consecutive_dx) and np.sign(data[index]["dy"]) == np.sign(consecutive_dy):
+ last_scroll = data[index]
+ consecutive_dx += data[index]["dx"]
+ consecutive_dy += data[index]["dy"]
+ index += 1
+ last_scroll["dx"], last_scroll["dy"] = consecutive_dx, consecutive_dy
+ return last_scroll, index
+
+ ### Converting ###
+ def ducktrack_event_file_to_action(self, ducktrack_event_file: str, out_file: str, compress_move: bool = True, compress_scroll: bool = True, compress_click: bool = True,compress_drag: bool = True, compress_press_key: bool = True, compress_typing: bool = True):
"""Converts DuckTrack event data to a list of actions and saves them to a file."""
if not os.path.exists(ducktrack_event_file):
raise FileNotFoundError(ducktrack_event_file)
- # set to default
- if compress_move is None:
- compress_move = self.compress_move
-
with open(ducktrack_event_file, 'r') as file:
- data = [json.loads(line) for line in file]
+ events = [json.loads(line) for line in file]
- result = {"action": [], "event": []}
+ # Save the compressed actions in a list
+ result = []
index = 0
+ presses_to_skip = 0
+ releases_to_skip = 0
+ move_to_skip = 0
+ keys_pressed = []
# Compress the mouse move events
- while index < len(data):
- event = data[index]
- if event["action"] == "move" and compress_move:
- first_move, last_move, index = self.compress_mouse_move(data, index)
- result["action"].extend([self.event_to_action(last_move)])
- result["event"].extend([last_move])
- else:
- result["action"].append(self.event_to_action(event))
- result["event"].append(event)
+ while index < len(events):
+
+ event = events[index]
+
+ def do_mouse_press(button: str, _index: int):
+
+ num_clicks = 0
+ mouse_pressed = True
+ skip_move = 0
+ click_x, click_y = event["x"], event["y"]
+
+ for j, next_event in enumerate(events[index + 1:]):
+ # make sure the time between mouse clicks is less than 500ms
+ if next_event["time_stamp"] - event["time_stamp"] > 0.5:
+ if num_clicks > 0:
+ if result[-1:][0]["action_type"] == "MOVE_TO":
+ result.pop()
+ result.append({
+ "action_type": "CLICK",
+ "parameters": {
+ "button": button,
+ "x" : click_x,
+ "y" : click_y,
+ "num_clicks": num_clicks
+ }
+ })
+ return num_clicks-1, num_clicks, _index, skip_move
+ break
+
+ if "x" in next_event and "y" in next_event:
+ # if the mouse moves out of the click radius/rectangle, it is not a click sequence
+ if math.sqrt((next_event["y"] - event["y"]) ** 2 +
+ (next_event["x"] - event["x"]) ** 2) > 4:
+ if num_clicks > 0:
+ if result[-1:][0]["action_type"] == "MOVE_TO":
+ result.pop()
+ result.append({
+ "action_type": "CLICK",
+ "parameters": {
+ "button": button,
+ "x" : click_x,
+ "y" : click_y,
+ "num_clicks": num_clicks
+ }
+ })
+ return num_clicks-1, num_clicks, _index, skip_move
+ break
+
+ if next_event["action"] == "click" and compress_click:
+ if not next_event["pressed"]:
+ num_clicks += 1
+ mouse_pressed = False
+ if num_clicks == 3:
+ if result[-1:][0]["action_type"] == "MOVE_TO":
+ result.pop()
+ result.append({
+ "action_type": "CLICK",
+ "parameters": {
+ "button": button,
+ "x" : click_x,
+ "y" : click_y,
+ "num_clicks": 3
+ }
+ })
+ return 2, 3, _index, skip_move
+ elif next_event["pressed"]:
+ mouse_pressed = True
+ else:
+ raise NotImplementedError(next_event["pressed"])
+ elif next_event["action"] != "click" and not mouse_pressed:
+ if next_event["action"] == "move":
+ if next_event["x"] == click_x and next_event["y"] == click_y:
+ skip_move += 1
+ continue
+ if result[-1:][0]["action_type"] == "MOVE_TO":
+ result.pop()
+ result.append({
+ "action_type": "CLICK",
+ "parameters": {
+ "button": button,
+ "x" : click_x,
+ "y" : click_y,
+ "num_clicks": num_clicks
+ }
+ })
+ return num_clicks-1, num_clicks, _index, skip_move
+
+ # Compress {MOUSE_DOWN, MOVE, MOUSE_UP} into DRAG_TO event
+ elif next_event["action"] == "move" and compress_drag:
+ if next_event["x"] == click_x and next_event["y"] == click_y:
+ skip_move += 1
+ continue
+ last_move, _index = self.compress_mouse_move(events, _index+1)
+ result.append({
+ "action_type": "DRAG_TO",
+ "parameters": {
+ "x": last_move["x"],
+ "y": last_move["y"]
+ }
+ })
+ return 0, 1, _index, skip_move
+
+ result.append({
+ "action_type": "MOUSE_DOWN",
+ "parameters": {
+ "button": button
+ }
+ })
+ return 0, 0, _index, skip_move
+
+ if event["action"] == "move":
+ if move_to_skip > 0:
+ move_to_skip -= 1
+ index += 1
+ continue
+ if compress_move:
+ last_move, index = self.compress_mouse_move(events, index)
+ result.extend([self.event_to_action(last_move)])
+
+ elif event["action"] == "scroll" and compress_scroll:
+ last_scroll, index = self.compress_scroll(events, index)
+ result.extend([self.event_to_action(last_scroll)])
+
+ elif event["action"] == "click":
+ button = event["button"]
+
+ if event["pressed"]:
+ if presses_to_skip == 0:
+ presses, releases, index, moves = do_mouse_press(button, index)
+ presses_to_skip += presses
+ releases_to_skip += releases
+ move_to_skip += moves
+ else:
+ presses_to_skip -= 1
+ else:
+ if releases_to_skip == 0:
+ result.append({
+ "action_type": "MOUSE_UP",
+ "parameters": {
+ "button": button
+ }
+ })
+ else:
+ releases_to_skip -= 1
index += 1
-
- # Compress the key down and key up actions
- # todo: handling the key down and key up events
- _new_actions = []
- _action = list(result["action"])
- idx = 0
-
- while True:
- if idx >= len(_action):
- break
-
- if _action[idx]["action_type"] == self.enum_to_str(Action.KEY_DOWN):
- typed_text = []
- while idx < len(_action) and _action[idx]["action_type"] in [self.enum_to_str(Action.KEY_DOWN), self.enum_to_str(Action.KEY_UP)] and len(_action[idx]["key"]) == 1:
- if _action[idx]["action_type"] == self.enum_to_str(Action.KEY_DOWN):
- typed_text.append(chr(_action[idx]["key"][0]))
- idx += 1
- if typed_text:
- _new_actions.append({"action_type": self.enum_to_str(Action.TYPE), "text": typed_text})
+ elif event["action"] == "press" and event["name"] not in COMMAND_KEYS and compress_typing:
+ typing_words = ""
+ while index < len(events) and events[index]["action"] in ["press", "release"] and events[index]["name"] not in COMMAND_KEYS:
+ if events[index]["action"] == "press":
+ keys_pressed.append(events[index]["name"])
+ typing_words += events[index]["name"] if events[index]["name"] not in typingkey2str else typingkey2str[events[index]["name"]]
+ elif events[index]["action"] == "release":
+ keys_pressed.remove(events[index]["name"])
+ index += 1
+ if len(typing_words) > 1:
+ result.append({
+ "action_type": "TYPING",
+ "parameters": {
+ "text": typing_words
+ }
+ })
else:
- _new_actions.append(_action[idx])
- idx += 1
+ result.append({
+ "action_type": "PRESS",
+ "parameters": {
+ "key": typing_words
+ }
+ })
+ elif event["action"] == "press" and compress_press_key:
+ keys_pressed.append(event["name"])
+ result.append({
+ "action_type": "PRESS",
+ "parameters": {
+ "key": event["name"] if event["name"] not in pynput2pyautogui_key else pynput2pyautogui_key[
+ event["name"]]
+ }
+ })
+ index += 1
+ elif event["action"] == "release" and compress_press_key:
+ keys_pressed.remove(event["name"])
+ index += 1
else:
- _new_actions.append(_action[idx])
- idx += 1
-
- result["action"] = _new_actions
-
- # Compress the scroll up and scroll down events
- # todo: handling the key down and key up events
- _new_actions = []
- _action = list(result["action"])
- idx = 0
-
- while True:
- if idx >= len(_action):
- break
-
- if _action[idx]["action_type"] == self.enum_to_str(Action.CLICK) and _action[idx]["click_type"] in [self.enum_to_str(MouseClick.WHEEL_UP), self.enum_to_str(MouseClick.WHEEL_DOWN)]:
- typed_text = []
- while idx < len(_action) and _action[idx]["action_type"] == self.enum_to_str(Action.CLICK) and _action[idx]["click_type"] in [self.enum_to_str(MouseClick.WHEEL_UP), self.enum_to_str(MouseClick.WHEEL_DOWN)]:
- if _action[idx]["click_type"] == self.enum_to_str(MouseClick.WHEEL_UP):
- typed_text.append("UP")
- idx += 1
- elif _action[idx]["click_type"] == self.enum_to_str(MouseClick.WHEEL_DOWN):
- typed_text.append("DOWN")
- idx += 1
- _new_actions.append({"action_type": self.enum_to_str(Action.CLICK), "click_type": "SCROLL", "text": typed_text})
- else:
- _new_actions.append(_action[idx])
- idx += 1
-
- result["action"] = _new_actions
-
- # Compress the mouse down and mouse up actions
- # todo: handling the key down and key up events
- _new_actions = []
- _action = list(result["action"])
- idx = 0
-
- while True:
- if idx >= len(_action):
- break
- if _action[idx]["action_type"] == self.enum_to_str(Action.MOUSE_DOWN):
- if idx + 1 < len(_action) and _action[idx+1]["action_type"] == self.enum_to_str(Action.MOUSE_UP):
- _new_actions.append({"action_type": self.enum_to_str(Action.CLICK), "click_type": _action[idx]["click_type"]})
- idx += 2
- else:
- _new_actions.append(_action[idx])
- idx += 1
- else:
- _new_actions.append(_action[idx])
- idx += 1
-
- result["action"] = _new_actions
+ result.append(self.event_to_action(event))
+ index += 1
with open(out_file, "w") as f:
json.dump(result, f)
if __name__ == "__main__":
- converter = DuckTrackEventActionConverter(human_readable=True)
- converter.ducktrack_event_file_to_action(ducktrack_event_file="sample.jsonl",
- out_file="output.json",
- compress_move=True)
+ converter = DuckTrackEventActionConverter()
+ converter.ducktrack_event_file_to_action(
+ ducktrack_event_file="complex_clicking.jsonl",
+ out_file="complex_clicking5.json",
+ compress_move=True,
+ compress_scroll=True,
+ compress_click=True,
+ compress_drag=True,
+ compress_press_key=True,
+ compress_typing=True,
+ )
diff --git a/utils/events_calc.json b/utils/events_calc.json
new file mode 100644
index 0000000..9aedf7c
--- /dev/null
+++ b/utils/events_calc.json
@@ -0,0 +1,111 @@
+[
+ {
+ "action_type": "MOVE_TO",
+ "parameters": {
+ "x": 152,
+ "y": 259
+ }
+ },
+ {
+ "action_type": "MOUSE_DOWN",
+ "parameters": {
+ "button": "left"
+ }
+ },
+ {
+ "action_type": "MOVE_TO",
+ "parameters": {
+ "x": 464,
+ "y": 317
+ }
+ },
+ {
+ "action_type": "MOUSE_UP",
+ "parameters": {
+ "button": "left"
+ }
+ },
+ {
+ "action_type": "MOVE_TO",
+ "parameters": {
+ "x": 466,
+ "y": 317
+ }
+ },
+ {
+ "action_type": "KEY_DOWN",
+ "parameters": {
+ "key": "altleft"
+ }
+ },
+ {
+ "action_type": "KEY_DOWN",
+ "parameters": {
+ "key": "="
+ }
+ },
+ {
+ "action_type": "KEY_UP",
+ "parameters": {
+ "key": "="
+ }
+ },
+ {
+ "action_type": "KEY_UP",
+ "parameters": {
+ "key": "altleft"
+ }
+ },
+ {
+ "action_type": "MOVE_TO",
+ "parameters": {
+ "x": 709,
+ "y": 1047
+ }
+ },
+ {
+ "action_type": "MOUSE_DOWN",
+ "parameters": {
+ "button": "left"
+ }
+ },
+ {
+ "action_type": "MOVE_TO",
+ "parameters": {
+ "x": 709,
+ "y": 1047
+ }
+ },
+ {
+ "action_type": "MOUSE_UP",
+ "parameters": {
+ "button": "left"
+ }
+ },
+ {
+ "action_type": "MOVE_TO",
+ "parameters": {
+ "x": 717,
+ "y": 304
+ }
+ },
+ {
+ "action_type": "MOUSE_DOWN",
+ "parameters": {
+ "button": "left"
+ }
+ },
+ {
+ "action_type": "MOVE_TO",
+ "parameters": {
+ "x": 717,
+ "y": 304
+ }
+ },
+ {
+ "action_type": "MOUSE_UP",
+ "parameters": {
+ "button": "left"
+ }
+ }
+]
\ No newline at end of file
diff --git a/utils/events_calc.jsonl b/utils/events_calc.jsonl
new file mode 100644
index 0000000..27125e8
--- /dev/null
+++ b/utils/events_calc.jsonl
@@ -0,0 +1,423 @@
+{"time_stamp": 21028.2899763, "action": "move", "x": 686, "y": 306}
+{"time_stamp": 21028.2965794, "action": "move", "x": 684, "y": 306}
+{"time_stamp": 21028.3046644, "action": "move", "x": 678, "y": 306}
+{"time_stamp": 21028.3126807, "action": "move", "x": 670, "y": 306}
+{"time_stamp": 21028.3208329, "action": "move", "x": 661, "y": 306}
+{"time_stamp": 21028.3288313, "action": "move", "x": 645, "y": 306}
+{"time_stamp": 21028.336626, "action": "move", "x": 625, "y": 306}
+{"time_stamp": 21028.3445457, "action": "move", "x": 603, "y": 305}
+{"time_stamp": 21028.3527487, "action": "move", "x": 574, "y": 303}
+{"time_stamp": 21028.3606394, "action": "move", "x": 544, "y": 301}
+{"time_stamp": 21028.3688565, "action": "move", "x": 508, "y": 300}
+{"time_stamp": 21028.3768381, "action": "move", "x": 471, "y": 298}
+{"time_stamp": 21028.3848709, "action": "move", "x": 430, "y": 296}
+{"time_stamp": 21028.3926563, "action": "move", "x": 389, "y": 296}
+{"time_stamp": 21028.4009164, "action": "move", "x": 348, "y": 296}
+{"time_stamp": 21028.4089388, "action": "move", "x": 313, "y": 296}
+{"time_stamp": 21028.4171707, "action": "move", "x": 280, "y": 296}
+{"time_stamp": 21028.4245847, "action": "move", "x": 252, "y": 294}
+{"time_stamp": 21028.4328148, "action": "move", "x": 225, "y": 294}
+{"time_stamp": 21028.4406678, "action": "move", "x": 208, "y": 294}
+{"time_stamp": 21028.4486998, "action": "move", "x": 192, "y": 294}
+{"time_stamp": 21028.4568529, "action": "move", "x": 177, "y": 294}
+{"time_stamp": 21028.4647334, "action": "move", "x": 163, "y": 293}
+{"time_stamp": 21028.4729702, "action": "move", "x": 153, "y": 293}
+{"time_stamp": 21028.4808044, "action": "move", "x": 143, "y": 293}
+{"time_stamp": 21028.4889062, "action": "move", "x": 135, "y": 293}
+{"time_stamp": 21028.4967676, "action": "move", "x": 130, "y": 293}
+{"time_stamp": 21028.5050544, "action": "move", "x": 124, "y": 293}
+{"time_stamp": 21028.5127317, "action": "move", "x": 120, "y": 293}
+{"time_stamp": 21028.520827, "action": "move", "x": 117, "y": 293}
+{"time_stamp": 21028.5289378, "action": "move", "x": 114, "y": 293}
+{"time_stamp": 21028.5371078, "action": "move", "x": 111, "y": 293}
+{"time_stamp": 21028.545514, "action": "move", "x": 107, "y": 293}
+{"time_stamp": 21028.5527022, "action": "move", "x": 104, "y": 292}
+{"time_stamp": 21028.5605384, "action": "move", "x": 100, "y": 292}
+{"time_stamp": 21028.5686583, "action": "move", "x": 96, "y": 291}
+{"time_stamp": 21028.5766951, "action": "move", "x": 90, "y": 291}
+{"time_stamp": 21028.5847502, "action": "move", "x": 85, "y": 291}
+{"time_stamp": 21028.5926223, "action": "move", "x": 79, "y": 290}
+{"time_stamp": 21028.6007454, "action": "move", "x": 74, "y": 290}
+{"time_stamp": 21028.6088707, "action": "move", "x": 70, "y": 289}
+{"time_stamp": 21028.6166501, "action": "move", "x": 67, "y": 289}
+{"time_stamp": 21028.6249259, "action": "move", "x": 66, "y": 289}
+{"time_stamp": 21028.6647889, "action": "move", "x": 66, "y": 289}
+{"time_stamp": 21028.6728642, "action": "move", "x": 68, "y": 288}
+{"time_stamp": 21028.6807781, "action": "move", "x": 70, "y": 286}
+{"time_stamp": 21028.6888295, "action": "move", "x": 74, "y": 285}
+{"time_stamp": 21028.6971027, "action": "move", "x": 77, "y": 284}
+{"time_stamp": 21028.7046499, "action": "move", "x": 81, "y": 282}
+{"time_stamp": 21028.7129405, "action": "move", "x": 86, "y": 281}
+{"time_stamp": 21028.7205325, "action": "move", "x": 91, "y": 279}
+{"time_stamp": 21028.7285422, "action": "move", "x": 98, "y": 278}
+{"time_stamp": 21028.7366509, "action": "move", "x": 104, "y": 275}
+{"time_stamp": 21028.7448279, "action": "move", "x": 110, "y": 275}
+{"time_stamp": 21028.7527897, "action": "move", "x": 116, "y": 273}
+{"time_stamp": 21028.7609718, "action": "move", "x": 120, "y": 272}
+{"time_stamp": 21028.7688693, "action": "move", "x": 124, "y": 271}
+{"time_stamp": 21028.7766846, "action": "move", "x": 128, "y": 270}
+{"time_stamp": 21028.7848371, "action": "move", "x": 131, "y": 270}
+{"time_stamp": 21028.7927773, "action": "move", "x": 133, "y": 268}
+{"time_stamp": 21028.8007498, "action": "move", "x": 134, "y": 268}
+{"time_stamp": 21028.8088143, "action": "move", "x": 136, "y": 268}
+{"time_stamp": 21028.8168157, "action": "move", "x": 137, "y": 268}
+{"time_stamp": 21028.8246469, "action": "move", "x": 139, "y": 268}
+{"time_stamp": 21028.8327817, "action": "move", "x": 140, "y": 268}
+{"time_stamp": 21028.8408239, "action": "move", "x": 141, "y": 268}
+{"time_stamp": 21028.8488115, "action": "move", "x": 142, "y": 267}
+{"time_stamp": 21028.8571578, "action": "move", "x": 143, "y": 267}
+{"time_stamp": 21028.8646641, "action": "move", "x": 144, "y": 267}
+{"time_stamp": 21028.8741985, "action": "move", "x": 145, "y": 267}
+{"time_stamp": 21028.8809717, "action": "move", "x": 146, "y": 267}
+{"time_stamp": 21028.8888646, "action": "move", "x": 146, "y": 267}
+{"time_stamp": 21028.961049, "action": "move", "x": 146, "y": 266}
+{"time_stamp": 21029.0249854, "action": "move", "x": 147, "y": 265}
+{"time_stamp": 21029.0328138, "action": "move", "x": 147, "y": 264}
+{"time_stamp": 21029.0407582, "action": "move", "x": 147, "y": 264}
+{"time_stamp": 21029.0487772, "action": "move", "x": 148, "y": 263}
+{"time_stamp": 21029.0569372, "action": "move", "x": 148, "y": 263}
+{"time_stamp": 21029.065073, "action": "move", "x": 149, "y": 262}
+{"time_stamp": 21029.0729933, "action": "move", "x": 150, "y": 262}
+{"time_stamp": 21029.0888149, "action": "move", "x": 150, "y": 261}
+{"time_stamp": 21029.0971595, "action": "move", "x": 151, "y": 260}
+{"time_stamp": 21029.10458, "action": "move", "x": 151, "y": 260}
+{"time_stamp": 21029.1126284, "action": "move", "x": 151, "y": 260}
+{"time_stamp": 21029.1208764, "action": "move", "x": 151, "y": 259}
+{"time_stamp": 21029.1287413, "action": "move", "x": 152, "y": 259}
+{"time_stamp": 21029.1611214, "action": "move", "x": 152, "y": 259}
+{"time_stamp": 21029.1614723, "action": "click", "x": 152, "y": 259, "button": "left", "pressed": true}
+{"time_stamp": 21029.2168134, "action": "move", "x": 152, "y": 259}
+{"time_stamp": 21029.2248681, "action": "move", "x": 154, "y": 259}
+{"time_stamp": 21029.2327317, "action": "move", "x": 156, "y": 260}
+{"time_stamp": 21029.2408222, "action": "move", "x": 158, "y": 262}
+{"time_stamp": 21029.2487515, "action": "move", "x": 163, "y": 263}
+{"time_stamp": 21029.2568152, "action": "move", "x": 169, "y": 266}
+{"time_stamp": 21029.2649126, "action": "move", "x": 174, "y": 270}
+{"time_stamp": 21029.2727425, "action": "move", "x": 183, "y": 273}
+{"time_stamp": 21029.2807226, "action": "move", "x": 190, "y": 276}
+{"time_stamp": 21029.2887741, "action": "move", "x": 200, "y": 279}
+{"time_stamp": 21029.296883, "action": "move", "x": 209, "y": 282}
+{"time_stamp": 21029.304834, "action": "move", "x": 220, "y": 285}
+{"time_stamp": 21029.3131548, "action": "move", "x": 233, "y": 287}
+{"time_stamp": 21029.3207916, "action": "move", "x": 244, "y": 290}
+{"time_stamp": 21029.3290871, "action": "move", "x": 256, "y": 292}
+{"time_stamp": 21029.3366508, "action": "move", "x": 268, "y": 293}
+{"time_stamp": 21029.3445108, "action": "move", "x": 279, "y": 294}
+{"time_stamp": 21029.3529213, "action": "move", "x": 288, "y": 297}
+{"time_stamp": 21029.3607282, "action": "move", "x": 298, "y": 297}
+{"time_stamp": 21029.3691604, "action": "move", "x": 307, "y": 297}
+{"time_stamp": 21029.3769931, "action": "move", "x": 316, "y": 298}
+{"time_stamp": 21029.3850192, "action": "move", "x": 324, "y": 300}
+{"time_stamp": 21029.3927881, "action": "move", "x": 331, "y": 301}
+{"time_stamp": 21029.4007925, "action": "move", "x": 336, "y": 302}
+{"time_stamp": 21029.4088638, "action": "move", "x": 342, "y": 304}
+{"time_stamp": 21029.4167924, "action": "move", "x": 346, "y": 304}
+{"time_stamp": 21029.4251047, "action": "move", "x": 349, "y": 304}
+{"time_stamp": 21029.4328699, "action": "move", "x": 352, "y": 306}
+{"time_stamp": 21029.4409293, "action": "move", "x": 355, "y": 306}
+{"time_stamp": 21029.4487136, "action": "move", "x": 356, "y": 307}
+{"time_stamp": 21029.4568755, "action": "move", "x": 358, "y": 308}
+{"time_stamp": 21029.4647053, "action": "move", "x": 361, "y": 309}
+{"time_stamp": 21029.4728173, "action": "move", "x": 363, "y": 310}
+{"time_stamp": 21029.4806011, "action": "move", "x": 365, "y": 311}
+{"time_stamp": 21029.4889321, "action": "move", "x": 367, "y": 312}
+{"time_stamp": 21029.4967544, "action": "move", "x": 370, "y": 313}
+{"time_stamp": 21029.5049087, "action": "move", "x": 374, "y": 314}
+{"time_stamp": 21029.5129759, "action": "move", "x": 377, "y": 316}
+{"time_stamp": 21029.5210278, "action": "move", "x": 381, "y": 317}
+{"time_stamp": 21029.5286154, "action": "move", "x": 386, "y": 317}
+{"time_stamp": 21029.5371491, "action": "move", "x": 390, "y": 318}
+{"time_stamp": 21029.5449815, "action": "move", "x": 393, "y": 319}
+{"time_stamp": 21029.5526305, "action": "move", "x": 397, "y": 319}
+{"time_stamp": 21029.5604721, "action": "move", "x": 400, "y": 319}
+{"time_stamp": 21029.5690371, "action": "move", "x": 402, "y": 319}
+{"time_stamp": 21029.5772927, "action": "move", "x": 405, "y": 319}
+{"time_stamp": 21029.5846161, "action": "move", "x": 406, "y": 319}
+{"time_stamp": 21029.5928399, "action": "move", "x": 407, "y": 319}
+{"time_stamp": 21029.6007032, "action": "move", "x": 408, "y": 319}
+{"time_stamp": 21029.609118, "action": "move", "x": 409, "y": 319}
+{"time_stamp": 21029.6166036, "action": "move", "x": 411, "y": 320}
+{"time_stamp": 21029.6249215, "action": "move", "x": 412, "y": 320}
+{"time_stamp": 21029.6327262, "action": "move", "x": 414, "y": 320}
+{"time_stamp": 21029.6408018, "action": "move", "x": 415, "y": 320}
+{"time_stamp": 21029.649463, "action": "move", "x": 418, "y": 320}
+{"time_stamp": 21029.6575693, "action": "move", "x": 420, "y": 320}
+{"time_stamp": 21029.6650956, "action": "move", "x": 423, "y": 320}
+{"time_stamp": 21029.6729346, "action": "move", "x": 426, "y": 320}
+{"time_stamp": 21029.6808747, "action": "move", "x": 429, "y": 320}
+{"time_stamp": 21029.688616, "action": "move", "x": 432, "y": 320}
+{"time_stamp": 21029.6970675, "action": "move", "x": 435, "y": 320}
+{"time_stamp": 21029.7049324, "action": "move", "x": 438, "y": 320}
+{"time_stamp": 21029.7130458, "action": "move", "x": 439, "y": 320}
+{"time_stamp": 21029.7207522, "action": "move", "x": 440, "y": 320}
+{"time_stamp": 21029.7289775, "action": "move", "x": 442, "y": 320}
+{"time_stamp": 21029.7366577, "action": "move", "x": 443, "y": 320}
+{"time_stamp": 21029.7444825, "action": "move", "x": 445, "y": 320}
+{"time_stamp": 21029.7526551, "action": "move", "x": 447, "y": 320}
+{"time_stamp": 21029.7604951, "action": "move", "x": 448, "y": 320}
+{"time_stamp": 21029.7686569, "action": "move", "x": 450, "y": 319}
+{"time_stamp": 21029.7775496, "action": "move", "x": 451, "y": 319}
+{"time_stamp": 21029.7849685, "action": "move", "x": 451, "y": 319}
+{"time_stamp": 21029.7929356, "action": "move", "x": 452, "y": 319}
+{"time_stamp": 21029.8007005, "action": "move", "x": 452, "y": 319}
+{"time_stamp": 21029.8170717, "action": "move", "x": 453, "y": 319}
+{"time_stamp": 21029.8248574, "action": "move", "x": 453, "y": 318}
+{"time_stamp": 21029.8330359, "action": "move", "x": 454, "y": 318}
+{"time_stamp": 21029.8407804, "action": "move", "x": 454, "y": 318}
+{"time_stamp": 21029.8487615, "action": "move", "x": 455, "y": 318}
+{"time_stamp": 21029.8648369, "action": "move", "x": 455, "y": 318}
+{"time_stamp": 21029.8726477, "action": "move", "x": 456, "y": 318}
+{"time_stamp": 21029.8809607, "action": "move", "x": 457, "y": 317}
+{"time_stamp": 21029.8888473, "action": "move", "x": 457, "y": 317}
+{"time_stamp": 21029.9048933, "action": "move", "x": 458, "y": 317}
+{"time_stamp": 21029.9129577, "action": "move", "x": 458, "y": 317}
+{"time_stamp": 21029.9208533, "action": "move", "x": 459, "y": 317}
+{"time_stamp": 21029.9286645, "action": "move", "x": 459, "y": 317}
+{"time_stamp": 21029.9368461, "action": "move", "x": 461, "y": 317}
+{"time_stamp": 21029.9448712, "action": "move", "x": 461, "y": 317}
+{"time_stamp": 21029.953212, "action": "move", "x": 462, "y": 317}
+{"time_stamp": 21029.9608238, "action": "move", "x": 463, "y": 317}
+{"time_stamp": 21029.9686821, "action": "move", "x": 463, "y": 317}
+{"time_stamp": 21029.9768342, "action": "move", "x": 464, "y": 317}
+{"time_stamp": 21030.361149, "action": "move", "x": 464, "y": 317}
+{"time_stamp": 21030.3613383, "action": "click", "x": 464, "y": 317, "button": "left", "pressed": false}
+{"time_stamp": 21030.9690893, "action": "move", "x": 465, "y": 317}
+{"time_stamp": 21030.9770331, "action": "move", "x": 465, "y": 317}
+{"time_stamp": 21030.9933165, "action": "move", "x": 466, "y": 317}
+{"time_stamp": 21031.8410512, "action": "press", "name": "alt_l"}
+{"time_stamp": 21032.1375784, "action": "press", "name": "="}
+{"time_stamp": 21032.2331653, "action": "release", "name": "="}
+{"time_stamp": 21032.4009051, "action": "release", "name": "alt_l"}
+{"time_stamp": 21033.1212821, "action": "move", "x": 466, "y": 317}
+{"time_stamp": 21033.1289659, "action": "move", "x": 467, "y": 320}
+{"time_stamp": 21033.1370348, "action": "move", "x": 471, "y": 325}
+{"time_stamp": 21033.1456134, "action": "move", "x": 475, "y": 332}
+{"time_stamp": 21033.1531721, "action": "move", "x": 482, "y": 340}
+{"time_stamp": 21033.1605014, "action": "move", "x": 490, "y": 349}
+{"time_stamp": 21033.1692663, "action": "move", "x": 498, "y": 359}
+{"time_stamp": 21033.1771117, "action": "move", "x": 508, "y": 371}
+{"time_stamp": 21033.1850449, "action": "move", "x": 521, "y": 383}
+{"time_stamp": 21033.1929826, "action": "move", "x": 535, "y": 399}
+{"time_stamp": 21033.201192, "action": "move", "x": 546, "y": 415}
+{"time_stamp": 21033.2089185, "action": "move", "x": 555, "y": 434}
+{"time_stamp": 21033.216848, "action": "move", "x": 563, "y": 452}
+{"time_stamp": 21033.2246769, "action": "move", "x": 570, "y": 469}
+{"time_stamp": 21033.2328685, "action": "move", "x": 574, "y": 485}
+{"time_stamp": 21033.2407514, "action": "move", "x": 577, "y": 503}
+{"time_stamp": 21033.2488102, "action": "move", "x": 578, "y": 518}
+{"time_stamp": 21033.2569003, "action": "move", "x": 578, "y": 534}
+{"time_stamp": 21033.2654896, "action": "move", "x": 580, "y": 552}
+{"time_stamp": 21033.2730147, "action": "move", "x": 580, "y": 571}
+{"time_stamp": 21033.2808888, "action": "move", "x": 582, "y": 592}
+{"time_stamp": 21033.2890461, "action": "move", "x": 583, "y": 617}
+{"time_stamp": 21033.2968868, "action": "move", "x": 586, "y": 643}
+{"time_stamp": 21033.3050093, "action": "move", "x": 588, "y": 665}
+{"time_stamp": 21033.3129685, "action": "move", "x": 591, "y": 694}
+{"time_stamp": 21033.3210515, "action": "move", "x": 592, "y": 716}
+{"time_stamp": 21033.3289082, "action": "move", "x": 594, "y": 735}
+{"time_stamp": 21033.3368274, "action": "move", "x": 598, "y": 751}
+{"time_stamp": 21033.3446464, "action": "move", "x": 601, "y": 761}
+{"time_stamp": 21033.3532343, "action": "move", "x": 604, "y": 773}
+{"time_stamp": 21033.3607161, "action": "move", "x": 606, "y": 783}
+{"time_stamp": 21033.3687129, "action": "move", "x": 608, "y": 794}
+{"time_stamp": 21033.3769088, "action": "move", "x": 611, "y": 804}
+{"time_stamp": 21033.3846615, "action": "move", "x": 614, "y": 816}
+{"time_stamp": 21033.3927661, "action": "move", "x": 617, "y": 826}
+{"time_stamp": 21033.4008999, "action": "move", "x": 619, "y": 837}
+{"time_stamp": 21033.408732, "action": "move", "x": 621, "y": 846}
+{"time_stamp": 21033.4169038, "action": "move", "x": 623, "y": 856}
+{"time_stamp": 21033.4250181, "action": "move", "x": 623, "y": 865}
+{"time_stamp": 21033.4329144, "action": "move", "x": 624, "y": 875}
+{"time_stamp": 21033.4410593, "action": "move", "x": 624, "y": 883}
+{"time_stamp": 21033.448994, "action": "move", "x": 626, "y": 891}
+{"time_stamp": 21033.4570193, "action": "move", "x": 626, "y": 899}
+{"time_stamp": 21033.4648038, "action": "move", "x": 627, "y": 906}
+{"time_stamp": 21033.4730101, "action": "move", "x": 628, "y": 913}
+{"time_stamp": 21033.4815421, "action": "move", "x": 631, "y": 920}
+{"time_stamp": 21033.4891275, "action": "move", "x": 635, "y": 926}
+{"time_stamp": 21033.4970011, "action": "move", "x": 639, "y": 930}
+{"time_stamp": 21033.5047772, "action": "move", "x": 647, "y": 935}
+{"time_stamp": 21033.5132552, "action": "move", "x": 653, "y": 939}
+{"time_stamp": 21033.5211245, "action": "move", "x": 659, "y": 943}
+{"time_stamp": 21033.5292347, "action": "move", "x": 665, "y": 947}
+{"time_stamp": 21033.5373088, "action": "move", "x": 671, "y": 950}
+{"time_stamp": 21033.5447875, "action": "move", "x": 677, "y": 955}
+{"time_stamp": 21033.5529495, "action": "move", "x": 684, "y": 960}
+{"time_stamp": 21033.5609559, "action": "move", "x": 690, "y": 965}
+{"time_stamp": 21033.5689335, "action": "move", "x": 696, "y": 971}
+{"time_stamp": 21033.5768783, "action": "move", "x": 700, "y": 977}
+{"time_stamp": 21033.5846548, "action": "move", "x": 703, "y": 981}
+{"time_stamp": 21033.5931357, "action": "move", "x": 705, "y": 985}
+{"time_stamp": 21033.6009205, "action": "move", "x": 707, "y": 988}
+{"time_stamp": 21033.6088781, "action": "move", "x": 708, "y": 991}
+{"time_stamp": 21033.6169713, "action": "move", "x": 709, "y": 994}
+{"time_stamp": 21033.6249134, "action": "move", "x": 709, "y": 997}
+{"time_stamp": 21033.6328882, "action": "move", "x": 710, "y": 999}
+{"time_stamp": 21033.6412016, "action": "move", "x": 711, "y": 1003}
+{"time_stamp": 21033.648939, "action": "move", "x": 711, "y": 1007}
+{"time_stamp": 21033.6572201, "action": "move", "x": 713, "y": 1010}
+{"time_stamp": 21033.6647348, "action": "move", "x": 715, "y": 1013}
+{"time_stamp": 21033.6730325, "action": "move", "x": 716, "y": 1017}
+{"time_stamp": 21033.6810552, "action": "move", "x": 717, "y": 1021}
+{"time_stamp": 21033.6890871, "action": "move", "x": 719, "y": 1024}
+{"time_stamp": 21033.6969594, "action": "move", "x": 720, "y": 1026}
+{"time_stamp": 21033.7048284, "action": "move", "x": 720, "y": 1028}
+{"time_stamp": 21033.7126425, "action": "move", "x": 720, "y": 1028}
+{"time_stamp": 21033.7610156, "action": "move", "x": 720, "y": 1029}
+{"time_stamp": 21033.7693689, "action": "move", "x": 720, "y": 1029}
+{"time_stamp": 21033.7772628, "action": "move", "x": 720, "y": 1030}
+{"time_stamp": 21033.7847737, "action": "move", "x": 720, "y": 1031}
+{"time_stamp": 21033.7929223, "action": "move", "x": 719, "y": 1031}
+{"time_stamp": 21033.801029, "action": "move", "x": 719, "y": 1032}
+{"time_stamp": 21033.808944, "action": "move", "x": 718, "y": 1033}
+{"time_stamp": 21033.8169394, "action": "move", "x": 717, "y": 1035}
+{"time_stamp": 21033.8248771, "action": "move", "x": 716, "y": 1035}
+{"time_stamp": 21033.8334548, "action": "move", "x": 716, "y": 1036}
+{"time_stamp": 21033.8410779, "action": "move", "x": 715, "y": 1037}
+{"time_stamp": 21033.8486117, "action": "move", "x": 715, "y": 1039}
+{"time_stamp": 21033.8568906, "action": "move", "x": 713, "y": 1039}
+{"time_stamp": 21033.8649249, "action": "move", "x": 712, "y": 1040}
+{"time_stamp": 21033.8729566, "action": "move", "x": 712, "y": 1042}
+{"time_stamp": 21033.8810286, "action": "move", "x": 711, "y": 1043}
+{"time_stamp": 21033.8888454, "action": "move", "x": 711, "y": 1044}
+{"time_stamp": 21033.8970736, "action": "move", "x": 709, "y": 1045}
+{"time_stamp": 21033.9051884, "action": "move", "x": 709, "y": 1046}
+{"time_stamp": 21033.91297, "action": "move", "x": 709, "y": 1047}
+{"time_stamp": 21033.9210518, "action": "move", "x": 709, "y": 1047}
+{"time_stamp": 21033.9770341, "action": "move", "x": 709, "y": 1047}
+{"time_stamp": 21033.9932821, "action": "move", "x": 709, "y": 1047}
+{"time_stamp": 21033.9933595, "action": "click", "x": 709, "y": 1047, "button": "left", "pressed": true}
+{"time_stamp": 21034.0734669, "action": "move", "x": 709, "y": 1047}
+{"time_stamp": 21034.0737272, "action": "click", "x": 709, "y": 1047, "button": "left", "pressed": false}
+{"time_stamp": 21034.1450402, "action": "move", "x": 709, "y": 1047}
+{"time_stamp": 21034.1608305, "action": "move", "x": 709, "y": 1047}
+{"time_stamp": 21034.1690642, "action": "move", "x": 709, "y": 1046}
+{"time_stamp": 21034.1770086, "action": "move", "x": 709, "y": 1045}
+{"time_stamp": 21034.1849649, "action": "move", "x": 709, "y": 1044}
+{"time_stamp": 21034.1927171, "action": "move", "x": 709, "y": 1043}
+{"time_stamp": 21034.2008052, "action": "move", "x": 709, "y": 1040}
+{"time_stamp": 21034.2088854, "action": "move", "x": 709, "y": 1038}
+{"time_stamp": 21034.2167939, "action": "move", "x": 709, "y": 1034}
+{"time_stamp": 21034.224882, "action": "move", "x": 709, "y": 1029}
+{"time_stamp": 21034.2327267, "action": "move", "x": 711, "y": 1023}
+{"time_stamp": 21034.2408131, "action": "move", "x": 711, "y": 1016}
+{"time_stamp": 21034.2502186, "action": "move", "x": 712, "y": 1005}
+{"time_stamp": 21034.256732, "action": "move", "x": 713, "y": 991}
+{"time_stamp": 21034.2646169, "action": "move", "x": 716, "y": 976}
+{"time_stamp": 21034.2729272, "action": "move", "x": 719, "y": 955}
+{"time_stamp": 21034.2813953, "action": "move", "x": 722, "y": 929}
+{"time_stamp": 21034.2889074, "action": "move", "x": 723, "y": 899}
+{"time_stamp": 21034.2971538, "action": "move", "x": 725, "y": 871}
+{"time_stamp": 21034.3049341, "action": "move", "x": 727, "y": 838}
+{"time_stamp": 21034.3130394, "action": "move", "x": 727, "y": 805}
+{"time_stamp": 21034.3208269, "action": "move", "x": 728, "y": 771}
+{"time_stamp": 21034.3289492, "action": "move", "x": 728, "y": 742}
+{"time_stamp": 21034.3367866, "action": "move", "x": 728, "y": 714}
+{"time_stamp": 21034.3446895, "action": "move", "x": 728, "y": 686}
+{"time_stamp": 21034.3528319, "action": "move", "x": 728, "y": 662}
+{"time_stamp": 21034.3606113, "action": "move", "x": 728, "y": 643}
+{"time_stamp": 21034.3686987, "action": "move", "x": 727, "y": 620}
+{"time_stamp": 21034.3766536, "action": "move", "x": 725, "y": 605}
+{"time_stamp": 21034.3847084, "action": "move", "x": 722, "y": 589}
+{"time_stamp": 21034.3930586, "action": "move", "x": 719, "y": 576}
+{"time_stamp": 21034.4009346, "action": "move", "x": 716, "y": 565}
+{"time_stamp": 21034.4090089, "action": "move", "x": 712, "y": 554}
+{"time_stamp": 21034.416996, "action": "move", "x": 710, "y": 544}
+{"time_stamp": 21034.4246653, "action": "move", "x": 708, "y": 536}
+{"time_stamp": 21034.4331124, "action": "move", "x": 707, "y": 527}
+{"time_stamp": 21034.4410156, "action": "move", "x": 706, "y": 519}
+{"time_stamp": 21034.4488925, "action": "move", "x": 705, "y": 509}
+{"time_stamp": 21034.4568042, "action": "move", "x": 705, "y": 500}
+{"time_stamp": 21034.4650783, "action": "move", "x": 704, "y": 492}
+{"time_stamp": 21034.472962, "action": "move", "x": 703, "y": 483}
+{"time_stamp": 21034.4809251, "action": "move", "x": 703, "y": 475}
+{"time_stamp": 21034.4889399, "action": "move", "x": 703, "y": 467}
+{"time_stamp": 21034.4968154, "action": "move", "x": 703, "y": 460}
+{"time_stamp": 21034.505111, "action": "move", "x": 703, "y": 454}
+{"time_stamp": 21034.5128327, "action": "move", "x": 703, "y": 446}
+{"time_stamp": 21034.5211697, "action": "move", "x": 704, "y": 439}
+{"time_stamp": 21034.5291453, "action": "move", "x": 704, "y": 432}
+{"time_stamp": 21034.53683, "action": "move", "x": 704, "y": 428}
+{"time_stamp": 21034.5453754, "action": "move", "x": 705, "y": 423}
+{"time_stamp": 21034.5531997, "action": "move", "x": 705, "y": 419}
+{"time_stamp": 21034.5610828, "action": "move", "x": 705, "y": 417}
+{"time_stamp": 21034.568917, "action": "move", "x": 705, "y": 414}
+{"time_stamp": 21034.5768693, "action": "move", "x": 705, "y": 412}
+{"time_stamp": 21034.5849601, "action": "move", "x": 706, "y": 409}
+{"time_stamp": 21034.5930116, "action": "move", "x": 706, "y": 406}
+{"time_stamp": 21034.6006017, "action": "move", "x": 706, "y": 404}
+{"time_stamp": 21034.6086777, "action": "move", "x": 706, "y": 402}
+{"time_stamp": 21034.6167229, "action": "move", "x": 706, "y": 400}
+{"time_stamp": 21034.6251342, "action": "move", "x": 706, "y": 398}
+{"time_stamp": 21034.6325694, "action": "move", "x": 706, "y": 396}
+{"time_stamp": 21034.6407476, "action": "move", "x": 706, "y": 393}
+{"time_stamp": 21034.6489079, "action": "move", "x": 707, "y": 390}
+{"time_stamp": 21034.6567719, "action": "move", "x": 707, "y": 388}
+{"time_stamp": 21034.6648437, "action": "move", "x": 707, "y": 386}
+{"time_stamp": 21034.6735978, "action": "move", "x": 707, "y": 383}
+{"time_stamp": 21034.6808034, "action": "move", "x": 707, "y": 381}
+{"time_stamp": 21034.6887831, "action": "move", "x": 707, "y": 379}
+{"time_stamp": 21034.6968931, "action": "move", "x": 707, "y": 377}
+{"time_stamp": 21034.7048123, "action": "move", "x": 707, "y": 375}
+{"time_stamp": 21034.7127621, "action": "move", "x": 706, "y": 373}
+{"time_stamp": 21034.7208214, "action": "move", "x": 706, "y": 372}
+{"time_stamp": 21034.7289712, "action": "move", "x": 705, "y": 371}
+{"time_stamp": 21034.7366015, "action": "move", "x": 705, "y": 370}
+{"time_stamp": 21034.7449792, "action": "move", "x": 705, "y": 369}
+{"time_stamp": 21034.7528215, "action": "move", "x": 705, "y": 368}
+{"time_stamp": 21034.7611243, "action": "move", "x": 705, "y": 367}
+{"time_stamp": 21034.7689338, "action": "move", "x": 705, "y": 366}
+{"time_stamp": 21034.7768638, "action": "move", "x": 705, "y": 365}
+{"time_stamp": 21034.7849091, "action": "move", "x": 705, "y": 364}
+{"time_stamp": 21034.792848, "action": "move", "x": 705, "y": 363}
+{"time_stamp": 21034.8010344, "action": "move", "x": 705, "y": 362}
+{"time_stamp": 21034.809155, "action": "move", "x": 704, "y": 362}
+{"time_stamp": 21034.8166183, "action": "move", "x": 704, "y": 359}
+{"time_stamp": 21034.8249556, "action": "move", "x": 704, "y": 358}
+{"time_stamp": 21034.8333238, "action": "move", "x": 704, "y": 356}
+{"time_stamp": 21034.8410045, "action": "move", "x": 703, "y": 354}
+{"time_stamp": 21034.8486685, "action": "move", "x": 703, "y": 352}
+{"time_stamp": 21034.857368, "action": "move", "x": 703, "y": 350}
+{"time_stamp": 21034.8647224, "action": "move", "x": 703, "y": 347}
+{"time_stamp": 21034.8730798, "action": "move", "x": 703, "y": 346}
+{"time_stamp": 21034.8809692, "action": "move", "x": 703, "y": 342}
+{"time_stamp": 21034.8889165, "action": "move", "x": 703, "y": 341}
+{"time_stamp": 21034.8969094, "action": "move", "x": 704, "y": 339}
+{"time_stamp": 21034.9052672, "action": "move", "x": 704, "y": 337}
+{"time_stamp": 21034.9145868, "action": "move", "x": 704, "y": 335}
+{"time_stamp": 21034.9208561, "action": "move", "x": 704, "y": 334}
+{"time_stamp": 21034.928931, "action": "move", "x": 704, "y": 333}
+{"time_stamp": 21034.9374176, "action": "move", "x": 704, "y": 332}
+{"time_stamp": 21034.9451258, "action": "move", "x": 704, "y": 330}
+{"time_stamp": 21034.9528709, "action": "move", "x": 704, "y": 329}
+{"time_stamp": 21034.9611476, "action": "move", "x": 704, "y": 328}
+{"time_stamp": 21034.968991, "action": "move", "x": 704, "y": 327}
+{"time_stamp": 21034.9768394, "action": "move", "x": 705, "y": 325}
+{"time_stamp": 21034.9848553, "action": "move", "x": 705, "y": 324}
+{"time_stamp": 21034.993121, "action": "move", "x": 705, "y": 323}
+{"time_stamp": 21035.0007992, "action": "move", "x": 706, "y": 322}
+{"time_stamp": 21035.0088762, "action": "move", "x": 707, "y": 320}
+{"time_stamp": 21035.0166123, "action": "move", "x": 707, "y": 320}
+{"time_stamp": 21035.0247724, "action": "move", "x": 708, "y": 318}
+{"time_stamp": 21035.0335071, "action": "move", "x": 708, "y": 317}
+{"time_stamp": 21035.0411458, "action": "move", "x": 709, "y": 317}
+{"time_stamp": 21035.0491997, "action": "move", "x": 709, "y": 316}
+{"time_stamp": 21035.0569637, "action": "move", "x": 711, "y": 314}
+{"time_stamp": 21035.06496, "action": "move", "x": 711, "y": 313}
+{"time_stamp": 21035.0726588, "action": "move", "x": 712, "y": 312}
+{"time_stamp": 21035.0807214, "action": "move", "x": 713, "y": 311}
+{"time_stamp": 21035.0888078, "action": "move", "x": 713, "y": 309}
+{"time_stamp": 21035.0972443, "action": "move", "x": 713, "y": 309}
+{"time_stamp": 21035.1048868, "action": "move", "x": 714, "y": 308}
+{"time_stamp": 21035.1127551, "action": "move", "x": 715, "y": 307}
+{"time_stamp": 21035.1208842, "action": "move", "x": 715, "y": 306}
+{"time_stamp": 21035.1285261, "action": "move", "x": 715, "y": 306}
+{"time_stamp": 21035.1366862, "action": "move", "x": 715, "y": 305}
+{"time_stamp": 21035.1446592, "action": "move", "x": 716, "y": 305}
+{"time_stamp": 21035.1528109, "action": "move", "x": 716, "y": 305}
+{"time_stamp": 21035.1848109, "action": "move", "x": 716, "y": 304}
+{"time_stamp": 21035.208994, "action": "move", "x": 717, "y": 304}
+{"time_stamp": 21035.2571327, "action": "move", "x": 717, "y": 304}
+{"time_stamp": 21035.2573543, "action": "click", "x": 717, "y": 304, "button": "left", "pressed": true}
+{"time_stamp": 21035.3377191, "action": "move", "x": 717, "y": 304}
+{"time_stamp": 21035.3379572, "action": "click", "x": 717, "y": 304, "button": "left", "pressed": false}
diff --git a/utils/image_processing/contour.py b/utils/image_processing/contour.py
new file mode 100644
index 0000000..f7f50e6
--- /dev/null
+++ b/utils/image_processing/contour.py
@@ -0,0 +1,34 @@
+import cv2
+from matplotlib import pyplot as plt
+
+# Load the image
+image = cv2.imread('../../mm_agents/stackoverflow.png')
+
+# Convert to grayscale
+gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
+
+# Apply adaptive thresholding to get a binary image
+thresh = cv2.adaptiveThreshold(
+ gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY_INV, 11, 2
+)
+
+# Find contours
+contours, _ = cv2.findContours(thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+
+# Filter out contours that are not of cell size
+# This is done by assuming that cells will have a relatively standard size
+# The size filter is just a placeholder, real values depend on the actual image size
+min_cell_size = 500
+max_cell_size = 5000
+cell_contours = [cnt for cnt in contours if min_cell_size < cv2.contourArea(cnt) < max_cell_size]
+
+# Draw contours on the image
+contour_output = image.copy()
+cv2.drawContours(contour_output, cell_contours, -1, (0, 255, 0), 2)
+
+# Display the image with cell contours
+plt.figure(figsize=(12,6))
+plt.imshow(cv2.cvtColor(contour_output, cv2.COLOR_BGR2RGB))
+plt.title('Spreadsheet with Cell Contours')
+plt.axis('off')
+plt.show()
diff --git a/utils/image_processing/point_marking.py b/utils/image_processing/point_marking.py
new file mode 100644
index 0000000..8cfd519
--- /dev/null
+++ b/utils/image_processing/point_marking.py
@@ -0,0 +1,32 @@
+from PIL import Image, ImageDraw
+
+
+def mark_point(image_path: str, x: int, y: int, radius: int = 5, color: str = 'red') -> str:
+ """
+ Mark a point on an image and save the image.
+ """
+ # Load the image
+ image = Image.open(image_path)
+
+ # Create a draw object
+ draw = ImageDraw.Draw(image)
+
+ # Draw a small circle to mark the point
+ draw.ellipse((x - radius, y - radius, x + radius, y + radius), fill=color, outline=color)
+
+ # Save the image with the point marked
+ marked_image_path = image_path[:-4] + '_marked' + image_path[-4:]
+ image.save(marked_image_path)
+
+ return marked_image_path
+
+
+if __name__ == '__main__':
+ image_path = 'chrome_start.png'
+ x = 100
+ y = 200
+ radius = 30
+ color = 'red'
+
+ marked_image_path = mark_point(image_path, x, y, radius, color)
+ print(f"Marked image saved to {marked_image_path}")
\ No newline at end of file