lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

compress scroll, click(x,y), drag, press and type

Browse Source
This commit is contained in:
tsuky_chen
2023-12-06 21:35:30 +08:00
committed by GitHub
parent b9c317f0f5
commit f545e4ebed
1 changed files with 165 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

202
utils/ducktrack.py
View File

@@ -5,6 +5,7 @@ sys.path.append(str(pathlib.Path(__file__).parents[1]))
import os import os
import math import math
import json import json
import numpy as np
from typing import List from typing import List
from copy import deepcopy from copy import deepcopy
@@ -12,7 +13,10 @@ pynput2pyautogui_key = {
"alt_l": "altleft", "alt_l": "altleft",
"alt_r": "altright", "alt_r": "altright",
} }
COMMAND_KEYS = ['accept', 'add', 'alt', 'altleft', 'altright', 'apps', 'backspace', 'browserback', 'browserfavorites', 'browserforward', 'browserhome', 'browserrefresh', 'browsersearch', 'browserstop', 'capslock', 'clear', 'convert', 'ctrl', 'ctrlleft', 'ctrlright', 'decimal', 'del', 'delete', 'divide', 'down', 'end', 'enter', 'esc', 'escape', 'execute', 'f1', 'f10', 'f11', 'f12', 'f13', 'f14', 'f15', 'f16', 'f17', 'f18', 'f19', 'f2', 'f20', 'f21', 'f22', 'f23', 'f24', 'f3', 'f4', 'f5', 'f6', 'f7', 'f8', 'f9', 'final', 'fn', 'hanguel', 'hangul', 'hanja', 'help', 'home', 'insert', 'junja', 'kana', 'kanji', 'launchapp1', 'launchapp2', 'launchmail', 'launchmediaselect', 'left', 'modechange', 'multiply', 'nexttrack', 'nonconvert', 'num0', 'num1', 'num2', 'num3', 'num4', 'num5', 'num6', 'num7', 'num8', 'num9', 'numlock', 'pagedown', 'pageup', 'pause', 'pgdn', 'pgup', 'playpause', 'prevtrack', 'print', 'printscreen', 'prntscrn', 'prtsc', 'prtscr', 'return', 'right', 'scrolllock', 'select', 'separator', 'shift', 'shiftleft', 'shiftright', 'sleep', 'stop', 'subtract', 'tab', 'up', 'volumedown', 'volumemute', 'volumeup', 'win', 'winleft', 'winright', 'yen', 'command', 'option', 'optionleft', 'optionright', 'alt_l', 'alt_r']
typingkey2str = {
"space" : " ",
}
class DuckTrackEventActionConverter: class DuckTrackEventActionConverter:
def __init__(self, ): def __init__(self, ):
@@ -106,15 +110,27 @@ class DuckTrackEventActionConverter:
### Compressing ### ### Compressing ###
def compress_mouse_move(self, data: List[dict], index: int): def compress_mouse_move(self, data: List[dict], index: int):
"""Compresses consecutive mouse move events into first and last move events.""" """Compresses consecutive mouse move events into the last move events."""
first_move, last_move = data[index], data[index] last_move = data[index]
while index < len(data) and data[index]["action"] == "move": while index < len(data) and data[index]["action"] == "move":
last_move = data[index] last_move = data[index]
index += 1 index += 1
return first_move, last_move, index return last_move, index
def compress_scroll(self, data: List[dict], index: int):
"""Compresses consecutive scroll events into a single scroll event."""
last_scroll = data[index]
consecutive_dx, consecutive_dy = data[index]["dx"], data[index]["dy"]
while index < len(data) and data[index]["action"] == "scroll" and np.sign(data[index]["dx"]) == np.sign(consecutive_dx) and np.sign(data[index]["dy"]) == np.sign(consecutive_dy):
last_scroll = data[index]
consecutive_dx += data[index]["dx"]
consecutive_dy += data[index]["dy"]
index += 1
last_scroll["dx"], last_scroll["dy"] = consecutive_dx, consecutive_dy
return last_scroll, index
### Converting ### ### Converting ###
def ducktrack_event_file_to_action(self, ducktrack_event_file: str, out_file: str, compress_move: bool = True): def ducktrack_event_file_to_action(self, ducktrack_event_file: str, out_file: str, compress_move: bool = True, compress_scroll: bool = True, compress_click: bool = True,compress_drag: bool = True, compress_press_key: bool = True, compress_typing: bool = True):
"""Converts DuckTrack event data to a list of actions and saves them to a file.""" """Converts DuckTrack event data to a list of actions and saves them to a file."""
if not os.path.exists(ducktrack_event_file): if not os.path.exists(ducktrack_event_file):
raise FileNotFoundError(ducktrack_event_file) raise FileNotFoundError(ducktrack_event_file)
@@ -127,52 +143,113 @@ class DuckTrackEventActionConverter:
index = 0 index = 0
presses_to_skip = 0 presses_to_skip = 0
releases_to_skip = 0 releases_to_skip = 0
move_to_skip = 0
keys_pressed = []
# Compress the mouse move events # Compress the mouse move events
while index < len(events): while index < len(events):
event = events[index] event = events[index]
def do_mouse_press(button: str): def do_mouse_press(button: str, _index: int):
for j, second_event in enumerate(events[index + 1:]):
num_clicks = 0
mouse_pressed = True
skip_move = 0
click_x, click_y = event["x"], event["y"]
for j, next_event in enumerate(events[index + 1:]):
# make sure the time between mouse clicks is less than 500ms # make sure the time between mouse clicks is less than 500ms
if second_event["time_stamp"] - event["time_stamp"] > 0.5: if next_event["time_stamp"] - event["time_stamp"] > 0.5:
if num_clicks > 0:
if result[-1:][0]["action_type"] == "MOVE_TO":
result.pop()
result.append({
"action_type": "CLICK",
"parameters": {
"button": button,
"x" : click_x,
"y" : click_y,
"num_clicks": num_clicks
}
})
return num_clicks-1, num_clicks, _index, skip_move
break break
if "x" in second_event and "y" in second_event: if "x" in next_event and "y" in next_event:
# if the mouse moves out of the click radius/rectangle, it is not a click sequence # if the mouse moves out of the click radius/rectangle, it is not a click sequence
if math.sqrt((second_event["y"] - event["y"]) ** 2 + if math.sqrt((next_event["y"] - event["y"]) ** 2 +
(second_event["x"] - event["x"]) ** 2) > 4: (next_event["x"] - event["x"]) ** 2) > 4:
break if num_clicks > 0:
if result[-1:][0]["action_type"] == "MOVE_TO":
if second_event["action"] == "click" and second_event["pressed"]: result.pop()
for k, third_event in enumerate(events[index + j + 2:]):
if third_event["time_stamp"] - second_event["time_stamp"] > 0.5:
break
if "x" in third_event and "y" in third_event:
if math.sqrt((third_event["y"] - event["y"]) ** 2 +
(third_event["x"] - event["x"]) ** 2) > 5:
break
if third_event["action"] == "click" and third_event["pressed"]:
result.append({ result.append({
"action_type": "CLICK", "action_type": "CLICK",
"parameters": { "parameters": {
"button": button, "button": button,
"num_clicks": 3 "x" : click_x,
"y" : click_y,
"num_clicks": num_clicks
} }
}) })
return 2, 2 return num_clicks-1, num_clicks, _index, skip_move
break
if next_event["action"] == "click" and compress_click:
if not next_event["pressed"]:
num_clicks += 1
mouse_pressed = False
if num_clicks == 3:
if result[-1:][0]["action_type"] == "MOVE_TO":
result.pop()
result.append({
"action_type": "CLICK",
"parameters": {
"button": button,
"x" : click_x,
"y" : click_y,
"num_clicks": 3
}
})
return 2, 3, _index, skip_move
elif next_event["pressed"]:
mouse_pressed = True
else:
raise NotImplementedError(next_event["pressed"])
elif next_event["action"] != "click" and not mouse_pressed:
if next_event["action"] == "move":
if next_event["x"] == click_x and next_event["y"] == click_y:
skip_move += 1
continue
if result[-1:][0]["action_type"] == "MOVE_TO":
result.pop()
result.append({ result.append({
"action_type": "CLICK", "action_type": "CLICK",
"parameters": { "parameters": {
"button": button, "button": button,
"num_clicks": 2 "x" : click_x,
"y" : click_y,
"num_clicks": num_clicks
} }
}) })
return 1, 1 return num_clicks-1, num_clicks, _index, skip_move
# Compress {MOUSE_DOWN, MOVE, MOUSE_UP} into DRAG_TO event
elif next_event["action"] == "move" and compress_drag:
if next_event["x"] == click_x and next_event["y"] == click_y:
skip_move += 1
continue
last_move, _index = self.compress_mouse_move(events, _index+1)
if result[-1:][0]["action_type"] == "MOVE_TO":
result.pop()
result.append({
"action_type": "DRAG_TO",
"parameters": {
"x": last_move["x"],
"y": last_move["y"]
}
})
return 0, 1, _index, skip_move
result.append({ result.append({
"action_type": "MOUSE_DOWN", "action_type": "MOUSE_DOWN",
@@ -180,20 +257,30 @@ class DuckTrackEventActionConverter:
"button": button "button": button
} }
}) })
return 0, 0 return 0, 0, _index, skip_move
if event["action"] == "move" and compress_move: if event["action"] == "move":
first_move, last_move, index = self.compress_mouse_move(events, index) if move_to_skip > 0:
result.extend([self.event_to_action(last_move)]) move_to_skip -= 1
index += 1
continue
if compress_move:
last_move, index = self.compress_mouse_move(events, index)
result.extend([self.event_to_action(last_move)])
elif event["action"] == "scroll" and compress_scroll:
last_scroll, index = self.compress_scroll(events, index)
result.extend([self.event_to_action(last_scroll)])
elif event["action"] == "click": elif event["action"] == "click":
button = event["button"] button = event["button"]
if event["pressed"]: if event["pressed"]:
if presses_to_skip == 0: if presses_to_skip == 0:
presses, releases = do_mouse_press(button) presses, releases, index, moves = do_mouse_press(button, index)
presses_to_skip += presses presses_to_skip += presses
releases_to_skip += releases releases_to_skip += releases
move_to_skip += moves
else: else:
presses_to_skip -= 1 presses_to_skip -= 1
else: else:
@@ -207,6 +294,42 @@ class DuckTrackEventActionConverter:
else: else:
releases_to_skip -= 1 releases_to_skip -= 1
index += 1 index += 1
elif event["action"] == "press" and event["name"] not in COMMAND_KEYS and compress_typing:
typing_words = ""
while index < len(events) and events[index]["action"] in ["press", "release"] and events[index]["name"] not in COMMAND_KEYS:
if events[index]["action"] == "press":
keys_pressed.append(events[index]["name"])
typing_words += events[index]["name"] if events[index]["name"] not in typingkey2str else typingkey2str[events[index]["name"]]
elif events[index]["action"] == "release":
keys_pressed.remove(events[index]["name"])
index += 1
if len(typing_words) > 1:
result.append({
"action_type": "TYPING",
"parameters": {
"text": typing_words
}
})
else:
result.append({
"action_type": "PRESS",
"parameters": {
"key": typing_words
}
})
elif event["action"] == "press" and compress_press_key:
keys_pressed.append(event["name"])
result.append({
"action_type": "PRESS",
"parameters": {
"key": event["name"] if event["name"] not in pynput2pyautogui_key else pynput2pyautogui_key[
event["name"]]
}
})
index += 1
elif event["action"] == "release" and compress_press_key:
keys_pressed.remove(event["name"])
index += 1
else: else:
result.append(self.event_to_action(event)) result.append(self.event_to_action(event))
index += 1 index += 1
@@ -218,7 +341,12 @@ class DuckTrackEventActionConverter:
if __name__ == "__main__": if __name__ == "__main__":
converter = DuckTrackEventActionConverter() converter = DuckTrackEventActionConverter()
converter.ducktrack_event_file_to_action( converter.ducktrack_event_file_to_action(
ducktrack_event_file="events_calc.jsonl", ducktrack_event_file="complex_clicking.jsonl",
out_file="events_calc.json", out_file="complex_clicking5.json",
compress_move=True compress_move=True,
compress_scroll=True,
compress_click=True,
compress_drag=True,
compress_press_key=True,
compress_typing=True,
) )