lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update compressor for data annotation

Browse Source
This commit is contained in:
Timothyxxx
2023-12-04 00:51:33 +08:00
parent 8cb31f1bb0
commit 8818779329
8 changed files with 2519 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
desktop_env/controllers/python.py
View File

@@ -66,14 +66,26 @@ class PythonController:
button = parameters["button"]
x = parameters["x"]
y = parameters["y"]
self.execute_python_command(f"pyautogui.click(button='{button}', x={x}, y={y})")
if "num_clicks" in parameters:
num_clicks = parameters["num_clicks"]
self.execute_python_command(f"pyautogui.click(button='{button}', x={x}, y={y}, clicks={num_clicks})")
else:
self.execute_python_command(f"pyautogui.click(button='{button}', x={x}, y={y})")
elif "button" in parameters and "x" not in parameters and "y" not in parameters:
button = parameters["button"]
self.execute_python_command(f"pyautogui.click(button='{button}')")
if "num_clicks" in parameters:
num_clicks = parameters["num_clicks"]
self.execute_python_command(f"pyautogui.click(button='{button}', clicks={num_clicks})")
else:
self.execute_python_command(f"pyautogui.click(button='{button}')")
elif "button" not in parameters and "x" in parameters and "y" in parameters:
x = parameters["x"]
y = parameters["y"]
self.execute_python_command(f"pyautogui.click(x={x}, y={y})")
if "num_clicks" in parameters:
num_clicks = parameters["num_clicks"]
self.execute_python_command(f"pyautogui.click(x={x}, y={y}, clicks={num_clicks})")
else:
self.execute_python_command(f"pyautogui.click(x={x}, y={y})")
else:
raise Exception(f"Unknown parameters: {parameters}")

9
desktop_env/envs/actions.py
View File

@@ -3,7 +3,7 @@ Y_MAX = 1080
KEYBOARD_KEYS = ['\t', '\n', '\r', ' ', '!', '"', '#', '$', '%', '&', "'", '(', ')', '*', '+', ',', '-', '.', '/', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ':', ';', '<', '=', '>', '?', '@', '[', '\\', ']', '^', '_', '`', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '{', '|', '}', '~', 'accept', 'add', 'alt', 'altleft', 'altright', 'apps', 'backspace', 'browserback', 'browserfavorites', 'browserforward', 'browserhome', 'browserrefresh', 'browsersearch', 'browserstop', 'capslock', 'clear', 'convert', 'ctrl', 'ctrlleft', 'ctrlright', 'decimal', 'del', 'delete', 'divide', 'down', 'end', 'enter', 'esc', 'escape', 'execute', 'f1', 'f10', 'f11', 'f12', 'f13', 'f14', 'f15', 'f16', 'f17', 'f18', 'f19', 'f2', 'f20', 'f21', 'f22', 'f23', 'f24', 'f3', 'f4', 'f5', 'f6', 'f7', 'f8', 'f9', 'final', 'fn', 'hanguel', 'hangul', 'hanja', 'help', 'home', 'insert', 'junja', 'kana', 'kanji', 'launchapp1', 'launchapp2', 'launchmail', 'launchmediaselect', 'left', 'modechange', 'multiply', 'nexttrack', 'nonconvert', 'num0', 'num1', 'num2', 'num3', 'num4', 'num5', 'num6', 'num7', 'num8', 'num9', 'numlock', 'pagedown', 'pageup', 'pause', 'pgdn', 'pgup', 'playpause', 'prevtrack', 'print', 'printscreen', 'prntscrn', 'prtsc', 'prtscr', 'return', 'right', 'scrolllock', 'select', 'separator', 'shift', 'shiftleft', 'shiftright', 'sleep', 'stop', 'subtract', 'tab', 'up', 'volumedown', 'volumemute', 'volumeup', 'win', 'winleft', 'winright', 'yen', 'command', 'option', 'optionleft', 'optionright']
ACTIONS = [
ACTION_SPACE = [
{
"action_type": "MOVE_TO",
"note": "move the cursor to the specified position",
@@ -38,7 +38,12 @@ ACTIONS = [
"type": float,
"range": [0, Y_MAX],
"optional": True,
}
},
"num_clicks": {
"type": int,
"range": [1, 2, 3],
"optional": True,
},
}
},
{