import json from desktop_env.envs.desktop_env import DesktopEnv def human_agent(): """ Runs the Gym environment with human input. """ with open("evaluation_examples/examples/37608790-6147-45d0-9f20-1137bb35703d.json", "r") as f: example = json.load(f) env = DesktopEnv( # path_to_vm=r"""C:\Users\tianbaox\Downloads\Windows 10 x64\Windows 10 x64.vmx""", path_to_vm=r"""C:\Users\tianbaox\Documents\Virtual Machines\Win10\Win10.vmx""", # path_to_vm="/home/yuri/vmware/Ubuntu 64-bit/Ubuntu 64-bit.vmx", snapshot_path="base_setup", config=example["config"], ) # reset the environment to certain snapshot observation = env.reset() done = False trajectory = [{"action_type": "MOVE_TO", "parameters": {"x": 501, "y": 382}}, {"action_type": "CLICK", "parameters": {"button": "right", "num_clicks": 1}}, {"action_type": "MOVE_TO", "parameters": {"x": 501, "y": 382}}, {"action_type": "MOVE_TO", "parameters": {"x": 501, "y": 382}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 560, "y": 385}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 560, "y": 385}}, {"action_type": "MOVE_TO", "parameters": {"x": 560, "y": 385}}, {"action_type": "MOVE_TO", "parameters": {"x": 560, "y": 385}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 623, "y": 386}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 623, "y": 386}}, {"action_type": "MOVE_TO", "parameters": {"x": 623, "y": 386}}, {"action_type": "MOVE_TO", "parameters": {"x": 623, "y": 386}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 716, "y": 389}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 716, "y": 389}}, {"action_type": "MOVE_TO", "parameters": {"x": 716, "y": 389}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 639, "y": 409}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 639, "y": 409}}, {"action_type": "MOVE_TO", "parameters": {"x": 639, "y": 409}}, {"action_type": "MOVE_TO", "parameters": {"x": 639, "y": 409}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 711, "y": 409}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 711, "y": 409}}, {"action_type": "MOVE_TO", "parameters": {"x": 711, "y": 409}}, {"action_type": "MOVE_TO", "parameters": {"x": 711, "y": 409}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 644, "y": 448}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 644, "y": 448}}, {"action_type": "MOVE_TO", "parameters": {"x": 644, "y": 449}}, {"action_type": "MOVE_TO", "parameters": {"x": 644, "y": 449}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 720, "y": 446}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 720, "y": 446}}, {"action_type": "MOVE_TO", "parameters": {"x": 720, "y": 446}}, {"action_type": "MOVE_TO", "parameters": {"x": 720, "y": 446}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 660, "y": 458}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 660, "y": 458}}, {"action_type": "MOVE_TO", "parameters": {"x": 660, "y": 458}}, {"action_type": "MOVE_TO", "parameters": {"x": 660, "y": 458}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 692, "y": 459}}, {"action_type": "KEY_DOWN", "parameters": {"key": "s"}}, {"action_type": "MOVE_TO", "parameters": {"x": 690, "y": 459}}, {"action_type": "KEY_DOWN", "parameters": {"key": "f"}}, {"action_type": "MOVE_TO", "parameters": {"x": 675, "y": 460}}, {"action_type": "KEY_UP", "parameters": {"key": "s"}}, {"action_type": "KEY_DOWN", "parameters": {"key": "g"}}, {"action_type": "MOVE_TO", "parameters": {"x": 674, "y": 460}}, {"action_type": "KEY_UP", "parameters": {"key": "f"}}, {"action_type": "KEY_UP", "parameters": {"key": "g"}}, {"action_type": "MOVE_TO", "parameters": {"x": 637, "y": 482}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 637, "y": 482 }}, {"action_type": "MOVE_TO", "parameters": {"x": 637, "y": 483}}, {"action_type": "MOVE_TO", "parameters": {"x": 637, "y": 483}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 601, "y": 501}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 601, "y": 501}}, {"action_type": "MOVE_TO", "parameters": {"x": 601, "y": 501}}, {"action_type": "MOVE_TO", "parameters": {"x": 601, "y": 501}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 557, "y": 501}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 557, "y": 501}}, {"action_type": "MOVE_TO", "parameters": {"x": 557, "y": 501}}, {"action_type": "MOVE_TO", "parameters": {"x": 557, "y": 501}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 568, "y": 509}}, {"action_type": "KEY_DOWN", "parameters": {"key": "g"}}, {"action_type": "KEY_DOWN", "parameters": {"key": "s"}}, {"action_type": "KEY_UP", "parameters": {"key": "g"}}, {"action_type": "KEY_DOWN", "parameters": {"key": "d"}}, {"action_type": "KEY_DOWN", "parameters": {"key": "g"}}, {"action_type": "MOVE_TO", "parameters": {"x": 570, "y": 511}}, {"action_type": "KEY_UP", "parameters": {"key": "d"}}, {"action_type": "MOVE_TO", "parameters": {"x": 613, "y": 516}}, {"action_type": "KEY_UP", "parameters": {"key": "g"}}, {"action_type": "MOVE_TO", "parameters": {"x": 616, "y": 513}}, {"action_type": "KEY_UP", "parameters": {"key": "s"}}, {"action_type": "MOVE_TO", "parameters": {"x": 600, "y": 337}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 600, "y": 337}}, {"action_type": "MOVE_TO", "parameters": {"x": 600, "y": 337}}, {"action_type": "MOVE_TO", "parameters": {"x": 600, "y": 337}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "KEY_DOWN", "parameters": {"key": "g"}}, {"action_type": "KEY_DOWN", "parameters": {"key": "s"}}, {"action_type": "MOVE_TO", "parameters": {"x": 600, "y": 341}}, {"action_type": "KEY_DOWN", "parameters": {"key": "d"}}, {"action_type": "MOVE_TO", "parameters": {"x": 600, "y": 342}}, {"action_type": "KEY_UP", "parameters": {"key": "g"}}, {"action_type": "KEY_DOWN", "parameters": {"key": "f"}}, {"action_type": "MOVE_TO", "parameters": {"x": 600, "y": 350}}, {"action_type": "KEY_DOWN", "parameters": {"key": "g"}}, {"action_type": "MOVE_TO", "parameters": {"x": 601, "y": 352}}, {"action_type": "KEY_UP", "parameters": {"key": "f"}}, {"action_type": "MOVE_TO", "parameters": {"x": 602, "y": 355}}, {"action_type": "KEY_UP", "parameters": {"key": "d"}}, {"action_type": "MOVE_TO", "parameters": {"x": 617, "y": 453}}, {"action_type": "KEY_UP", "parameters": {"key": "g"}}, {"action_type": "MOVE_TO", "parameters": {"x": 610, "y": 489}}, {"action_type": "KEY_UP", "parameters": {"key": "s"}}, {"action_type": "MOVE_TO", "parameters": {"x": 533, "y": 586}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 533, "y": 586}}, {"action_type": "MOVE_TO", "parameters": {"x": 533, "y": 586}}, {"action_type": "MOVE_TO", "parameters": {"x": 533, "y": 586}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 538, "y": 588}}, {"action_type": "KEY_DOWN", "parameters": {"key": "s"}}, {"action_type": "KEY_DOWN", "parameters": {"key": "g"}}, {"action_type": "MOVE_TO", "parameters": {"x": 541, "y": 590}}, {"action_type": "KEY_DOWN", "parameters": {"key": "f"}}, {"action_type": "MOVE_TO", "parameters": {"x": 541, "y": 591}}, {"action_type": "KEY_UP", "parameters": {"key": "g"}}, {"action_type": "MOVE_TO", "parameters": {"x": 544, "y": 595}}, {"action_type": "KEY_DOWN", "parameters": {"key": "g"}}, {"action_type": "MOVE_TO", "parameters": {"x": 545, "y": 597}}, {"action_type": "KEY_UP", "parameters": {"key": "f"}}, {"action_type": "MOVE_TO", "parameters": {"x": 536, "y": 605}}, {"action_type": "KEY_DOWN", "parameters": {"key": "f"}}, {"action_type": "MOVE_TO", "parameters": {"x": 526, "y": 606}}, {"action_type": "KEY_UP", "parameters": {"key": "g"}}, {"action_type": "MOVE_TO", "parameters": {"x": 513, "y": 606}}, {"action_type": "KEY_UP", "parameters": {"key": "f"}}, {"action_type": "MOVE_TO", "parameters": {"x": 490, "y": 604}}, {"action_type": "KEY_UP", "parameters": {"key": "s"}}, {"action_type": "MOVE_TO", "parameters": {"x": 489, "y": 604}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 451, "y": 604}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "right"}}, {"action_type": "MOVE_TO", "parameters": {"x": 451, "y": 604}}, {"action_type": "MOUSE_UP", "parameters": {"button": "right"}}, {"action_type": "MOVE_TO", "parameters": {"x": 563, "y": 590}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 563, "y": 590}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 570, "y": 579}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "right"}}, {"action_type": "MOVE_TO", "parameters": {"x": 570, "y": 579}}, {"action_type": "MOUSE_UP", "parameters": {"button": "right"}}, {"action_type": "MOVE_TO", "parameters": {"x": 615, "y": 612}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 615, "y": 612}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 623, "y": 458}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 623, "y": 458}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 626, "y": 455}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "right"}}, {"action_type": "MOVE_TO", "parameters": {"x": 626, "y": 455}}, {"action_type": "MOUSE_UP", "parameters": {"button": "right"}}, {"action_type": "MOVE_TO", "parameters": {"x": 647, "y": 456}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "right"}}, {"action_type": "MOVE_TO", "parameters": {"x": 647, "y": 456}}, {"action_type": "MOUSE_UP", "parameters": {"button": "right"}}, {"action_type": "MOVE_TO", "parameters": {"x": 647, "y": 458}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 559, "y": 458}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 557, "y": 459}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 557, "y": 459}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 570, "y": 427}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 570, "y": 427}}, {"action_type": "MOVE_TO", "parameters": {"x": 570, "y": 427}}, {"action_type": "MOVE_TO", "parameters": {"x": 570, "y": 427}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 628, "y": 423}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 628, "y": 423}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 553, "y": 375}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 647, "y": 504}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 649, "y": 504}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 649, "y": 504}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 573, "y": 371}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 573, "y": 371}}, {"action_type": "MOVE_TO", "parameters": {"x": 573, "y": 371}}, {"action_type": "MOVE_TO", "parameters": {"x": 573, "y": 371}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 564, "y": 335}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 3}}, {"action_type": "MOVE_TO", "parameters": {"x": 564, "y": 335}}, {"action_type": "MOVE_TO", "parameters": {"x": 567, "y": 333}}, {"action_type": "MOVE_TO", "parameters": {"x": 567, "y": 333}}, {"action_type": "MOVE_TO", "parameters": {"x": 567, "y": 333}}, {"action_type": "MOVE_TO", "parameters": {"x": 567, "y": 333}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 573, "y": 332}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 573, "y": 332}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 618, "y": 411}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 618, "y": 411}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 755, "y": 1057}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 754, "y": 1057}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 701, "y": 295}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 701, "y": 295}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}] for i in range(len(trajectory)): # action = get_human_action() # action = { # "action_type": 0, # "click_type": 3, # } print(trajectory[i]) observation, reward, done, info = env.step(trajectory[i], pause=1) print("Observation:", observation) print("Reward:", reward) print("Info:", info) print("================================\n") if done: print("The episode is done.") break env.close() print("Environment closed.") if __name__ == "__main__": human_agent()