lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update the script on server for ubuntu

Browse Source
This commit is contained in:
Timothyxxx
2023-12-18 19:53:22 +08:00
parent facfcf1b3c
commit 05d3b5863d
3 changed files with 18 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
desktop_env/server/main.py
View File

@@ -38,13 +38,17 @@ def execute_command():
@app.route('/screenshot', methods=['GET']) @app.route('/screenshot', methods=['GET'])
def capture_screen_with_cursor(): def capture_screen_with_cursor():
# fixme: when running on virtual machines, the cursor is not captured, don't know why
file_path = os.path.join("screenshots", "screenshot.png") file_path = os.path.join("screenshots", "screenshot.png")
user_platform = platform.system() user_platform = platform.system()
# Ensure the screenshots directory exists # Ensure the screenshots directory exists
os.makedirs(os.path.dirname(file_path), exist_ok=True) os.makedirs(os.path.dirname(file_path), exist_ok=True)
if user_platform == "Windows":
# fixme: This is a temporary fix for the cursor not being captured on Windows and Linux
if user_platform == "Windows" or user_platform == "Linux":
def _download_image(url, path): def _download_image(url, path):
response = requests.get(url) response = requests.get(url)
with open(path, 'wb') as file: with open(path, 'wb') as file:
@@ -57,14 +61,16 @@ def capture_screen_with_cursor():
screenshot = pyautogui.screenshot() screenshot = pyautogui.screenshot()
cursor_x, cursor_y = pyautogui.position() cursor_x, cursor_y = pyautogui.position()
cursor = Image.open(cursor_path) cursor = Image.open(cursor_path)
# make the cursor smaller
cursor = cursor.resize((int(cursor.width / 1.5), int(cursor.height / 1.5)))
screenshot.paste(cursor, (cursor_x, cursor_y), cursor) screenshot.paste(cursor, (cursor_x, cursor_y), cursor)
screenshot.save(file_path) screenshot.save(file_path)
elif user_platform == "Linux": # elif user_platform == "Linux":
# Use xlib to prevent scrot dependency for Linux # # Use xlib to prevent scrot dependency for Linux
screen = Xlib.display.Display().screen() # screen = Xlib.display.Display().screen()
size = screen.width_in_pixels, screen.height_in_pixels # size = screen.width_in_pixels, screen.height_in_pixels
screenshot = ImageGrab.grab(bbox=(0, 0, size[0], size[1])) # screenshot = ImageGrab.grab(bbox=(0, 0, size[0], size[1]))
screenshot.save(file_path) # screenshot.save(file_path)
elif user_platform == "Darwin": # (Mac OS) elif user_platform == "Darwin": # (Mac OS)
# Use the screencapture utility to capture the screen with the cursor # Use the screencapture utility to capture the screen with the cursor
subprocess.run(["screencapture", "-C", file_path]) subprocess.run(["screencapture", "-C", file_path])

1
desktop_env/server/requirements.txt
View File

@@ -3,3 +3,4 @@ PyAutoGUI==0.9.54
Pillow==10.1.0 Pillow==10.1.0
git+https://github.com/moses-palmer/pynput.git@refs/pull/541/head # to make sure that it works on Apple Silicon git+https://github.com/moses-palmer/pynput.git@refs/pull/541/head # to make sure that it works on Apple Silicon
requests requests
flask

8
main.py
View File

@@ -12,13 +12,13 @@ def human_agent():
env = DesktopEnv( env = DesktopEnv(
# path_to_vm=r"""C:\Users\tianbaox\Downloads\Windows 10 x64\Windows 10 x64.vmx""", # path_to_vm=r"""C:\Users\tianbaox\Downloads\Windows 10 x64\Windows 10 x64.vmx""",
path_to_vm=r"""C:\Users\tianbaox\Documents\Virtual Machines\Win10\Win10.vmx""", path_to_vm=r"""C:\Users\tianbaox\Documents\Virtual Machines\Ubuntu\Ubuntu.vmx""",
# path_to_vm="/home/yuri/vmware/Ubuntu 64-bit/Ubuntu 64-bit.vmx", # path_to_vm="/home/yuri/vmware/Ubuntu 64-bit/Ubuntu 64-bit.vmx",
action_space="computer_13", action_space="computer_13",
snapshot_path="base_setup3", snapshot_path="base_setup",
instruction=example["instruction"], instruction=example["instruction"],
config=example["config"], # config=example["config"],
evaluator=example["evaluator"] # evaluator=example["evaluator"]
) )
# reset the environment to certain snapshot # reset the environment to certain snapshot