lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ver Nov23rd

Browse Source 
trying to set up the environment
This commit is contained in:
David Chang
2023-11-23 22:25:32 +08:00
parent 203fae5798
commit 857f73f61d
3 changed files with 20 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@@ -161,3 +161,5 @@ instructor-embedding
frontend/node_modules/ frontend/node_modules/
frontend/.next/ frontend/.next/
frontend/.idea frontend/.idea
tags

12
desktop_env/envs/desktop_env.py
View File

@@ -74,7 +74,7 @@ class DesktopEnv(gym.Env):
def _wait_for_emulator_load(self): def _wait_for_emulator_load(self):
while True: while True:
try: try:
output = subprocess.check_output(f"vmrun -T ws list", shell=True, stderr=subprocess.STDOUT) output = subprocess.check_output("vmrun -T ws list", shell=True, stderr=subprocess.STDOUT)
output = output.decode() output = output.decode()
if self.path_to_vm.lstrip("~/") in output: if self.path_to_vm.lstrip("~/") in output:
print("VM is running.") print("VM is running.")
@@ -127,16 +127,18 @@ class DesktopEnv(gym.Env):
return image_path return image_path
def _get_obs(self): def _get_obs(self):
print("OBS 1")
screenshot_image_path = self._get_screenshot() screenshot_image_path = self._get_screenshot()
print("OBS 2")
with Image.open(screenshot_image_path) as img: with Image.open(screenshot_image_path) as img:
return np.array(img) return np.array(img)
def reset(self): def reset(self):
input() input("Reset #1 PE")
self._execute_command(["vmrun", "-T", "ws", "revertToSnapshot", self.path_to_vm, self.snapshot_path]) #self._execute_command(["vmrun", "-T", "ws", "revertToSnapshot", self.path_to_vm, self.snapshot_path])
input() input("Revert to snapshot #2 PE")
self._start_emulator() self._start_emulator()
input() input("Started emulator #3 PE")
self._wait_for_emulator_load() self._wait_for_emulator_load()
observation = self._get_obs() observation = self._get_obs()

17
main.py
View File

@@ -1,4 +1,4 @@
from pprint import pprint #from pprint import pprint
from desktop_env.envs.desktop_env import DesktopEnv, Action, MouseClick from desktop_env.envs.desktop_env import DesktopEnv, Action, MouseClick
def get_human_action(): def get_human_action():
@@ -38,13 +38,16 @@ def human_agent():
""" """
Runs the Gym environment with human input. Runs the Gym environment with human input.
""" """
env = DesktopEnv(path_to_vm="/home/yuri/vmware/Windows 10 x64/Windows 10 x64.vmx", #env = DesktopEnv( path_to_vm="/home/yuri/vmware/Windows 10 x64/Windows 10 x64.vmx"
# path_to_vm="/home/yuri/vmware/Ubuntu 64-bit/Ubuntu 64-bit.vmx", # path_to_vm="/home/yuri/vmware/Ubuntu 64-bit/Ubuntu 64-bit.vmx",
username="user", env = DesktopEnv( path_to_vm="/home/david/vmware/KUbuntu 64-bit/KUbuntu 64-bit.vmx"
password="password", , username="david"
# host="192.168.7.128", , password="123456"
host="http://192.168.7.129:5000", , host="192.168.174.129"
vm_os="windows") #host="http://192.168.7.129:5000",
#vm_os="windows")
, vm_os="ubuntu"
)
observation = env.reset() observation = env.reset()
done = False done = False