lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve on agent and tasks configs

Browse Source
This commit is contained in:
Timothyxxx
2024-01-26 23:30:04 +08:00
parent 96bcce27ae
commit 6952b45de4
36 changed files with 425 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
desktop_env/controllers/python.py
View File

@@ -11,7 +11,7 @@ logger = logging.getLogger("desktopenv.pycontroller")
class PythonController:
def __init__(self, vm_ip: str, pkgs_prefix: str = "import pyautogui; {command}"):
def __init__(self, vm_ip: str, pkgs_prefix: str = "import pyautogui; import time; {command}"):
self.vm_ip = vm_ip
self.http_server = f"http://{vm_ip}:5000"
self.pkgs_prefix = pkgs_prefix # fixme: this is a hacky way to execute python commands. fix it and combine it with installation of packages

10
evaluation_examples/examples/os/13584542-872b-42d8-b299-866967b5c3ef.json
View File

@@ -5,6 +5,16 @@
"source": "https://superuser.com/questions/72176/linux-set-default-terminal-size-and-screen-position",
"trajectory": "trajectories/",
"config": [
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"related_apps": [
"os"

10
evaluation_examples/examples/os/23393935-50c7-4a86-aeea-2b78fd089c5c.json
View File

@@ -80,6 +80,16 @@
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"related_apps": [

13
evaluation_examples/examples/os/28cc3b7e-b194-4bc9-8353-d04c0f4d56d2.json
View File

@@ -4,7 +4,18 @@
"instruction": "The volume of my system is too small. Can you help me turn up to the max volume?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/sound-volume.html.en",
"trajectory": "trajectories/",
"config": [],
"config": [
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"related_apps": [
"os"
],

13
evaluation_examples/examples/os/3ce045a0-877b-42aa-8d2c-b4a863336ab8.json
View File

@@ -4,7 +4,18 @@
"instruction": "My glasses are broken, and I'm having trouble seeing small things clearly. Could you help me enlarge the text on my screen so it's easier to read?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/a11y-font-size.html.en",
"trajectory": "trajectories/",
"config": [],
"config": [
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"related_apps": [
"os"
],

15
evaluation_examples/examples/os/43c2d64c-bab5-4dcb-a30c-b888321c319a.json
View File

@@ -3,7 +3,18 @@
"snapshot": "os",
"instruction": "Could you please help me create a dir named 'test' in the root directory of current computer?",
"source": "https://ubuntu.com/tutorials/command-line-for-beginners#4-creating-folders-and-files",
"config": [],
"config": [
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"os"
@@ -12,7 +23,7 @@
"func": "exact_match",
"result": {
"type": "vm_command_line",
"command": "[ -d '/home/user/Desktop/test' ] && echo 'Directory exists.' || echo 'Directory does not exist.'",
"command": "[ -d '/test' ] && echo 'Directory exists.' || echo 'Directory does not exist.'",
"shell": true
},
"expected": {

10
evaluation_examples/examples/os/5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57.json
View File

@@ -22,6 +22,16 @@
"command": "mv ~/poster_party_night.webp ~/.local/share/Trash/files/",
"shell": true
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"related_apps": [

13
evaluation_examples/examples/os/765d2b74-88a7-4d50-bf51-34e4106fd24a.json
View File

@@ -3,7 +3,18 @@
"snapshot": "os",
"instruction": "Can you help me delete the \"test\" file on my desktop?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/files-delete.html.en",
"config": [],
"config": [
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"os"

10
evaluation_examples/examples/os/7688b85f-87a4-4e4a-b2f8-f3d6c3f29b82.json
View File

@@ -17,7 +17,17 @@
"command": "echo 'password' | sudo -S chmod 777 ~/Desktop/todo.txt && sudo chmod 777 ~/Desktop/done",
"shell": true
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [

13
evaluation_examples/examples/os/94d95f96-9699-4208-98ba-3c3119edf9c2.json
View File

@@ -4,7 +4,18 @@
"instruction": "I want to install Spotify on my current system. Could you please help me?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/addremove-install.html.en",
"trajectory": "trajectories/",
"config": [],
"config": [
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"related_apps": [
"os"
],

10
evaluation_examples/examples/os/a462a795-fdc7-4b23-b689-e8b6df786b78.json
View File

@@ -10,6 +10,16 @@
"command": "echo password | sudo -S su - charles",
"shell": true
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",

12
evaluation_examples/examples/os/a4d98375-215b-4a4d-aee9-3d4370fccc41.json
View File

@@ -4,6 +4,18 @@
"instruction": "I want to have my computer automatically locked after I leaved. Can you help me?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/privacy-screen-lock.html.en",
"trajectory": "trajectories/",
"config": [
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"related_apps": [
"os"
],

10
evaluation_examples/examples/os/ae039631-2b12-4637-84f6-c67d51511be3.json
View File

@@ -9,6 +9,16 @@
"parameters": {
"command": "xdg-settings set default-web-browser firefox.desktop"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",

10
evaluation_examples/examples/os/b6781586-6346-41cd-935a-a6b1487918fc.json
View File

@@ -5,6 +5,16 @@
"source": "https://help.ubuntu.com/lts/ubuntu-help/clock-timezone.html.en",
"trajectory": "trajectories/",
"config": [
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"related_apps": [
"os"

13
evaluation_examples/examples/os/bedcedc4-4d72-425e-ad62-21960b11fe0d.json
View File

@@ -4,7 +4,18 @@
"instruction": "Could you set the 'Dim screen when inactive' to on in setting?",
"source": "https://www.youtube.com/watch?v=D4WyNjt_hbQ&t=2s",
"trajectory": "trajectories/",
"config": [],
"config": [
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"related_apps": [
"os"
],

12
evaluation_examples/examples/os/ddc75b62-7311-4af8-bfb3-859558542b36.json
View File

@@ -4,6 +4,18 @@
"instruction": "I want to uninstall the Mahjongg on my system. Can you help me?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/addremove-remove.html.en",
"trajectory": "trajectories/",
"config": [
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"related_apps": [
"os"
],

10
evaluation_examples/examples/os/e0df059f-28a6-4169-924f-b9623e7184cc.json
View File

@@ -11,6 +11,16 @@
"command": "echo 'password' | sudo -S mkdir ~/Desktop/todo_list_Jan_1",
"shell": true
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"related_apps": [

10
evaluation_examples/examples/os/ec4e3f68-9ea4-4c18-a5c9-69f89d1178b3.json
View File

@@ -21,6 +21,16 @@
"parameters": {
"command": "gsettings set org.gnome.shell favorite-apps \"['thunderbird.desktop', 'vim.desktop', 'google-chrome.desktop']\""
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",

10
evaluation_examples/examples/os/f9be0997-4b7c-45c5-b05c-4612b44a6118.json
View File

@@ -10,7 +10,17 @@
"command": "gsettings set org.gnome.desktop.notifications show-banners true",
"shell": true
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [

54
evaluation_examples/examples/os/fe41f596-a71b-4c2f-9b2f-9dcd40b568c3.json
View File

@@ -1,24 +1,36 @@
{
"id": "fe41f596-a71b-4c2f-9b2f-9dcd40b568c3",
"snapshot": "os",
"instruction": "I want to see the battery percentage. Can you help me display it on my screen?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/power-percentage.html.en",
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "exact_match",
"result": {
"type": "vm_command_line",
"command": "gsettings get org.gnome.desktop.interface show-battery-percentage",
"shell": true
},
"expected": {
"type": "rule",
"rules":{
"expected": "true\n"
}
"id": "fe41f596-a71b-4c2f-9b2f-9dcd40b568c3",
"snapshot": "os",
"instruction": "I want to see the battery percentage. Can you help me display it on my screen?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/power-percentage.html.en",
"trajectory": "trajectories/",
"config": [
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
}
],
"related_apps": [
"os"
],
"evaluator": {
"func": "exact_match",
"result": {
"type": "vm_command_line",
"command": "gsettings get org.gnome.desktop.interface show-battery-percentage",
"shell": true
},
"expected": {
"type": "rule",
"rules": {
"expected": "true\n"
}
}
}
}

10
evaluation_examples/examples/vlc/215dfd39-f493-4bc3-a027-8a97d72c61bf.json
View File

@@ -9,6 +9,16 @@
"parameters": {
"command": "vlc"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",

10
evaluation_examples/examples/vlc/386dbd0e-0241-4a0a-b6a2-6704fba26b1c.json
View File

@@ -64,6 +64,16 @@
"import pyautogui; pyautogui.press('f11');"
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",

10
evaluation_examples/examples/vlc/59f21cfb-0120-4326-b255-a5b827b38967.json
View File

@@ -20,6 +20,16 @@
"parameters": {
"command": "vlc"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",

10
evaluation_examples/examples/vlc/8ba5ae7a-5ae5-4eab-9fcc-5dd4fe3abf89.json
View File

@@ -9,6 +9,16 @@
"parameters": {
"command": "vlc"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",

27
evaluation_examples/examples/vlc/8d9fd4e2-6fdb-46b0-b9b9-02f06495c62f.json
View File

@@ -4,10 +4,35 @@
"instruction": "Can you make the video fill up the whole screen? It's a bit too small to see right now, and I'd like to see it better.",
"source": "https://www.youtube.com/watch?v=XHprwDJ0-fU&t=436s",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=13OhDRuINzQ-w0qmzdGJ8ZSWnKyKnTRje&export=download&authuser=0&confirm=t&uuid=39cf0297-23eb-47df-a642-fd6c2f981eff&at=APZUnTUR83eI1tenMxVdyFxNeBo7:1706275408621",
"path": "Desktop/Gen 2.mp4"
}
]
}
},
{
"type": "launch",
"parameters": {
"command": "vlc"
"command": [
"vlc",
"--start-time=15",
"Desktop/Gen 2.mp4"
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],

10
evaluation_examples/examples/vlc/8f080098-ddb1-424c-b438-4e96e5e4786e.json
View File

@@ -24,6 +24,16 @@
"Desktop/Baby Justin Bieber.mp4"
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",

10
evaluation_examples/examples/vlc/9195653c-f4aa-453d-aa95-787f6ccfaae9.json
View File

@@ -9,6 +9,16 @@
"parameters": {
"command": "vlc"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",

10
evaluation_examples/examples/vlc/a5bbbcd5-b398-4c91-83d4-55e1e31bbb81.json
View File

@@ -9,6 +9,16 @@
"parameters": {
"command": "vlc"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",

10
evaluation_examples/examples/vlc/aa4b5023-aef6-4ed9-bdc9-705f59ab9ad6.json
View File

@@ -20,6 +20,16 @@
"parameters": {
"command": ["vlc", "Desktop/flipped_1984_Apple_Macintosh_Commercial.mp4"]
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",

10
evaluation_examples/examples/vlc/bba3381f-b5eb-4439-bd9e-80c22218d5a7.json
View File

@@ -9,6 +9,16 @@
"parameters": {
"command": "vlc"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",

10
evaluation_examples/examples/vlc/d06f0d4d-2cd5-4ede-8de9-598629438c6e.json
View File

@@ -9,6 +9,16 @@
"parameters": {
"command": "vlc"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",

10
evaluation_examples/examples/vlc/efcf0d81-0835-4880-b2fd-d866e8bc2294.json
View File

@@ -26,6 +26,16 @@
"Desktop/Interstellar Movie - Official Trailer.mp4"
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",

10
evaluation_examples/examples/vlc/f3977615-2b45-4ac5-8bba-80c17dbe2a37.json
View File

@@ -9,6 +9,16 @@
"parameters": {
"command": "vlc"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",

22
evaluation_examples/examples/vlc/fba2c100-79e8-42df-ae74-b592418d54f4.json
View File

@@ -26,6 +26,28 @@
"Desktop/Interstellar Movie - Official Trailer.mp4"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 1
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Interstellar Movie - Interstellar Movie - Official Trailer - VLC media player"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(500, 500); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",

12
mm_agents/gpt_4v_agent.py
View File

@@ -225,8 +225,8 @@ class GPT4v_Agent:
})
# Append trajectory
assert len(self.observations) == len(self.actions) and len(self.actions) == len(self.thoughts)\
, "The number of observations and actions should be the same."
assert len(self.observations) == len(self.actions) and len(self.actions) == len(self.thoughts) \
, "The number of observations and actions should be the same."
if len(self.observations) > self.max_trajectory_length:
_observations = self.observations[-self.max_trajectory_length:]
@@ -255,7 +255,7 @@ class GPT4v_Agent:
{
"type": "image_url",
"image_url": {
"url": f"data:image/jpeg;base64,{_screenshot}",
"url": f"data:image/png;base64,{_screenshot}",
"detail": "high"
}
}
@@ -315,14 +315,14 @@ class GPT4v_Agent:
]
})
else:
raise ValueError("Invalid experiment type: " + self.exp) # 1}}}
raise ValueError("Invalid experiment type: " + self.exp) # 1}}}
messages.append({
"role": "assistant",
"content": [
{
"type": "text",
"text": previous_thought.stip() if len(previous_thought)>0 else "No valid action"
"text": previous_thought.strip() if len(previous_thought) > 0 else "No valid action"
},
]
})
@@ -436,7 +436,7 @@ class GPT4v_Agent:
]
})
else:
raise ValueError("Invalid experiment type: " + self.exp) # 1}}}
raise ValueError("Invalid experiment type: " + self.exp) # 1}}}
with open("messages.json", "w") as f:
f.write(json.dumps(messages, indent=4))

30
mm_agents/prompts.py
View File

@@ -3,8 +3,9 @@ You are an agent which follow my instruction and perform desktop computer tasks
You have good knowledge of computer and good internet connection and assume your code will run on a computer for controlling the mouse and keyboard.
For each step, you will get an observation of an image, which is the screenshot of the computer screen and you will predict the action of the computer based on the image.
You are required to use `pyautogui` to perform the action, but don't use the `pyautogui.locateCenterOnScreen` function to locate the element you want to operate with since we have no image of the element you want to operate with.
Return one line or multiple lines of python code to perform the action each time, be time efficient.
You are required to use `pyautogui` to perform the action grounded to the observation, but DONOT use the `pyautogui.locateCenterOnScreen` function to locate the element you want to operate with since we have no image of the element you want to operate with. DONOT USE `pyautogui.screenshot()` to make screenshot.
Return one line or multiple lines of python code to perform the action each time, be time efficient. When predicting multiple lines of code, make some small sleep like `time.sleep(0.5);` interval so that the machine could take
You need to to specify the coordinates of by yourself based on your observation of current observation, but you should be careful to ensure that the coordinates are correct.
You ONLY need to return the code inside a code block, like this:
```python
# your code here
@@ -14,6 +15,7 @@ When you think you have to wait for some time, return ```WAIT```;
When you think the task can not be done, return ```FAIL```, don't easily say ```FAIL```, try your best to do the task;
When you think the task is done, return ```DONE```.
My computer's password is 'password', feel free to use it when you need sudo rights.
First give the current screenshot and previous things we did a short reflection, then RETURN ME THE CODE OR SPECIAL CODE I ASKED FOR. NEVER EVER RETURN ME ANYTHING ELSE.
""".strip()
@@ -267,8 +269,9 @@ You are an agent which follow my instruction and perform desktop computer tasks
You have good knowledge of computer and good internet connection and assume your code will run on a computer for controlling the mouse and keyboard.
For each step, you will get an observation of the desktop by accessibility tree, which is based on AT-SPI library. And you will predict the action of the computer based on the accessibility tree.
You are required to use `pyautogui` to perform the action, but don't use the `pyautogui.locateCenterOnScreen` function to locate the element you want to operate with since we have no image of the element you want to operate with.
Return one line or multiple lines of python code to perform the action each time, be time efficient.
You are required to use `pyautogui` to perform the action grounded to the observation, but DONOT use the `pyautogui.locateCenterOnScreen` function to locate the element you want to operate with since we have no image of the element you want to operate with. DONOT USE `pyautogui.screenshot()` to make screenshot.
Return one line or multiple lines of python code to perform the action each time, be time efficient. When predicting multiple lines of code, make some small sleep like `time.sleep(0.5);` interval so that the machine could take
You need to to specify the coordinates of by yourself based on your observation of current observation, but you should be careful to ensure that the coordinates are correct.
You ONLY need to return the code inside a code block, like this:
```python
# your code here
@@ -278,6 +281,7 @@ When you think you have to wait for some time, return ```WAIT```;
When you think the task can not be done, return ```FAIL```, don't easily say ```FAIL```, try your best to do the task;
When you think the task is done, return ```DONE```.
My computer's password is 'password', feel free to use it when you need sudo rights.
First give the current screenshot and previous things we did a short reflection, then RETURN ME THE CODE OR SPECIAL CODE I ASKED FOR. NEVER EVER RETURN ME ANYTHING ELSE.
""".strip()
@@ -532,8 +536,9 @@ You have good knowledge of computer and good internet connection and assume your
For each step, you will get an observation of the desktop by 1) a screenshot; and 2) accessibility tree, which is based on AT-SPI library.
And you will predict the action of the computer based on the screenshot and accessibility tree.
You are required to use `pyautogui` to perform the action, but don't use the `pyautogui.locateCenterOnScreen` function to locate the element you want to operate with since we have no image of the element you want to operate with.
Return one line or multiple lines of python code to perform the action each time, be time efficient.
You are required to use `pyautogui` to perform the action grounded to the observation, but DONOT use the `pyautogui.locateCenterOnScreen` function to locate the element you want to operate with since we have no image of the element you want to operate with. DONOT USE `pyautogui.screenshot()` to make screenshot.
Return one line or multiple lines of python code to perform the action each time, be time efficient. When predicting multiple lines of code, make some small sleep like `time.sleep(0.5);` interval so that the machine could take
You need to to specify the coordinates of by yourself based on your observation of current observation, but you should be careful to ensure that the coordinates are correct.
You ONLY need to return the code inside a code block, like this:
```python
# your code here
@@ -543,6 +548,7 @@ When you think you have to wait for some time, return ```WAIT```;
When you think the task can not be done, return ```FAIL```, don't easily say ```FAIL```, try your best to do the task;
When you think the task is done, return ```DONE```.
My computer's password is 'password', feel free to use it when you need sudo rights.
First give the current screenshot and previous things we did a short reflection, then RETURN ME THE CODE OR SPECIAL CODE I ASKED FOR. NEVER EVER RETURN ME ANYTHING ELSE.
""".strip()
@@ -797,7 +803,7 @@ You are an agent which follow my instruction and perform desktop computer tasks
You have good knowledge of computer and good internet connection and assume your code will run on a computer for controlling the mouse and keyboard.
For each step, you will get an observation of the desktop by 1) a screenshot; and 2) accessibility tree, which is based on AT-SPI library.
You are required to use `pyautogui` to perform the action, but don't use the `pyautogui.locateCenterOnScreen` function to locate the element you want to operate with since we have no image of the element you want to operate with.
You are required to use `pyautogui` to perform the action grounded to the observation, but DONOT use the `pyautogui.locateCenterOnScreen` function to locate the element you want to operate with since we have no image of the element you want to operate with. DONOT USE `pyautogui.screenshot()` to make screenshot.
You can replace x, y in the code with the tag of the element you want to operate with. such as:
```python
pyautogui.moveTo(tag#3)
@@ -806,7 +812,8 @@ pyautogui.dragTo(tag#1, button='left')
```
When you think you can directly output precise x and y coordinates or there is no tag on which you want to interact, you can also use them directly.
But you should be careful to ensure that the coordinates are correct.
Return one line or multiple lines of python code to perform the action each time, be time efficient.
Return one line or multiple lines of python code to perform the action each time, be time efficient. When predicting multiple lines of code, make some small sleep like `time.sleep(0.5);` interval so that the machine could take
You need to to specify the coordinates of by yourself based on your observation of current observation, but you should be careful to ensure that the coordinates are correct.
You ONLY need to return the code inside a code block, like this:
```python
# your code here
@@ -816,6 +823,7 @@ When you think you have to wait for some time, return ```WAIT```;
When you think the task can not be done, return ```FAIL```, don't easily say ```FAIL```, try your best to do the task;
When you think the task is done, return ```DONE```.
My computer's password is 'password', feel free to use it when you need sudo rights.
First give the current screenshot and previous things we did a short reflection, then RETURN ME THE CODE OR SPECIAL CODE I ASKED FOR. NEVER EVER RETURN ME ANYTHING ELSE.
""".strip()
@@ -845,7 +853,7 @@ Then, based on your analysis, in conjunction with human desktop using habits and
"""
ACTION_GROUNDING_PROMPT_SEEACT = """
You are required to use `pyautogui` to perform the action, but don't use the `pyautogui.locateCenterOnScreen` function to locate the element you want to operate with since we have no image of the element you want to operate with.
You are required to use `pyautogui` to perform the action grounded to the observation, but DONOT use the `pyautogui.locateCenterOnScreen` function to locate the element you want to operate with since we have no image of the element you want to operate with. DONOT USE `pyautogui.screenshot()` to make screenshot.
You can replace x, y in the code with the tag of the element you want to operate with. such as:
```python
pyautogui.moveTo(tag#3)
@@ -854,7 +862,8 @@ pyautogui.dragTo(tag#1, button='left')
```
When you think you can directly output precise x and y coordinates or there is no tag on which you want to interact, you can also use them directly.
But you should be careful to ensure that the coordinates are correct.
Return one line or multiple lines of python code to perform the action each time, be time efficient.
Return one line or multiple lines of python code to perform the action each time, be time efficient. When predicting multiple lines of code, make some small sleep like `time.sleep(0.5);` interval so that the machine could take
You need to to specify the coordinates of by yourself based on your observation of current observation, but you should be careful to ensure that the coordinates are correct.
You ONLY need to return the code inside a code block, like this:
```python
# your code here
@@ -864,5 +873,6 @@ When you think you have to wait for some time, return ```WAIT```;
When you think the task can not be done, return ```FAIL```, don't easily say ```FAIL```, try your best to do the task;
When you think the task is done, return ```DONE```.
My computer's password is 'password', feel free to use it when you need sudo rights.
First give the current screenshot and previous things we did a short reflection, then RETURN ME THE CODE OR SPECIAL CODE I ASKED FOR. NEVER EVER RETURN ME ANYTHING ELSE.
"""