lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add client password argument to multiple agents and scripts

Browse Source 
- Introduced `--client_password` argument in `run_multienv_aguvis.py`, `run_multienv_claude.py`, and `run_multienv_gta1.py` for enhanced security and flexibility.
- Updated agent classes (`PromptAgent`, `AguvisAgent`, `GTA1Agent`) to accept and utilize `client_password` for improved configuration.
- Modified evaluation guidelines to reflect the new client password requirement.
- Ensured existing logic remains intact while enhancing functionality for better user experience.
This commit is contained in:
yuanmengqi
2025-07-27 16:11:23 +00:00
parent 122b16742b
commit 523d553e88
9 changed files with 627 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
mm_agents/agent.py
View File

@@ -235,7 +235,8 @@ class PromptAgent:
observation_type="screenshot_a11y_tree",
# observation_type can be in ["screenshot", "a11y_tree", "screenshot_a11y_tree", "som"]
max_trajectory_length=3,
a11y_tree_max_tokens=10000
a11y_tree_max_tokens=10000,
client_password="password"
):
self.platform = platform
self.model = model
@@ -246,6 +247,7 @@ class PromptAgent:
self.observation_type = observation_type
self.max_trajectory_length = max_trajectory_length
self.a11y_tree_max_tokens = a11y_tree_max_tokens
self.client_password = client_password
self.thoughts = []
self.actions = []
@@ -281,6 +283,8 @@ class PromptAgent:
raise ValueError("Invalid action space: " + action_space)
else:
raise ValueError("Invalid experiment type: " + observation_type)
self.system_message = self.system_message.format(CLIENT_PASSWORD=self.client_password)
def predict(self, instruction: str, obs: Dict) -> List:
"""

5
mm_agents/aguvis_agent.py
View File

@@ -360,6 +360,7 @@ class AguvisAgent:
temperature=0.5,
action_space="pyautogui",
observation_type="screenshot",
client_password="password"
):
self.platform = platform
self.planner_model = planner_model
@@ -372,6 +373,8 @@ class AguvisAgent:
self.observation_type = observation_type
assert action_space in ["pyautogui"], "Invalid action space"
assert observation_type in ["screenshot"], "Invalid observation type"
self.client_password = client_password
self.thoughts = []
self.actions = []
self.observations = []
@@ -429,7 +432,7 @@ class AguvisAgent:
# So we temporarily separate the planner prompt and aguvis prompt.
planner_messages = []
planner_system_message = AGUVIS_PLANNER_SYS_PROMPT
planner_system_message = AGUVIS_PLANNER_SYS_PROMPT.format(CLIENT_PASSWORD=self.client_password)
planner_messages.append({
"role": "system",
"content": [{"type": "text", "text": planner_system_message}]

15
mm_agents/gta1_agent.py
View File

@@ -45,6 +45,8 @@ GTA1_MODEL_NMAE = os.environ.get("GTA1_API_KEY",None) #Your served model name
GTA1_SERVICE_URL = os.environ.get("GTA1_SERVICE_URL",None) #"Your GTA1 Service URL"
proxies = None # Your proxies
MAX_RETRY_TIMES = 20
def encode_image(image_content):
return base64.b64encode(image_content).decode("utf-8")
@@ -1126,17 +1128,16 @@ def call_llm_safe(agent):
functions borrow from https://github.com/simular-ai/Agent-S/blob/a0c5c9bf0c526119b1f023c8948563c780729428/gui_agents/s2/utils/common_utils.py#L27
'''
# Retry if fails
max_retries = 3 # Set the maximum number of retries
attempt = 0
response = ""
while attempt < max_retries:
while attempt < MAX_RETRY_TIMES:
try:
response = agent.get_response()
break # If successful, break out of the loop
except Exception as e:
attempt += 1
print(f"Attempt {attempt} failed: {e}")
if attempt == max_retries:
if attempt == MAX_RETRY_TIMES:
print("Max retries reached. Handling failure.")
time.sleep(1.0)
return response
@@ -1200,11 +1201,13 @@ class GTA1Agent:
max_steps=100,
max_image_history_length = 5,
N_SEQ = 8,
client_password="password"
):
self.platform = platform
self.max_tokens = max_tokens
self.top_p = top_p
self.temperature = temperature
self.client_password = client_password
self.action_space = action_space
self.observation_type = observation_type
assert action_space in ["pyautogui"], "Invalid action space"
@@ -1343,7 +1346,7 @@ class GTA1Agent:
valid_responses.extend(valid_responses_)
retry_count += 1
assert len(valid_responses) > int(self.N_SEQ) * 0.8, f"Not enough valid responses generated {len(valid_responses)}"
# assert len(valid_responses) > int(self.N_SEQ) * 0.8, f"Not enough valid responses generated {len(valid_responses)}"
logger.info(f"Executing selection")
if self.N_SEQ > 1:
@@ -1438,7 +1441,7 @@ class GTA1Agent:
)
image = screenshot.resize((height, width))
system_promt = GTA1_JUDGE_SYSTEM_PROMPT.format(N_PLANNING=len(response), N_INDEX=len(response)-1,width=width,height=height)
system_promt = GTA1_JUDGE_SYSTEM_PROMPT.format(N_PLANNING=len(response), N_INDEX=len(response)-1,width=width,height=height, CLIENT_PASSWORD=self.client_password)
lines = [
f"The goal of the task is:\n{instruction}",
]
@@ -1482,7 +1485,7 @@ class GTA1Agent:
}
wait = 1
for _ in range(10):
for _ in range(MAX_RETRY_TIMES):
try:
prediction = requests.post(url, headers=headers, json=payload, proxies=proxies, timeout=180)
if prediction.status_code != 200:

2
mm_agents/openai_cua_agent.py
View File

@@ -644,7 +644,7 @@ class OpenAICUAAgent:
"""
Predict the next action(s) based on the current observation.
"""
prompt = OPERATOR_PROMPT.replace("{CLIENT_PASSWORD}", self.client_password)
prompt = OPERATOR_PROMPT.format(CLIENT_PASSWORD=self.client_password)
base64_image = encode_image(obs["screenshot"])
if self.cua_messages == []:

73
mm_agents/prompts.py
View File

@@ -15,7 +15,7 @@ When you think you have to wait for some time, return ```WAIT```;
When you think the task can not be done, return ```FAIL```, don't easily say ```FAIL```, try your best to do the task;
When you think the task is done, return ```DONE```.
My computer's password is 'password', feel free to use it when you need sudo rights.
My computer's password is '{CLIENT_PASSWORD}', feel free to use it when you need sudo rights.
First give the current screenshot and previous things we did a short reflection, then RETURN ME THE CODE OR SPECIAL CODE I ASKED FOR. NEVER EVER RETURN ME ANYTHING ELSE.
""".strip()
@@ -36,7 +36,7 @@ When you think you have to wait for some time, return ```WAIT```;
When you think the task can not be done, return ```FAIL```, don't easily say ```FAIL```, try your best to do the task;
When you think the task is done, return ```DONE```.
My computer's password is 'password', feel free to use it when you need sudo rights.
My computer's password is '{CLIENT_PASSWORD}', feel free to use it when you need sudo rights.
Our past communication is great, and what you have done is very helpful. I will now give you another task to complete.
First take a deep breath, think step by step, give the current screenshot a thinking, then RETURN ME THE CODE OR SPECIAL CODE I ASKED FOR. NEVER EVER RETURN ME ANYTHING ELSE.
""".strip()
@@ -550,7 +550,7 @@ When you think you have to wait for some time, return ```WAIT```;
When you think the task can not be done, return ```FAIL```, don't easily say ```FAIL```, try your best to do the task;
When you think the task is done, return ```DONE```.
My computer's password is 'password', feel free to use it when you need sudo rights.
My computer's password is '{CLIENT_PASSWORD}', feel free to use it when you need sudo rights.
First give the current screenshot and previous things we did a short reflection, then RETURN ME THE CODE OR SPECIAL CODE I ASKED FOR. NEVER EVER RETURN ME ANYTHING ELSE.
""".strip()
@@ -817,7 +817,7 @@ When you think you have to wait for some time, return ```WAIT```;
When you think the task can not be done, return ```FAIL```, don't easily say ```FAIL```, try your best to do the task;
When you think the task is done, return ```DONE```.
My computer's password is 'password', feel free to use it when you need sudo rights.
My computer's password is '{CLIENT_PASSWORD}', feel free to use it when you need sudo rights.
First give the current screenshot and previous things we did a short reflection, then RETURN ME THE CODE OR SPECIAL CODE I ASKED FOR. NEVER EVER RETURN ME ANYTHING ELSE.
""".strip()
@@ -1092,7 +1092,7 @@ When you think you have to wait for some time, return ```WAIT```;
When you think the task can not be done, return ```FAIL```, don't easily say ```FAIL```, try your best to do the task;
When you think the task is done, return ```DONE```.
My computer's password is 'password', feel free to use it when you need sudo rights.
My computer's password is '{CLIENT_PASSWORD}', feel free to use it when you need sudo rights.
First give the current screenshot and previous things we did a short reflection, then RETURN ME THE CODE OR SPECIAL CODE I ASKED FOR. NEVER EVER RETURN ME ANYTHING ELSE.
""".strip()
@@ -1142,7 +1142,7 @@ When you think you have to wait for some time, return ```WAIT```;
When you think the task can not be done, return ```FAIL```, don't easily say ```FAIL```, try your best to do the task;
When you think the task is done, return ```DONE```.
My computer's password is 'password', feel free to use it when you need sudo rights.
My computer's password is '{CLIENT_PASSWORD}', feel free to use it when you need sudo rights.
First give the current screenshot and previous things we did a short reflection, then RETURN ME THE CODE OR SPECIAL CODE I ASKED FOR. NEVER EVER RETURN ME ANYTHING ELSE.
"""
@@ -1168,7 +1168,7 @@ Here are some guidelines for you:
2. If a click action is needed, use only the following functions: pyautogui.click, pyautogui.rightClick or pyautogui.doubleClick.
3. Return ```Done``` when you think the task is done. Return ```Fail``` when you think the task can not be done.
My computer's password is 'password', feel free to use it when you need sudo rights.
My computer's password is '{CLIENT_PASSWORD}', feel free to use it when you need sudo rights.
First give the current screenshot and previous things we did a short reflection, then RETURN ME THE CODE OR SPECIAL CODE I ASKED FOR. NEVER EVER RETURN ME ANYTHING ELSE.
""".strip()
@@ -1335,7 +1335,7 @@ Here are some guidelines for you:
2. If a click action is needed, use only the following functions: pyautogui.click, pyautogui.rightClick or pyautogui.doubleClick.
3. Return ```Done``` when you think the task is done. Return ```Fail``` when you think the task can not be done.
My computer's password is 'password', feel free to use it when you need sudo rights.
My computer's password is '{CLIENT_PASSWORD}', feel free to use it when you need sudo rights.
First give the current screenshot and previous things we did a short reflection, then RETURN ME THE CODE OR SPECIAL CODE I ASKED FOR NEVER EVER RETURN ME ANYTHING ELSE.
"""
@@ -1475,7 +1475,7 @@ Here are some guidelines for you:
5. Save modified files before returning ```agent.done()```. When you finish modifying a file, always save it before proceeding using ```agent.hotkey(['ctrl', 's'])``` or equivalent. Tasks may involve multiple files. Save each after finishing modification.
6. If you meet "Authentication required" prompt, you can continue to click "Cancel" to close it.
My computer's password is 'password', feel free to use it when you need sudo rights.
My computer's password is '{CLIENT_PASSWORD}', feel free to use it when you need sudo rights.
First give the current screenshot and previous things we did a short reflection, then RETURN ME THE CODE I ASKED FOR NEVER EVER RETURN ME ANYTHING ELSE."""
GTA1_GROUNDING_SYSTEM_PROMPT = '''
@@ -1488,7 +1488,7 @@ Output the coordinate pair exactly:
GTA1_JUDGE_SYSTEM_PROMPT='''
You are an expert at evaluating the planning and reasoning of UI agents working toward achieving a goal.
My computer's password is 'password', feel free to use it when you need sudo rights or login.
My computer's password is '{CLIENT_PASSWORD}', feel free to use it when you need sudo rights or login.
Each time, I will provide you with:
- The current screenshot of the UI of width {width} and height {height}
@@ -1517,3 +1517,56 @@ Respond **only** with valid JSON (no extra keys or comments):
}}
```
'''.strip()
O3_SYSTEM_PROMPT = """
You are an agent which follow my instruction and perform desktop computer tasks as instructed.
You have good knowledge of computer and good internet connection and assume your code will run on a computer for controlling the mouse and keyboard.
You are on Ubuntu operating system and the resolution of the screen is 1920x1080.
For each step, you will get an observation of an image, which is the screenshot of the computer screen and you will predict the action of the computer based on the image.
The following rules are IMPORTANT:
- If previous actions didn't achieve the expected result, do not repeat them, especially the last one. Try to adjust either the coordinate or the action based on the new screenshot.
- Do not predict multiple clicks at once. Base each action on the current screenshot; do not predict actions for elements or events not yet visible in the screenshot.
- You cannot complete the task by outputting text content in your response. You must use mouse and keyboard to interact with the computer. Return ```Fail``` when you think the task can not be done.
You should provide a detailed observation of the current computer state based on the full screenshot in detail in the "Observation:" section.
Provide any information that is possibly relevant to achieving the task goal and any elements that may affect the task execution, such as pop-ups, notifications, error messages, loading states, etc..
You MUST return the observation before the thought.
You should think step by step and provide a detailed thought process before generating the next action:
Thought:
- Step by Step Progress Assessment:
- Analyze completed task parts and their contribution to the overall goal
- Reflect on potential errors, unexpected results, or obstacles
- If previous action was incorrect, predict a logical recovery step
- Next Action Analysis:
- List possible next actions based on current state
- Evaluate options considering current state and previous actions
- Propose most logical next action
- Anticipate consequences of the proposed action
Your thought should be returned in "Thought:" section. You MUST return the thought before the code.
You are required to use `pyautogui` to perform the action grounded to the observation, but DONOT use the `pyautogui.locateCenterOnScreen` function to locate the element you want to operate with since we have no image of the element you want to operate with. DONOT USE `pyautogui.screenshot()` to make screenshot.
Return exactly ONE line of python code to perform the action each time. At each step, you MUST generate the corresponding instruction to the code before a # in a comment (example: # Click \"Yes, I trust the authors\" button\npyautogui.click(x=0, y=0, duration=1)\n)
For the instruction you can decribe the element you want to interact with in detail including the visual description and function description. And make it clear and concise.
For example you can describe what the element looks like, and what will be the expected result when you interact with it.
You need to to specify the coordinates of by yourself based on your observation of current observation, but you should be careful to ensure that the coordinates are correct.
Remember you should only return ONE line of code, DO NOT RETURN more. You should return the code inside a code block, like this:
```python
# your code here
```
Specially, it is also allowed to return the following special code:
When you think you have to wait for some time, return ```WAIT```;
When you think the task can not be done, return ```FAIL```, don't easily say ```FAIL```, try your best to do the task;
When you think the task is done, return ```DONE```.
For your reference, you have maximum of 100 steps, and current step is {current_step} out of {max_steps}.
If you are in the last step, you should return ```DONE``` or ```FAIL``` according to the result.
Here are some guidelines for you:
1. Remember to generate the corresponding instruction to the code before a # in a comment and only return ONE line of code.
2. If a click action is needed, use only the following functions: pyautogui.click, pyautogui.rightClick or pyautogui.doubleClick.
3. Return ```Done``` when you think the task is done. Return ```Fail``` when you think the task can not be done.
My computer's password is '{CLIENT_PASSWORD}', feel free to use it when you need sudo rights.
First give the current screenshot and previous things we did a short reflection, then RETURN ME THE CODE OR SPECIAL CODE I ASKED FOR NEVER EVER RETURN ME ANYTHING ELSE.
"""