Merge branch 'zdy'

This commit is contained in:
David Chang
2024-01-27 23:18:33 +08:00
6 changed files with 57 additions and 32 deletions

View File

@@ -22,6 +22,10 @@ from mm_agents.prompts import SYS_PROMPT_IN_SCREENSHOT_OUT_CODE, SYS_PROMPT_IN_S
SYS_PROMPT_IN_SOM_A11Y_OUT_TAG, \
SYS_PROMPT_SEEACT, ACTION_DESCRIPTION_PROMPT_SEEACT, ACTION_GROUNDING_PROMPT_SEEACT
import logging
logger = logging.getLogger("desktopenv.agent")
# Function to encode the image
def encode_image(image_path):
@@ -252,6 +256,7 @@ class GPT4v_Agent:
if self.exp == "both":
_screenshot = previous_obs["screenshot"]
_linearized_accessibility_tree = previous_obs["accessibility_tree"]
logger.debug("LINEAR AT: %s", _linearized_accessibility_tree)
messages.append({
"role": "user",
@@ -456,7 +461,7 @@ class GPT4v_Agent:
"max_tokens": self.max_tokens
})
print(response)
logger.debug("RESPONSE: %s", response)
if self.exp == "seeact":
messages.append({