diff --git a/mm_agents/agent.py b/mm_agents/agent.py index cb0ba85..263e5ee 100644 --- a/mm_agents/agent.py +++ b/mm_agents/agent.py @@ -5,20 +5,21 @@ import os import re import time import uuid -import openai import xml.etree.ElementTree as ET from http import HTTPStatus from io import BytesIO from typing import Dict, List -from google.api_core.exceptions import InvalidArgument + import backoff import dashscope import google.generativeai as genai +import openai import requests import wandb from PIL import Image +from google.api_core.exceptions import InvalidArgument -from mm_agents.accessibility_tree_wrap.heuristic_retrieve import find_leaf_nodes, filter_nodes, draw_bounding_boxes +from mm_agents.accessibility_tree_wrap.heuristic_retrieve import filter_nodes, draw_bounding_boxes from mm_agents.prompts import SYS_PROMPT_IN_SCREENSHOT_OUT_CODE, SYS_PROMPT_IN_SCREENSHOT_OUT_ACTION, \ SYS_PROMPT_IN_A11Y_OUT_CODE, SYS_PROMPT_IN_A11Y_OUT_ACTION, \ SYS_PROMPT_IN_BOTH_OUT_CODE, SYS_PROMPT_IN_BOTH_OUT_ACTION, \ @@ -423,7 +424,6 @@ class PromptAgent: # with open("messages.json", "w") as f: # f.write(json.dumps(messages, indent=4)) - logger.info("Generating content with GPT model: %s", self.model) response = self.call_llm({ "model": self.model, "messages": messages, @@ -462,7 +462,7 @@ class PromptAgent: "Content-Type": "application/json", "Authorization": f"Bearer {os.environ['OPENAI_API_KEY']}" } - # logger.info("Generating content with GPT model: %s", self.model) + logger.info("Generating content with GPT model: %s", self.model) response = requests.post( "https://api.openai.com/v1/chat/completions", headers=headers, @@ -496,7 +496,7 @@ class PromptAgent: temperature = payload["temperature"] claude_messages = [] - + for i, message in enumerate(messages): claude_message = { "role": message["role"], @@ -504,17 +504,17 @@ class PromptAgent: } assert len(message["content"]) in [1, 2], "One text, or one text with one image" for part in message["content"]: - + if part['type'] == "image_url": image_source = {} image_source["type"] = "base64" image_source["media_type"] = "image/png" image_source["data"] = part['image_url']['url'].replace("data:image/png;base64,", "") claude_message['content'].append({"type": "image", "source": image_source}) - + if part['type'] == "text": claude_message['content'].append({"type": "text", "text": part['text']}) - + claude_messages.append(claude_message) # the claude not support system message in our endpoint, so we concatenate it at the first user message @@ -523,7 +523,6 @@ class PromptAgent: claude_messages[1]['content'].insert(0, claude_system_message_item) claude_messages.pop(0) - headers = { "x-api-key": os.environ["ANTHROPIC_API_KEY"], "anthropic-version": "2023-06-01", @@ -541,7 +540,7 @@ class PromptAgent: headers=headers, json=payload ) - + if response.status_code != 200: logger.error("Failed to call LLM: " + response.text) @@ -551,55 +550,101 @@ class PromptAgent: return response.json()['content'][0]['text'] - # elif self.model.startswith("mistral"): - # print("Call mistral") - # messages = payload["messages"] - # max_tokens = payload["max_tokens"] - # - # misrtal_messages = [] - # - # for i, message in enumerate(messages): - # mistral_message = { - # "role": message["role"], - # "content": [] - # } - # - # for part in message["content"]: - # mistral_message['content'] = part['text'] if part['type'] == "text" else None - # - # misrtal_messages.append(mistral_message) - # - # # the mistral not support system message in our endpoint, so we concatenate it at the first user message - # if misrtal_messages[0]['role'] == "system": - # misrtal_messages[1]['content'] = misrtal_messages[0]['content'] + "\n" + misrtal_messages[1]['content'] - # misrtal_messages.pop(0) - # - # # openai.api_base = "http://localhost:8000/v1" - # # openai.api_key = "test" - # # response = openai.ChatCompletion.create( - # # messages=misrtal_messages, - # # model="Mixtral-8x7B-Instruct-v0.1" - # # ) - # - # from openai import OpenAI - # TOGETHER_API_KEY = "d011650e7537797148fb6170ec1e0be7ae75160375686fae02277136078e90d2" - # - # client = OpenAI(api_key=TOGETHER_API_KEY, - # base_url='https://api.together.xyz', - # ) - # logger.info("Generating content with Mistral model: %s", self.model) - # response = client.chat.completions.create( - # messages=misrtal_messages, - # model="mistralai/Mixtral-8x7B-Instruct-v0.1", - # max_tokens=1024 - # ) - # - # try: - # # return response['choices'][0]['message']['content'] - # return response.choices[0].message.content - # except Exception as e: - # print("Failed to call LLM: " + str(e)) - # return "" + elif self.model.startswith("mistral"): + print("Call mistral") + messages = payload["messages"] + max_tokens = payload["max_tokens"] + top_p = payload["top_p"] + temperature = payload["temperature"] + + misrtal_messages = [] + + for i, message in enumerate(messages): + mistral_message = { + "role": message["role"], + "content": "" + } + + for part in message["content"]: + mistral_message['content'] = part['text'] if part['type'] == "text" else "" + + + misrtal_messages.append(mistral_message) + + + # openai.api_base = "http://localhost:8000/v1" + # response = openai.ChatCompletion.create( + # messages=misrtal_messages, + # model="Mixtral-8x7B-Instruct-v0.1" + # ) + + from openai import OpenAI + + client = OpenAI(api_key=os.environ["TOGETHER_API_KEY"], + base_url='https://api.together.xyz', + ) + logger.info("Generating content with Mistral model: %s", self.model) + + response = client.chat.completions.create( + messages=misrtal_messages, + model=self.model, + max_tokens=max_tokens + ) + + try: + return response.choices[0].message.content + except Exception as e: + print("Failed to call LLM: " + str(e)) + return "" + + elif self.model.startswith("THUDM"): + # THUDM/cogagent-chat-hf + print("Call CogAgent") + messages = payload["messages"] + max_tokens = payload["max_tokens"] + top_p = payload["top_p"] + temperature = payload["temperature"] + + cog_messages = [] + + for i, message in enumerate(messages): + cog_message = { + "role": message["role"], + "content": [] + } + + for part in message["content"]: + if part['type'] == "image_url": + cog_message['content'].append({"type": "image_url", "image_url": {"url": part['image_url']['url'] } }) + + if part['type'] == "text": + cog_message['content'].append({"type": "text", "text": part['text']}) + + cog_messages.append(cog_message) + + # the cogagent not support system message in our endpoint, so we concatenate it at the first user message + if cog_messages[0]['role'] == "system": + cog_system_message_item = cog_messages[0]['content'][0] + cog_messages[1]['content'].insert(0, cog_system_message_item) + cog_messages.pop(0) + + payload = { + "model": self.model, + "max_tokens": max_tokens, + "messages": cog_messages + } + + base_url = "http://127.0.0.1:8000" + + response = requests.post(f"{base_url}/v1/chat/completions", json=payload, stream=False) + if response.status_code == 200: + decoded_line = response.json() + content = decoded_line.get("choices", [{}])[0].get("message", "").get("content", "") + return content + else: + print("Failed to call LLM: ", response.status_code) + return "" + elif self.model.startswith("gemini"): def encoded_img_to_pil_img(data_str): @@ -675,6 +720,7 @@ class PromptAgent: try: return response.text except Exception as e: + logger.error("Meet exception when calling Gemini API, " + str(e)) return "" elif self.model.startswith("qwen"): messages = payload["messages"] diff --git a/run.py b/run.py index 28563c8..4284169 100644 --- a/run.py +++ b/run.py @@ -6,6 +6,7 @@ import datetime import json import logging import os +import random import sys import wandb @@ -75,7 +76,7 @@ def config() -> argparse.Namespace: "screenshot_a11y_tree", "som" ], - default="som", + default="a11y_tree", help="Observation type", ) parser.add_argument("--screen_width", type=int, default=1920) @@ -88,7 +89,7 @@ def config() -> argparse.Namespace: parser.add_argument("--test_config_base_dir", type=str, default="evaluation_examples") # lm config - parser.add_argument("--model", type=str, default="gpt-4-vision-preview") + parser.add_argument("--model", type=str, default="gpt-4-0125-preview") parser.add_argument("--temperature", type=float, default=1.0) parser.add_argument("--top_p", type=float, default=0.9) parser.add_argument("--max_tokens", type=int, default=1500) @@ -231,15 +232,13 @@ def get_unfinished(action_space, use_model, observation_type, result_dir, total_ def get_result(action_space, use_model, observation_type, result_dir, total_file_json): target_dir = os.path.join(result_dir, action_space, observation_type, use_model) + if not os.path.exists(target_dir): + print("New experiment, no result yet.") + return None all_result = [] - if not os.path.exists(target_dir): - return total_file_json - - finished = {} for domain in os.listdir(target_dir): - finished[domain] = [] domain_path = os.path.join(target_dir, domain) if os.path.isdir(domain_path): for example_id in os.listdir(domain_path): @@ -247,10 +246,17 @@ def get_result(action_space, use_model, observation_type, result_dir, total_file if os.path.isdir(example_path): if "result.txt" in os.listdir(example_path): # empty all files under example_id - all_result.append(float(open(os.path.join(example_path, "result.txt"), "r").read())) + try: + all_result.append(float(open(os.path.join(example_path, "result.txt"), "r").read())) + except: + all_result.append(0.0) - print("Success Rate:", sum(all_result) / len(all_result) * 100, "%") - return all_result + if not all_result: + print("New experiment, no result yet.") + return None + else: + print("Current Success Rate:", sum(all_result) / len(all_result) * 100, "%") + return all_result if __name__ == '__main__':