ver Mar18th

checked Claude agent

branch-config/filelist

@@ -5,5 +5,8 @@ evaluation_examples
 logs
 
 mm_agents
+run.py
+lib_run_single.py
+settings.json
 
 quick_evaluate.py

branch_flag

@@ -1 +1 @@
-test
+claude

mm_agents/agent.py

@@ -256,7 +256,6 @@ class PromptAgent:
             if self.observation_type == "screenshot_a11y_tree":
                 _screenshot = previous_obs["screenshot"]
                 _linearized_accessibility_tree = previous_obs["accessibility_tree"]
-                logger.debug("LINEAR AT: %s", _linearized_accessibility_tree)
 
                 messages.append({
                     "role": "user",
@@ -343,6 +342,7 @@ class PromptAgent:
         if self.observation_type in ["screenshot", "screenshot_a11y_tree"]:
             base64_image = encode_image(obs["screenshot"])
             linearized_accessibility_tree = linearize_accessibility_tree(accessibility_tree=obs["accessibility_tree"])
+            logger.debug("LINEAR AT: %s", linearized_accessibility_tree)
 
             if self.observation_type == "screenshot_a11y_tree":
                 self.observations.append({
@@ -376,6 +376,7 @@ class PromptAgent:
             })
         elif self.observation_type == "a11y_tree":
             linearized_accessibility_tree = linearize_accessibility_tree(accessibility_tree=obs["accessibility_tree"])
+            logger.debug("LINEAR AT: %s", linearized_accessibility_tree)
 
             self.observations.append({
                 "screenshot": None,
@@ -423,6 +424,8 @@ class PromptAgent:
         # with open("messages.json", "w") as f:
         #     f.write(json.dumps(messages, indent=4))
 
+        #logger.info("PROMPT: %s", messages)
+
         response = self.call_llm({
             "model": self.model,
             "messages": messages,
@@ -522,6 +525,8 @@ class PromptAgent:
                 claude_messages[1]['content'].insert(0, claude_system_message_item)
                 claude_messages.pop(0)
 
+            logger.debug("CLAUDE MESSAGE: %s", repr(claude_messages))
+
             # headers = {
             #     "x-api-key": os.environ["ANTHROPIC_API_KEY"],
             #     "anthropic-version": "2023-06-01",

run.py

@@ -140,6 +140,7 @@ def test(
 
     env = DesktopEnv(
         path_to_vm=args.path_to_vm,
+        snapshot_name="Snapshot 35",
         action_space=agent.action_space,
         screen_size=(args.screen_width, args.screen_height),
         headless=args.headless,