fix(os_symphony):prompt (#402)

* add_os_symphony

* fix(os_symphony)

* fix(os_symphony):prompt

---------

Co-authored-by: Tianbao Xie <47296835+Timothyxxx@users.noreply.github.com>

lib_run_single.py

@@ -463,6 +463,87 @@ def run_single_example_uipath(agent, env, example, max_steps, instruction, args,
     env.controller.end_recording(os.path.join(example_result_dir, "recording.mp4"))
 
 
+from mm_agents.os_symphony.utils.common_utils import draw_coordinates
+from mm_agents.os_symphony.utils.process_context import set_current_result_dir
+
+
+logger = logging.getLogger("desktopenv.experiment")
+
+def run_single_example_os_symphony(agent, env, example, max_steps, instruction, args, example_result_dir, scores):
+    set_current_result_dir(example_result_dir)
+    
+    agent.reset(result_dir=example_result_dir)
+    env.reset(task_config=example)
+    time.sleep(30) # Wait for the environment to be ready
+    obs = env._get_obs() # Get the initial observation
+    done = False
+    step_idx = 0
+    # env.controller.start_recording()
+    start_time = time.time()
+
+    while not done and step_idx < max_steps:
+        response, actions = agent.predict(
+            instruction,
+            obs,
+            step_idx == max_steps - 1
+        )
+        for action in actions:
+            # Save screenshot and trajectory information
+            if "reflection" in response and response["reflection"].get("is_milestone"):
+                img_name = f"step_{step_idx + 1}_milestone.png"
+            else:
+                img_name = f"step_{step_idx + 1}.png"
+                
+            with open(os.path.join(example_result_dir, img_name),
+                      "wb") as _f:
+                _f.write(obs['screenshot'])
+            if "coordinates" in response and response["coordinates"]:
+                draw_coordinates(
+                    image_bytes=obs['screenshot'], 
+                    coordinates=response["coordinates"], 
+                    save_path=os.path.join(example_result_dir, img_name[:-4] + "_draw.png")
+                )
+
+            logger.info("Step %d: %s", step_idx + 1, action)
+            obs, reward, done, info = env.step(action, args.sleep_after_execution)
+            logger.info("Done: %s", done)
+
+            with open(os.path.join(example_result_dir, "traj.jsonl"), "a", encoding="utf-8") as f:
+                f.write(json.dumps({
+                    "instruction": instruction,
+                    "step_num": step_idx + 1,
+                    "action": action,
+                    "response": response,
+                    "done": done,
+                    "info": info,
+                    "screenshot_file": img_name
+                }))
+                f.write("\n")
+            with open(os.path.join(example_result_dir, f"traj_{step_idx+1}.json"), "w", encoding="utf-8") as f:
+                json.dump({
+                    "step_num": step_idx + 1,
+                    "action": action,
+                    "response": response,
+                    "done": done,
+                    "info": info,
+                    "screenshot_file": img_name
+                }, f, indent=4, ensure_ascii=False)
+            if done:
+                logger.info("The episode is done.")
+                time.sleep(60)
+                break
+        step_idx += 1
+    end_time = time.time()
+    result = float(env.evaluate())
+    logger.info("Result: %.2f", result)
+    scores.append(result)
+    with open(os.path.join(example_result_dir, "result.txt"), "w", encoding="utf-8") as f:
+        f.write(f"{result}\n")
+
+    with open(os.path.join(example_result_dir, "time.txt"), "w", encoding="utf-8") as f:
+        f.write(f"{end_time-start_time:.2f}\n")
+
+
 def run_single_example_evocua(agent, env, example, max_steps, instruction, args, example_result_dir, scores):
     """
     Unified run function for EvoCUAAgent (supporting both S1 and S2 modes).
@@ -561,3 +642,4 @@ def run_single_example_evocua(agent, env, example, max_steps, instruction, args,
     log_task_completion(example, result, example_result_dir, args)
 
     env.controller.end_recording(os.path.join(example_result_dir, "recording.mp4"))
+

mm_agents/os_symphony/memory/procedural_memory.py

@@ -192,6 +192,8 @@ class PROCEDURAL_MEMORY:
                 - **Use One Provided Action at a Time**: Execute only one grounded action per turn. Only use the methods provided in the Agent class. Do not invent new methods.
                 - **No Interaction with User**: You MUST complete the task individually. There is **NO** additional input from someone else.
                 - **Password**: Your sudo password is "CLIENT_PASSWORD".
+                - **User**: Your username is "user".
+                - **Home**: Your home path is "/home/user".
 
                 ## 2.2 Interaction & Input Guidelines
                 - **Guideline for Clicks**: 
@@ -990,4 +992,4 @@ class PROCEDURAL_MEMORY:
                 - The generated action(s) must exist within the defined action space.
                 - The generated action(s) should be enclosed within <action></action> tags.'''
             )
-        return system_prompt, user_message
+        return system_prompt, user_message