feat: 增强任务步骤注入与a11y状态表达，提升树形交互稳定性

- 打通 metadata.steps 传递链路，将任务步骤注入 agent 预测上下文

- 优化 a11y tree 线性化输出：使用中心坐标并新增 states 列（expanded/collapsed/selected 等）

- 放宽可保留节点条件，保留无文本输入类控件（edit/textfield/searchbox 等）

- 强化输出约束：单轮仅允许动作代码或 WAIT/DONE/FAIL，禁止动作与 DONE 同轮返回

- 补充 avogadro 示例步骤：展开 aromatics 并选择 benzene.cjson

.gitignore

@@ -208,7 +208,17 @@ quick_start.py
 result_multi_apps_pengxiang_transformers12evaluation_examples/settings/proxy/dataimpulse.json
 evaluation_examples/settings/proxy/dataimpulse.json
 
+# Benchmark input data (large binary files - share via cloud storage or Git LFS)
+evaluation_examples/inputs/
+
+# Temporary data processing workspace (scraped docs, intermediate scripts)
+evaluation_examples/sandbox/
+
+# Image cache
+evaluation_examples/inputs/.img_cache/
+
 # Local test configurations (not for public repo)
 evaluation_examples/spiderman.json
 evaluation_examples/test_50_random_proportional.json
 evaluation_examples/test_chrome.json
+evaluation_examples/prepare_input_files.py

desktop_env/desktop_env.py

@@ -20,42 +20,42 @@ Metric = Callable[[Any, Any], float]
 Getter = Callable[[gym.Env, Dict[str, Any]], Any]
 
 MAX_RETRIES = 5 # Maximum retries for environment setup
-            
+
 
 
 def _fix_pyautogui_less_than_bug(command: str) -> str:
     """
     Fix PyAutoGUI '<' character bug by converting it to hotkey("shift", ',') calls.
-    
+
     This fixes the known PyAutoGUI issue where typing '<' produces '>' instead.
     References:
     - https://github.com/asweigart/pyautogui/issues/198
     - https://github.com/xlang-ai/OSWorld/issues/257
-    
+
     Args:
         command (str): The original pyautogui command
-        
+
     Returns:
         str: The fixed command with '<' characters handled properly
     """
-    # Pattern to match press('<') or press('\u003c') calls  
+    # Pattern to match press('<') or press('\u003c') calls
     press_pattern = r'pyautogui\.press\(["\'](?:<|\\u003c)["\']\)'
 
     # Handle press('<') calls
     def replace_press_less_than(match):
         return 'pyautogui.hotkey("shift", ",")'
-    
+
     # First handle press('<') calls
     command = re.sub(press_pattern, replace_press_less_than, command)
 
     # Pattern to match typewrite calls with quoted strings
     typewrite_pattern = r'pyautogui\.typewrite\((["\'])(.*?)\1\)'
-    
+
     # Then handle typewrite calls
     def process_typewrite_match(match):
         quote_char = match.group(1)
         content = match.group(2)
-        
+
         # Preprocess: Try to decode Unicode escapes like \u003c to actual '<'
         # This handles cases where '<' is represented as escaped Unicode
         try:
@@ -65,15 +65,15 @@ def _fix_pyautogui_less_than_bug(command: str) -> str:
         except UnicodeDecodeError:
             # If decoding fails, proceed with original content to avoid breaking existing logic
             pass  # English comment: Graceful degradation - fall back to original content if decoding fails
-        
+
         # Check if content contains '<'
         if '<' not in content:
             return match.group(0)
-        
+
         # Split by '<' and rebuild
         parts = content.split('<')
         result_parts = []
-        
+
         for i, part in enumerate(parts):
             if i == 0:
                 # First part
@@ -84,11 +84,11 @@ def _fix_pyautogui_less_than_bug(command: str) -> str:
                 result_parts.append('pyautogui.hotkey("shift", ",")')
                 if part:
                     result_parts.append(f"pyautogui.typewrite({quote_char}{part}{quote_char})")
-        
+
         return '; '.join(result_parts)
-    
+
     command = re.sub(typewrite_pattern, process_typewrite_match, command)
-    
+
     return command
 
 
@@ -145,12 +145,12 @@ class DesktopEnv(gym.Env):
         self.screen_width = screen_size[0]
         self.screen_height = screen_size[1]
 
-        # Default 
+        # Default
         self.server_port = 5000
         self.chromium_port = 9222
         self.vnc_port = 8006
         self.vlc_port = 8080
-        
+
         # Initialize with default (no proxy) provider
         self.current_use_proxy = False
         self.manager, self.provider = create_vm_manager_and_provider(provider_name, region, use_proxy=False)
@@ -173,7 +173,7 @@ class DesktopEnv(gym.Env):
                 if provider_name in {"vmware", "virtualbox"} else path_to_vm
         else:
             self.path_to_vm = self.manager.get_vm_path(os_type=self.os_type, region=region, screen_size=(self.screen_width, self.screen_height))
-        
+
         self.snapshot_name = snapshot_name
         self.cache_dir_base: str = cache_dir
         # todo: add the logic to get the screen size from the VM
@@ -229,8 +229,8 @@ class DesktopEnv(gym.Env):
         # due to the fact it could be changed when implemented by cloud services
         path_to_vm = self.provider.revert_to_snapshot(self.path_to_vm, self.snapshot_name)
         if path_to_vm and not path_to_vm == self.path_to_vm:
-            # path_to_vm has to be a new path 
-            
+            # path_to_vm has to be a new path
+
             self.manager.delete_vm(self.path_to_vm, self.region)
             self.manager.add_vm(path_to_vm, self.region)
             self.manager.occupy_vm(path_to_vm, os.getpid(), self.region)
@@ -245,7 +245,7 @@ class DesktopEnv(gym.Env):
         self.provider.stop_emulator(self.path_to_vm)
 
     def reset(self, task_config: Optional[Dict[str, Any]] = None, seed=None, options=None) -> Dict[str, Any]:
-        
+
         # Reset to certain task in OSWorld
         logger.info("Resetting environment...")
         logger.info("Switching task...")
@@ -258,17 +258,17 @@ class DesktopEnv(gym.Env):
             # Only revert to snapshot if environment has been used (step/setup)
             # This optimization is especially important for cloud providers like AWS
             # where unnecessary snapshot operations are costly and time-consuming
-            
+
             if task_config is not None:
                 # Only consider task proxy requirement if proxy is enabled at system level
                 task_use_proxy = task_config.get("proxy", False) and self.enable_proxy
                 if not self.enable_proxy and task_config.get("proxy", False):
                     logger.info("Task requires proxy but proxy is disabled at system level, ignoring proxy requirement.")
-                
+
                 if task_use_proxy != self.current_use_proxy:
                     # keep because get_info_from_website depend on this
                     self.current_use_proxy = task_use_proxy
-            
+
             if self.is_environment_used:
                 logger.info("Environment has been used, reverting to snapshot: {}...".format(self.snapshot_name))
                 self._revert_to_snapshot()
@@ -302,7 +302,7 @@ class DesktopEnv(gym.Env):
                     time.sleep(5)
             else:
                 break
-            
+
         logger.info("Environment setup complete.")
 
         observation = self._get_obs()
@@ -333,7 +333,8 @@ class DesktopEnv(gym.Env):
         os.makedirs(self.cache_dir, exist_ok=True)
         self.instruction = task_config["instruction"]
         self.config = task_config["config"] if "config" in task_config else []
-        
+        self.metadata = task_config.get("metadata", {})
+
         self._set_evaluator_info(task_config)
 
     def _set_evaluator_info(self, task_config: Dict[str, Any]):
@@ -386,7 +387,7 @@ class DesktopEnv(gym.Env):
     def step(self, action, pause=2):
         self._step_no += 1
         self.action_history.append(action)
-        
+
         # Mark environment as used when step is called
         self.is_environment_used = True
 
@@ -461,12 +462,16 @@ class DesktopEnv(gym.Env):
             self.metric_options["instruction"] = self.instruction
             self.metric_options["eval_model"] = self.eval_model
 
+            # Pass pre-configured environment info and expected steps
+            self.metric_options["config"] = self.config
+            self.metric_options["metadata"] = self.metadata
+
             if result_dir:
                 self.metric_options["result_dir"] = result_dir
                 logger.info(f"Using result_dir for vllm_eval: {result_dir}")
 
             logger.info(f"Evaluation options prepared: {self.metric_options.keys()}")
-            
+
         if type(self.metric) == list:
             # Multiple metrics to evaluate whether the task is successfully completed
             results = []

desktop_env/evaluators/metrics/vllm_eval.py

@@ -287,7 +287,7 @@ class UnifiedLLM:
             raise ValueError(f"Unsupported provider: {self.provider}")
 
 
-def _load_screenshots_from_dir(result_dir: str, compress: bool = True, max_size: int = 800, quality: int = 85) -> List[str]:
+def _load_screenshots_from_dir(result_dir: str, compress: bool = True, max_size: int = 800, quality: int = 85) -> tuple:
     """
     Load all step screenshots from result directory and convert to base64
 
@@ -298,9 +298,10 @@ def _load_screenshots_from_dir(result_dir: str, compress: bool = True, max_size:
         quality: JPEG quality for compression (default: 85)
 
     Returns:
-        List of base64 encoded screenshot strings
+        Tuple of (list of base64 encoded screenshot strings, list of short filenames like 'step_1', 'step_2', ...)
     """
     screenshots = []
+    filenames = []
 
     # Find all step screenshot files (e.g., step_1_20240101@120000.png)
     pattern = os.path.join(result_dir, "step_*.png")
@@ -308,8 +309,9 @@ def _load_screenshots_from_dir(result_dir: str, compress: bool = True, max_size:
 
     if not screenshot_files:
         logger.warning(f"No screenshot files found in {result_dir}")
-        return screenshots
+        return screenshots, filenames
 
+    import re as _re
     for filepath in screenshot_files:
         try:
             with open(filepath, "rb") as f:
@@ -321,11 +323,16 @@ def _load_screenshots_from_dir(result_dir: str, compress: bool = True, max_size:
                     img_b64 = _compress_image(img_b64, max_size=max_size, quality=quality)
 
                 screenshots.append(img_b64)
+                # Extract short name like 'step_1' from 'step_1_20240101@120000.png'
+                basename = os.path.basename(filepath)
+                match = _re.match(r'(step_\d+)', basename)
+                short_name = match.group(1) if match else basename
+                filenames.append(short_name)
         except Exception as e:
             logger.error(f"Error loading screenshot {filepath}: {e}")
 
-    logger.info(f"Loaded {len(screenshots)} screenshots from {result_dir}")
-    return screenshots
+    logger.info(f"Loaded {len(screenshots)} screenshots from {result_dir}: {filenames}")
+    return screenshots, filenames
 
 
 def vllm_eval(result_state, **options) -> float:
@@ -358,8 +365,10 @@ def vllm_eval(result_state, **options) -> float:
     max_image_size = options.get("max_image_size", 800)
     image_quality = options.get("image_quality", 85)
 
+    screenshot_filenames = []  # Short names like 'step_1', 'step_2', ...
+
     if result_dir and not screenshots:
-        screenshots = _load_screenshots_from_dir(
+        screenshots, screenshot_filenames = _load_screenshots_from_dir(
             result_dir,
             compress=compress_images,
             max_size=max_image_size,
@@ -368,6 +377,7 @@ def vllm_eval(result_state, **options) -> float:
         logger.info(f"Loaded {len(screenshots)} screenshots from result_dir: {result_dir}")
     elif screenshots:
         logger.info(f"Using {len(screenshots)} screenshots from options")
+        screenshot_filenames = [f"step_{i+1}" for i in range(len(screenshots))]
         # Compress screenshots if needed
         if compress_images:
             logger.info("Compressing provided screenshots...")
@@ -375,6 +385,8 @@ def vllm_eval(result_state, **options) -> float:
 
     instruction = options.get("instruction", "")
     eval_model = options.get("eval_model", "gpt-4-vision-preview")
+    config = options.get("config", [])
+    metadata = options.get("metadata", {})
 
     params = {
         "temperature": options.get("temperature", 0.7),
@@ -384,32 +396,91 @@ def vllm_eval(result_state, **options) -> float:
 
     llm = UnifiedLLM(eval_model)
 
-    prompt = f"""You are an expert evaluator for desktop environment tasks.
+    # Build pre-configured environment description from config
+    preconfig_items = []
+    for cfg in config:
+        if cfg.get("type") == "launch":
+            cmds = cfg.get("parameters", {}).get("command", [])
+            if cmds:
+                app_name = os.path.basename(cmds[0]) if cmds else "unknown"
+                preconfig_items.append(f"Application '{app_name}' was automatically launched before the agent started.")
+        elif cfg.get("type") == "sleep":
+            pass  # not relevant to scoring
+        elif cfg.get("type") == "open":
+            path = cfg.get("parameters", {}).get("path", "")
+            preconfig_items.append(f"File/URL '{path}' was automatically opened before the agent started.")
+
+    preconfig_section = ""
+    if preconfig_items:
+        preconfig_desc = "\n".join(f"  - {item}" for item in preconfig_items)
+        preconfig_section = f"""
+PRE-CONFIGURED ENVIRONMENT (done BEFORE the agent started, NOT the agent's work):
+{preconfig_desc}
+IMPORTANT: The above actions were performed automatically as part of environment setup. The agent did NOT perform these actions. Do NOT give ANY credit for them. For example, if the application was pre-launched, the agent merely having the application open is worth 0 points - that was the starting state."""
+
+    # Build expected steps section from metadata
+    expected_steps_section = ""
+    if metadata.get("steps"):
+        expected_steps_section = f"""
+EXPECTED STEPS for this task (use as reference for what the agent should have done):
+{metadata['steps']}
+NOTE: Evaluate the screenshots against these expected steps. Only give credit for steps that show VISIBLE evidence of completion BEYOND the pre-configured starting state."""
+
+    # Build image list description for the prompt
+    if screenshot_filenames:
+        img_list_str = ", ".join(screenshot_filenames)
+        img_info = f"""\nYou are provided with exactly {len(screenshot_filenames)} screenshots in chronological order: {img_list_str}
+The FIRST screenshot is: {screenshot_filenames[0]}
+The LAST screenshot (final state): {screenshot_filenames[-1]}
+IMPORTANT: Only reference screenshots from the list above. Do NOT reference any screenshot that is not listed."""
+    else:
+        img_info = "\nNo screenshots were provided."
+
+    prompt = f"""You are a STRICT and RIGOROUS evaluator for desktop environment tasks. Your job is to score ONLY based on concrete, visible evidence of task completion in the screenshots.
 
 Task Instruction: {instruction}
+{preconfig_section}
+{expected_steps_section}
+{img_info}
 
-I will provide you with screenshot(s) showing the current state of the desktop environment. Please analyze the task execution step by step and provide a detailed evaluation.
+Analyze ONLY the FINAL screenshot ({screenshot_filenames[-1] if screenshot_filenames else 'N/A'}) to determine the end state, while using earlier screenshots for context.
+
+CRITICAL SCORING RULES:
+1. Score ONLY based on what the AGENT actually accomplished. The pre-configured environment (application already launched, files already opened, etc.) is the STARTING STATE and worth 0 points.
+2. Score ONLY based on what is ACTUALLY VISIBLE in the screenshots. Do NOT give credit for assumed or potential progress.
+3. If the screenshots show NO meaningful action beyond the initial pre-configured state, the score MUST be 0.
+4. Do NOT give partial credit for "having the system on", "desktop being visible", "the application being open" (if it was pre-launched), or "the application being installed". These are prerequisites or pre-configured state, NOT progress.
+5. Each point must correspond to a SPECIFIC, VERIFIABLE action that was successfully completed BY THE AGENT toward the task goal.
+
+SCORING GUIDE (0-10):
+- 0: No progress beyond the pre-configured starting state. If the app was pre-launched, merely having it open is 0. If the screenshots only show the desktop or the initial app state without any agent action, score is 0.
+- 1-2: The agent performed one minor action (e.g., clicked on a menu) but did not make meaningful progress toward the task goal.
+- 3-4: Some initial steps toward the task have been taken but the task is far from complete.
+- 5-6: Significant progress - about half the required steps are completed with visible evidence.
+- 7-8: Most steps are completed but the final result is not fully achieved or has minor issues.
+- 9: The task is essentially complete with very minor cosmetic differences.
+- 10: The task is perfectly and completely finished with clear evidence in the final screenshot.
 
 IMPORTANT: You must respond with ONLY a valid JSON object (no additional text before or after). Use the following exact format:
 
 {{
   "steps_analysis": [
-    {{"step": "Step description", "status": "Success/Fail", "evidence_img": "step_X.png", "reason": "Brief explanation"}},
-    {{"step": "Another step", "status": "Success/Fail", "evidence_img": "step_Y.png", "reason": "Brief explanation"}}
+    {{"step": "Step description", "status": "Success/Fail", "evidence_img": "step_X.png", "reason": "Brief explanation of VISIBLE evidence"}},
+    {{"step": "Another step", "status": "Success/Fail", "evidence_img": "step_Y.png", "reason": "Brief explanation of VISIBLE evidence"}}
   ],
   "final_completion": "True/False",
   "score": 0-10
 }}
 
 Where:
-- "steps_analysis": Array of steps you identified from the screenshots (reference screenshot filenames like step_1.png, step_2.png, etc.)
+- "steps_analysis": Array of steps you identified from the screenshots. Each step must cite VISIBLE evidence from a specific screenshot. Do NOT include pre-configured actions as agent steps.
 - "status": Either "Success" or "Fail" for each step
 - "evidence_img": The screenshot filename that shows evidence for this step (e.g., "step_2.png")
-- "reason": Brief explanation of why this step succeeded or failed
-- "final_completion": "True" if the overall task is completed, "False" otherwise
-- "score": Integer from 0 to 10, where 10 means perfectly completed and 0 means not completed at all
+- "reason": Explanation of what is VISUALLY observed in the screenshot as evidence
+- "final_completion": "True" ONLY if the overall task is fully completed with clear visual proof, "False" otherwise
+- "score": Integer from 0 to 10, following the strict scoring guide above
 
-Remember: Return ONLY the JSON object, no additional text."""
+Remember: Return ONLY the JSON object, no additional text. Be STRICT - when in doubt, score LOWER."""
 
     try:
         result = llm.generate_with_images(

evaluation_examples/examples/avogadro/building-metal-complexes_task1.json

@@ -0,0 +1,44 @@
+{
+  "id": "building-metal-complexes_task1",
+  "snapshot": "avogadro",
+  "instruction": "在 Avogadro 2 中，使用 Template Tool 创建 [Co(NH3)6]3+ 配位化合物，设置为八面体配位几何。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\Avogadro2\\bin\\avogadro2.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "avogadro"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 打开 Template Tool（快捷键 Ctrl+3 或点击工具栏图标）。\n2. 切换到 Centers 选项卡。\n3. 输入 'Co' 或从弹出菜单中选择钴元素。\n4. 点击三次 '+' 符号，将正电荷设置为 +3。\n5. 按键 '6' 或选择八面体几何形状。\n6. 点击空白区域，放置钴中心，六个氢原子会显示在配位位置。"
+  }
+}

evaluation_examples/examples/avogadro/building-metal-complexes_task3.json

@@ -0,0 +1,44 @@
+{
+  "id": "building-metal-complexes_task3",
+  "snapshot": "avogadro",
+  "instruction": "在 Avogadro 2 中，使用 Template Tool 创建 [Ni(en)(NH3)2]2+ 配位化合物，设置为平面四方配位几何。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\Avogadro2\\bin\\avogadro2.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "avogadro"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 打开 Template Tool。\n2. 切换到 Centers 选项卡。\n3. 输入 'Ni' 或从弹出菜单中选择镍元素。\n4. 点击两次 '+' 符号，将正电荷设置为 +2。\n5. 按键 '44' 或选择平面四方几何形状。\n6. 点击空白区域，放置镍中心，四个氢原子会显示在配位位置。"
+  }
+}

evaluation_examples/examples/avogadro/building-metal-complexes_task7.json

@@ -0,0 +1,44 @@
+{
+  "id": "building-metal-complexes_task7",
+  "snapshot": "avogadro",
+  "instruction": "在 Avogadro 2 中，创建具有两个环戊二烯基 (Cp) 和两个氯配体的 ZrCp2Cl2 配合物。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\Avogadro2\\bin\\avogadro2.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "avogadro"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 打开 Template Tool，点击 Centers 选项卡。\n2. 输入 'Zr' 或选择锆元素。\n3. 点击四次 '+'，将正电荷设置为 +4。\n4. 按键 '4'，选择四面体几何形状。\n5. 在空白区域放置锆中心。\n6. 切换到 Ligands 选项卡，输入 'cp' 或选择环戊二烯基。\n7. 点击一个氢原子，添加第一个 Cp 配体。\n8. 点击相邻氢，添加第二个 Cp 配体。\n9. 切换到 Draw Tool（快捷键 Ctrl+2）。\n10. 选择 Cl 元素。\n11. 点击两个剩余氢原子，每次点击替换为氯配体。"
+  }
+}

evaluation_examples/examples/avogadro/building-organic-molecules_task1.json

@@ -0,0 +1,44 @@
+{
+  "id": "building-organic-molecules_task1",
+  "snapshot": "avogadro",
+  "instruction": "在 Avogadro 2 中，使用软件的 Build 工具插入一个苯环。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\Avogadro2\\bin\\avogadro2.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "avogadro"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 点击菜单栏 Build(构建) → Insert(插入) → Molecule(分子…)，打开\"插入片段\"对话框。\n2. 在\"筛选\"输入框中输入 benzene（注意：需要先切换到英文输入法再输入）。\n3. 筛选结果会显示一个 aromatics 文件夹（树形结构），需要双击或点击展开该文件夹。\n4. 展开后选中列表中的 benzene.cjson 文件。\n5. 点击\"插入\"按钮将苯环插入到工作区。\n6. 关闭\"插入片段\"对话框，确认苯环已显示在主工作界面中。"
+  }
+}

evaluation_examples/examples/avogadro/building-organic-molecules_task3.json

@@ -0,0 +1,44 @@
+{
+  "id": "building-organic-molecules_task3",
+  "snapshot": "avogadro",
+  "instruction": "在 Avogadro 2 中，在甲苯分子的对位添加一硝基（-NO2），生成 4-硝基甲苯。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\Avogadro2\\bin\\avogadro2.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "avogadro"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 按 'N' 键选择硝基。\n2. 点击甲基对位（苯环上的一个氢原子），将其替换为 -NO2。\n3. 确保分子结构正确。"
+  }
+}

evaluation_examples/examples/avogadro/building-organic-molecules_task4.json

@@ -0,0 +1,44 @@
+{
+  "id": "building-organic-molecules_task4",
+  "snapshot": "avogadro",
+  "instruction": "在 Avogadro 2 中，为甲苯分子执行几何优化。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\Avogadro2\\bin\\avogadro2.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "avogadro"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 按 Ctrl+Alt+O 或点击 Auto Optimize 工具执行几何优化。\n2. 检查分子是否获得合乎逻辑的几何结构。"
+  }
+}

evaluation_examples/examples/avogadro/building-organic-molecules_task5.json

@@ -0,0 +1,44 @@
+{
+  "id": "building-organic-molecules_task5",
+  "snapshot": "avogadro",
+  "instruction": "在 Avogadro 2 中，使用 Draw Tool 创建一个单碳结构，然后添加一个羧基（-COOH）。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\Avogadro2\\bin\\avogadro2.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "avogadro"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 使用 Draw Tool 在界面中绘制一个单碳。\n2. 激活 Template Tool，通过按 Ctrl+3 或点击工具栏上的图标进入 Groups。\n3. 按 'C' 或 'co' 选择羧基。\n4. 点击单碳结构上的一个氢原子，将其替换为羧基。"
+  }
+}

evaluation_examples/examples/avogadro/building-organic-molecules_task9.json

@@ -0,0 +1,44 @@
+{
+  "id": "building-organic-molecules_task9",
+  "snapshot": "avogadro",
+  "instruction": "在 Avogadro 2 中，创建一个 4-甲氧基-3-硝基苯甲酸分子，包含苯环、羧基、硝基和甲氧基。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\Avogadro2\\bin\\avogadro2.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "avogadro"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 插入苯环。\n2. 按 'C' 键选择羧基，并添加到苯环的第 1 个位置。\n3. 按 'N' 键选择硝基，并添加到苯环的第 3 个位置。\n4. 按 'om' 键选择甲氧基，并添加到苯环的第 4 个位置。\n5. 使用优化工具进行几何优化并检查分子是否正确。"
+  }
+}

evaluation_examples/examples/avogadro/naming-a-molecule_task1.json

@@ -0,0 +1,44 @@
+{
+  "id": "naming-a-molecule_task1",
+  "snapshot": "avogadro",
+  "instruction": "在 Avogadro 中通过 Analysis → Properties → Molecular... 查看当前分子的 IUPAC 名称及相关性质。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\Avogadro2\\bin\\avogadro2.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "avogadro"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 打开 Avogadro 软件。\n2. 点击菜单栏中的 Analysis。\n3. 从下拉菜单选择 Properties。\n4. 点击 Molecular...。\n5. 在弹出的 'Molecular Properties' 窗口中查看分子的名字和相关信息，例如分子质量、化学式、原子数和键数。"
+  }
+}

evaluation_examples/examples/avogadro/viewing-electrostatic-potential_task1.json

@@ -0,0 +1,44 @@
+{
+  "id": "viewing-electrostatic-potential_task1",
+  "snapshot": "avogadro",
+  "instruction": "在 Avogadro 中通过 Analyze → Create Surfaces 菜单创建 Van der Waals 表面并设置电荷分布可视化。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\Avogadro2\\bin\\avogadro2.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "avogadro"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 打开 Avogadro 软件并加载目标分子的模型。\n2. 通过菜单栏选择 Analyze → Create Surfaces。\n3. 在弹出的 Create Surfaces 对话框中，将 Surface 设置为 'Van der Waals'。\n4. 将 Color By 设置为 'Electrostatic Potential'。\n5. 选择一个电荷模型（例如 'EEM'）。\n6. 选择色阶为 'Balance'。\n7. 点击 'Calculate' 按钮开始计算表面。\n8. 等待软件完成计算，点击 'Close' 关闭对话框。"
+  }
+}

evaluation_examples/examples/imagej/user-guide_task1.json

@@ -0,0 +1,44 @@
+{
+  "id": "user-guide_task1",
+  "snapshot": "imagej",
+  "instruction": "在 ImageJ 中，通过 File → New → Image 创建一个名为 'Text Image' 的新图像，设置图像类型为 8-bit，背景填充为 White，并设置宽度为 40 像素，高度为 40 像素。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\ImageJ\\ImageJ.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "imagej"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 在菜单栏点击 File → New → Image。\n2. 在弹出的对话框中输入名称 'Text Image'。\n3. 从 Type 下拉菜单中选择 '8-bit'。\n4. 从 Fill With 下拉菜单中选择 'White'。\n5. 在宽度（Width）框中输入 40。\n6. 在高度（Height）框中输入 40。\n7. 点击 OK 按钮完成操作。"
+  }
+}

evaluation_examples/examples/imagej/user-guide_task10.json

@@ -0,0 +1,44 @@
+{
+  "id": "user-guide_task10",
+  "snapshot": "imagej",
+  "instruction": "在 ImageJ 中，通过 Edit → Selection → Restore Selection 恢复之前存储的选区。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\ImageJ\\ImageJ.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "imagej"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 在菜单栏点击 Edit → Selection → Restore Selection。\n2. 在图像上确保选区可见。\n3. 查看并确认选区正确恢复。"
+  }
+}

evaluation_examples/examples/imagej/user-guide_task2.json

@@ -0,0 +1,57 @@
+{
+  "id": "user-guide_task2",
+  "snapshot": "imagej",
+  "instruction": "在 ImageJ 中，通过 Process → Find Maxima 对 blobs.gif 图像执行 Maxima 寻找，设置噪声容忍值为 50，并选择 Output Type 为 'Single Points'。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "upload_file",
+      "parameters": {
+        "files": [
+          {
+            "local_path": "evaluation_examples/data/imagej/blobs.gif",
+            "path": "C:\\Users\\user\\Desktop\\blobs.gif"
+          }
+        ]
+      }
+    },
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\ImageJ\\ImageJ.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "imagej"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [
+      "blobs.gif"
+    ],
+    "steps": "1. 打开 blobs.gif 文件。\n2. 在菜单栏点击 Process → Find Maxima。\n3. 在弹出的对话框中，将 Noise Tolerance 设置为 50。\n4. 从 Output Type 下拉菜单中选择 'Single Points'。\n5. 点击 OK 按钮完成操作。"
+  }
+}

evaluation_examples/examples/imagej/user-guide_task3.json

@@ -0,0 +1,44 @@
+{
+  "id": "user-guide_task3",
+  "snapshot": "imagej",
+  "instruction": "在 ImageJ 中，通过 Plugins → Utilities → Find Commands 查找关键字 'threshold' 的相关命令，显示完整信息并运行 'Adaptive3DThreshold'。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\ImageJ\\ImageJ.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "imagej"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 在菜单栏点击 Plugins → Utilities → Find Commands。\n2. 在弹出的 Command Finder 窗口中输入 'threshold'。\n3. 勾选 'Show full information'。\n4. 在列表中选择 'Adaptive3DThreshold' 并双击运行命令。"
+  }
+}

evaluation_examples/examples/imagej/user-guide_task4.json

@@ -0,0 +1,44 @@
+{
+  "id": "user-guide_task4",
+  "snapshot": "imagej",
+  "instruction": "在 ImageJ 中，通过 Image → Adjust → Threshold 使用 'Default' 自动阈值法对当前图像进行阈值分割，并设置显示模式为 Over/Under。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\ImageJ\\ImageJ.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "imagej"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 在菜单栏点击 Image → Adjust → Threshold。\n2. 在弹出的对话框中，从 Method 下拉菜单选择 'Default'。\n3. 确保 Display 模式设置为 'Over/Under'。\n4. 点击 Apply 按钮完成操作。"
+  }
+}

evaluation_examples/examples/imagej/user-guide_task5.json

@@ -0,0 +1,44 @@
+{
+  "id": "user-guide_task5",
+  "snapshot": "imagej",
+  "instruction": "在 ImageJ 中，通过 Analyze → Tools → Curve Fitting 对数据拟合二次多项式，并将最大迭代次数设为 100。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\ImageJ\\ImageJ.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "imagej"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 在菜单栏点击 Analyze → Tools → Curve Fitting。\n2. 在弹出的对话框中，从 Function 下拉菜单选择 '2nd Degree Polynomial'。\n3. 点击 Fit 按钮。\n4. 在 Simplex Fitting Options 中，将 Maximum number of iterations 设置为 100。\n5. 点击 OK 按钮完成拟合。"
+  }
+}

evaluation_examples/examples/imagej/user-guide_task6.json

@@ -0,0 +1,57 @@
+{
+  "id": "user-guide_task6",
+  "snapshot": "imagej",
+  "instruction": "在 ImageJ 中，通过 Image → Transform → Rotate 90 Degrees Right 旋转 mri-stack.tif 图像 90°。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "upload_file",
+      "parameters": {
+        "files": [
+          {
+            "local_path": "evaluation_examples/data/imagej/mri-stack.tif",
+            "path": "C:\\Users\\user\\Desktop\\mri-stack.tif"
+          }
+        ]
+      }
+    },
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\ImageJ\\ImageJ.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "imagej"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [
+      "mri-stack.tif"
+    ],
+    "steps": "1. 打开 mri-stack.tif 文件。\n2. 在菜单栏点击 Image → Transform → Rotate 90 Degrees Right。\n3. 确保图像正确旋转后保存或查看结果。"
+  }
+}

evaluation_examples/examples/imagej/user-guide_task7.json

@@ -0,0 +1,44 @@
+{
+  "id": "user-guide_task7",
+  "snapshot": "imagej",
+  "instruction": "在 ImageJ 中，通过 Process → Binary → Options 设置黑色背景选项为开启状态并进行预览。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\ImageJ\\ImageJ.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "imagej"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 在菜单栏点击 Process → Binary → Options。\n2. 在弹出的对话框中勾选 'Black Background'。\n3. 点击 Preview 查看效果。\n4. 点击 OK 按钮保存选项。"
+  }
+}

evaluation_examples/examples/imagej/user-guide_task8.json

@@ -0,0 +1,44 @@
+{
+  "id": "user-guide_task8",
+  "snapshot": "imagej",
+  "instruction": "在 ImageJ 中，通过 File → Save As → PNG 保存当前图像为 PNG 格式，设置透明索引为 255。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\ImageJ\\ImageJ.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "imagej"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 在菜单栏点击 File → Save As → PNG。\n2. 在弹出的对话框中，将透明索引设置为 255。\n3. 输入文件名并指定保存路径。\n4. 点击 OK 按钮完成保存。"
+  }
+}

evaluation_examples/examples/imagej/user-guide_task9.json

@@ -0,0 +1,44 @@
+{
+  "id": "user-guide_task9",
+  "snapshot": "imagej",
+  "instruction": "在 ImageJ 中，通过 Analyze → Measure 测量当前选区的面积和灰度值。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\ImageJ\\ImageJ.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "imagej"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 创建或选择一个区域选区。\n2. 在菜单栏点击 Analyze → Measure。\n3. 在弹出的 Results 窗口中查看面积和灰度值等测量结果。"
+  }
+}

evaluation_examples/examples/jade/MDIJade6.5使用手册_task1.json

@@ -0,0 +1,46 @@
+{
+  "id": "MDIJade6.5使用手册_task1",
+  "snapshot": "jade",
+  "instruction": "在 MDI Jade 中通过菜单 File → Patterns 加载衍射数据文件 DEMO001.MDI。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\JADE\\jade 6.5\\MDI Jade 6.5\\jade6.5.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "jade"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [
+      "DEMO001.MDI"
+    ],
+    "steps": "1. 在桌面找到 MDI Jade 图标，双击打开软件。\n2. 点击菜单 File → Patterns。\n3. 在弹出的对话框中选择 DEMO001.MDI 并点击 Open。"
+  }
+}

evaluation_examples/examples/jade/MDIJade6.5使用手册_task10.json

@@ -0,0 +1,44 @@
+{
+  "id": "MDIJade6.5使用手册_task10",
+  "snapshot": "jade",
+  "instruction": "从菜单 Options → Cell Refinement 打开晶胞点阵参数对话框并精修点阵常数。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\JADE\\jade 6.5\\MDI Jade 6.5\\jade6.5.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "jade"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 点击菜单 Options → Cell Refinement。\n2. 在弹出的对话框中检查点阵参数。\n3. 点击 Refine 按钮进行精修。\n4. 检查精修结果并保存。"
+  }
+}

evaluation_examples/examples/jade/MDIJade6.5使用手册_task2.json

@@ -0,0 +1,44 @@
+{
+  "id": "MDIJade6.5使用手册_task2",
+  "snapshot": "jade",
+  "instruction": "在 JADE 中将当前打开的衍射图谱通过 File → Save-Primary Pattern as *.txt 导出为 ASCII 格式，保存为 DEMO001.txt。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\JADE\\jade 6.5\\MDI Jade 6.5\\jade6.5.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "jade"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 点击菜单 File → Save-Primary Pattern as *.txt。\n2. 在弹出的保存对话框中，设置文件名为 DEMO001.txt。\n3. 点击 Save 按钮保存文件。"
+  }
+}

evaluation_examples/examples/jade/MDIJade6.5使用手册_task3.json

@@ -0,0 +1,44 @@
+{
+  "id": "MDIJade6.5使用手册_task3",
+  "snapshot": "jade",
+  "instruction": "使用 Search/Match 功能进行物相检索，并限制元素范围为 Al, Sn, O。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\JADE\\jade 6.5\\MDI Jade 6.5\\jade6.5.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "jade"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 点击菜单 S/M 按钮。\n2. 在 Search/Match 对话框中勾选 Use Chemistry Filter。\n3. 输入限定元素 Al, Sn, O，点击 OK。\n4. 等待物相检索完成，检查结果列表。"
+  }
+}

evaluation_examples/examples/jade/MDIJade6.5使用手册_task4.json

@@ -0,0 +1,44 @@
+{
+  "id": "MDIJade6.5使用手册_task4",
+  "snapshot": "jade",
+  "instruction": "在 JADE 中通过 Options → Report-Peak ID Extended 计算 RIR 方法的物相质量分数，并保存结果为 PDF 格式。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\JADE\\jade 6.5\\MDI Jade 6.5\\jade6.5.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "jade"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 打开菜单 Options → Report-Peak ID Extended。\n2. 确认结果数据，确保内容显示完整。\n3. 点击 Save，选择文件类型为 PDF。\n4. 输入文件名并点击保存。"
+  }
+}

evaluation_examples/examples/jade/MDIJade6.5使用手册_task5.json

@@ -0,0 +1,44 @@
+{
+  "id": "MDIJade6.5使用手册_task5",
+  "snapshot": "jade",
+  "instruction": "通过 Report → Peak Search Report 菜单计算晶粒大小及微观应变，设置 D 值为 1。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\JADE\\jade 6.5\\MDI Jade 6.5\\jade6.5.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "jade"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 点击菜单 Report → Peak Search Report。\n2. 在弹出的对话框中选择 Size/strain 选项。\n3. 设置反卷积参数 D 值为 1。\n4. 点击 Save 按钮保存计算结果。"
+  }
+}

evaluation_examples/examples/jade/MDIJade6.5使用手册_task6.json

@@ -0,0 +1,44 @@
+{
+  "id": "MDIJade6.5使用手册_task6",
+  "snapshot": "jade",
+  "instruction": "通过 File → Save 菜单保存当前仪器半高宽校正曲线到 Si_hw_curve.fwhm。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\JADE\\jade 6.5\\MDI Jade 6.5\\jade6.5.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "jade"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 点击菜单 File。\n2. 选择 Save → FWHM Curve of Peaks。\n3. 在保存对话框中输入文件名 Si_hw_curve.fwhm。\n4. 点击 Save 按钮完成保存。"
+  }
+}

evaluation_examples/examples/jade/MDIJade6.5使用手册_task7.json

@@ -0,0 +1,44 @@
+{
+  "id": "MDIJade6.5使用手册_task7",
+  "snapshot": "jade",
+  "instruction": "调用 Options → D-Spacing 菜单计算已知结构的衍射谱，加权强度公式为 Z = 12。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\JADE\\jade 6.5\\MDI Jade 6.5\\jade6.5.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "jade"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 打开菜单 Options → D-Spacing。\n2. 在计算衍射谱对话框中，设置加权强度公式参数 Z 值为 12。\n3. 点击 Calculate 按钮以生成计算结果。\n4. 检查结果并关闭窗口。"
+  }
+}

evaluation_examples/examples/jade/MDIJade6.5使用手册_task8.json

@@ -0,0 +1,44 @@
+{
+  "id": "MDIJade6.5使用手册_task8",
+  "snapshot": "jade",
+  "instruction": "通过 Options → Calculate Stress 菜单计算残余应力，使用 Fit All 功能拟合曲线。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\JADE\\jade 6.5\\MDI Jade 6.5\\jade6.5.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "jade"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 点击菜单 Options → Calculate Stress。\n2. 在弹出的对话框中，选择以 Fit All 功能拟合所有数据。\n3. 检查拟合结果图。\n4. 点击 Save 按钮以保存拟合结果文件。"
+  }
+}

evaluation_examples/examples/jade/jade_test.json

@@ -1,29 +0,0 @@
-{
- "id": "jade_test",
- "snapshot": "snapshot",
- "instruction": "请打开桌面上的 JADE 6.5 软件",
- "source": "custom",
- "config": [],
- "trajectory": "trajectories/",
- "related_apps": [
-   "jade"
- ],
- "evaluator": {
-   "postconfig": [
-     {
-       "type": "sleep",
-       "parameters": {
-         "seconds": 3
-       }
-     }
-   ],
-   "func": "vllm_eval",
-   "result": {
-     "type": "vm_command_line",
-     "command": "tasklist | findstr /i jade"
-   }
- },
- "proxy": false,
- "fixed_ip": false,
- "possibility_of_env_change": "low"
-}

evaluation_examples/examples/origin/Origin_User_Guide_2025b_E_task1.json

@@ -0,0 +1,57 @@
+{
+  "id": "Origin_User_Guide_2025b_E_task1",
+  "snapshot": "origin",
+  "instruction": "在 Origin 中通过 Data → Connect to File 导入一个本地 Excel 文件 example.xlsx",
+  "source": "custom",
+  "config": [
+    {
+      "type": "upload_file",
+      "parameters": {
+        "files": [
+          {
+            "local_path": "evaluation_examples/data/origin/example.xlsx",
+            "path": "C:\\Users\\user\\Desktop\\example.xlsx"
+          }
+        ]
+      }
+    },
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OriginLab\\Origin2025b\\Origin64.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "origin"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [
+      "example.xlsx"
+    ],
+    "steps": "1. 在 Origin 的主菜单中选择 Data → Connect to File。\n2. 点击 Connect to File 菜单中的按钮。\n3. 选择文件 example.xlsx 并点击 Open。\n4. 数据将被加载到当前的工作表中。"
+  }
+}

evaluation_examples/examples/origin/Origin_User_Guide_2025b_E_task11.json

@@ -0,0 +1,44 @@
+{
+  "id": "Origin_User_Guide_2025b_E_task11",
+  "snapshot": "origin",
+  "instruction": "在 Origin 中通过 Graph → Adding Error Bars 添加误差条到现有图表",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OriginLab\\Origin2025b\\Origin64.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "origin"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 打开一个现有图表并右键点击图表元素。\n2. 选择 Graph → Adding Error Bars。\n3. 选择误差数据列并点击 OK 应用。\n4. 查看图表是否正确添加了误差条。"
+  }
+}

evaluation_examples/examples/origin/Origin_User_Guide_2025b_E_task12.json

@@ -0,0 +1,44 @@
+{
+  "id": "Origin_User_Guide_2025b_E_task12",
+  "snapshot": "origin",
+  "instruction": "在 Origin 中通过 Tools → Pick Data Points 工具拾取数据点并保存到新的工作表中",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OriginLab\\Origin2025b\\Origin64.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "origin"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 打开一个包含数据点的图表。\n2. 在主菜单中选择 Tools → Pick Data Points。\n3. 使用交叉标记在图中选择数据点。\n4. 点击 Done 按钮以保存选择的数据点到新的工作表。"
+  }
+}

evaluation_examples/examples/origin/Origin_User_Guide_2025b_E_task2.json

@@ -0,0 +1,44 @@
+{
+  "id": "Origin_User_Guide_2025b_E_task2",
+  "snapshot": "origin",
+  "instruction": "在 Origin 中通过 View → Formula Bar 打开公式栏，并在公式栏输入 =stdev(B1:B10)",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OriginLab\\Origin2025b\\Origin64.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "origin"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 在主菜单中选择 View → Formula Bar。\n2. 在出现的公式栏中，点击当前单元格内并输入 =stdev(B1:B10)。\n3. 按 Enter 键以应用公式并计算结果。\n4. 检查公式栏输出的结果是否正确。"
+  }
+}

evaluation_examples/examples/origin/Origin_User_Guide_2025b_E_task3.json

@@ -0,0 +1,44 @@
+{
+  "id": "Origin_User_Guide_2025b_E_task3",
+  "snapshot": "origin",
+  "instruction": "在 Origin 中通过 Axis Dialog 修改 X 轴的范围为 20 到 180",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OriginLab\\Origin2025b\\Origin64.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "origin"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 在图层的 X 轴区域右键点击并选择 Axis Dialog。\n2. 在左侧选择 Scale 标签。\n3. 将 From 值修改为 20，将 To 值修改为 180。\n4. 点击 Apply To 按钮以应用更改，然后点击 OK 完成。"
+  }
+}

evaluation_examples/examples/origin/Origin_User_Guide_2025b_E_task4.json

@@ -0,0 +1,44 @@
+{
+  "id": "Origin_User_Guide_2025b_E_task4",
+  "snapshot": "origin",
+  "instruction": "在 Origin 中通过 Graph → Rescale to Show All 重设比例以显示所有数据",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OriginLab\\Origin2025b\\Origin64.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "origin"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 打开一个包含数据的图表。\n2. 在主菜单选择 Graph → Rescale to Show All。\n3. 图表比例重设以显示所有数据点。"
+  }
+}

evaluation_examples/examples/origin/Origin_User_Guide_2025b_E_task5.json

@@ -0,0 +1,44 @@
+{
+  "id": "Origin_User_Guide_2025b_E_task5",
+  "snapshot": "origin",
+  "instruction": "在 Origin 中通过 Tools → Data Slicer 激活数据切片器并设置切片条件为 X=50",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OriginLab\\Origin2025b\\Origin64.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "origin"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 在主菜单中选择 Tools → Data Slicer。\n2. 数据切片器面板将被激活。\n3. 在切片器的条件中选择 X=50 并应用切片。\n4. 图表中将显示切片后的数据点。"
+  }
+}

evaluation_examples/examples/origin/Origin_User_Guide_2025b_E_task8.json

@@ -0,0 +1,44 @@
+{
+  "id": "Origin_User_Guide_2025b_E_task8",
+  "snapshot": "origin",
+  "instruction": "在 Origin 中通过 Worksheet → Convert to Matrix 将活动表格转换成矩阵",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OriginLab\\Origin2025b\\Origin64.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "origin"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 打开一个包含数据的活动表格。\n2. 在主菜单中选择 Worksheet → Convert to Matrix。\n3. 根据对话框选择矩阵转换选项（例如 X Across Columns）。\n4. 点击 OK 完成转换，生成矩阵数据。"
+  }
+}

evaluation_examples/examples/origin/Origin_User_Guide_2025b_E_task9.json

@@ -0,0 +1,44 @@
+{
+  "id": "Origin_User_Guide_2025b_E_task9",
+  "snapshot": "origin",
+  "instruction": "在 Origin 中通过 Object Edit Toolbar 对齐选中的图表对象",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OriginLab\\Origin2025b\\Origin64.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "origin"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 使用鼠标选择需要对齐的对象。\n2. 打开 Object Edit Toolbar。\n3. 点击对齐按钮，例如 Align Left 或 Align Center。\n4. 所选对象将以统一对齐样式排列。"
+  }
+}

evaluation_examples/examples/ovito/animation_task3.json

@@ -0,0 +1,44 @@
+{
+  "id": "animation_task3",
+  "snapshot": "ovito",
+  "instruction": "在 OVITO 中将动画帧数从默认设置改为 10 帧每秒。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OVITO Basic\\ovito.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "ovito"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 在 OVITO 的顶部菜单栏中选择 'Animation settings'。\n2. 在弹出的 'Animation settings' 窗口中，找到 'Frames per second' 输入框。\n3. 将帧速率设置为 10。\n4. 点击 'OK' 以保存更改。"
+  }
+}

evaluation_examples/examples/ovito/aspherical_particles_task1.json

@@ -0,0 +1,44 @@
+{
+  "id": "aspherical_particles_task1",
+  "snapshot": "ovito",
+  "instruction": "在 OVITO 中，为粒子指定球形形状。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OVITO Basic\\ovito.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "ovito"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 打开 OVITO 软件。\n2. 选择 File → New 或打开一个现有的粒子数据文件。\n3. 在 'Pipeline' 界面选择 'Particles' 可视化元素。\n4. 转到 'Particle types' 面板，将粒子形状调整为 Sphere（球形）。\n5. 确认并应用更改，确保形状为球形并更新可视化。"
+  }
+}

evaluation_examples/examples/ovito/clone_pipeline_task1.json

@@ -0,0 +1,44 @@
+{
+  "id": "clone_pipeline_task1",
+  "snapshot": "ovito",
+  "instruction": "在 OVITO 中，通过主工具栏中的 Pipeline 下拉菜单，选择 'Clone current pipeline...' 选项来克隆当前数据通道。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OVITO Basic\\ovito.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "ovito"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 在 OVITO 的主工具栏中找到 'Pipelines' 下拉菜单。\n2. 点击下拉菜单并选择 'Clone current pipeline...'。\n3. 在打开的 'Clone pipeline' 对话框中，选择克隆模式（如 Copy 或 Share）。\n4. 点击 'OK' 按钮完成克隆操作。\n5. 确认在可视化场景中同时显示原始通道和克隆通道的输出。"
+  }
+}

evaluation_examples/examples/ovito/code_generation_task1.json

@@ -0,0 +1,44 @@
+{
+  "id": "code_generation_task1",
+  "snapshot": "ovito",
+  "instruction": "在 OVITO 中，通过 File → Generate Python Script 打开代码生成器窗口。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OVITO Basic\\ovito.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "ovito"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 启动 OVITO 软件。\n2. 点击顶部菜单栏中的 File。\n3. 在下拉菜单中选择 Generate Python Script。\n4. 确保代码生成器窗口正常打开（可见 Python 代码编辑界面）。"
+  }
+}

evaluation_examples/examples/ovito/customize_init_state_task1.json

@@ -0,0 +1,44 @@
+{
+  "id": "customize_init_state_task1",
+  "snapshot": "ovito",
+  "instruction": "在 OVITO 中创建一个名为 defaults.ovito 的文件，以保存空的初始会话状态。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OVITO Basic\\ovito.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "ovito"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 打开 OVITO。\n2. 点击菜单栏中的 File → Save Session State As。\n3. 在弹出的文件保存对话框中，将文件命名为 defaults.ovito。\n4. 确保会话为空（即不包含数据集和管道）。\n5. 点击保存按钮。"
+  }
+}

evaluation_examples/examples/ovito/data_model_task1.json

@@ -0,0 +1,44 @@
+{
+  "id": "data_model_task1",
+  "snapshot": "ovito",
+  "instruction": "在 OVITO 中，通过 Data Inspector 检查导入的粒子属性表，包括 Position 和 Potential Energy 列。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OVITO Basic\\ovito.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "ovito"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 打开 OVITO 软件。\n2. 导入一个包含粒子属性的模拟文件（如 .xyz 格式）。\n3. 点击顶部工具栏中的 Data Inspector 按钮。\n4. 在 Data Inspector 面板查看粒子属性表，包括 Position 和 Potential Energy 列。"
+  }
+}

evaluation_examples/examples/ovito/export_task1.json

@@ -0,0 +1,44 @@
+{
+  "id": "export_task1",
+  "snapshot": "ovito",
+  "instruction": "在 OVITO 中，将当前数据管道导出为粒子及其属性的数据表，保存为文件 particle_data.csv。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OVITO Basic\\ovito.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "ovito"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 在菜单栏中，点击 File → Export File。\n2. 在弹出的对话框中，选择导出格式为 'Table of Particles'.\n3. 指定文件名为 particle_data.csv，并选择保存位置。\n4. 点击 'Save' 按钮以完成导出。"
+  }
+}

evaluation_examples/examples/ovito/marker_particles_task2.json

@@ -0,0 +1,44 @@
+{
+  "id": "marker_particles_task2",
+  "snapshot": "ovito",
+  "instruction": "在 OVITO 中，调整动画播放速度为每秒 15 帧。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OVITO Basic\\ovito.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "ovito"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 在 OVITO 界面上，点击 Animation Settings 按钮（小钟图标）。\n2. 在 Animation Settings 窗口中，找到 'Frames per second'（帧率）选项。\n3. 将 Frames per second 的值设置为 15。\n4. 点击 'OK' 确认设置。"
+  }
+}

evaluation_examples/examples/ovito/miscellaneous_task1.json

@@ -0,0 +1,44 @@
+{
+  "id": "miscellaneous_task1",
+  "snapshot": "ovito",
+  "instruction": "在 OVITO 中，通过 File → Save Session State 保存当前会话为 'session.ovitostate'。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OVITO Basic\\ovito.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "ovito"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 点击菜单栏中的 File。\n2. 选择 Save Session State。\n3. 在弹出的保存对话框中选择目标路径并输入文件名 'session.ovitostate'。\n4. 点击 Save 保存文件。"
+  }
+}

evaluation_examples/examples/ovito/python_extensions_task1.json

@@ -0,0 +1,44 @@
+{
+  "id": "python_extensions_task1",
+  "snapshot": "ovito",
+  "instruction": "在 OVITO Pro 中，通过 Edit → Python Extensions 打开扩展目录。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OVITO Basic\\ovito.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "ovito"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 打开 OVITO Pro 软件。\n2. 点击顶部菜单栏中的 Edit 菜单。\n3. 从下拉菜单中选择 Python Extensions。\n4. 查看扩展目录窗口，确认已打开。"
+  }
+}

evaluation_examples/examples/ovito/remote_file_access_task1.json

@@ -0,0 +1,44 @@
+{
+  "id": "remote_file_access_task1",
+  "snapshot": "ovito",
+  "instruction": "在 OVITO 中，通过 File → Load Remote File 打开远程 SSH 文件 sftp://user@hostname/path/file",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OVITO Basic\\ovito.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "ovito"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 打开 OVITO 软件。\n2. 点击菜单 File → Load Remote File。\n3. 在弹出的对话框中填写 Remote URL 字段，例如：sftp://user@hostname/path/file。\n4. 在 File type 下选择 Auto-detect file format。\n5. 在 SSH connection method 下选择 Integrated client (default)。\n6. 点击 Open 完成连接并加载文件。"
+  }
+}

evaluation_examples/examples/ovito/remote_rendering_task1.json

@@ -0,0 +1,44 @@
+{
+  "id": "remote_rendering_task1",
+  "snapshot": "ovito",
+  "instruction": "在 OVITO Pro 中设置远程渲染任务的导出目录并配置 CPU 核心数。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OVITO Basic\\ovito.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "ovito"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 打开 OVITO Pro 软件。\n2. 点击顶部菜单中的 Utilities 标签。\n3. 选择 Render On Remote Computer 工具。\n4. 在弹出的对话框中，点击 'Choose' 按钮为 Bundle Export Directory 设置一个本地导出目录。\n5. 在 CPU cores per task 选项框中输入渲染任务所需的 CPU 核心数量（可为空，默认使用所有核心）。"
+  }
+}

evaluation_examples/examples/ovito/rendering_task1.json

@@ -0,0 +1,44 @@
+{
+  "id": "rendering_task1",
+  "snapshot": "ovito",
+  "instruction": "在 OVITO 中，通过 Render Settings 面板渲染主动观察窗口为分辨率 1024x768 的图像，背景为透明色。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OVITO Basic\\ovito.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "ovito"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 打开 OVITO 软件。\n2. 确保观察窗口激活（黄色边框）。\n3. 点击右侧命令面板上的 Render 图标。\n4. 在弹出的 Render Settings 面板中，选择 'Single frame'。\n5. 设置输出图像大小为 Width: 1024 和 Height: 768。\n6. 选择背景为 'Transparent'。\n7. 点击 'Render active viewport' 按钮完成渲染。"
+  }
+}

evaluation_examples/examples/ovito/transparent_particles_task1.json

@@ -0,0 +1,44 @@
+{
+  "id": "transparent_particles_task1",
+  "snapshot": "ovito",
+  "instruction": "在软件中，将所有粒子的 Transparency 属性设置为 0.5，使粒子半透明。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OVITO Basic\\ovito.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "ovito"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 打开软件并加载需要的粒子数据。\n2. 插入 Compute 属性修正器到数据管道。\n3. 在 Compute 属性修正器中找到 Transparency 属性。\n4. 在表达式字段中输入透明度值 0.5。\n5. 应用设置，确保 Transparency 属性被分配给所有粒子。"
+  }
+}

evaluation_examples/examples/pymol/MovieSchool_1_task1.json

@@ -0,0 +1,44 @@
+{
+  "id": "MovieSchool_1_task1",
+  "snapshot": "pymol",
+  "instruction": "在 PyMOL 中，通过命令行制作一个简单动画，播放 NMR ensemble。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\PYMOL\\PyMOLWin.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "pymol"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 在 PyMOL 命令行中输入 `fetch 1nmr` 来加载 NMR ensemble。\n2. 输入 `mplay` 命令开始播放动画。\n3. 如果需要停止播放动画，输入 `mstop` 。"
+  }
+}

evaluation_examples/examples/pymol/MovieSchool_1_task2.json

@@ -0,0 +1,44 @@
+{
+  "id": "MovieSchool_1_task2",
+  "snapshot": "pymol",
+  "instruction": "在 PyMOL 中制作一个场景绕 Y 轴 360 度旋转的动画。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\PYMOL\\PyMOLWin.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "pymol"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 从菜单中选择适当选项，用于创建场景绕 Y 轴旋转的动画。\n2. 按下“Pressplay”开始播放动画。"
+  }
+}

evaluation_examples/examples/pymol/MovieSchool_1_task3.json

@@ -0,0 +1,44 @@
+{
+  "id": "MovieSchool_1_task3",
+  "snapshot": "pymol",
+  "instruction": "在 PyMOL 中制作场景摇摆动画，可选择 30, 60, 90, 120 或 180 度摇摆角度。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\PYMOL\\PyMOLWin.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "pymol"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 从菜单中选择用于设置场景摇摆动画的选项。\n2. 选择摇摆角度（例如 30、60、90、120 或 180 度）。\n3. 按下“Pressplay”启动动画。"
+  }
+}

evaluation_examples/examples/pymol/MovieSchool_1_task4.json

@@ -0,0 +1,44 @@
+{
+  "id": "MovieSchool_1_task4",
+  "snapshot": "pymol",
+  "instruction": "在 PyMOL 中制作一个简单的场景“摇摆（Nutate）”动画。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\PYMOL\\PyMOLWin.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "pymol"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 从菜单中选择制作摇摆动画的选项。\n2. 设置摇摆效果参数。\n3. 按下“Pressplay”启动摇摆动画。"
+  }
+}

evaluation_examples/examples/pymol/MovieSchool_1_task5.json

@@ -0,0 +1,44 @@
+{
+  "id": "MovieSchool_1_task5",
+  "snapshot": "pymol",
+  "instruction": "在 PyMOL 中使用 Scene Loop 制作一个从原子缩放并返回的动画。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\PYMOL\\PyMOLWin.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "pymol"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 创建一个场景并设置为缩放到特定原子。\n2. 保存该场景。\n3. 使用 PyMOL 中的 Scene Loop 功能连接多个保存的场景。\n4. 播放动画以观察缩放效果。"
+  }
+}

evaluation_examples/examples/pymol/MovieSchool_3_task1.json

@@ -0,0 +1,44 @@
+{
+  "id": "MovieSchool_3_task1",
+  "snapshot": "pymol",
+  "instruction": "在 PyMOL 中，通过 Movie 菜单添加 2 秒到当前视频的尾部。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\PYMOL\\PyMOLWin.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "pymol"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 点击菜单栏中的 Movie。\n2. 从下拉菜单中选择 Append。\n3. 在 Append 子菜单中选择 2 seconds。"
+  }
+}

evaluation_examples/examples/pymol/MovieSchool_3_task10.json

@@ -0,0 +1,44 @@
+{
+  "id": "MovieSchool_3_task10",
+  "snapshot": "pymol",
+  "instruction": "在 PyMOL 中，通过 ALA Motions 菜单查看 ALA fragment 的运动选项。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\PYMOL\\PyMOLWin.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "pymol"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 在 PyMOL 的底部工具栏上点击 All。\n2. 从下拉菜单中选择 ALA。\n3. 在 ALA 菜单中选择 Motions。\n4. 浏览显示的运动/位置选项。"
+  }
+}

evaluation_examples/examples/pymol/MovieSchool_3_task2.json

@@ -0,0 +1,44 @@
+{
+  "id": "MovieSchool_3_task2",
+  "snapshot": "pymol",
+  "instruction": "在 PyMOL 中，通过 Movie 菜单设置视频帧率为 15 FPS。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\PYMOL\\PyMOLWin.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "pymol"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 点击菜单栏中的 Movie。\n2. 从下拉菜单中选择 Frame Rate。\n3. 在 Frame Rate 子菜单中选择 15 FPS。"
+  }
+}

evaluation_examples/examples/pymol/MovieSchool_3_task3.json

@@ -0,0 +1,44 @@
+{
+  "id": "MovieSchool_3_task3",
+  "snapshot": "pymol",
+  "instruction": "在 PyMOL 中，通过 Scene 菜单将当前场景存储为名称 'my_scene'。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\PYMOL\\PyMOLWin.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "pymol"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 点击菜单栏中的 Scene。\n2. 从下拉菜单中选择 Store。\n3. 在弹出的窗口中输入 'my_scene' 作为场景名称。\n4. 点击确认按钮存储场景。"
+  }
+}

evaluation_examples/examples/pymol/MovieSchool_3_task4.json

@@ -0,0 +1,44 @@
+{
+  "id": "MovieSchool_3_task4",
+  "snapshot": "pymol",
+  "instruction": "在 PyMOL 中，通过 Scene 菜单清除所有存储的场景。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\PYMOL\\PyMOLWin.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "pymol"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 点击菜单栏中的 Scene。\n2. 从下拉菜单中选择 Clear。\n3. 在确认对话框中点击是以清除所有场景。"
+  }
+}

evaluation_examples/examples/pymol/MovieSchool_3_task5.json

@@ -0,0 +1,44 @@
+{
+  "id": "MovieSchool_3_task5",
+  "snapshot": "pymol",
+  "instruction": "在 PyMOL 中，通过 Mouse 菜单将鼠标模式设置为 3 Button Motions。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\PYMOL\\PyMOLWin.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "pymol"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 点击菜单栏中的 Mouse。\n2. 从下拉菜单中选择 Edit。\n3. 在 Edit 菜单中选择 Motions。\n4. 在弹出的子菜单中选择 3 Button Motions。"
+  }
+}

evaluation_examples/examples/pymol/Mutagenesis_task4.json

@@ -0,0 +1,44 @@
+{
+  "id": "Mutagenesis_task4",
+  "snapshot": "pymol",
+  "instruction": "解释 PyMOL Mutagenesis 工具中的颜色代码，以理解范德瓦尔斯半径重叠情况。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\PYMOL\\PyMOLWin.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "pymol"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 打开 PyMOL 软件并加载任意结构。\n2. 通过 Wizard → Mutagenesis 打开 Mutagenesis 工具。\n3. 查看 Mutagenesis 工具中指定区域的颜色提示。\n4. 确认颜色解释：绿色表示轻微重叠，红色表示显著重叠。\n5. 使用颜色信息选择适当的操作来优化结构。"
+  }
+}

evaluation_examples/examples/pymol/Practical_Pymol_for_Beginners_task6.json

@@ -0,0 +1,44 @@
+{
+  "id": "Practical_Pymol_for_Beginners_task6",
+  "snapshot": "pymol",
+  "instruction": "在 PyMOL 中，通过 File → Save Session 保存当前会话为 .pse 文件。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\PYMOL\\PyMOLWin.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "pymol"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 确保所有需要的对象和场景已设置好。\n2. 点击菜单栏 File → Save Session。\n3. 在弹出的窗口中命名文件并保存为 .pse 格式。\n4. 确认会话被成功保存。"
+  }
+}

evaluation_examples/examples/vesta/VESTA_Manual_task1.json

@@ -0,0 +1,57 @@
+{
+  "id": "VESTA_Manual_task1",
+  "snapshot": "vesta",
+  "instruction": "在 VESTA 中启动软件并加载结构文件 example_structure.cif。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "upload_file",
+      "parameters": {
+        "files": [
+          {
+            "local_path": "evaluation_examples/data/vesta/example_structure.cif",
+            "path": "C:\\Users\\user\\Desktop\\example_structure.cif"
+          }
+        ]
+      }
+    },
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\VESTA-win64\\VESTA.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "vesta"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [
+      "example_structure.cif"
+    ],
+    "steps": "1. 启动 VESTA 软件。\n2. 点击 File → Open。\n3. 在文件浏览窗口中选择 example_structure.cif 文件。\n4. 点击 Open 按钮加载文件。\n5. 确认结构已显示在视图窗口中。"
+  }
+}

evaluation_examples/examples/vesta/VESTA_Manual_task10.json

@@ -0,0 +1,57 @@
+{
+  "id": "VESTA_Manual_task10",
+  "snapshot": "vesta",
+  "instruction": "在 VESTA 中导入 Crystallographic Information File (CIF) 并查看其对称性。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "upload_file",
+      "parameters": {
+        "files": [
+          {
+            "local_path": "evaluation_examples/data/vesta/sample.cif",
+            "path": "C:\\Users\\user\\Desktop\\sample.cif"
+          }
+        ]
+      }
+    },
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\VESTA-win64\\VESTA.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "vesta"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [
+      "sample.cif"
+    ],
+    "steps": "1. 启动 VESTA 软件。\n2. 点击 File → Open，打开文件浏览器。\n3. 选择 sample.cif 文件并点击 Open。\n4. 加载文件后，点击 Edit → Data。\n5. 选择 Unit Cell 标签。\n6. 查看 Symmetry 选项卡中显示的对称性信息。"
+  }
+}

evaluation_examples/examples/vesta/VESTA_Manual_task11.json

@@ -0,0 +1,57 @@
+{
+  "id": "VESTA_Manual_task11",
+  "snapshot": "vesta",
+  "instruction": "在 VESTA 中生成 polyhedra 并调整其透明度。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "upload_file",
+      "parameters": {
+        "files": [
+          {
+            "local_path": "evaluation_examples/data/vesta/example_structure.cif",
+            "path": "C:\\Users\\user\\Desktop\\example_structure.cif"
+          }
+        ]
+      }
+    },
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\VESTA-win64\\VESTA.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "vesta"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [
+      "example_structure.cif"
+    ],
+    "steps": "1. 打开 VESTA 软件并加载 example_structure.vesta 文件。\n2. 点击 Edit → Properties。\n3. 在 Properties 对话框中选择 Polyhedra 标签。\n4. 勾选 Enable Polyhedra 绘图。\n5. 调整 Transparency 滑块到所需透明度值，例如 50%。\n6. 点击 OK 按钮保存设置。\n7. 验证主视图窗口中 Polyhedra 的更新显示。"
+  }
+}

evaluation_examples/examples/vesta/VESTA_Manual_task2.json

@@ -0,0 +1,57 @@
+{
+  "id": "VESTA_Manual_task2",
+  "snapshot": "vesta",
+  "instruction": "在 VESTA 中设置显示模式为 Ball-and-Stick，用于 example_structure.cif 文件。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "upload_file",
+      "parameters": {
+        "files": [
+          {
+            "local_path": "evaluation_examples/data/vesta/example_structure.cif",
+            "path": "C:\\Users\\user\\Desktop\\example_structure.cif"
+          }
+        ]
+      }
+    },
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\VESTA-win64\\VESTA.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "vesta"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [
+      "example_structure.cif"
+    ],
+    "steps": "1. 打开 VESTA 软件并加载文件 loaded_structure.vesta。\n2. 在顶部菜单中选择 View → Display Style。\n3. 在弹出的对话框中选择 Ball-and-Stick 模式。\n4. 点击 OK 按钮应用设置。\n5. 查看主视图窗口以确认显示模式已改变。"
+  }
+}

evaluation_examples/examples/vesta/VESTA_Manual_task3.json

@@ -0,0 +1,44 @@
+{
+  "id": "VESTA_Manual_task3",
+  "snapshot": "vesta",
+  "instruction": "在 VESTA 中测量当前晶体中两个原子之间的距离。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\VESTA-win64\\VESTA.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "vesta"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 在 VESTA 软件中打开任何结构文件。\n2. 点击垂直工具栏中的 Measure Distance 工具。\n3. 在主视图窗口中选择两个要测量距离的原子。\n4. 在 Measure Distance 工具下确认显示两个原子之间的距离。\n5. 验证输出的距离值是否正确显示。"
+  }
+}

evaluation_examples/examples/vesta/VESTA_Manual_task4.json

@@ -0,0 +1,57 @@
+{
+  "id": "VESTA_Manual_task4",
+  "snapshot": "vesta",
+  "instruction": "在 VESTA 中定义自定义绘图边界。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "upload_file",
+      "parameters": {
+        "files": [
+          {
+            "local_path": "evaluation_examples/data/vesta/MgB2.cif",
+            "path": "C:\\Users\\user\\Desktop\\MgB2.cif"
+          }
+        ]
+      }
+    },
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\VESTA-win64\\VESTA.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "vesta"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [
+      "MgB2.cif"
+    ],
+    "steps": "1. 打开 VESTA 软件并加载文件 MgB2.cif。\n2. 点击左侧侧边栏的 Objects → Boundary 按钮，打开 Boundary 对话框。\n3. 在对话框中调整范围 (x[min], x[max], y[min], y[max], z[min], z[max]) 为自定义值，例如 0 到 1。\n4. 点击 OK 或 Apply 按钮。\n5. 查看修改后的晶体绘图边界显示在主视图中。"
+  }
+}

evaluation_examples/examples/vesta/VESTA_Manual_task5.json

@@ -0,0 +1,57 @@
+{
+  "id": "VESTA_Manual_task5",
+  "snapshot": "vesta",
+  "instruction": "通过 VESTA 的 Properties 对话框调整晶体键的颜色和半径。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "upload_file",
+      "parameters": {
+        "files": [
+          {
+            "local_path": "evaluation_examples/data/vesta/xTiO2.cif",
+            "path": "C:\\Users\\user\\Desktop\\xTiO2.cif"
+          }
+        ]
+      }
+    },
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\VESTA-win64\\VESTA.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "vesta"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [
+      "xTiO2.cif"
+    ],
+    "steps": "1. 打开 VESTA 软件并加载 xTiO2.vesta 文件。\n2. 点击 Edit → Properties。\n3. 在对话框中导航到 Bonds 页面。\n4. 调整 Radius (cylinder) 输入框值，例如更改为 0.3。\n5. 修改颜色设置为 RGB 值 (100, 150, 200)。\n6. 点击 OK 按钮保存更改并关闭对话框。\n7. 确保更改在主视图中可见。"
+  }
+}

evaluation_examples/examples/vesta/VESTA_Manual_task6.json

@@ -0,0 +1,57 @@
+{
+  "id": "VESTA_Manual_task6",
+  "snapshot": "vesta",
+  "instruction": "在 VESTA 中切换晶体投影为 [110] 方向。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "upload_file",
+      "parameters": {
+        "files": [
+          {
+            "local_path": "evaluation_examples/data/vesta/Si.cif",
+            "path": "C:\\Users\\user\\Desktop\\Si.cif"
+          }
+        ]
+      }
+    },
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\VESTA-win64\\VESTA.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "vesta"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [
+      "Si.cif"
+    ],
+    "steps": "1. 打开 VESTA 软件并加载文件 Si.cif。\n2. 在顶部菜单中选择 View → Lattice Planes。\n3. 在对话框中选择 [110] 方向作为投影。\n4. 点击 OK 按钮应用更改。\n5. 确认主视图窗口中显示的是 [110] 方向的晶体投影。"
+  }
+}

evaluation_examples/examples/vesta/VESTA_Manual_task7.json

@@ -0,0 +1,57 @@
+{
+  "id": "VESTA_Manual_task7",
+  "snapshot": "vesta",
+  "instruction": "在 VESTA 中生成晶体的二维 (2D) 投影视图。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "upload_file",
+      "parameters": {
+        "files": [
+          {
+            "local_path": "evaluation_examples/data/vesta/rutile_TiO2.cif",
+            "path": "C:\\Users\\user\\Desktop\\rutile_TiO2.cif"
+          }
+        ]
+      }
+    },
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\VESTA-win64\\VESTA.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "vesta"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [
+      "rutile_TiO2.cif"
+    ],
+    "steps": "1. 打开 VESTA 软件并加载 rutile_TiO2.cif 文件。\n2. 在顶部菜单中选择 File → Export → 2D Image。\n3. 在弹出的对话框中设置输出格式为 PNG，并选择合适的分辨率 (例如 300 dpi)。\n4. 设置保存路径为桌面并命名文件为 projection.png。\n5. 点击 Save 以导出图像。\n6. 验证桌面的 PNG 文件是否正确生成。"
+  }
+}

evaluation_examples/examples/vesta/VESTA_Manual_task8.json

@@ -0,0 +1,57 @@
+{
+  "id": "VESTA_Manual_task8",
+  "snapshot": "vesta",
+  "instruction": "在 VESTA 中查看倒易晶格的详细几何参数。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "upload_file",
+      "parameters": {
+        "files": [
+          {
+            "local_path": "evaluation_examples/data/vesta/YBa2Cu3O7.cif",
+            "path": "C:\\Users\\user\\Desktop\\YBa2Cu3O7.cif"
+          }
+        ]
+      }
+    },
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\VESTA-win64\\VESTA.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "vesta"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [
+      "YBa2Cu3O7.cif"
+    ],
+    "steps": "1. 打开 VESTA 软件并加载文件 YBa2Cu3O7.vesta。\n2. 在顶部菜单中选择 Edit → Data → Reciprocal Lattice Parameters。\n3. 查看弹出的对话框中的倒易晶格详细数据。\n4. 点击 OK 关闭对话框。\n5. 验证数据是否已在 Text Area 中正确显示。"
+  }
+}

evaluation_examples/examples/vesta/VESTA_Manual_task9.json

@@ -0,0 +1,57 @@
+{
+  "id": "VESTA_Manual_task9",
+  "snapshot": "vesta",
+  "instruction": "在 VESTA 中使用 Fourier Synthesis 生成电子密度图。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "upload_file",
+      "parameters": {
+        "files": [
+          {
+            "local_path": "evaluation_examples/data/vesta/monazite.cif",
+            "path": "C:\\Users\\user\\Desktop\\monazite.cif"
+          }
+        ]
+      }
+    },
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\VESTA-win64\\VESTA.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "vesta"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [
+      "monazite.cif"
+    ],
+    "steps": "1. 打开 VESTA 软件并加载文件 monazite.vesta。\n2. 在顶部菜单中选择 Utilities → Fourier Synthesis。\n3. 在弹出的对话框中设置分辨率值为 0.05。\n4. 点击 Calculate 按钮开始生成电子密度图。\n5. 验证生成的图形是否出现在主视图中。"
+  }
+}

evaluation_examples/extract_instructions_v2.py

@@ -0,0 +1,565 @@
+import os
+import sys
+import asyncio
+import aiohttp
+import base64
+import logging
+from pathlib import Path
+from typing import List, Optional
+import tempfile
+import shutil
+from dataclasses import dataclass
+from datetime import datetime
+import json
+import re
+
+# Configuration
+SCRIPT_DIR = Path(__file__).parent
+PROJECT_ROOT = SCRIPT_DIR.parent
+
+API_BASE_URL = os.getenv("OPENAI_BASE_URL", "https://api.openai.com/v1")
+API_URL = f"{API_BASE_URL}/chat/completions"
+API_KEY = os.getenv("OPENAI_API_KEY")
+MODEL_NAME = os.getenv("EXTRACT_MODEL", "gpt-4o")  # Configurable via env var
+MAX_CONCURRENT_REQUESTS = 5
+
+# Input folder where PDFs/Docs are stored, organized by software name
+# e.g. evaluation_examples/inputs/vesta/tutorial.pdf
+INPUT_FOLDER = PROJECT_ROOT / "evaluation_examples" / "inputs"
+EXAMPLES_FOLDER = PROJECT_ROOT / "evaluation_examples" / "examples"
+TEST_ALL_JSON = PROJECT_ROOT / "evaluation_examples" / "test_all.json"
+
+# Retry configuration
+MAX_RETRY_ATTEMPTS = 3
+RETRY_DELAY = 5
+RETRY_BACKOFF = 2
+
+# Image limit - keep low to avoid 413 payload too large errors
+MAX_IMAGES_PER_REQUEST = 20
+
+# Supported file extensions
+SUPPORTED_EXTENSIONS = {'.docx', '.doc', '.ppt', '.pptx', '.pdf', '.mp4', '.avi', '.mov', '.mkv'}
+
+# Software-specific launch config and snapshot mapping
+# Maps software folder name -> {"snapshot": ..., "config": [...]}
+SOFTWARE_CONFIG = {
+    "avogadro": {
+        "snapshot": "avogadro",
+        "config": [
+            {"type": "launch", "parameters": {"command": ["C:\\Avogadro2\\bin\\avogadro2.exe"]}},
+            {"type": "sleep", "parameters": {"seconds": 5}}
+        ]
+    },
+    "imagej": {
+        "snapshot": "imagej",
+        "config": [
+            {"type": "launch", "parameters": {"command": ["C:\\ImageJ\\ImageJ.exe"]}},
+            {"type": "sleep", "parameters": {"seconds": 5}}
+        ]
+    },
+    "origin": {
+        "snapshot": "origin",
+        "config": [
+            {"type": "launch", "parameters": {"command": ["C:\\OriginLab\\Origin2025b\\Origin64.exe"]}},
+            {"type": "sleep", "parameters": {"seconds": 5}}
+        ]
+    },
+    "ovito": {
+        "snapshot": "ovito",
+        "config": [
+            {"type": "launch", "parameters": {"command": ["C:\\OVITO Basic\\ovito.exe"]}},
+            {"type": "sleep", "parameters": {"seconds": 5}}
+        ]
+    },
+    "pymol": {
+        "snapshot": "pymol",
+        "config": [
+            {"type": "launch", "parameters": {"command": ["C:\\PYMOL\\PyMOLWin.exe"]}},
+            {"type": "sleep", "parameters": {"seconds": 5}}
+        ]
+    },
+    "vesta": {
+        "snapshot": "vesta",
+        "config": [
+            {"type": "launch", "parameters": {"command": ["C:\\VESTA-win64\\VESTA.exe"]}},
+            {"type": "sleep", "parameters": {"seconds": 5}}
+        ]
+    },
+}
+
+# Default config for unknown software
+DEFAULT_SOFTWARE_CONFIG = {
+    "snapshot": "snapshot",
+    "config": []
+}
+
+SYSTEM_PROMPT = """你是一个科研软件 GUI 自动化测试专家。你的任务是从教程文档中提取出多个**具体的、可执行的、可验证的** GUI 操作任务。
+
+## 核心要求
+这些任务将被用于测试 AI Agent 操控桌面软件的能力。每个任务必须足够具体，让 Agent 明确知道要做什么，做完后能通过截图判断是否成功。
+
+## 任务粒度要求（非常重要）
+- **每个任务应该是 3-8 步 GUI 操作就能完成的小任务**
+- **task_goal 必须包含具体的参数值、文件名、菜单路径等细节**
+- **绝对不要写模糊的指令**
+
+### ❌ 错误示例（太模糊）：
+- "Perform phase identification" — Agent 不知道用哪个文件、选什么参数
+- "Export data" — 导出什么格式？保存到哪里？
+- "Calculate crystallite size" — 选哪个峰？什么参数？
+
+### ✅ 正确示例（具体可执行）：
+- "在 ImageJ 中，通过 File → Open 打开桌面上的 cell_image.tif 文件"
+- "在 ImageJ 中，使用 Image → Adjust → Threshold 对当前图像进行阈值分割，选择 Default 方法并点击 Apply"
+- "在 ImageJ 中，通过 Analyze → Measure 测量当前选区的面积和平均灰度值"
+- "在 ImageJ 中，使用 Process → Filters → Gaussian Blur 对图像施加半径为 2.0 像素的高斯模糊"
+- "在 Avogadro 2 中，通过 Build → Insert → Molecule 搜索并插入一个 benzene 分子"
+- "在 VESTA 中通过 File → Open 打开桌面上的 Si.cif 文件，然后将视角旋转到 [110] 方向"
+
+## 输出格式
+返回严格的 JSON 对象：
+{
+    "tasks": [
+        {
+            "task_goal": "一句话具体描述要做什么（包含软件名、菜单路径、文件名、参数值等具体信息）。用中文。",
+            "input_files": ["涉及的文件名列表，如 'sample.raw'。如果不需要输入文件则为空列表 []"],
+            "steps": "详细的 GUI 操作步骤，带编号，用换行分隔"
+        }
+    ]
+}
+
+## 任务提取规则
+1. **独立性**：每个任务都能独立完成（假设软件已打开或从头启动）
+2. **具体性**：task_goal 中必须包含教程中提到的具体文件名、参数值、菜单名称
+3. **可验证性**：完成后应该能从屏幕截图看出任务是否成功（例如：文件已打开、图表已显示、对话框已出现等）
+4. **忠实性**：只描述教程中实际出现的操作，不要编造功能
+5. **数量**：从一份教程中提取 10-15 个不同的任务，覆盖教程的各个章节。优先选择最常用、最有代表性的操作
+6. **软件名称**：task_goal 必须以「在 XXX 中，」开头，明确指出软件名称
+7. **难度分布**：包含简单（2-3步）、中等（4-5步）、较难（6-8步）的任务各占三分之一
+
+"""
+
+# Logging configuration
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(message)s',
+    handlers=[
+        logging.StreamHandler(sys.stdout)
+    ]
+)
+logger = logging.getLogger(__name__)
+
+stats = None # Will be initialized in main
+
+@dataclass
+class ProcessingStats:
+    """Processing statistics tracker"""
+    total_files: int = 0
+    completed_files: int = 0
+    failed_files: int = 0
+    retried_files: int = 0
+    generated_tasks: int = 0
+    start_time: datetime = None
+    failed_list: List[tuple] = None
+
+    def __post_init__(self):
+        if self.start_time is None:
+            self.start_time = datetime.now()
+        if self.failed_list is None:
+            self.failed_list = []
+
+    def add_completed(self, num_tasks=1):
+        self.completed_files += 1
+        self.generated_tasks += num_tasks
+        self._log_progress()
+
+    def add_failed(self, file_path: str, error: str):
+        self.failed_files += 1
+        self.failed_list.append((file_path, error))
+        self._log_progress()
+
+    def add_retry(self):
+        self.retried_files += 1
+
+    def _log_progress(self):
+        processed = self.completed_files + self.failed_files
+        percentage = (processed / self.total_files * 100) if self.total_files > 0 else 0
+        elapsed = (datetime.now() - self.start_time).total_seconds()
+
+        logger.info(f"Progress: {processed}/{self.total_files} ({percentage:.1f}%) | "
+                   f"Tasks Gen: {self.generated_tasks} | Failed: {self.failed_files}")
+
+# -----------------------------------------------------------------------------
+# Dependency Checks & File Conversion (Copied & Adapted from original script)
+# -----------------------------------------------------------------------------
+
+def check_dependencies():
+    """Check and prompt for missing dependencies"""
+    missing = []
+    try: import pdf2image
+    except ImportError: missing.append("pdf2image")
+    try: import PIL
+    except ImportError: missing.append("Pillow")
+    try: import cv2
+    except ImportError: missing.append("opencv-python")
+
+    if not shutil.which("soffice") and not shutil.which("libreoffice"):
+        logger.warning("LibreOffice not detected (needed for .doc/.ppt)")
+
+    if missing:
+        logger.error(f"Missing dependencies: {', '.join(missing)}")
+        return False
+    return True
+
+def convert_pdf_to_images(pdf_path: str) -> List[str]:
+    try:
+        from pdf2image import convert_from_path
+        import io
+
+        # First, get total page count at very low DPI
+        quick_check = convert_from_path(pdf_path, dpi=36, fmt='jpeg')
+        total_pages = len(quick_check)
+        del quick_check
+
+        # For large PDFs: lower DPI + sample pages evenly
+        if total_pages > MAX_IMAGES_PER_REQUEST:
+            dpi = 100  # lower DPI for large docs
+            quality = 80
+            # Sample pages evenly across the document
+            step = total_pages / MAX_IMAGES_PER_REQUEST
+            selected_pages = [int(step * i) + 1 for i in range(MAX_IMAGES_PER_REQUEST)]
+            logger.info(f"Large PDF ({total_pages} pages): sampling {len(selected_pages)} pages at {dpi} DPI")
+            base64_images = []
+            for page_num in selected_pages:
+                imgs = convert_from_path(pdf_path, dpi=dpi, fmt='jpeg',
+                                        first_page=page_num, last_page=page_num)
+                if imgs:
+                    buffer = io.BytesIO()
+                    imgs[0].save(buffer, format='JPEG', quality=quality)
+                    base64_images.append(base64.b64encode(buffer.getvalue()).decode('utf-8'))
+            return base64_images
+        else:
+            # Small PDF: convert all pages at normal quality
+            dpi = 150
+            quality = 90
+            logger.info(f"PDF ({total_pages} pages) at {dpi} DPI")
+            images = convert_from_path(pdf_path, dpi=dpi, fmt='jpeg')
+            base64_images = []
+            for img in images:
+                buffer = io.BytesIO()
+                img.save(buffer, format='JPEG', quality=quality)
+                base64_images.append(base64.b64encode(buffer.getvalue()).decode('utf-8'))
+            return base64_images
+    except Exception as e:
+        logger.error(f"PDF conversion failed: {e}")
+        return []
+
+def convert_office_to_pdf(input_path: str) -> Optional[str]:
+    try:
+        import subprocess
+        temp_dir = tempfile.mkdtemp()
+        soffice_cmd = "soffice" if shutil.which("soffice") else "libreoffice"
+        if not soffice_cmd: return None
+
+        cmd = [soffice_cmd, "--headless", "--convert-to", "pdf", "--outdir", temp_dir, input_path]
+        subprocess.run(cmd, capture_output=True, timeout=60)
+
+        pdf_name = Path(input_path).stem + ".pdf"
+        pdf_path = os.path.join(temp_dir, pdf_name)
+        return pdf_path if os.path.exists(pdf_path) else None
+    except Exception:
+        return None
+
+def extract_video_frames(video_path: str, num_frames=10) -> List[str]:
+    try:
+        import cv2
+        cap = cv2.VideoCapture(video_path)
+        total = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
+        if total == 0: return []
+        indices = [int(total * i / (num_frames + 1)) for i in range(1, num_frames + 1)]
+        frames = []
+        for idx in indices:
+            cap.set(cv2.CAP_PROP_POS_FRAMES, idx)
+            ret, frame = cap.read()
+            if ret:
+                h, w = frame.shape[:2]
+                if w > 1280:
+                    scale = 1280/w
+                    frame = cv2.resize(frame, (1280, int(h*scale)))
+                _, buf = cv2.imencode('.jpg', frame)
+                frames.append(base64.b64encode(buf).decode('utf-8'))
+        cap.release()
+        return frames
+    except Exception:
+        return []
+
+def convert_document_to_images(file_path: str) -> List[str]:
+    path = Path(file_path)
+    ext = path.suffix.lower()
+    if ext == '.pdf':
+        return convert_pdf_to_images(file_path)
+    elif ext in ['.docx', '.doc', '.ppt', '.pptx']:
+        pdf = convert_office_to_pdf(file_path)
+        if pdf:
+            imgs = convert_pdf_to_images(pdf)
+            shutil.rmtree(os.path.dirname(pdf), ignore_errors=True)
+            return imgs
+    elif ext in ['.mp4', '.avi', '.mov', '.mkv']:
+        return extract_video_frames(file_path)
+    return []
+
+# -----------------------------------------------------------------------------
+# API Interaction
+# -----------------------------------------------------------------------------
+
+async def call_multimodal_api(images_b64: List[str], session: aiohttp.ClientSession) -> tuple[str, bool]:
+    messages = [{"role": "system", "content": SYSTEM_PROMPT}]
+
+    content = [{"type": "text", "text": "Analyze these tutorial pages and extract benchmark tasks as JSON."}]
+
+    # Cap images to avoid huge payloads
+    subset_images = images_b64[:MAX_IMAGES_PER_REQUEST]
+    for img in subset_images:
+        content.append({
+            "type": "image_url",
+            "image_url": {"url": f"data:image/jpeg;base64,{img}"}
+        })
+
+    messages.append({"role": "user", "content": content})
+
+    for attempt in range(1, MAX_RETRY_ATTEMPTS + 1):
+        try:
+            headers = {
+                "Authorization": f"Bearer {API_KEY}",
+                "Content-Type": "application/json"
+            }
+            # Add site specific headers if using openrouter or others if needed
+
+            payload = {
+                "model": MODEL_NAME,
+                "messages": messages,
+                "max_tokens": 4096,
+            }
+
+            async with session.post(API_URL, headers=headers, json=payload, timeout=180) as response:
+                if response.status == 200:
+                    res_json = await response.json()
+                    return res_json['choices'][0]['message']['content'], True
+                else:
+                    err = await response.text()
+                    logger.warning(f"API Error ({response.status}): {err}")
+                    if attempt < MAX_RETRY_ATTEMPTS:
+                        await asyncio.sleep(RETRY_DELAY)
+                    else:
+                        return f"API Error: {err}", False
+        except Exception as e:
+            logger.warning(f"Exception: {e}")
+            if attempt < MAX_RETRY_ATTEMPTS:
+                await asyncio.sleep(RETRY_DELAY)
+            else:
+                return str(e), False
+    return "Max retries", False
+
+# -----------------------------------------------------------------------------
+# Main Logic
+# -----------------------------------------------------------------------------
+
+software_tests = {}  # Global dict to track software -> [test_ids]
+FORCE_REGENERATE = False  # Set via --force flag
+
+async def process_file(file_path: str, session: aiohttp.ClientSession, semaphore: asyncio.Semaphore):
+    async with semaphore:
+        file_path_obj = Path(file_path)
+        file_stem = file_path_obj.stem
+
+        # Infer software name from folder structure
+        try:
+            rel_path = file_path_obj.relative_to(INPUT_FOLDER)
+            software_name = rel_path.parts[0] if len(rel_path.parts) > 1 else "unknown"
+        except ValueError:
+            software_name = "unknown"
+
+        # Skip if already processed (check if task1 json exists)
+        existing_task1 = EXAMPLES_FOLDER / software_name / f"{file_stem}_task1.json"
+        if existing_task1.exists() and not FORCE_REGENERATE:
+            logger.info(f"Skipping (already processed): {file_path_obj.name} → use --force to regenerate")
+            # Still register existing tasks in software_tests for test_all.json
+            import glob as g
+            existing_tasks = g.glob(str(EXAMPLES_FOLDER / software_name / f"{file_stem}_task*.json"))
+            for t in existing_tasks:
+                tid = Path(t).stem
+                if software_name not in software_tests:
+                    software_tests[software_name] = []
+                software_tests[software_name].append(tid)
+            stats.add_completed(num_tasks=len(existing_tasks))
+            return
+
+        logger.info(f"Processing: {file_path_obj.name}")
+
+        # 1. Convert to images
+        images = convert_document_to_images(file_path)
+        if not images:
+            stats.add_failed(file_path, "No images extracted")
+            return
+
+        # 2. Call API
+        content, success = await call_multimodal_api(images, session)
+        if not success:
+            stats.add_failed(file_path, content)
+            return
+
+        # 3. Parse JSON
+        try:
+            # Try to find JSON block if mixed with text
+            json_match = re.search(r'\{.*\}', content, re.DOTALL)
+            if json_match:
+                json_str = json_match.group(0)
+            else:
+                json_str = content
+
+            api_result = json.loads(json_str)
+            tasks = api_result.get("tasks", [])
+            if not tasks:
+                logger.warning(f"No tasks found in JSON for {file_path}")
+                return
+
+        except json.JSONDecodeError as e:
+            stats.add_failed(file_path, f"JSON Parse Error: {e}")
+            logger.error(f"Raw content: {content[:200]}...")
+            return
+
+        # 4. Generate Output Files
+        for i, task in enumerate(tasks, 1):
+            test_id = f"{file_stem}_task{i}"
+            output_file = EXAMPLES_FOLDER / software_name / f"{test_id}.json"
+            output_file.parent.mkdir(parents=True, exist_ok=True)
+
+            # Get software-specific config
+            sw_cfg = SOFTWARE_CONFIG.get(software_name, DEFAULT_SOFTWARE_CONFIG)
+
+            # Construct the OSWorld/Jade Benchmark Standard JSON
+            task_json = {
+                "id": test_id,
+                "snapshot": sw_cfg["snapshot"],
+                "instruction": task.get("task_goal", ""),
+                "source": "custom",
+                "config": sw_cfg["config"],
+                "trajectory": "trajectories/",
+                "related_apps": [software_name],
+                "evaluator": {
+                    "postconfig": [
+                        {
+                            "type": "sleep",
+                            "parameters": {
+                                "seconds": 3
+                            }
+                        }
+                    ],
+                    "func": "vllm_eval"
+                    # "result" field is NOT needed for vllm_eval
+                },
+                "proxy": False,
+                "fixed_ip": False,
+                "possibility_of_env_change": "low",
+                "metadata": {
+                    "input_files": task.get("input_files", []),
+                    "steps": task.get("steps", "")
+                }
+            }
+
+            with open(output_file, 'w', encoding='utf-8') as f:
+                json.dump(task_json, f, ensure_ascii=False, indent=2)
+
+            # Register to global index
+            if software_name not in software_tests:
+                software_tests[software_name] = []
+            software_tests[software_name].append(test_id)
+
+        stats.add_completed(num_tasks=len(tasks))
+        logger.info(f"Generated {len(tasks)} tasks for {file_path_obj.name}")
+
+def save_test_all_json():
+    """Update test_all.json with new tests"""
+    test_all_meta_path = Path(TEST_ALL_JSON)
+    existing_data = {}
+    if test_all_meta_path.exists():
+        try:
+            with open(test_all_meta_path, 'r', encoding='utf-8') as f:
+                existing_data = json.load(f)
+        except: pass
+
+    # Merge new tests
+    for software, test_ids in software_tests.items():
+        current_list = existing_data.get(software, [])
+        # Append unique
+        updated_list = sorted(list(set(current_list + test_ids)))
+        existing_data[software] = updated_list
+
+    with open(test_all_meta_path, 'w', encoding='utf-8') as f:
+        json.dump(existing_data, f, ensure_ascii=False, indent=2)
+
+    # Also save a 'test_custom.json' that ONLY contains the softwares we just processed/have in our inputs
+    # This is useful for running ONLY your custom benchmarks without OSWorld defaults
+    custom_data = {}
+
+    # We scan the INPUT_FOLDER to see which softwares are "ours"
+    custom_softwares = set()
+    if INPUT_FOLDER.exists():
+        for item in os.listdir(INPUT_FOLDER):
+            if (INPUT_FOLDER / item).is_dir():
+                custom_softwares.add(item)
+
+    for software in custom_softwares:
+        if software in existing_data:
+            custom_data[software] = existing_data[software]
+
+    test_custom_path = PROJECT_ROOT / "evaluation_examples" / "test_custom.json"
+    with open(test_custom_path, 'w', encoding='utf-8') as f:
+        json.dump(custom_data, f, ensure_ascii=False, indent=2)
+    logger.info(f"Custom test index saved to: {test_custom_path}")
+
+async def main():
+    global stats, FORCE_REGENERATE
+    stats = ProcessingStats()
+
+    # Parse --force flag
+    FORCE_REGENERATE = "--force" in sys.argv
+
+    if not API_KEY:
+        logger.error("OPENAI_API_KEY environment variable not set.")
+        return
+
+    # Check/Create Input Folder
+    if not INPUT_FOLDER.exists():
+        logger.warning(f"Input folder {INPUT_FOLDER} does not exist. Creating it.")
+        INPUT_FOLDER.mkdir(parents=True, exist_ok=True)
+        logger.info(f"Please put software PDF tutorials into subfolders in: {INPUT_FOLDER}")
+        return
+
+    # Find files
+    files = []
+    for root, _, filenames in os.walk(INPUT_FOLDER):
+        for f in filenames:
+            if Path(f).suffix.lower() in SUPPORTED_EXTENSIONS:
+                files.append(os.path.join(root, f))
+
+    stats.total_files = len(files)
+    logger.info(f"Found {len(files)} files in {INPUT_FOLDER}")
+
+    if not files:
+        logger.info("No files to process.")
+        return
+
+    # Process
+    semaphore = asyncio.Semaphore(MAX_CONCURRENT_REQUESTS)
+    async with aiohttp.ClientSession() as session:
+        tasks = [process_file(f, session, semaphore) for f in files]
+        await asyncio.gather(*tasks)
+
+    # Save Index
+    save_test_all_json()
+
+    logger.info("Done.")
+
+if __name__ == "__main__":
+    asyncio.run(main())

evaluation_examples/test_all.json

@@ -389,6 +389,314 @@
     "971cbb5b-3cbf-4ff7-9e24-b5c84fcebfa6"
   ],
   "jade": [
+    "MDIJade6.5使用手册_task1",
+    "MDIJade6.5使用手册_task10",
+    "MDIJade6.5使用手册_task2",
+    "MDIJade6.5使用手册_task3",
+    "MDIJade6.5使用手册_task4",
+    "MDIJade6.5使用手册_task5",
+    "MDIJade6.5使用手册_task6",
+    "MDIJade6.5使用手册_task7",
+    "MDIJade6.5使用手册_task8",
+    "MDIJade6.5使用手册_task9",
     "jade_test"
+  ],
+  "avogadro": [
+    "building-metal-complexes_task1",
+    "building-metal-complexes_task2",
+    "building-metal-complexes_task3",
+    "building-metal-complexes_task4",
+    "building-metal-complexes_task5",
+    "building-metal-complexes_task6",
+    "building-metal-complexes_task7",
+    "building-organic-molecules_task1",
+    "building-organic-molecules_task10",
+    "building-organic-molecules_task2",
+    "building-organic-molecules_task3",
+    "building-organic-molecules_task4",
+    "building-organic-molecules_task5",
+    "building-organic-molecules_task6",
+    "building-organic-molecules_task7",
+    "building-organic-molecules_task8",
+    "building-organic-molecules_task9",
+    "learning-avogadro_task1",
+    "learning-avogadro_task2",
+    "learning-avogadro_task3",
+    "learning-avogadro_task4",
+    "learning-avogadro_task5",
+    "learning-avogadro_task6",
+    "learning-avogadro_task7",
+    "learning-avogadro_task8",
+    "learning-avogadro_task9",
+    "naming-a-molecule_task1",
+    "naming-a-molecule_task2",
+    "using-qtaim-and-wfn_task1",
+    "using-qtaim-and-wfn_task2",
+    "using-qtaim-and-wfn_task3",
+    "viewing-electrostatic-potential_task1",
+    "viewing-electrostatic-potential_task2",
+    "viewing-molecular-orbitals_task1",
+    "viewing-molecular-orbitals_task2",
+    "viewing-molecular-orbitals_task3",
+    "viewing-vibrations_task1",
+    "viewing-vibrations_task2",
+    "viewing-vibrations_task3",
+    "viewing-vibrations_task4",
+    "viewing-vibrations_task5"
+  ],
+  "imagej": [
+    "user-guide_task1",
+    "user-guide_task10",
+    "user-guide_task2",
+    "user-guide_task3",
+    "user-guide_task4",
+    "user-guide_task5",
+    "user-guide_task6",
+    "user-guide_task7",
+    "user-guide_task8",
+    "user-guide_task9"
+  ],
+  "origin": [
+    "Origin_User_Guide_2025b_E_task1",
+    "Origin_User_Guide_2025b_E_task10",
+    "Origin_User_Guide_2025b_E_task11",
+    "Origin_User_Guide_2025b_E_task12",
+    "Origin_User_Guide_2025b_E_task2",
+    "Origin_User_Guide_2025b_E_task3",
+    "Origin_User_Guide_2025b_E_task4",
+    "Origin_User_Guide_2025b_E_task5",
+    "Origin_User_Guide_2025b_E_task6",
+    "Origin_User_Guide_2025b_E_task7",
+    "Origin_User_Guide_2025b_E_task8",
+    "Origin_User_Guide_2025b_E_task9"
+  ],
+  "ovito": [
+    "animation_task1",
+    "animation_task10",
+    "animation_task2",
+    "animation_task3",
+    "animation_task4",
+    "animation_task5",
+    "animation_task6",
+    "animation_task7",
+    "animation_task8",
+    "animation_task9",
+    "aspherical_particles_task1",
+    "aspherical_particles_task10",
+    "aspherical_particles_task2",
+    "aspherical_particles_task3",
+    "aspherical_particles_task4",
+    "aspherical_particles_task5",
+    "aspherical_particles_task6",
+    "aspherical_particles_task7",
+    "aspherical_particles_task8",
+    "aspherical_particles_task9",
+    "clone_pipeline_task1",
+    "clone_pipeline_task2",
+    "clone_pipeline_task3",
+    "clone_pipeline_task4",
+    "clone_pipeline_task5",
+    "clone_pipeline_task6",
+    "clone_pipeline_task7",
+    "clone_pipeline_task8",
+    "code_generation_task1",
+    "code_generation_task2",
+    "code_generation_task3",
+    "code_generation_task4",
+    "code_generation_task5",
+    "code_generation_task6",
+    "code_generation_task7",
+    "code_generation_task8",
+    "customize_init_state_task1",
+    "customize_init_state_task2",
+    "customize_init_state_task3",
+    "customize_init_state_task4",
+    "customize_init_state_task5",
+    "data_model_task1",
+    "data_model_task10",
+    "data_model_task2",
+    "data_model_task3",
+    "data_model_task4",
+    "data_model_task5",
+    "data_model_task6",
+    "data_model_task7",
+    "data_model_task8",
+    "data_model_task9",
+    "export_task1",
+    "export_task2",
+    "export_task3",
+    "export_task4",
+    "export_task5",
+    "import_task1",
+    "import_task10",
+    "import_task2",
+    "import_task3",
+    "import_task4",
+    "import_task5",
+    "import_task6",
+    "import_task7",
+    "import_task8",
+    "import_task9",
+    "marker_particles_task1",
+    "marker_particles_task2",
+    "marker_particles_task3",
+    "marker_particles_task4",
+    "marker_particles_task5",
+    "marker_particles_task6",
+    "marker_particles_task7",
+    "marker_particles_task8",
+    "marker_particles_task9",
+    "miscellaneous_task1",
+    "miscellaneous_task10",
+    "miscellaneous_task2",
+    "miscellaneous_task3",
+    "miscellaneous_task4",
+    "miscellaneous_task5",
+    "miscellaneous_task6",
+    "miscellaneous_task7",
+    "miscellaneous_task8",
+    "miscellaneous_task9",
+    "pipeline_task1",
+    "pipeline_task2",
+    "pipeline_task3",
+    "pipeline_task4",
+    "pipeline_task5",
+    "pipeline_task6",
+    "pipeline_task7",
+    "pipeline_task8",
+    "pipeline_task9",
+    "python_extensions_task1",
+    "python_extensions_task10",
+    "python_extensions_task2",
+    "python_extensions_task3",
+    "python_extensions_task4",
+    "python_extensions_task5",
+    "python_extensions_task6",
+    "python_extensions_task7",
+    "python_extensions_task8",
+    "python_extensions_task9",
+    "remote_file_access_task1",
+    "remote_file_access_task10",
+    "remote_file_access_task2",
+    "remote_file_access_task3",
+    "remote_file_access_task4",
+    "remote_file_access_task5",
+    "remote_file_access_task6",
+    "remote_file_access_task7",
+    "remote_file_access_task8",
+    "remote_file_access_task9",
+    "remote_rendering_task1",
+    "remote_rendering_task2",
+    "remote_rendering_task3",
+    "remote_rendering_task4",
+    "remote_rendering_task5",
+    "remote_rendering_task6",
+    "remote_rendering_task7",
+    "remote_rendering_task8",
+    "remote_rendering_task9",
+    "rendering_task1",
+    "rendering_task2",
+    "rendering_task3",
+    "rendering_task4",
+    "rendering_task5",
+    "rendering_task6",
+    "rendering_task7",
+    "rendering_task8",
+    "rendering_task9",
+    "transparent_particles_task1",
+    "transparent_particles_task2",
+    "turntable_animation_task1",
+    "turntable_animation_task2",
+    "turntable_animation_task3",
+    "turntable_animation_task4",
+    "turntable_animation_task5",
+    "turntable_animation_task6",
+    "viewport_layouts_task1",
+    "viewport_layouts_task10",
+    "viewport_layouts_task2",
+    "viewport_layouts_task3",
+    "viewport_layouts_task4",
+    "viewport_layouts_task5",
+    "viewport_layouts_task6",
+    "viewport_layouts_task7",
+    "viewport_layouts_task8",
+    "viewport_layouts_task9",
+    "viewports_task1",
+    "viewports_task10",
+    "viewports_task11",
+    "viewports_task2",
+    "viewports_task3",
+    "viewports_task4",
+    "viewports_task5",
+    "viewports_task6",
+    "viewports_task7",
+    "viewports_task8",
+    "viewports_task9"
+  ],
+  "pymol": [
+    "Biochemistry_student_intro_task1",
+    "Biochemistry_student_intro_task10",
+    "Biochemistry_student_intro_task2",
+    "Biochemistry_student_intro_task3",
+    "Biochemistry_student_intro_task4",
+    "Biochemistry_student_intro_task5",
+    "Biochemistry_student_intro_task6",
+    "Biochemistry_student_intro_task7",
+    "Biochemistry_student_intro_task8",
+    "Biochemistry_student_intro_task9",
+    "MovieSchool_1_task1",
+    "MovieSchool_1_task2",
+    "MovieSchool_1_task3",
+    "MovieSchool_1_task4",
+    "MovieSchool_1_task5",
+    "MovieSchool_1_task6",
+    "MovieSchool_1_task7",
+    "MovieSchool_3_task1",
+    "MovieSchool_3_task10",
+    "MovieSchool_3_task2",
+    "MovieSchool_3_task3",
+    "MovieSchool_3_task4",
+    "MovieSchool_3_task5",
+    "MovieSchool_3_task6",
+    "MovieSchool_3_task7",
+    "MovieSchool_3_task8",
+    "MovieSchool_3_task9",
+    "Mutagenesis_task1",
+    "Mutagenesis_task2",
+    "Mutagenesis_task3",
+    "Mutagenesis_task4",
+    "Mutagenesis_task5",
+    "Mutagenesis_task6",
+    "Mutagenesis_task7",
+    "Practical_Pymol_for_Beginners_task1",
+    "Practical_Pymol_for_Beginners_task10",
+    "Practical_Pymol_for_Beginners_task11",
+    "Practical_Pymol_for_Beginners_task12",
+    "Practical_Pymol_for_Beginners_task13",
+    "Practical_Pymol_for_Beginners_task2",
+    "Practical_Pymol_for_Beginners_task3",
+    "Practical_Pymol_for_Beginners_task4",
+    "Practical_Pymol_for_Beginners_task5",
+    "Practical_Pymol_for_Beginners_task6",
+    "Practical_Pymol_for_Beginners_task7",
+    "Practical_Pymol_for_Beginners_task8",
+    "Practical_Pymol_for_Beginners_task9",
+    "Visualizing_a_computed_structure_-_a_commented_example_task1",
+    "Visualizing_a_computed_structure_-_a_commented_example_task2",
+    "Visualizing_a_computed_structure_-_a_commented_example_task3",
+    "Visualizing_a_computed_structure_-_a_commented_example_task4"
+  ],
+  "vesta": [
+    "VESTA_Manual_task1",
+    "VESTA_Manual_task10",
+    "VESTA_Manual_task11",
+    "VESTA_Manual_task2",
+    "VESTA_Manual_task3",
+    "VESTA_Manual_task4",
+    "VESTA_Manual_task5",
+    "VESTA_Manual_task6",
+    "VESTA_Manual_task7",
+    "VESTA_Manual_task8",
+    "VESTA_Manual_task9"
   ]
 }

evaluation_examples/test_curated.json

@@ -0,0 +1,93 @@
+{
+  "avogadro": [
+    "building-metal-complexes_task1",
+    "building-metal-complexes_task3",
+    "building-metal-complexes_task7",
+    "building-organic-molecules_task1",
+    "building-organic-molecules_task3",
+    "building-organic-molecules_task4",
+    "building-organic-molecules_task5",
+    "building-organic-molecules_task9",
+    "naming-a-molecule_task1",
+    "viewing-electrostatic-potential_task1"
+  ],
+  "imagej": [
+    "user-guide_task1",
+    "user-guide_task10",
+    "user-guide_task2",
+    "user-guide_task3",
+    "user-guide_task4",
+    "user-guide_task5",
+    "user-guide_task6",
+    "user-guide_task7",
+    "user-guide_task8",
+    "user-guide_task9"
+  ],
+  "jade": [
+    "MDIJade6.5使用手册_task1",
+    "MDIJade6.5使用手册_task10",
+    "MDIJade6.5使用手册_task2",
+    "MDIJade6.5使用手册_task3",
+    "MDIJade6.5使用手册_task4",
+    "MDIJade6.5使用手册_task5",
+    "MDIJade6.5使用手册_task6",
+    "MDIJade6.5使用手册_task7",
+    "MDIJade6.5使用手册_task8"
+
+  ],
+  "origin": [
+    "Origin_User_Guide_2025b_E_task1",
+    "Origin_User_Guide_2025b_E_task11",
+    "Origin_User_Guide_2025b_E_task12",
+    "Origin_User_Guide_2025b_E_task2",
+    "Origin_User_Guide_2025b_E_task3",
+    "Origin_User_Guide_2025b_E_task4",
+    "Origin_User_Guide_2025b_E_task5",
+    "Origin_User_Guide_2025b_E_task8",
+    "Origin_User_Guide_2025b_E_task9"
+  ],
+  "ovito": [
+    "animation_task3",
+    "aspherical_particles_task1",
+    "clone_pipeline_task1",
+    "code_generation_task1",
+    "customize_init_state_task1",
+    "data_model_task1",
+    "export_task1",
+    "marker_particles_task2",
+    "miscellaneous_task1",
+    "python_extensions_task1",
+    "remote_file_access_task1",
+    "remote_rendering_task1",
+    "rendering_task1",
+    "transparent_particles_task1"
+  ],
+  "pymol": [
+    "MovieSchool_1_task1",
+    "MovieSchool_1_task2",
+    "MovieSchool_1_task3",
+    "MovieSchool_1_task4",
+    "MovieSchool_1_task5",
+    "MovieSchool_3_task1",
+    "MovieSchool_3_task10",
+    "MovieSchool_3_task2",
+    "MovieSchool_3_task3",
+    "MovieSchool_3_task4",
+    "MovieSchool_3_task5",
+    "Mutagenesis_task4",
+    "Practical_Pymol_for_Beginners_task6"
+  ],
+  "vesta": [
+    "VESTA_Manual_task1",
+    "VESTA_Manual_task10",
+    "VESTA_Manual_task11",
+    "VESTA_Manual_task2",
+    "VESTA_Manual_task3",
+    "VESTA_Manual_task4",
+    "VESTA_Manual_task5",
+    "VESTA_Manual_task6",
+    "VESTA_Manual_task7",
+    "VESTA_Manual_task8",
+    "VESTA_Manual_task9"
+  ]
+}

evaluation_examples/test_custom.json

@@ -0,0 +1,93 @@
+{
+  "avogadro": [
+    "building-metal-complexes_task1",
+    "building-metal-complexes_task3",
+    "building-metal-complexes_task7",
+    "building-organic-molecules_task1",
+    "building-organic-molecules_task3",
+    "building-organic-molecules_task4",
+    "building-organic-molecules_task5",
+    "building-organic-molecules_task9",
+    "naming-a-molecule_task1",
+    "viewing-electrostatic-potential_task1"
+  ],
+  "imagej": [
+    "user-guide_task1",
+    "user-guide_task10",
+    "user-guide_task2",
+    "user-guide_task3",
+    "user-guide_task4",
+    "user-guide_task5",
+    "user-guide_task6",
+    "user-guide_task7",
+    "user-guide_task8",
+    "user-guide_task9"
+  ],
+  "jade": [
+    "MDIJade6.5使用手册_task1",
+    "MDIJade6.5使用手册_task10",
+    "MDIJade6.5使用手册_task2",
+    "MDIJade6.5使用手册_task3",
+    "MDIJade6.5使用手册_task4",
+    "MDIJade6.5使用手册_task5",
+    "MDIJade6.5使用手册_task6",
+    "MDIJade6.5使用手册_task7",
+    "MDIJade6.5使用手册_task8",
+    "jade_test"
+  ],
+  "origin": [
+    "Origin_User_Guide_2025b_E_task1",
+    "Origin_User_Guide_2025b_E_task11",
+    "Origin_User_Guide_2025b_E_task12",
+    "Origin_User_Guide_2025b_E_task2",
+    "Origin_User_Guide_2025b_E_task3",
+    "Origin_User_Guide_2025b_E_task4",
+    "Origin_User_Guide_2025b_E_task5",
+    "Origin_User_Guide_2025b_E_task8",
+    "Origin_User_Guide_2025b_E_task9"
+  ],
+  "ovito": [
+    "animation_task3",
+    "aspherical_particles_task1",
+    "clone_pipeline_task1",
+    "code_generation_task1",
+    "customize_init_state_task1",
+    "data_model_task1",
+    "export_task1",
+    "marker_particles_task2",
+    "miscellaneous_task1",
+    "python_extensions_task1",
+    "remote_file_access_task1",
+    "remote_rendering_task1",
+    "rendering_task1",
+    "transparent_particles_task1"
+  ],
+  "pymol": [
+    "MovieSchool_1_task1",
+    "MovieSchool_1_task2",
+    "MovieSchool_1_task3",
+    "MovieSchool_1_task4",
+    "MovieSchool_1_task5",
+    "MovieSchool_3_task1",
+    "MovieSchool_3_task10",
+    "MovieSchool_3_task2",
+    "MovieSchool_3_task3",
+    "MovieSchool_3_task4",
+    "MovieSchool_3_task5",
+    "Mutagenesis_task4",
+    "Practical_Pymol_for_Beginners_task6"
+  ],
+  "vesta": [
+    "VESTA_Manual_task1",
+    "VESTA_Manual_task10",
+    "VESTA_Manual_task11",
+    "VESTA_Manual_task2",
+    "VESTA_Manual_task3",
+    "VESTA_Manual_task4",
+    "VESTA_Manual_task5",
+    "VESTA_Manual_task6",
+    "VESTA_Manual_task7",
+    "VESTA_Manual_task8",
+    "VESTA_Manual_task9"
+  ]
+}

evaluation_examples/test_each_domain_a11y_tree.json

@@ -0,0 +1,9 @@
+{
+    "avogadro": ["building-organic-molecules_task1"],
+    "imagej": ["user-guide_task1"],
+    "jade": ["MDIJade6.5使用手册_task1"],
+    "origin": ["Origin_User_Guide_2025b_E_task1"],
+    "ovito": ["animation_task3"],
+    "pymol": ["MovieSchool_1_task1"],
+    "vesta": ["VESTA_Manual_task1"]
+}

evaluation_examples/test_single.json

@@ -0,0 +1 @@
+{"avogadro": ["building-organic-molecules_task1"]}

lib_run_single.py

@@ -9,7 +9,7 @@ from lib_results_logger import log_task_completion
 logger = logging.getLogger("desktopenv.experiment")
 
 
-def run_single_example(agent, env, example, max_steps, instruction, args, example_result_dir, scores):
+def run_single_example(agent, env, example, max_steps, instruction, args, example_result_dir, scores, metadata_steps=""):
     runtime_logger = setup_logger(example, example_result_dir)
 
     # Reset environment first to get fresh VM IP
@@ -22,7 +22,7 @@ def run_single_example(agent, env, example, max_steps, instruction, args, exampl
     # except Exception as e:
     #     agent.reset(vm_ip=env.vm_ip)
 
-    # time.sleep(10) # Wait for the environment to be ready
+    time.sleep(15) # Wait for the environment to be ready (apps like Avogadro need time to fully load)
 
     # get initial observation
     logger.info("Getting initial observation...")
@@ -36,21 +36,22 @@ def run_single_example(agent, env, example, max_steps, instruction, args, exampl
         logger.info(f"Step {step_idx + 1} prediction...")
         response, actions = agent.predict(
             instruction,
-            obs
+            obs,
+            metadata_steps=metadata_steps,
         )
         logger.info(f"Response: {response}")
         logger.info(f"Actions: {actions}")
-        
+
         logger.info(f"Executing actions...")
         for action in actions:
             # Capture the timestamp before executing the action
             action_timestamp = datetime.datetime.now().strftime("%Y%m%d@%H%M%S%f")
             logger.info("Step %d: %s", step_idx + 1, action)
-            
+
             logger.info("执行动作中...")
             obs, reward, done, info = env.step(action, args.sleep_after_execution)
             logger.info("动作执行完成。")
-            
+
             logger.info("Reward: %.2f", reward)
             logger.info("Done: %s", done)
             # Save screenshot and trajectory information
@@ -78,10 +79,10 @@ def run_single_example(agent, env, example, max_steps, instruction, args, exampl
     scores.append(result)
     with open(os.path.join(example_result_dir, "result.txt"), "w", encoding="utf-8") as f:
         f.write(f"{result}\n")
-    
+
     # Log task completion to results.json
     log_task_completion(example, result, example_result_dir, args)
-    
+
     if getattr(args, 'enable_recording', False):
         env.controller.end_recording(os.path.join(example_result_dir, "recording.mp4"))
 
@@ -97,11 +98,11 @@ def run_single_example_human(env, example, max_steps, instruction, args, example
     env.reset(task_config=example)
     time.sleep(60) # Wait for the environment to be ready
     obs = env._get_obs() # Get the initial observation
-    
+
     # Save initial screenshot
     with open(os.path.join(example_result_dir, "initial_state.png"), "wb") as _f:
         _f.write(obs['screenshot'])
-    
+
     # Save trajectory information
     with open(os.path.join(example_result_dir, "traj.jsonl"), "a") as f:
         f.write(json.dumps({
@@ -109,7 +110,7 @@ def run_single_example_human(env, example, max_steps, instruction, args, example
             "initial_state": "initial_state.png"
         }))
         f.write("\n")
-    
+
     # Evaluate the result
     result = env.evaluate(result_dir=example_result_dir)
     logger.info("Result: %.2f", result)
@@ -254,14 +255,14 @@ def run_single_example_opencua(agent, env, example, max_steps, instruction, args
 
         logger.info(f"Got Action: {actions}")
         # Breack if no actions
-        if not actions or len(actions)==0 or actions[0]=="" or actions[0].lower().startswith("error"): 
+        if not actions or len(actions)==0 or actions[0]=="" or actions[0].lower().startswith("error"):
             break
 
         for action in actions:
             # Capture the timestamp before executing the action
             action_timestamp = datetime.datetime.now().strftime("%Y%m%d@%H%M%S")
             logger.info("Step %d: %s", step_idx + 1, action)
-            
+
             obs, reward, done, info = env.step(action, args.sleep_after_execution)
 
             logger.info(f"Action {action} executed, reward: {reward}, done: {done}")
@@ -304,7 +305,7 @@ def run_single_example_autoglm(agent, env, example, max_steps, instruction, args
         agent.reset()
 
     env.reset(task_config=example)
-    
+
     time.sleep(60) # Wait for the environment to be ready
     obs = env._get_obs() # Get the initial observation
     done = False
@@ -339,20 +340,20 @@ def run_single_example_autoglm(agent, env, example, max_steps, instruction, args
                     "screenshot_file": f"step_{step_idx + 1}_{action_timestamp}.png"
                 }))
                 f.write("\n")
-                
+
             if done:
                 logger.info("The episode is done.")
                 break
-        
+
         # Invalid Action
         if not actions:
             obs = env._get_obs() # update observation
-            
+
         step_idx += 1
-    
+
     if not done: # not completed the task yet
         env.action_history.append('FAIL')
-    
+
     result = env.evaluate()
     logger.info("Result: %.2f", result)
     scores.append(result)
@@ -369,7 +370,7 @@ def run_single_example_mano(agent, env, example, max_steps, instruction, args, e
     done = False
     step_idx = 0
     env.controller.start_recording()
-    
+
     with open(os.path.join(example_result_dir, f"step_0.png"),
       "wb") as _f:
         _f.write(obs['screenshot'])
@@ -379,12 +380,12 @@ def run_single_example_mano(agent, env, example, max_steps, instruction, args, e
             obs
         )
         if len(actions) > 1:
-            if (("pyautogui.hotkey('shift')" in actions[0] or "pyautogui.hotkey('ctrl')" in actions[0]) 
+            if (("pyautogui.hotkey('shift')" in actions[0] or "pyautogui.hotkey('ctrl')" in actions[0])
                 and "pyautogui.click" in actions[1]):
                 hotkey_type = 'shift' if "shift" in actions[0] else 'ctrl'
                 action = f"pyautogui.keyDown('{hotkey_type}')\n{actions[1]}\npyautogui.keyUp('{hotkey_type}')"
-                actions = [action]  
-                
+                actions = [action]
+
         for action in actions:
             # Capture the timestamp before executing the action
             action_timestamp = datetime.datetime.now().strftime("%Y%m%d@%H%M%S")
@@ -419,7 +420,7 @@ def run_single_example_mano(agent, env, example, max_steps, instruction, args, e
     with open(os.path.join(example_result_dir, "result.txt"), "w", encoding="utf-8") as f:
         f.write(f"{result}\n")
     env.controller.end_recording(os.path.join(example_result_dir, "recording.mp4"))
-    
+
 def run_single_example_uipath(agent, env, example, max_steps, instruction, args, example_result_dir, scores):
     runtime_logger = setup_logger(example, example_result_dir)
     try:
@@ -485,7 +486,7 @@ logger = logging.getLogger("desktopenv.experiment")
 
 def run_single_example_os_symphony(agent, env, example, max_steps, instruction, args, example_result_dir, scores):
     set_current_result_dir(example_result_dir)
-    
+
     agent.reset(result_dir=example_result_dir)
     env.reset(task_config=example)
     time.sleep(30) # Wait for the environment to be ready
@@ -507,14 +508,14 @@ def run_single_example_os_symphony(agent, env, example, max_steps, instruction,
                 img_name = f"step_{step_idx + 1}_milestone.png"
             else:
                 img_name = f"step_{step_idx + 1}.png"
-                
+
             with open(os.path.join(example_result_dir, img_name),
                       "wb") as _f:
                 _f.write(obs['screenshot'])
             if "coordinates" in response and response["coordinates"]:
                 draw_coordinates(
-                    image_bytes=obs['screenshot'], 
-                    coordinates=response["coordinates"], 
+                    image_bytes=obs['screenshot'],
+                    coordinates=response["coordinates"],
                     save_path=os.path.join(example_result_dir, img_name[:-4] + "_draw.png")
                 )
 
@@ -563,10 +564,10 @@ def run_single_example_evocua(agent, env, example, max_steps, instruction, args,
     Unified run function for EvoCUAAgent (supporting both S1 and S2 modes).
     """
     runtime_logger = setup_logger(example, example_result_dir)
-    
+
     # Reset Environment
     env.reset(task_config=example)
-    
+
     # Reset Agent
     # Handle agent reset signature differences if any
     try:
@@ -587,7 +588,7 @@ def run_single_example_evocua(agent, env, example, max_steps, instruction, args,
         # EvoCUAAgent.predict unified signature: returns (response, actions)
         # It handles both modes internally.
         predict_res = agent.predict(instruction, obs)
-        
+
         # Check return signature logic
         if len(predict_res) == 3:
             # Compatibility with S1 original signature if agent was updated to match
@@ -597,7 +598,7 @@ def run_single_example_evocua(agent, env, example, max_steps, instruction, args,
             info_dict = {}
 
         logger.info(f"Step {step_idx + 1} Actions: {actions}")
-        
+
         # Break if no actions (fail-safe)
         if not actions or (len(actions) == 1 and (actions[0] == "" or "error" in actions[0].lower())):
              # Allow "FAIL" or "DONE" to process through execution loop if agent outputs them as actions
@@ -608,18 +609,18 @@ def run_single_example_evocua(agent, env, example, max_steps, instruction, args,
         for action in actions:
             action_timestamp = datetime.datetime.now().strftime("%Y%m%d@%H%M%S%f")
             logger.info("Executing action: %s", action)
-            
+
             # Execute
             obs, reward, done, info = env.step(action, args.sleep_after_execution)
-            
+
             logger.info("Reward: %.2f", reward)
             logger.info("Done: %s", done)
-            
+
             # Save screenshot
             screenshot_file = f"step_{step_idx + 1}_{action_timestamp}.png"
             with open(os.path.join(example_result_dir, screenshot_file), "wb") as _f:
                 _f.write(obs['screenshot'])
-            
+
             # Log Trajectory
             log_entry = {
                 "step_num": step_idx + 1,
@@ -634,25 +635,25 @@ def run_single_example_evocua(agent, env, example, max_steps, instruction, args,
             # Add natural language info if available (S1 style)
             if info_dict:
                 log_entry["natural_language_action"] = info_dict.get("action")
-            
+
             with open(os.path.join(example_result_dir, "traj.jsonl"), "a", encoding="utf-8") as f:
                 f.write(json.dumps(log_entry, ensure_ascii=False))
                 f.write("\n")
-                
+
             if done:
                 logger.info("The episode is done.")
                 break
-        
+
         step_idx += 1
-        
+
     time.sleep(20) # Wait for environment to settle
     result = env.evaluate(result_dir=example_result_dir)
     logger.info("Result: %.2f", result)
     scores.append(result)
-    
+
     with open(os.path.join(example_result_dir, "result.txt"), "w", encoding="utf-8") as f:
         f.write(f"{result}\n")
-    
+
     log_task_completion(example, result, example_result_dir, args)
 
     env.controller.end_recording(os.path.join(example_result_dir, "recording.mp4"))

mm_agents/accessibility_tree_wrap/heuristic_retrieve.py

@@ -46,6 +46,15 @@ def judge_node(node: ET, platform="ubuntu", check_image=False) -> bool:
         raise ValueError("Invalid platform, must be 'ubuntu' or 'windows'")
 
     keeps: bool = node.tag.startswith("document") \
+                  or node.tag.startswith("sunawt") \
+                  or node.tag.startswith("qt5q") \
+                  or node.tag.startswith("qt6q") \
+                  or node.tag.startswith("ovito") \
+                  or node.tag.startswith("pymol") \
+                  or node.tag.startswith("contentspanel") \
+                  or node.tag.startswith("wx") \
+                  or node.tag.startswith("afx") \
+                  or node.tag.startswith("thunderrt") \
                   or node.tag.endswith("item") \
                   or node.tag.endswith("button") \
                   or node.tag.endswith("heading") \
@@ -58,6 +67,18 @@ def judge_node(node: ET, platform="ubuntu", check_image=False) -> bool:
                   or node.tag.endswith("textfield") \
                   or node.tag.endswith("textarea") \
                   or node.tag.endswith("menu") \
+                  or node.tag.endswith("menuitem") \
+                  or node.tag.endswith("menubar") \
+                  or node.tag.endswith("toolbar") \
+                  or node.tag.endswith("tabitem") \
+                  or node.tag.endswith("treeitem") \
+                  or node.tag.endswith("window") \
+                  or node.tag.endswith("edit") \
+                  or node.tag.endswith("widget") \
+                  or node.tag.endswith("box") \
+                  or node.tag.endswith("dialog") \
+                  or node.tag.endswith("view") \
+                  or node.tag.endswith("frame") \
                   or node.tag in {"alert", "canvas", "check-box"
                       , "combo-box", "entry", "icon"
                       , "image", "paragraph", "scroll-bar"
@@ -66,6 +87,16 @@ def judge_node(node: ET, platform="ubuntu", check_image=False) -> bool:
                       , "netuiribbontab", "start", "trayclockwclass"
                       , "traydummysearchcontrol", "uiimage", "uiproperty"
                       , "uiribboncommandbar"
+                      , "qt5qwindowicon", "textblock", "listview"
+                      , "chrome_widgetwin_1", "chrome_renderwidgethosthwnd"
+                      , "unknown", "pane", "tree", "tab"
+                      , "datagrid", "dataitem", "group"
+                      , "statusbar", "titlebar", "tooltip"
+                      , "toolbarwindow32", "richedit50w"
+                      , "msctls_statusbar32", "qaction"
+                      , "qsplitter", "qsplitterhandle"
+                      , "qtoolbarseparator", "qtextbrowser"
+                      , "qtabbar", "qopenglwidget"
                                   }
     keeps = keeps and (
             platform == "ubuntu"
@@ -83,6 +114,12 @@ def judge_node(node: ET, platform="ubuntu", check_image=False) -> bool:
             and (
                     node.get("name", "") != "" or node.text is not None and len(node.text) > 0 \
                     or check_image and node.get("image", "false") == "true"
+                    # Keep empty input fields (edit/textfield) - they are important interactive elements
+                    # even without name/text (e.g., search boxes, filter inputs)
+                    or node.tag.endswith("edit") or node.tag.endswith("textfield")
+                    or node.tag.endswith("textarea") or node.tag.endswith("textbox")
+                    or node.tag.endswith("searchbox") or node.tag.endswith("combobox")
+                    or node.tag in {"entry", "combo-box", "check-box", "slider"}
             )
 
     coordinates: Tuple[int, int] = eval(node.get("{{{:}}}screencoord".format(_component_ns), "(-1, -1)"))

mm_agents/agent.py

@@ -126,7 +126,7 @@ def linearize_accessibility_tree(accessibility_tree, platform="ubuntu"):
         raise ValueError("Invalid platform, must be 'ubuntu' or 'windows'")
 
     filtered_nodes = filter_nodes(ET.fromstring(accessibility_tree), platform)
-    linearized_accessibility_tree = ["tag\tname\ttext\tclass\tdescription\tposition (top-left x&y)\tsize (w&h)"]
+    linearized_accessibility_tree = ["tag\tname\ttext\tposition (center x&y)\tsize (w&h)\tstates"]
 
     # Linearize the accessibility tree nodes into a table format
     for node in filtered_nodes:
@@ -145,14 +145,36 @@ def linearize_accessibility_tree(accessibility_tree, platform="ubuntu"):
         else:
             text = '""'
 
+        # Compute center coordinates from top-left + size/2
+        coords_str = node.get('{{{:}}}screencoord'.format(_component_ns), "")
+        size_str = node.get('{{{:}}}size'.format(_component_ns), "")
+        if coords_str and size_str:
+            try:
+                cx, cy = coords_str.strip('()').split(', ')
+                sw, sh = size_str.strip('()').split(', ')
+                center_x = int(cx) + int(sw) // 2
+                center_y = int(cy) + int(sh) // 2
+                center_str = "({:d}, {:d})".format(center_x, center_y)
+            except (ValueError, IndexError):
+                center_str = coords_str
+        else:
+            center_str = coords_str
+
+        # Extract useful UI states (expanded/collapsed/checked/selected/focused)
+        state_flags = []
+        for state_name in ["expanded", "collapsed", "checked", "selected", "focused", "pressed"]:
+            val = node.get("{{{:}}}{:}".format(_state_ns, state_name), "")
+            if val == "true":
+                state_flags.append(state_name)
+        states_str = ",".join(state_flags) if state_flags else ""
+
         linearized_accessibility_tree.append(
-            "{:}\t{:}\t{:}\t{:}\t{:}\t{:}\t{:}".format(
+            "{:}\t{:}\t{:}\t{:}\t{:}\t{:}".format(
                 node.tag, node.get("name", ""),
                 text,
-                node.get("{{{:}}}class".format(_attributes_ns), "") if platform == "ubuntu" else node.get("{{{:}}}class".format(class_ns_windows), ""),
-                node.get("{{{:}}}description".format(_attributes_ns), ""),
-                node.get('{{{:}}}screencoord'.format(_component_ns), ""),
-                node.get('{{{:}}}size'.format(_component_ns), "")
+                center_str,
+                size_str,
+                states_str
             )
         )
 
@@ -329,14 +351,16 @@ class PromptAgent:
                 raise ValueError("Invalid action space: " + action_space)
         else:
             raise ValueError("Invalid experiment type: " + observation_type)
-        
+
         self.system_message = self.system_message.format(CLIENT_PASSWORD=self.client_password, SCREEN_WIDTH=self.screen_width, SCREEN_HEIGHT=self.screen_height)
 
-    def predict(self, instruction: str, obs: Dict) -> List:
+    def predict(self, instruction: str, obs: Dict, metadata_steps: str = "") -> List:
         """
         Predict the next action(s) based on the current observation.
         """
         system_message = self.system_message + "\nYou are asked to complete the following task: {}".format(instruction)
+        if metadata_steps:
+            system_message += "\n\nHere are the reference steps from the software tutorial, which may help you complete the task:\n{}".format(metadata_steps)
 
         # Prepare the payload for the API call
         messages = []
@@ -502,9 +526,30 @@ class PromptAgent:
                 ]
             })
         elif self.observation_type == "a11y_tree":
+            # Debug: log raw a11y tree XML to help diagnose missing elements
+            raw_tree = obs["accessibility_tree"]
+            if raw_tree:
+                # Log first 2000 chars of raw XML and count total nodes
+                root = ET.fromstring(raw_tree)
+                all_tags = set()
+                total_nodes = 0
+                for node in root.iter():
+                    all_tags.add(node.tag)
+                    total_nodes += 1
+                logger.info("Raw a11y tree: %d total nodes, unique tags: %s", total_nodes, all_tags)
+                logger.debug("Raw a11y tree XML (first 2000 chars): %s", raw_tree[:2000])
+                # Also log nodes containing 'avogadro' or 'qt5' in their attributes
+                for node in root.iter():
+                    node_str = ET.tostring(node, encoding="unicode")
+                    if 'avogadro' in node_str.lower() or 'qt5' in node_str.lower():
+                        logger.info("Avogadro/Qt5 node: tag=%s, name=%s, visible=%s, enabled=%s",
+                                    node.tag, node.get("name", ""),
+                                    node.get("{https://accessibility.windows.example.org/ns/state}visible", "?"),
+                                    node.get("{https://accessibility.windows.example.org/ns/state}enabled", "?"))
             linearized_accessibility_tree = linearize_accessibility_tree(accessibility_tree=obs["accessibility_tree"],
                                                                          platform=self.platform)
             logger.debug("LINEAR AT: %s", linearized_accessibility_tree)
+            logger.info("Linearized a11y tree lines: %d", len(linearized_accessibility_tree.split('\n')) if linearized_accessibility_tree else 0)
 
             if linearized_accessibility_tree:
                 linearized_accessibility_tree = trim_accessibility_tree(linearized_accessibility_tree,
@@ -670,7 +715,7 @@ class PromptAgent:
                 return response.json()['choices'][0]['message']['content']
         elif self.model.startswith("gpt"):
             # Support custom OpenAI base URL via environment variable
-            base_url = os.environ.get('OPENAI_BASE_URL', 'https://api.openai.com')
+            base_url = os.environ.get('OPENAI_BASE_URL', os.environ.get('OPENAI_API_BASE', 'https://api.openai.com'))
             # Smart handling: avoid duplicate /v1 if base_url already ends with /v1
             api_url = f"{base_url}/chat/completions" if base_url.endswith('/v1') else f"{base_url}/v1/chat/completions"
             headers = {
@@ -1153,7 +1198,7 @@ class PromptAgent:
             except Exception as e:
                 print("Failed to call LLM: " + str(e))
                 return ""
-        
+
         else:
             raise ValueError("Invalid model: " + self.model)
 
@@ -1192,4 +1237,4 @@ class PromptAgent:
 
         self.thoughts = []
         self.actions = []
-        self.observations = []
+        self.observations = []

mm_agents/prompts.py

@@ -15,6 +15,8 @@ When you think you have to wait for some time, return ```WAIT```;
 When you think the task can not be done, return ```FAIL```, don't easily say ```FAIL```, try your best to do the task;
 When you think the task is done, return ```DONE```.
 
+IMPORTANT: You must return EITHER a python code block OR a special code (WAIT/DONE/FAIL) in each response, NEVER BOTH. Do NOT include ```DONE``` in the same response as a ```python``` code block. After executing an action, wait for the next observation to verify the result before returning ```DONE```.
+
 My computer's password is '{{CLIENT_PASSWORD}}', feel free to use it when you need sudo rights.
 First give the current screenshot and previous things we did a short reflection, then RETURN ME THE CODE OR SPECIAL CODE I ASKED FOR. NEVER EVER RETURN ME ANYTHING ELSE.
 """.strip()
@@ -36,6 +38,8 @@ When you think you have to wait for some time, return ```WAIT```;
 When you think the task can not be done, return ```FAIL```, don't easily say ```FAIL```, try your best to do the task;
 When you think the task is done, return ```DONE```.
 
+IMPORTANT: You must return EITHER a python code block OR a special code (WAIT/DONE/FAIL) in each response, NEVER BOTH. Do NOT include ```DONE``` in the same response as a ```python``` code block. After executing an action, wait for the next observation to verify the result before returning ```DONE```.
+
 My computer's password is '{{CLIENT_PASSWORD}}', feel free to use it when you need sudo rights.
 Our past communication is great, and what you have done is very helpful. I will now give you another task to complete.
 First take a deep breath, think step by step, give the current screenshot a thinking, then RETURN ME THE CODE OR SPECIAL CODE I ASKED FOR. NEVER EVER RETURN ME ANYTHING ELSE.
@@ -550,6 +554,8 @@ When you think you have to wait for some time, return ```WAIT```;
 When you think the task can not be done, return ```FAIL```, don't easily say ```FAIL```, try your best to do the task;
 When you think the task is done, return ```DONE```.
 
+IMPORTANT: You must return EITHER a python code block OR a special code (WAIT/DONE/FAIL) in each response, NEVER BOTH. Do NOT include ```DONE``` in the same response as a ```python``` code block. After executing an action, wait for the next observation to verify the result before returning ```DONE```.
+
 My computer's password is '{{CLIENT_PASSWORD}}', feel free to use it when you need sudo rights.
 First give the current screenshot and previous things we did a short reflection, then RETURN ME THE CODE OR SPECIAL CODE I ASKED FOR. NEVER EVER RETURN ME ANYTHING ELSE.
 """.strip()
@@ -817,6 +823,8 @@ When you think you have to wait for some time, return ```WAIT```;
 When you think the task can not be done, return ```FAIL```, don't easily say ```FAIL```, try your best to do the task;
 When you think the task is done, return ```DONE```.
 
+IMPORTANT: You must return EITHER a python code block OR a special code (WAIT/DONE/FAIL) in each response, NEVER BOTH. Do NOT include ```DONE``` in the same response as a ```python``` code block. After executing an action, wait for the next observation to verify the result before returning ```DONE```.
+
 My computer's password is '{{CLIENT_PASSWORD}}', feel free to use it when you need sudo rights.
 First give the current screenshot and previous things we did a short reflection, then RETURN ME THE CODE OR SPECIAL CODE I ASKED FOR. NEVER EVER RETURN ME ANYTHING ELSE.
 """.strip()

run.py

@@ -142,6 +142,7 @@ def test(args: argparse.Namespace, test_all_meta: dict) -> None:
     }
 
     agent = PromptAgent(
+        platform="windows",
         model=args.model,
         max_tokens=args.max_tokens,
         top_p=args.top_p,
@@ -197,8 +198,11 @@ def test(args: argparse.Namespace, test_all_meta: dict) -> None:
             logger.info(f"[Example ID]: {example_id}")
 
             instruction = example["instruction"]
+            metadata_steps = example.get("metadata", {}).get("steps", "")
 
             logger.info(f"[Instruction]: {instruction}")
+            if metadata_steps:
+                logger.info(f"[Metadata Steps]: {metadata_steps}")
             # wandb each example config settings
             cfg_args["instruction"] = instruction
             cfg_args["start_time"] = datetime.datetime.now().strftime(
@@ -226,6 +230,7 @@ def test(args: argparse.Namespace, test_all_meta: dict) -> None:
                     args,
                     example_result_dir,
                     scores,
+                    metadata_steps=metadata_steps,
                 )
             except Exception as e:
                 logger.error(f"Exception in {domain}/{example_id}: {e}")
@@ -321,8 +326,19 @@ def get_result(action_space, use_model, observation_type, result_dir, total_file
 if __name__ == "__main__":
     ####### The complete version of the list of examples #######
     os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+    # Bypass proxy for local/VM IPs so Clash global mode doesn't break VM communication
+    no_proxy = os.environ.get("NO_PROXY", os.environ.get("no_proxy", ""))
+    extra = "localhost,127.0.0.1,192.168.*,10.*,172.16.*,172.17.*,172.18.*,172.19.*,172.20.*,172.21.*,172.22.*,172.23.*,172.24.*,172.25.*,172.26.*,172.27.*,172.28.*,172.29.*,172.30.*,172.31.*"
+    if no_proxy:
+        no_proxy = f"{no_proxy},{extra}"
+    else:
+        no_proxy = extra
+    os.environ["NO_PROXY"] = no_proxy
+    os.environ["no_proxy"] = no_proxy
+
     args = config()
-    
+
     # save args to json in result_dir/action_space/observation_type/model/args.json
     path_to_args = os.path.join(
         args.result_dir,