refine jade metadata steps: add shortcuts & merge menu operations to avoid timeout

- 打通 metadata.steps 传递链路，将任务步骤注入 agent 预测上下文

- 优化 a11y tree 线性化输出：使用中心坐标并新增 states 列（expanded/collapsed/selected 等）

- 放宽可保留节点条件，保留无文本输入类控件（edit/textfield/searchbox 等）

- 强化输出约束：单轮仅允许动作代码或 WAIT/DONE/FAIL，禁止动作与 DONE 同轮返回

- 补充 avogadro 示例步骤：展开 aromatics 并选择 benzene.cjson

.gitignore

@@ -208,7 +208,17 @@ quick_start.py
 result_multi_apps_pengxiang_transformers12evaluation_examples/settings/proxy/dataimpulse.json
 evaluation_examples/settings/proxy/dataimpulse.json
 
+# Benchmark input data (large binary files - share via cloud storage or Git LFS)
+evaluation_examples/inputs/
+
+# Temporary data processing workspace (scraped docs, intermediate scripts)
+evaluation_examples/sandbox/
+
+# Image cache
+evaluation_examples/inputs/.img_cache/
+
 # Local test configurations (not for public repo)
 evaluation_examples/spiderman.json
 evaluation_examples/test_50_random_proportional.json
 evaluation_examples/test_chrome.json
+evaluation_examples/prepare_input_files.py

desktop_env/desktop_env.py

@@ -20,42 +20,42 @@ Metric = Callable[[Any, Any], float]
 Getter = Callable[[gym.Env, Dict[str, Any]], Any]
 
 MAX_RETRIES = 5 # Maximum retries for environment setup
-            
+
 
 
 def _fix_pyautogui_less_than_bug(command: str) -> str:
     """
     Fix PyAutoGUI '<' character bug by converting it to hotkey("shift", ',') calls.
-    
+
     This fixes the known PyAutoGUI issue where typing '<' produces '>' instead.
     References:
     - https://github.com/asweigart/pyautogui/issues/198
     - https://github.com/xlang-ai/OSWorld/issues/257
-    
+
     Args:
         command (str): The original pyautogui command
-        
+
     Returns:
         str: The fixed command with '<' characters handled properly
     """
-    # Pattern to match press('<') or press('\u003c') calls  
+    # Pattern to match press('<') or press('\u003c') calls
     press_pattern = r'pyautogui\.press\(["\'](?:<|\\u003c)["\']\)'
 
     # Handle press('<') calls
     def replace_press_less_than(match):
         return 'pyautogui.hotkey("shift", ",")'
-    
+
     # First handle press('<') calls
     command = re.sub(press_pattern, replace_press_less_than, command)
 
     # Pattern to match typewrite calls with quoted strings
     typewrite_pattern = r'pyautogui\.typewrite\((["\'])(.*?)\1\)'
-    
+
     # Then handle typewrite calls
     def process_typewrite_match(match):
         quote_char = match.group(1)
         content = match.group(2)
-        
+
         # Preprocess: Try to decode Unicode escapes like \u003c to actual '<'
         # This handles cases where '<' is represented as escaped Unicode
         try:
@@ -65,15 +65,15 @@ def _fix_pyautogui_less_than_bug(command: str) -> str:
         except UnicodeDecodeError:
             # If decoding fails, proceed with original content to avoid breaking existing logic
             pass  # English comment: Graceful degradation - fall back to original content if decoding fails
-        
+
         # Check if content contains '<'
         if '<' not in content:
             return match.group(0)
-        
+
         # Split by '<' and rebuild
         parts = content.split('<')
         result_parts = []
-        
+
         for i, part in enumerate(parts):
             if i == 0:
                 # First part
@@ -84,11 +84,11 @@ def _fix_pyautogui_less_than_bug(command: str) -> str:
                 result_parts.append('pyautogui.hotkey("shift", ",")')
                 if part:
                     result_parts.append(f"pyautogui.typewrite({quote_char}{part}{quote_char})")
-        
+
         return '; '.join(result_parts)
-    
+
     command = re.sub(typewrite_pattern, process_typewrite_match, command)
-    
+
     return command
 
 
@@ -111,6 +111,7 @@ class DesktopEnv(gym.Env):
             os_type: str = "Ubuntu",
             enable_proxy: bool = False,
             client_password: str = "",
+            eval_model: str = "gpt-5.2-chat-latest"
     ):
         """
         Args:
@@ -127,6 +128,7 @@ class DesktopEnv(gym.Env):
             require_terminal (bool): whether to require terminal output
             os_type (str): operating system type, default to "Ubuntu"
             enable_proxy (bool): whether to enable proxy support, default to False
+            eval_model (str): evaluation model to use, default to "gpt-5.2-chat-latest"
         """
         # Initialize VM manager and vitualization provider
         self.region = region
@@ -143,12 +145,12 @@ class DesktopEnv(gym.Env):
         self.screen_width = screen_size[0]
         self.screen_height = screen_size[1]
 
-        # Default 
+        # Default
         self.server_port = 5000
         self.chromium_port = 9222
         self.vnc_port = 8006
         self.vlc_port = 8080
-        
+
         # Initialize with default (no proxy) provider
         self.current_use_proxy = False
         self.manager, self.provider = create_vm_manager_and_provider(provider_name, region, use_proxy=False)
@@ -171,7 +173,7 @@ class DesktopEnv(gym.Env):
                 if provider_name in {"vmware", "virtualbox"} else path_to_vm
         else:
             self.path_to_vm = self.manager.get_vm_path(os_type=self.os_type, region=region, screen_size=(self.screen_width, self.screen_height))
-        
+
         self.snapshot_name = snapshot_name
         self.cache_dir_base: str = cache_dir
         # todo: add the logic to get the screen size from the VM
@@ -179,6 +181,9 @@ class DesktopEnv(gym.Env):
         self.require_a11y_tree = require_a11y_tree
         self.require_terminal = require_terminal
 
+        # Evaluation model
+        self.eval_model = eval_model
+
         # Initialize emulator and controller
         logger.info("Initializing...")
         self._start_emulator()
@@ -224,8 +229,8 @@ class DesktopEnv(gym.Env):
         # due to the fact it could be changed when implemented by cloud services
         path_to_vm = self.provider.revert_to_snapshot(self.path_to_vm, self.snapshot_name)
         if path_to_vm and not path_to_vm == self.path_to_vm:
-            # path_to_vm has to be a new path 
-            
+            # path_to_vm has to be a new path
+
             self.manager.delete_vm(self.path_to_vm, self.region)
             self.manager.add_vm(path_to_vm, self.region)
             self.manager.occupy_vm(path_to_vm, os.getpid(), self.region)
@@ -240,7 +245,7 @@ class DesktopEnv(gym.Env):
         self.provider.stop_emulator(self.path_to_vm)
 
     def reset(self, task_config: Optional[Dict[str, Any]] = None, seed=None, options=None) -> Dict[str, Any]:
-        
+
         # Reset to certain task in OSWorld
         logger.info("Resetting environment...")
         logger.info("Switching task...")
@@ -253,17 +258,17 @@ class DesktopEnv(gym.Env):
             # Only revert to snapshot if environment has been used (step/setup)
             # This optimization is especially important for cloud providers like AWS
             # where unnecessary snapshot operations are costly and time-consuming
-            
+
             if task_config is not None:
                 # Only consider task proxy requirement if proxy is enabled at system level
                 task_use_proxy = task_config.get("proxy", False) and self.enable_proxy
                 if not self.enable_proxy and task_config.get("proxy", False):
                     logger.info("Task requires proxy but proxy is disabled at system level, ignoring proxy requirement.")
-                
+
                 if task_use_proxy != self.current_use_proxy:
                     # keep because get_info_from_website depend on this
                     self.current_use_proxy = task_use_proxy
-            
+
             if self.is_environment_used:
                 logger.info("Environment has been used, reverting to snapshot: {}...".format(self.snapshot_name))
                 self._revert_to_snapshot()
@@ -297,7 +302,7 @@ class DesktopEnv(gym.Env):
                     time.sleep(5)
             else:
                 break
-            
+
         logger.info("Environment setup complete.")
 
         observation = self._get_obs()
@@ -328,7 +333,8 @@ class DesktopEnv(gym.Env):
         os.makedirs(self.cache_dir, exist_ok=True)
         self.instruction = task_config["instruction"]
         self.config = task_config["config"] if "config" in task_config else []
-        
+        self.metadata = task_config.get("metadata", {})
+
         self._set_evaluator_info(task_config)
 
     def _set_evaluator_info(self, task_config: Dict[str, Any]):
@@ -381,7 +387,7 @@ class DesktopEnv(gym.Env):
     def step(self, action, pause=2):
         self._step_no += 1
         self.action_history.append(action)
-        
+
         # Mark environment as used when step is called
         self.is_environment_used = True
 
@@ -425,7 +431,7 @@ class DesktopEnv(gym.Env):
 
         return observation, reward, done, info
 
-    def evaluate(self):
+    def evaluate(self, result_dir: Optional[str] = None):
         """
         Evaluate whether the task is successfully completed.
         """
@@ -448,6 +454,24 @@ class DesktopEnv(gym.Env):
                 if last_action == "FAIL" or (type(last_action) == dict and last_action.get('action_type') == 'FAIL'):
                     return 0
 
+        if self.evaluator['func'] == "vllm_eval":
+            logger.info("Preparing vllm_eval metric options...")
+            screenshot_bytes = self.controller.get_screenshot()
+
+            import base64
+            self.metric_options["instruction"] = self.instruction
+            self.metric_options["eval_model"] = self.eval_model
+
+            # Pass pre-configured environment info and expected steps
+            self.metric_options["config"] = self.config
+            self.metric_options["metadata"] = self.metadata
+
+            if result_dir:
+                self.metric_options["result_dir"] = result_dir
+                logger.info(f"Using result_dir for vllm_eval: {result_dir}")
+
+            logger.info(f"Evaluation options prepared: {self.metric_options.keys()}")
+
         if type(self.metric) == list:
             # Multiple metrics to evaluate whether the task is successfully completed
             results = []
@@ -455,13 +479,18 @@ class DesktopEnv(gym.Env):
             if "expected" in self.evaluator:
                 assert len(self.metric) == len(self.expected_getter), "The number of metrics and expected getters must be the same"
             for idx, metric in enumerate(self.metric):
-                try:
-                    config = self.evaluator["result"][idx]
-                    result_state = self.result_getter[idx](self, config)
-                except FileNotFoundError:
-                    logger.error("File not found!")
-                    if self.metric_conj == 'and':
-                        return 0
+                # Skip result state extraction if result_getter is None (e.g., for vllm_eval)
+                if self.result_getter[idx] is not None:
+                    try:
+                        config = self.evaluator["result"][idx]
+                        result_state = self.result_getter[idx](self, config)
+                    except FileNotFoundError:
+                        logger.error("File not found!")
+                        if self.metric_conj == 'and':
+                            return 0
+                else:
+                    # For evaluators that don't need result state (e.g., vllm_eval)
+                    result_state = None
 
                 if "expected" in self.evaluator and self.expected_getter and self.evaluator["expected"]:
                     expected_state = self.expected_getter[idx](self, self.evaluator["expected"][idx])
@@ -479,11 +508,16 @@ class DesktopEnv(gym.Env):
             return sum(results) / len(results) if self.metric_conj == 'and' else max(results)
         else:
             # Single metric to evaluate whether the task is successfully completed
-            try:
-                result_state = self.result_getter(self, self.evaluator["result"])
-            except FileNotFoundError:
-                logger.error("File not found!")
-                return 0
+            # For evaluators like vllm_eval that don't need result_getter, skip result state extraction
+            if self.result_getter is not None:
+                try:
+                    result_state = self.result_getter(self, self.evaluator["result"])
+                except FileNotFoundError:
+                    logger.error("File not found!")
+                    return 0
+            else:
+                # For evaluators that don't need result state (e.g., vllm_eval)
+                result_state = None
 
             if "expected" in self.evaluator and self.expected_getter and self.evaluator["expected"]:
                 expected_state = self.expected_getter(self, self.evaluator["expected"])

desktop_env/evaluators/metrics/__init__.py

@@ -158,3 +158,5 @@ from .vscode import (
 
 def infeasible():
     pass
+
+from .vllm_eval import vllm_eval

desktop_env/evaluators/metrics/vllm_eval.py

@@ -0,0 +1,600 @@
+import os
+from typing import Optional, List, Dict, Any
+from dotenv import load_dotenv
+import logging
+import base64
+import glob
+from io import BytesIO
+from PIL import Image
+
+logger = logging.getLogger("desktopenv.vllm_eval")
+load_dotenv()
+
+
+def _compress_image(img_b64: str, max_size: int = 800, quality: int = 85) -> str:
+    """
+    Compress base64 encoded image to reduce size
+
+    Args:
+        img_b64: Base64 encoded image string
+        max_size: Maximum dimension (width or height) in pixels
+        quality: JPEG quality (1-100), lower means smaller file size
+
+    Returns:
+        Compressed base64 encoded image string
+    """
+    try:
+        # Decode base64 to image
+        img_data = base64.b64decode(img_b64)
+        img = Image.open(BytesIO(img_data))
+
+        # Convert to RGB if necessary (for PNG with transparency)
+        if img.mode in ('RGBA', 'LA', 'P'):
+            background = Image.new('RGB', img.size, (255, 255, 255))
+            if img.mode == 'P':
+                img = img.convert('RGBA')
+            background.paste(img, mask=img.split()[-1] if img.mode in ('RGBA', 'LA') else None)
+            img = background
+
+        # Resize if image is too large
+        original_size = img.size
+        if max(img.size) > max_size:
+            ratio = max_size / max(img.size)
+            new_size = tuple(int(dim * ratio) for dim in img.size)
+            img = img.resize(new_size, Image.Resampling.LANCZOS)
+            logger.info(f"Resized image from {original_size} to {new_size}")
+
+        # Compress to JPEG
+        buffer = BytesIO()
+        img.save(buffer, format='JPEG', quality=quality, optimize=True)
+        compressed_data = buffer.getvalue()
+
+        # Encode back to base64
+        compressed_b64 = base64.b64encode(compressed_data).decode('utf-8')
+
+        # Log compression ratio
+        original_size_kb = len(img_b64) * 3 / 4 / 1024  # base64 to bytes to KB
+        compressed_size_kb = len(compressed_b64) * 3 / 4 / 1024
+        compression_ratio = (1 - compressed_size_kb / original_size_kb) * 100
+        logger.info(f"Compressed image: {original_size_kb:.1f}KB -> {compressed_size_kb:.1f}KB ({compression_ratio:.1f}% reduction)")
+
+        return compressed_b64
+
+    except Exception as e:
+        logger.warning(f"Failed to compress image: {e}, using original")
+        return img_b64
+
+
+class UnifiedLLM:
+
+    def __init__(self, model: str):
+        if model.startswith("gpt"):
+            self.provider = "openai"
+        elif model.startswith("claude"):
+            self.provider = "anthropic"
+        elif model.startswith("gemini"):
+            self.provider = "gemini"
+        else:
+            self.provider = "unknown"
+
+        self.model = model
+        self.client = self._init_client()
+
+    def _init_client(self):
+        """Initialize client"""
+        if self.provider == "openai":
+            from openai import OpenAI
+            return OpenAI(
+                base_url=os.getenv("OPENAI_BASE_URL"),
+                api_key=os.getenv("OPENAI_API_KEY")
+            )
+
+        elif self.provider == "anthropic":
+            from anthropic import Anthropic
+            return Anthropic(
+                base_url=os.getenv("ANTHROPIC_BASE_URL"),
+                api_key=os.getenv("ANTHROPIC_API_KEY")
+            )
+
+        elif self.provider == "gemini":
+            logger.warning("Using Google Gemini model, make sure your internet connection is working.")
+            import google.generativeai as genai
+            genai.configure(api_key=os.getenv("GOOGLE_API_KEY"))
+            return genai.GenerativeModel(self.model)
+
+        else:
+            logger.error(f"Unsupported LLM provider for model: {self.model}")
+            raise ValueError(f"Unsupported LLM provider for model: {self.model}")
+
+    def _get_supported_params(self, temperature: float, max_tokens: int, top_p: float) -> Dict[str, Any]:
+        """Get supported parameters for each provider"""
+        base_params = {
+            "temperature": temperature,
+            "max_tokens": max_tokens
+        }
+
+        # GPT-5.2 and newer models may not support top_p
+        if self.provider == "openai":
+            # Only add top_p for older models
+            if not self.model.startswith("gpt-5"):
+                base_params["top_p"] = top_p
+        elif self.provider == "anthropic":
+            base_params["top_p"] = top_p
+        elif self.provider == "gemini":
+            base_params["top_p"] = top_p
+
+        return base_params
+
+    def generate(
+        self,
+        prompt: str,
+        temperature: float = 0.7,
+        max_tokens: int = 16384,
+        top_p: float = 1.0,
+        **kwargs
+    ) -> str:
+        """
+        Args:
+            prompt: Input prompt
+            temperature: Temperature (0.0-2.0)
+            max_tokens: Maximum number of tokens
+            top_p: Top-p sampling (0.0-1.0)
+
+        Returns:
+            Generated text
+        """
+        params = self._get_supported_params(temperature, max_tokens, top_p)
+
+        if self.provider == "openai":
+            try:
+                response = self.client.chat.completions.create(
+                    model=self.model,
+                    messages=[{"role": "user", "content": prompt}],
+                    **params
+                )
+                return response.choices[0].message.content
+            except Exception as e:
+                logger.error(f"OpenAI API error: {e}")
+                raise e
+
+        elif self.provider == "anthropic":
+            try:
+                response = self.client.messages.create(
+                    model=self.model,
+                    messages=[{"role": "user", "content": prompt}],
+                    **params
+                )
+                return response.content[0].text
+            except Exception as e:
+                logger.error(f"Anthropic API error: {e}")
+                raise e
+
+        elif self.provider == "gemini":
+            try:
+                import google.generativeai as genai
+                config = genai.GenerationConfig(
+                    temperature=params["temperature"],
+                    max_output_tokens=params["max_tokens"],
+                    top_p=params.get("top_p", 1.0)
+                )
+                response = self.client.generate_content(prompt, generation_config=config)
+                return response.text
+            except Exception as e:
+                logger.error(f"Gemini API error: {e}")
+                raise e
+
+    def generate_with_images(
+        self,
+        prompt: str,
+        images_b64: List[str],
+        temperature: float = 0.7,
+        max_tokens: int = 16384,
+        top_p: float = 1.0,
+        **kwargs
+    ) -> str:
+        """
+        Generate with multiple images in a single request
+
+        Args:
+            prompt: Instruction prompt
+            images_b64: List of base64 encoded images
+            temperature: Temperature (0.0-2.0)
+            max_tokens: Maximum number of tokens
+            top_p: Top-p sampling (0.0-1.0)
+
+        Returns:
+            Generated text
+        """
+        if not images_b64:
+            logger.warning("No images provided, falling back to text-only generation")
+            return self.generate(prompt, temperature, max_tokens, top_p, **kwargs)
+
+        params = self._get_supported_params(temperature, max_tokens, top_p)
+
+        if self.provider == "openai":
+            # Build content with text and all images
+            content = [{"type": "text", "text": prompt}]
+
+            for img_b64 in images_b64:
+                content.append({
+                    "type": "image_url",
+                    "image_url": {
+                        "url": f"data:image/jpeg;base64,{img_b64}"
+                    }
+                })
+
+            try:
+                response = self.client.chat.completions.create(
+                    model=self.model,
+                    messages=[{"role": "user", "content": content}],
+                    **params
+                )
+                return response.choices[0].message.content
+            except Exception as e:
+                logger.error(f"OpenAI API error: {e}")
+                raise e
+
+        elif self.provider == "anthropic":
+            # Build content with text and all images
+            content = [{"type": "text", "text": prompt}]
+
+            for img_b64 in images_b64:
+                content.append({
+                    "type": "image",
+                    "source": {
+                        "type": "base64",
+                        "media_type": "image/jpeg",
+                        "data": img_b64
+                    }
+                })
+
+            try:
+                response = self.client.messages.create(
+                    model=self.model,
+                    messages=[{"role": "user", "content": content}],
+                    **params
+                )
+                return response.content[0].text
+            except Exception as e:
+                logger.error(f"Anthropic API error: {e}")
+                raise e
+
+        elif self.provider == "gemini":
+            import google.generativeai as genai
+
+            config = genai.GenerationConfig(
+                temperature=params["temperature"],
+                max_output_tokens=params["max_tokens"],
+                top_p=params.get("top_p", 1.0)
+            )
+
+            # Build content parts
+            content_parts = [prompt]
+
+            for img_b64 in images_b64:
+                img_data = base64.b64decode(img_b64)
+                img = Image.open(BytesIO(img_data))
+                content_parts.append(img)
+
+            try:
+                response = self.client.generate_content(content_parts, generation_config=config)
+                return response.text
+            except Exception as e:
+                logger.error(f"Gemini API error: {e}")
+                raise e
+
+        else:
+            raise ValueError(f"Unsupported provider: {self.provider}")
+
+
+def _load_screenshots_from_dir(result_dir: str, compress: bool = True, max_size: int = 800, quality: int = 85) -> tuple:
+    """
+    Load all step screenshots from result directory and convert to base64
+
+    Args:
+        result_dir: Path to result directory containing step_*.png files
+        compress: Whether to compress images (default: True)
+        max_size: Maximum dimension for compression (default: 800)
+        quality: JPEG quality for compression (default: 85)
+
+    Returns:
+        Tuple of (list of base64 encoded screenshot strings, list of short filenames like 'step_1', 'step_2', ...)
+    """
+    screenshots = []
+    filenames = []
+
+    # Find all step screenshot files (e.g., step_1_20240101@120000.png)
+    pattern = os.path.join(result_dir, "step_*.png")
+    screenshot_files = sorted(glob.glob(pattern))
+
+    if not screenshot_files:
+        logger.warning(f"No screenshot files found in {result_dir}")
+        return screenshots, filenames
+
+    import re as _re
+    for filepath in screenshot_files:
+        try:
+            with open(filepath, "rb") as f:
+                img_data = f.read()
+                img_b64 = base64.b64encode(img_data).decode('utf-8')
+
+                # Compress if enabled
+                if compress:
+                    img_b64 = _compress_image(img_b64, max_size=max_size, quality=quality)
+
+                screenshots.append(img_b64)
+                # Extract short name like 'step_1' from 'step_1_20240101@120000.png'
+                basename = os.path.basename(filepath)
+                match = _re.match(r'(step_\d+)', basename)
+                short_name = match.group(1) if match else basename
+                filenames.append(short_name)
+        except Exception as e:
+            logger.error(f"Error loading screenshot {filepath}: {e}")
+
+    logger.info(f"Loaded {len(screenshots)} screenshots from {result_dir}: {filenames}")
+    return screenshots, filenames
+
+
+def vllm_eval(result_state, **options) -> float:
+    """
+    Evaluate task completion using vision-language model
+
+    Args:
+        result_state: Current state description
+        **options: Additional options including:
+            - result_dir: Path to result directory containing step screenshots (recommended)
+            - screenshots: List of base64 encoded screenshots (deprecated, use result_dir instead)
+            - instruction: Task instruction
+            - eval_model: Model name to use
+            - compress_images: Whether to compress images (default: True)
+            - max_image_size: Maximum image dimension for compression (default: 800)
+            - image_quality: JPEG quality for compression (default: 85)
+            - temperature: Temperature parameter
+            - max_tokens: Maximum tokens
+            - top_p: Top-p parameter
+
+    Returns:
+        Score between 0.0 and 1.0
+    """
+    # Try to load screenshots from result_dir if provided
+    result_dir = options.get("result_dir", None)
+    screenshots = options.get("screenshots", [])
+
+    # Image compression options
+    compress_images = options.get("compress_images", True)
+    max_image_size = options.get("max_image_size", 800)
+    image_quality = options.get("image_quality", 85)
+
+    screenshot_filenames = []  # Short names like 'step_1', 'step_2', ...
+
+    if result_dir and not screenshots:
+        screenshots, screenshot_filenames = _load_screenshots_from_dir(
+            result_dir,
+            compress=compress_images,
+            max_size=max_image_size,
+            quality=image_quality
+        )
+        logger.info(f"Loaded {len(screenshots)} screenshots from result_dir: {result_dir}")
+    elif screenshots:
+        logger.info(f"Using {len(screenshots)} screenshots from options")
+        screenshot_filenames = [f"step_{i+1}" for i in range(len(screenshots))]
+        # Compress screenshots if needed
+        if compress_images:
+            logger.info("Compressing provided screenshots...")
+            screenshots = [_compress_image(img, max_size=max_image_size, quality=image_quality) for img in screenshots]
+
+    instruction = options.get("instruction", "")
+    eval_model = options.get("eval_model", "gpt-4-vision-preview")
+    config = options.get("config", [])
+    metadata = options.get("metadata", {})
+
+    params = {
+        "temperature": options.get("temperature", 0.7),
+        "max_tokens": options.get("max_tokens", 16384),
+        "top_p": options.get("top_p", 1.0)
+    }
+
+    llm = UnifiedLLM(eval_model)
+
+    # Build pre-configured environment description from config
+    preconfig_items = []
+    for cfg in config:
+        if cfg.get("type") == "launch":
+            cmds = cfg.get("parameters", {}).get("command", [])
+            if cmds:
+                app_name = os.path.basename(cmds[0]) if cmds else "unknown"
+                preconfig_items.append(f"Application '{app_name}' was automatically launched before the agent started.")
+        elif cfg.get("type") == "sleep":
+            pass  # not relevant to scoring
+        elif cfg.get("type") == "open":
+            path = cfg.get("parameters", {}).get("path", "")
+            preconfig_items.append(f"File/URL '{path}' was automatically opened before the agent started.")
+
+    preconfig_section = ""
+    if preconfig_items:
+        preconfig_desc = "\n".join(f"  - {item}" for item in preconfig_items)
+        preconfig_section = f"""
+PRE-CONFIGURED ENVIRONMENT (done BEFORE the agent started, NOT the agent's work):
+{preconfig_desc}
+IMPORTANT: The above actions were performed automatically as part of environment setup. The agent did NOT perform these actions. Do NOT give ANY credit for them. For example, if the application was pre-launched, the agent merely having the application open is worth 0 points - that was the starting state."""
+
+    # Build expected steps section from metadata
+    expected_steps_section = ""
+    if metadata.get("steps"):
+        expected_steps_section = f"""
+EXPECTED STEPS for this task (use as reference for what the agent should have done):
+{metadata['steps']}
+NOTE: Evaluate the screenshots against these expected steps. Only give credit for steps that show VISIBLE evidence of completion BEYOND the pre-configured starting state."""
+
+    # Build image list description for the prompt
+    if screenshot_filenames:
+        img_list_str = ", ".join(screenshot_filenames)
+        img_info = f"""\nYou are provided with exactly {len(screenshot_filenames)} screenshots in chronological order: {img_list_str}
+The FIRST screenshot is: {screenshot_filenames[0]}
+The LAST screenshot (final state): {screenshot_filenames[-1]}
+IMPORTANT: Only reference screenshots from the list above. Do NOT reference any screenshot that is not listed."""
+    else:
+        img_info = "\nNo screenshots were provided."
+
+    prompt = f"""You are a STRICT and RIGOROUS evaluator for desktop environment tasks. Your job is to score ONLY based on concrete, visible evidence of task completion in the screenshots.
+
+Task Instruction: {instruction}
+{preconfig_section}
+{expected_steps_section}
+{img_info}
+
+Analyze ONLY the FINAL screenshot ({screenshot_filenames[-1] if screenshot_filenames else 'N/A'}) to determine the end state, while using earlier screenshots for context.
+
+CRITICAL SCORING RULES:
+1. Score ONLY based on what the AGENT actually accomplished. The pre-configured environment (application already launched, files already opened, etc.) is the STARTING STATE and worth 0 points.
+2. Score ONLY based on what is ACTUALLY VISIBLE in the screenshots. Do NOT give credit for assumed or potential progress.
+3. If the screenshots show NO meaningful action beyond the initial pre-configured state, the score MUST be 0.
+4. Do NOT give partial credit for "having the system on", "desktop being visible", "the application being open" (if it was pre-launched), or "the application being installed". These are prerequisites or pre-configured state, NOT progress.
+5. Each point must correspond to a SPECIFIC, VERIFIABLE action that was successfully completed BY THE AGENT toward the task goal.
+
+SCORING GUIDE (0-10):
+- 0: No progress beyond the pre-configured starting state. If the app was pre-launched, merely having it open is 0. If the screenshots only show the desktop or the initial app state without any agent action, score is 0.
+- 1-2: The agent performed one minor action (e.g., clicked on a menu) but did not make meaningful progress toward the task goal.
+- 3-4: Some initial steps toward the task have been taken but the task is far from complete.
+- 5-6: Significant progress - about half the required steps are completed with visible evidence.
+- 7-8: Most steps are completed but the final result is not fully achieved or has minor issues.
+- 9: The task is essentially complete with very minor cosmetic differences.
+- 10: The task is perfectly and completely finished with clear evidence in the final screenshot.
+
+IMPORTANT: You must respond with ONLY a valid JSON object (no additional text before or after). Use the following exact format:
+
+{{
+  "steps_analysis": [
+    {{"step": "Step description", "status": "Success/Fail", "evidence_img": "step_X.png", "reason": "Brief explanation of VISIBLE evidence"}},
+    {{"step": "Another step", "status": "Success/Fail", "evidence_img": "step_Y.png", "reason": "Brief explanation of VISIBLE evidence"}}
+  ],
+  "final_completion": "True/False",
+  "score": 0-10
+}}
+
+Where:
+- "steps_analysis": Array of steps you identified from the screenshots. Each step must cite VISIBLE evidence from a specific screenshot. Do NOT include pre-configured actions as agent steps.
+- "status": Either "Success" or "Fail" for each step
+- "evidence_img": The screenshot filename that shows evidence for this step (e.g., "step_2.png")
+- "reason": Explanation of what is VISUALLY observed in the screenshot as evidence
+- "final_completion": "True" ONLY if the overall task is fully completed with clear visual proof, "False" otherwise
+- "score": Integer from 0 to 10, following the strict scoring guide above
+
+Remember: Return ONLY the JSON object, no additional text. Be STRICT - when in doubt, score LOWER."""
+
+    try:
+        result = llm.generate_with_images(
+            prompt=prompt,
+            images_b64=screenshots,
+            **params
+        )
+
+        # Parse score from result
+        score = _parse_score(result)
+        logger.info(f"Evaluation result: {result}")
+        logger.info(f"Parsed score: {score}")
+
+        # Save raw result to file for reference
+        if result_dir:
+            eval_output_path = os.path.join(result_dir, "vllm_evaluation_result.json")
+            with open(eval_output_path, "w", encoding="utf-8") as f:
+                f.write(result)
+            logger.info(f"Saved evaluation result to {eval_output_path}")
+
+        return score
+    except Exception as e:
+        logger.error(f"Error during evaluation: {e}")
+        return 0.0
+
+
+def _parse_evaluation_response(text: str) -> Dict[str, Any]:
+    """
+    Parse the JSON evaluation response from the model
+
+    Returns:
+        Dictionary containing steps_analysis, final_completion, and score
+    """
+    import re
+    import json
+
+    # Try to extract JSON from the response
+    # Sometimes models wrap JSON in markdown code blocks
+    text = text.strip()
+
+    # Remove markdown code blocks if present
+    if text.startswith("```"):
+        # Extract content between ``` markers
+        match = re.search(r'```(?:json)?\s*(\{.*?\})\s*```', text, re.DOTALL)
+        if match:
+            text = match.group(1)
+        else:
+            # Try to remove opening and closing ```
+            text = re.sub(r'^```(?:json)?\s*', '', text)
+            text = re.sub(r'\s*```$', '', text)
+
+    try:
+        result = json.loads(text)
+
+        # Validate required fields
+        if "steps_analysis" not in result:
+            logger.warning("Missing 'steps_analysis' field in response")
+            result["steps_analysis"] = []
+
+        if "final_completion" not in result:
+            logger.warning("Missing 'final_completion' field in response")
+            result["final_completion"] = "False"
+
+        if "score" not in result:
+            logger.warning("Missing 'score' field in response")
+            result["score"] = 0
+
+        return result
+
+    except json.JSONDecodeError as e:
+        logger.error(f"Failed to parse JSON response: {e}")
+        logger.error(f"Response text: {text[:500]}")
+
+        # Return a default structure
+        return {
+            "steps_analysis": [],
+            "final_completion": "False",
+            "score": 0
+        }
+
+
+def _parse_score(text: str) -> float:
+    """
+    Parse score from model response and convert to 0.0-1.0 range
+
+    Args:
+        text: Raw model response (expected to be JSON format)
+
+    Returns:
+        Score between 0.0 and 1.0
+    """
+    result = _parse_evaluation_response(text)
+
+    # Extract score (0-10) and convert to 0.0-1.0
+    score = result.get("score", 0)
+
+    try:
+        score = float(score)
+        # Clamp to [0, 10] then normalize to [0.0, 1.0]
+        score = max(0.0, min(10.0, score))
+        normalized_score = score / 10.0
+
+        logger.info(f"Final completion: {result.get('final_completion')}")
+        logger.info(f"Raw score (0-10): {score}, Normalized score (0-1): {normalized_score}")
+
+        # Log steps analysis if available
+        steps = result.get("steps_analysis", [])
+        if steps:
+            logger.info(f"Steps analysis ({len(steps)} steps):")
+            for i, step in enumerate(steps):
+                logger.info(f"  Step {i+1}: {step.get('step', 'N/A')} - {step.get('status', 'N/A')}")
+
+        return normalized_score
+
+    except (ValueError, TypeError) as e:
+        logger.warning(f"Could not parse score: {e}")
+        return 0.0

evaluation_examples/examples/avogadro/building-metal-complexes_task1.json

@@ -0,0 +1,44 @@
+{
+  "id": "building-metal-complexes_task1",
+  "snapshot": "avogadro",
+  "instruction": "在 Avogadro 2 中，使用 Template Tool 创建 [Co(NH3)6]3+ 配位化合物，设置为八面体配位几何。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\Avogadro2\\bin\\avogadro2.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "avogadro"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 打开 Template Tool（快捷键 Ctrl+3 或点击工具栏图标）。\n2. 切换到 Centers 选项卡。\n3. 输入 'Co' 或从弹出菜单中选择钴元素。\n4. 点击三次 '+' 符号，将正电荷设置为 +3。\n5. 按键 '6' 或选择八面体几何形状。\n6. 点击空白区域，放置钴中心，六个氢原子会显示在配位位置。"
+  }
+}

evaluation_examples/examples/avogadro/building-metal-complexes_task3.json

@@ -0,0 +1,44 @@
+{
+  "id": "building-metal-complexes_task3",
+  "snapshot": "avogadro",
+  "instruction": "在 Avogadro 2 中，使用 Template Tool 创建 [Ni(en)(NH3)2]2+ 配位化合物，设置为平面四方配位几何。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\Avogadro2\\bin\\avogadro2.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "avogadro"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 打开 Template Tool。\n2. 切换到 Centers 选项卡。\n3. 输入 'Ni' 或从弹出菜单中选择镍元素。\n4. 点击两次 '+' 符号，将正电荷设置为 +2。\n5. 按键 '44' 或选择平面四方几何形状。\n6. 点击空白区域，放置镍中心，四个氢原子会显示在配位位置。"
+  }
+}

evaluation_examples/examples/avogadro/building-metal-complexes_task7.json

@@ -0,0 +1,44 @@
+{
+  "id": "building-metal-complexes_task7",
+  "snapshot": "avogadro",
+  "instruction": "在 Avogadro 2 中，创建具有两个环戊二烯基 (Cp) 和两个氯配体的 ZrCp2Cl2 配合物。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\Avogadro2\\bin\\avogadro2.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "avogadro"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 打开 Template Tool，点击 Centers 选项卡。\n2. 输入 'Zr' 或选择锆元素。\n3. 点击四次 '+'，将正电荷设置为 +4。\n4. 按键 '4'，选择四面体几何形状。\n5. 在空白区域放置锆中心。\n6. 切换到 Ligands 选项卡，输入 'cp' 或选择环戊二烯基。\n7. 点击一个氢原子，添加第一个 Cp 配体。\n8. 点击相邻氢，添加第二个 Cp 配体。\n9. 切换到 Draw Tool（快捷键 Ctrl+2）。\n10. 选择 Cl 元素。\n11. 点击两个剩余氢原子，每次点击替换为氯配体。"
+  }
+}

evaluation_examples/examples/avogadro/building-organic-molecules_task1.json

@@ -0,0 +1,44 @@
+{
+  "id": "building-organic-molecules_task1",
+  "snapshot": "avogadro",
+  "instruction": "在 Avogadro 2 中，使用软件的 Build 工具插入一个苯环。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\Avogadro2\\bin\\avogadro2.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "avogadro"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 点击菜单栏 Build(构建) → Insert(插入) → Molecule(分子…)，打开\"插入片段\"对话框。\n2. 在\"筛选\"输入框中输入 benzene（注意：需要先切换到英文输入法再输入）。\n3. 筛选结果会显示一个 aromatics 文件夹（树形结构），需要双击或点击展开该文件夹。\n4. 展开后选中列表中的 benzene.cjson 文件。\n5. 点击\"插入\"按钮将苯环插入到工作区。\n6. 关闭\"插入片段\"对话框，确认苯环已显示在主工作界面中。"
+  }
+}

evaluation_examples/examples/avogadro/building-organic-molecules_task3.json

@@ -0,0 +1,44 @@
+{
+  "id": "building-organic-molecules_task3",
+  "snapshot": "avogadro",
+  "instruction": "在 Avogadro 2 中，在甲苯分子的对位添加一硝基（-NO2），生成 4-硝基甲苯。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\Avogadro2\\bin\\avogadro2.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "avogadro"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 按 'N' 键选择硝基。\n2. 点击甲基对位（苯环上的一个氢原子），将其替换为 -NO2。\n3. 确保分子结构正确。"
+  }
+}

evaluation_examples/examples/avogadro/building-organic-molecules_task4.json

@@ -0,0 +1,44 @@
+{
+  "id": "building-organic-molecules_task4",
+  "snapshot": "avogadro",
+  "instruction": "在 Avogadro 2 中，为甲苯分子执行几何优化。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\Avogadro2\\bin\\avogadro2.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "avogadro"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 按 Ctrl+Alt+O 或点击 Auto Optimize 工具执行几何优化。\n2. 检查分子是否获得合乎逻辑的几何结构。"
+  }
+}

evaluation_examples/examples/avogadro/building-organic-molecules_task5.json

@@ -0,0 +1,44 @@
+{
+  "id": "building-organic-molecules_task5",
+  "snapshot": "avogadro",
+  "instruction": "在 Avogadro 2 中，使用 Draw Tool 创建一个单碳结构，然后添加一个羧基（-COOH）。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\Avogadro2\\bin\\avogadro2.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "avogadro"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 使用 Draw Tool 在界面中绘制一个单碳。\n2. 激活 Template Tool，通过按 Ctrl+3 或点击工具栏上的图标进入 Groups。\n3. 按 'C' 或 'co' 选择羧基。\n4. 点击单碳结构上的一个氢原子，将其替换为羧基。"
+  }
+}

evaluation_examples/examples/avogadro/building-organic-molecules_task9.json

@@ -0,0 +1,44 @@
+{
+  "id": "building-organic-molecules_task9",
+  "snapshot": "avogadro",
+  "instruction": "在 Avogadro 2 中，创建一个 4-甲氧基-3-硝基苯甲酸分子，包含苯环、羧基、硝基和甲氧基。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\Avogadro2\\bin\\avogadro2.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "avogadro"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 插入苯环。\n2. 按 'C' 键选择羧基，并添加到苯环的第 1 个位置。\n3. 按 'N' 键选择硝基，并添加到苯环的第 3 个位置。\n4. 按 'om' 键选择甲氧基，并添加到苯环的第 4 个位置。\n5. 使用优化工具进行几何优化并检查分子是否正确。"
+  }
+}

evaluation_examples/examples/avogadro/naming-a-molecule_task1.json

@@ -0,0 +1,44 @@
+{
+  "id": "naming-a-molecule_task1",
+  "snapshot": "avogadro",
+  "instruction": "在 Avogadro 中通过 Analysis → Properties → Molecular... 查看当前分子的 IUPAC 名称及相关性质。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\Avogadro2\\bin\\avogadro2.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "avogadro"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 打开 Avogadro 软件。\n2. 点击菜单栏中的 Analysis。\n3. 从下拉菜单选择 Properties。\n4. 点击 Molecular...。\n5. 在弹出的 'Molecular Properties' 窗口中查看分子的名字和相关信息，例如分子质量、化学式、原子数和键数。"
+  }
+}

evaluation_examples/examples/avogadro/viewing-electrostatic-potential_task1.json

@@ -0,0 +1,44 @@
+{
+  "id": "viewing-electrostatic-potential_task1",
+  "snapshot": "avogadro",
+  "instruction": "在 Avogadro 中通过 Analyze → Create Surfaces 菜单创建 Van der Waals 表面并设置电荷分布可视化。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\Avogadro2\\bin\\avogadro2.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "avogadro"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 打开 Avogadro 软件并加载目标分子的模型。\n2. 通过菜单栏选择 Analyze → Create Surfaces。\n3. 在弹出的 Create Surfaces 对话框中，将 Surface 设置为 'Van der Waals'。\n4. 将 Color By 设置为 'Electrostatic Potential'。\n5. 选择一个电荷模型（例如 'EEM'）。\n6. 选择色阶为 'Balance'。\n7. 点击 'Calculate' 按钮开始计算表面。\n8. 等待软件完成计算，点击 'Close' 关闭对话框。"
+  }
+}

evaluation_examples/examples/imagej/user-guide_task1.json

@@ -0,0 +1,44 @@
+{
+  "id": "user-guide_task1",
+  "snapshot": "imagej",
+  "instruction": "在 ImageJ 中，通过 File → New → Image 创建一个名为 'Text Image' 的新图像，设置图像类型为 8-bit，背景填充为 White，并设置宽度为 40 像素，高度为 40 像素。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\ImageJ\\ImageJ.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "imagej"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 在菜单栏点击 File → New → Image。\n2. 在弹出的对话框中输入名称 'Text Image'。\n3. 从 Type 下拉菜单中选择 '8-bit'。\n4. 从 Fill With 下拉菜单中选择 'White'。\n5. 在宽度（Width）框中输入 40。\n6. 在高度（Height）框中输入 40。\n7. 点击 OK 按钮完成操作。"
+  }
+}

evaluation_examples/examples/imagej/user-guide_task10.json

@@ -0,0 +1,44 @@
+{
+  "id": "user-guide_task10",
+  "snapshot": "imagej",
+  "instruction": "在 ImageJ 中，通过 Edit → Selection → Restore Selection 恢复之前存储的选区。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\ImageJ\\ImageJ.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "imagej"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 在菜单栏点击 Edit → Selection → Restore Selection。\n2. 在图像上确保选区可见。\n3. 查看并确认选区正确恢复。"
+  }
+}

evaluation_examples/examples/imagej/user-guide_task2.json

@@ -0,0 +1,57 @@
+{
+  "id": "user-guide_task2",
+  "snapshot": "imagej",
+  "instruction": "在 ImageJ 中，通过 Process → Find Maxima 对 blobs.gif 图像执行 Maxima 寻找，设置噪声容忍值为 50，并选择 Output Type 为 'Single Points'。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "upload_file",
+      "parameters": {
+        "files": [
+          {
+            "local_path": "evaluation_examples/data/imagej/blobs.gif",
+            "path": "C:\\Users\\user\\Desktop\\blobs.gif"
+          }
+        ]
+      }
+    },
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\ImageJ\\ImageJ.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "imagej"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [
+      "blobs.gif"
+    ],
+    "steps": "1. 打开 blobs.gif 文件。\n2. 在菜单栏点击 Process → Find Maxima。\n3. 在弹出的对话框中，将 Noise Tolerance 设置为 50。\n4. 从 Output Type 下拉菜单中选择 'Single Points'。\n5. 点击 OK 按钮完成操作。"
+  }
+}

evaluation_examples/examples/imagej/user-guide_task3.json

@@ -0,0 +1,44 @@
+{
+  "id": "user-guide_task3",
+  "snapshot": "imagej",
+  "instruction": "在 ImageJ 中，通过 Plugins → Utilities → Find Commands 查找关键字 'threshold' 的相关命令，显示完整信息并运行 'Adaptive3DThreshold'。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\ImageJ\\ImageJ.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "imagej"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 在菜单栏点击 Plugins → Utilities → Find Commands。\n2. 在弹出的 Command Finder 窗口中输入 'threshold'。\n3. 勾选 'Show full information'。\n4. 在列表中选择 'Adaptive3DThreshold' 并双击运行命令。"
+  }
+}

evaluation_examples/examples/imagej/user-guide_task4.json

@@ -0,0 +1,44 @@
+{
+  "id": "user-guide_task4",
+  "snapshot": "imagej",
+  "instruction": "在 ImageJ 中，通过 Image → Adjust → Threshold 使用 'Default' 自动阈值法对当前图像进行阈值分割，并设置显示模式为 Over/Under。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\ImageJ\\ImageJ.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "imagej"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 在菜单栏点击 Image → Adjust → Threshold。\n2. 在弹出的对话框中，从 Method 下拉菜单选择 'Default'。\n3. 确保 Display 模式设置为 'Over/Under'。\n4. 点击 Apply 按钮完成操作。"
+  }
+}

evaluation_examples/examples/imagej/user-guide_task5.json

@@ -0,0 +1,44 @@
+{
+  "id": "user-guide_task5",
+  "snapshot": "imagej",
+  "instruction": "在 ImageJ 中，通过 Analyze → Tools → Curve Fitting 对数据拟合二次多项式，并将最大迭代次数设为 100。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\ImageJ\\ImageJ.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "imagej"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 在菜单栏点击 Analyze → Tools → Curve Fitting。\n2. 在弹出的对话框中，从 Function 下拉菜单选择 '2nd Degree Polynomial'。\n3. 点击 Fit 按钮。\n4. 在 Simplex Fitting Options 中，将 Maximum number of iterations 设置为 100。\n5. 点击 OK 按钮完成拟合。"
+  }
+}

evaluation_examples/examples/imagej/user-guide_task6.json

@@ -0,0 +1,57 @@
+{
+  "id": "user-guide_task6",
+  "snapshot": "imagej",
+  "instruction": "在 ImageJ 中，通过 Image → Transform → Rotate 90 Degrees Right 旋转 mri-stack.tif 图像 90°。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "upload_file",
+      "parameters": {
+        "files": [
+          {
+            "local_path": "evaluation_examples/data/imagej/mri-stack.tif",
+            "path": "C:\\Users\\user\\Desktop\\mri-stack.tif"
+          }
+        ]
+      }
+    },
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\ImageJ\\ImageJ.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "imagej"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [
+      "mri-stack.tif"
+    ],
+    "steps": "1. 打开 mri-stack.tif 文件。\n2. 在菜单栏点击 Image → Transform → Rotate 90 Degrees Right。\n3. 确保图像正确旋转后保存或查看结果。"
+  }
+}

evaluation_examples/examples/imagej/user-guide_task7.json

@@ -0,0 +1,44 @@
+{
+  "id": "user-guide_task7",
+  "snapshot": "imagej",
+  "instruction": "在 ImageJ 中，通过 Process → Binary → Options 设置黑色背景选项为开启状态并进行预览。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\ImageJ\\ImageJ.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "imagej"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 在菜单栏点击 Process → Binary → Options。\n2. 在弹出的对话框中勾选 'Black Background'。\n3. 点击 Preview 查看效果。\n4. 点击 OK 按钮保存选项。"
+  }
+}

evaluation_examples/examples/imagej/user-guide_task8.json

@@ -0,0 +1,44 @@
+{
+  "id": "user-guide_task8",
+  "snapshot": "imagej",
+  "instruction": "在 ImageJ 中，通过 File → Save As → PNG 保存当前图像为 PNG 格式，设置透明索引为 255。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\ImageJ\\ImageJ.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "imagej"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 在菜单栏点击 File → Save As → PNG。\n2. 在弹出的对话框中，将透明索引设置为 255。\n3. 输入文件名并指定保存路径。\n4. 点击 OK 按钮完成保存。"
+  }
+}

evaluation_examples/examples/imagej/user-guide_task9.json

@@ -0,0 +1,44 @@
+{
+  "id": "user-guide_task9",
+  "snapshot": "imagej",
+  "instruction": "在 ImageJ 中，通过 Analyze → Measure 测量当前选区的面积和灰度值。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\ImageJ\\ImageJ.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "imagej"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 创建或选择一个区域选区。\n2. 在菜单栏点击 Analyze → Measure。\n3. 在弹出的 Results 窗口中查看面积和灰度值等测量结果。"
+  }
+}

evaluation_examples/examples/jade/MDIJade6.5使用手册_task1.json

@@ -0,0 +1,47 @@
+{
+  "id": "MDIJade6.5使用手册_task1",
+  "snapshot": "jade",
+  "instruction": "在 MDI Jade 中通过菜单 File → Patterns 加载衍射数据文件 DEMO001.MDI。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\JADE\\jade 6.5\\MDI Jade 6.5\\jade6.5.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "jade"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [
+      "DEMO001.MDI"
+    ],
+    "steps": "1. 在 Windows 桌面找到名为 MDI Jade 的快捷方式图标。\n2. 双击该快捷方式图标启动 MDI Jade 软件。\n3. 在软件主界面顶部的菜单栏中，单击 File 菜单。\n4. 在弹出的下拉菜单中，单击 Patterns... 选项，打开读入文件对话框。\n5. 在弹出的 Read Pattern Files 对话框中，在中间的 File Name 文件列表区域，寻找并单击选中名为 DEMO001.MDI 的文件。\n6. 单击该对话框左上方的 Read 按钮，将选定的衍射数据文件加载到软件中。",
+    "steps_original": "1. 在桌面找到 MDI Jade 图标，双击打开软件。\n2. 点击菜单 File → Patterns。\n3. 在弹出的对话框中选择 DEMO001.MDI 并点击 Open。"
+  }
+}

evaluation_examples/examples/jade/MDIJade6.5使用手册_task10.json

@@ -0,0 +1,45 @@
+{
+  "id": "MDIJade6.5使用手册_task10",
+  "snapshot": "jade",
+  "instruction": "从菜单 Options → Cell Refinement 打开晶胞点阵参数对话框并精修点阵常数。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\JADE\\jade 6.5\\MDI Jade 6.5\\jade6.5.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "jade"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 使用键盘快捷键 Shift+Ctrl+F4 直接打开 Cell Refinement（晶胞精细化）对话框（等价于菜单 Options → Cell Refinement，但快捷键更可靠；也可以在同一个代码块中先点击 Options 菜单再立即点击 Cell Refinement 菜单项，中间仅 sleep 0.5 秒，注意必须在同一步完成以防菜单自动关闭）。\n2. 等待晶胞点阵参数对话框弹出，查看当前显示的点阵参数列表。\n3. 单击该对话框中的 \"Refine\" 按钮，开始执行点阵参数精修计算。\n4. 等待精修计算完成，并查看对话框中更新后的精修结果数值。\n5. 单击对话框中的 \"Save\" 按钮，保存当前的精修结果。\n6. 单击对话框底部或右上角的 \"Close\"（或关闭）按钮，关闭晶胞点阵参数对话框。",
+    "steps_original": "1. 点击菜单 Options → Cell Refinement。\n2. 在弹出的对话框中检查点阵参数。\n3. 点击 Refine 按钮进行精修。\n4. 检查精修结果并保存。"
+  }
+}

evaluation_examples/examples/jade/MDIJade6.5使用手册_task2.json

@@ -0,0 +1,45 @@
+{
+  "id": "MDIJade6.5使用手册_task2",
+  "snapshot": "jade",
+  "instruction": "在 JADE 中将选择DEMO001的衍射图谱,并通过 File → Save-Primary Pattern as *.txt 导出为 ASCII 格式，保存为 DEMO001.txt。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\JADE\\jade 6.5\\MDI Jade 6.5\\jade6.5.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "jade"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 单击顶部菜单栏中的 \"File\" 菜单\n2. 在展开的下拉菜单中，单击选中 \"Save-Primary Pattern as *.txt\" 菜单项\n3. 在弹出的保存文件对话框中，单击将光标定位到 \"文件名\"（或 \"File name\"）输入框\n4. 清空该输入框中的已有内容\n5. 在输入框中输入文字 \"DEMO001.txt\"\n6. 单击对话框右下方的 \"保存\"（或 \"Save\"）按钮",
+    "steps_original": "1. 点击菜单 File → Save-Primary Pattern as *.txt。\n2. 在弹出的保存对话框中，设置文件名为 DEMO001.txt。\n3. 点击 Save 按钮保存文件。"
+  }
+}

evaluation_examples/examples/jade/MDIJade6.5使用手册_task3.json

@@ -0,0 +1,45 @@
+{
+  "id": "MDIJade6.5使用手册_task3",
+  "snapshot": "jade",
+  "instruction": "使用 Search/Match 功能进行物相检索，并限制元素范围为 Al, Sn, O。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\JADE\\jade 6.5\\MDI Jade 6.5\\jade6.5.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "jade"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 在主窗口工具栏中，使用鼠标右键单击“S/M”按钮，打开“Search/Match”条件设置对话框。\n2. 在“Search/Match”对话框右侧的“Search/Match Filters:”区域中，单击勾选“Use Chemistry Filter”复选框，打开化学元素过滤界面。\n3. 在弹出的“Current Chemistry [Filter]”对话框的元素周期表面板中，单击“Al”按钮将其选中为限定元素。\n4. 在元素周期表面板中，单击“Sn”按钮将其选中为限定元素。\n5. 在元素周期表面板中，单击“O”按钮将其选中为限定元素。\n6. 单击“Current Chemistry [Filter]”对话框右上角的“OK”按钮，保存元素设置并返回“Search/Match”对话框。\n7. 单击“Search/Match”对话框底部的“OK”按钮，开始执行物相检索。\n8. 等待进度条结束，在弹出的“Search/Match Display”窗口的列表中查看检索到的物相结果。",
+    "steps_original": "1. 点击菜单 S/M 按钮。\n2. 在 Search/Match 对话框中勾选 Use Chemistry Filter。\n3. 输入限定元素 Al, Sn, O，点击 OK。\n4. 等待物相检索完成，检查结果列表。"
+  }
+}

evaluation_examples/examples/jade/MDIJade6.5使用手册_task4.json

@@ -0,0 +1,45 @@
+{
+  "id": "MDIJade6.5使用手册_task4",
+  "snapshot": "jade",
+  "instruction": "在 JADE 中通过 Options → Report-Peak ID Extended 计算 RIR 方法的物相质量分数，并保存结果为 PDF 格式。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\JADE\\jade 6.5\\MDI Jade 6.5\\jade6.5.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "jade"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 在顶部菜单栏中，单击选中 \"Options\" 菜单。\n2. 在展开的下拉菜单中，单击选中 \"Report-Peak ID Extended\"。\n3. 在弹出的 \"Peak ID Extended Report\" 对话框中，查看列表区域确认结果数据完整。\n4. 点击该对话框顶部工具栏中的 \"Save\" 按钮。\n5. 在弹出的 \"Enter File Name to Save\" 对话框中，找到底部的 \"保存类型(T)\" 下拉菜单。\n6. 点击 \"保存类型(T)\" 下拉菜单，选择 \"PDF Overlay List File (*.pdf)\"（或相应的 PDF 格式）。\n7. 在 \"文件名(N)\" 输入框中，单击定位光标并输入需要的文件名。\n8. 单击对话框右下角的 \"保存(S)\" 按钮完成保存。",
+    "steps_original": "1. 打开菜单 Options → Report-Peak ID Extended。\n2. 确认结果数据，确保内容显示完整。\n3. 点击 Save，选择文件类型为 PDF。\n4. 输入文件名并点击保存。"
+  }
+}

evaluation_examples/examples/jade/MDIJade6.5使用手册_task5.json

@@ -0,0 +1,45 @@
+{
+  "id": "MDIJade6.5使用手册_task5",
+  "snapshot": "jade",
+  "instruction": "通过 Report → Peak Search Report 菜单计算晶粒大小及微观应变，设置 D 值为 1。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\JADE\\jade 6.5\\MDI Jade 6.5\\jade6.5.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "jade"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 单击顶部主菜单栏中的 \"Report\" 菜单。\n2. 在展开的下拉菜单中，单击选中 \"Size & Strain Plot\" 菜单项。\n3. 在弹出的 \"Estimate Crystallite Size & Strain from Peak Broadening\" 对话框中，单击选中顶部的 \"Size/Strain\" 单选框。\n4. 将鼠标光标定位到该单选框右侧的 D 值输入框（位于 \"Origin(0,0)\" 选项的右侧）。\n5. 选中或清空输入框内的当前数值（如 2.0），通过键盘输入数字 \"1\"。\n6. 单击对话框左上方的 \"Save\" 按钮以保存计算结果图片。",
+    "steps_original": "1. 点击菜单 Report → Peak Search Report。\n2. 在弹出的对话框中选择 Size/strain 选项。\n3. 设置反卷积参数 D 值为 1。\n4. 点击 Save 按钮保存计算结果。"
+  }
+}

evaluation_examples/examples/jade/MDIJade6.5使用手册_task6.json

@@ -0,0 +1,45 @@
+{
+  "id": "MDIJade6.5使用手册_task6",
+  "snapshot": "jade",
+  "instruction": "通过 File → Save 菜单保存当前仪器半高宽校正曲线到 Si_hw_curve.fwhm。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\JADE\\jade 6.5\\MDI Jade 6.5\\jade6.5.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "jade"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 单击顶部菜单栏的 \"File\" 菜单项\n2. 在展开的下拉菜单中，单击 \"Save\" 菜单项以展开级联子菜单\n3. 在展开的子菜单中，单击选择 \"FWHM Curve of Peaks\" 菜单项，打开保存文件对话框\n4. 在弹出的保存对话框中，单击定位到 \"文件名\" (或 File name) 输入框\n5. 清空 \"文件名\" 输入框中的已有内容\n6. 在 \"文件名\" 输入框中输入文字 \"Si_hw_curve.fwhm\"\n7. 单击对话框底部的 \"保存\" (或 Save) 按钮完成保存操作",
+    "steps_original": "1. 点击菜单 File。\n2. 选择 Save → FWHM Curve of Peaks。\n3. 在保存对话框中输入文件名 Si_hw_curve.fwhm。\n4. 点击 Save 按钮完成保存。"
+  }
+}

evaluation_examples/examples/jade/MDIJade6.5使用手册_task7.json

@@ -0,0 +1,45 @@
+{
+  "id": "MDIJade6.5使用手册_task7",
+  "snapshot": "jade",
+  "instruction": "调用 Options → D-Spacing 菜单计算已知结构的衍射谱，加权强度公式为 Z = 12。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\JADE\\jade 6.5\\MDI Jade 6.5\\jade6.5.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "jade"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 单击顶部菜单栏的 \"Options\" 菜单\n2. 在弹出的下拉菜单中，单击 \"D-Spacing & hkl...\" 菜单项，打开 \"Calculate d-Spacing & Miller Indices\" 对话框\n3. 在弹出的对话框左下区域，找到 Z 值输入框（位于化学公式输入框的下方，\"Density(c)=\" 文本的左侧，截图中显示为 \"6.0\" 的位置）\n4. 单击该 Z 值输入框将光标定位其中\n5. 清空该输入框中的原有数值\n6. 在该输入框中输入数值 \"12\"\n7. 单击对话框顶部按钮栏中的 \"Calc\" 按钮以生成计算结果\n8. 单击对话框顶部按钮栏最左侧的 \"Close\" 按钮以关闭当前窗口",
+    "steps_original": "1. 打开菜单 Options → D-Spacing。\n2. 在计算衍射谱对话框中，设置加权强度公式参数 Z 值为 12。\n3. 点击 Calculate 按钮以生成计算结果。\n4. 检查结果并关闭窗口。"
+  }
+}

evaluation_examples/examples/jade/MDIJade6.5使用手册_task8.json

@@ -0,0 +1,45 @@
+{
+  "id": "MDIJade6.5使用手册_task8",
+  "snapshot": "jade",
+  "instruction": "通过 Options → Calculate Stress 菜单计算残余应力，使用 Fit All 功能拟合曲线。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\JADE\\jade 6.5\\MDI Jade 6.5\\jade6.5.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "jade"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 在顶部菜单栏中，单击 \"Options\" 菜单。\n2. 在展开的下拉菜单中，单击 \"Calculate Stress\" 菜单项。\n3. 在弹出的计算残余应力相关对话框中，单击名为 \"Fit All\" 的按钮。\n4. 观察主窗口中显示的拟合曲线图，确认拟合已完成。\n5. 单击名为 \"Save\" 的按钮，以保存拟合结果文件。",
+    "steps_original": "1. 点击菜单 Options → Calculate Stress。\n2. 在弹出的对话框中，选择以 Fit All 功能拟合所有数据。\n3. 检查拟合结果图。\n4. 点击 Save 按钮以保存拟合结果文件。"
+  }
+}

evaluation_examples/examples/jade/jade_test.json

@@ -1,36 +0,0 @@
-{
- "id": "jade_test",
- "snapshot": "snapshot",
- "instruction": "请打开桌面上的 JADE 6.5 软件",
- "source": "custom",
- "config": [],
- "trajectory": "trajectories/",
- "related_apps": [
-   "jade"
- ],
- "evaluator": {
-   "postconfig": [
-     {
-       "type": "sleep",
-       "parameters": {
-         "seconds": 3
-       }
-     }
-   ],
-   "func": "check_include_exclude",
-   "result": {
-     "type": "vm_command_line",
-     "command": "tasklist | findstr /i jade"
-   },
-   "expected": {
-     "type": "rule",
-     "rules": {
-       "include": ["jade"],
-       "exclude": []
-     }
-   }
- },
- "proxy": false,
- "fixed_ip": false,
- "possibility_of_env_change": "low"
-}

evaluation_examples/examples/origin/Origin_User_Guide_2025b_E_task1.json

@@ -0,0 +1,57 @@
+{
+  "id": "Origin_User_Guide_2025b_E_task1",
+  "snapshot": "origin",
+  "instruction": "在 Origin 中通过 Data → Connect to File 导入一个本地 Excel 文件 example.xlsx",
+  "source": "custom",
+  "config": [
+    {
+      "type": "upload_file",
+      "parameters": {
+        "files": [
+          {
+            "local_path": "evaluation_examples/data/origin/example.xlsx",
+            "path": "C:\\Users\\user\\Desktop\\example.xlsx"
+          }
+        ]
+      }
+    },
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OriginLab\\Origin2025b\\Origin64.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "origin"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [
+      "example.xlsx"
+    ],
+    "steps": "1. 在 Origin 的主菜单中选择 Data → Connect to File。\n2. 点击 Connect to File 菜单中的按钮。\n3. 选择文件 example.xlsx 并点击 Open。\n4. 数据将被加载到当前的工作表中。"
+  }
+}

evaluation_examples/examples/origin/Origin_User_Guide_2025b_E_task11.json

@@ -0,0 +1,44 @@
+{
+  "id": "Origin_User_Guide_2025b_E_task11",
+  "snapshot": "origin",
+  "instruction": "在 Origin 中通过 Graph → Adding Error Bars 添加误差条到现有图表",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OriginLab\\Origin2025b\\Origin64.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "origin"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 打开一个现有图表并右键点击图表元素。\n2. 选择 Graph → Adding Error Bars。\n3. 选择误差数据列并点击 OK 应用。\n4. 查看图表是否正确添加了误差条。"
+  }
+}

evaluation_examples/examples/origin/Origin_User_Guide_2025b_E_task12.json

@@ -0,0 +1,44 @@
+{
+  "id": "Origin_User_Guide_2025b_E_task12",
+  "snapshot": "origin",
+  "instruction": "在 Origin 中通过 Tools → Pick Data Points 工具拾取数据点并保存到新的工作表中",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OriginLab\\Origin2025b\\Origin64.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "origin"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 打开一个包含数据点的图表。\n2. 在主菜单中选择 Tools → Pick Data Points。\n3. 使用交叉标记在图中选择数据点。\n4. 点击 Done 按钮以保存选择的数据点到新的工作表。"
+  }
+}

evaluation_examples/examples/origin/Origin_User_Guide_2025b_E_task2.json

@@ -0,0 +1,44 @@
+{
+  "id": "Origin_User_Guide_2025b_E_task2",
+  "snapshot": "origin",
+  "instruction": "在 Origin 中通过 View → Formula Bar 打开公式栏，并在公式栏输入 =stdev(B1:B10)",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OriginLab\\Origin2025b\\Origin64.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "origin"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 在主菜单中选择 View → Formula Bar。\n2. 在出现的公式栏中，点击当前单元格内并输入 =stdev(B1:B10)。\n3. 按 Enter 键以应用公式并计算结果。\n4. 检查公式栏输出的结果是否正确。"
+  }
+}

evaluation_examples/examples/origin/Origin_User_Guide_2025b_E_task3.json

@@ -0,0 +1,44 @@
+{
+  "id": "Origin_User_Guide_2025b_E_task3",
+  "snapshot": "origin",
+  "instruction": "在 Origin 中通过 Axis Dialog 修改 X 轴的范围为 20 到 180",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OriginLab\\Origin2025b\\Origin64.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "origin"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 在图层的 X 轴区域右键点击并选择 Axis Dialog。\n2. 在左侧选择 Scale 标签。\n3. 将 From 值修改为 20，将 To 值修改为 180。\n4. 点击 Apply To 按钮以应用更改，然后点击 OK 完成。"
+  }
+}

evaluation_examples/examples/origin/Origin_User_Guide_2025b_E_task4.json

@@ -0,0 +1,44 @@
+{
+  "id": "Origin_User_Guide_2025b_E_task4",
+  "snapshot": "origin",
+  "instruction": "在 Origin 中通过 Graph → Rescale to Show All 重设比例以显示所有数据",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OriginLab\\Origin2025b\\Origin64.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "origin"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 打开一个包含数据的图表。\n2. 在主菜单选择 Graph → Rescale to Show All。\n3. 图表比例重设以显示所有数据点。"
+  }
+}

evaluation_examples/examples/origin/Origin_User_Guide_2025b_E_task5.json

@@ -0,0 +1,44 @@
+{
+  "id": "Origin_User_Guide_2025b_E_task5",
+  "snapshot": "origin",
+  "instruction": "在 Origin 中通过 Tools → Data Slicer 激活数据切片器并设置切片条件为 X=50",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OriginLab\\Origin2025b\\Origin64.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "origin"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 在主菜单中选择 Tools → Data Slicer。\n2. 数据切片器面板将被激活。\n3. 在切片器的条件中选择 X=50 并应用切片。\n4. 图表中将显示切片后的数据点。"
+  }
+}

evaluation_examples/examples/origin/Origin_User_Guide_2025b_E_task8.json

@@ -0,0 +1,44 @@
+{
+  "id": "Origin_User_Guide_2025b_E_task8",
+  "snapshot": "origin",
+  "instruction": "在 Origin 中通过 Worksheet → Convert to Matrix 将活动表格转换成矩阵",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OriginLab\\Origin2025b\\Origin64.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "origin"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 打开一个包含数据的活动表格。\n2. 在主菜单中选择 Worksheet → Convert to Matrix。\n3. 根据对话框选择矩阵转换选项（例如 X Across Columns）。\n4. 点击 OK 完成转换，生成矩阵数据。"
+  }
+}

evaluation_examples/examples/origin/Origin_User_Guide_2025b_E_task9.json

@@ -0,0 +1,44 @@
+{
+  "id": "Origin_User_Guide_2025b_E_task9",
+  "snapshot": "origin",
+  "instruction": "在 Origin 中通过 Object Edit Toolbar 对齐选中的图表对象",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OriginLab\\Origin2025b\\Origin64.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "origin"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 使用鼠标选择需要对齐的对象。\n2. 打开 Object Edit Toolbar。\n3. 点击对齐按钮，例如 Align Left 或 Align Center。\n4. 所选对象将以统一对齐样式排列。"
+  }
+}

evaluation_examples/examples/ovito/animation_task3.json

@@ -0,0 +1,44 @@
+{
+  "id": "animation_task3",
+  "snapshot": "ovito",
+  "instruction": "在 OVITO 中将动画帧数从默认设置改为 10 帧每秒。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OVITO Basic\\ovito.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "ovito"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 在 OVITO 的顶部菜单栏中选择 'Animation settings'。\n2. 在弹出的 'Animation settings' 窗口中，找到 'Frames per second' 输入框。\n3. 将帧速率设置为 10。\n4. 点击 'OK' 以保存更改。"
+  }
+}

evaluation_examples/examples/ovito/aspherical_particles_task1.json

@@ -0,0 +1,44 @@
+{
+  "id": "aspherical_particles_task1",
+  "snapshot": "ovito",
+  "instruction": "在 OVITO 中，为粒子指定球形形状。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OVITO Basic\\ovito.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "ovito"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 打开 OVITO 软件。\n2. 选择 File → New 或打开一个现有的粒子数据文件。\n3. 在 'Pipeline' 界面选择 'Particles' 可视化元素。\n4. 转到 'Particle types' 面板，将粒子形状调整为 Sphere（球形）。\n5. 确认并应用更改，确保形状为球形并更新可视化。"
+  }
+}

evaluation_examples/examples/ovito/clone_pipeline_task1.json

@@ -0,0 +1,44 @@
+{
+  "id": "clone_pipeline_task1",
+  "snapshot": "ovito",
+  "instruction": "在 OVITO 中，通过主工具栏中的 Pipeline 下拉菜单，选择 'Clone current pipeline...' 选项来克隆当前数据通道。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OVITO Basic\\ovito.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "ovito"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 在 OVITO 的主工具栏中找到 'Pipelines' 下拉菜单。\n2. 点击下拉菜单并选择 'Clone current pipeline...'。\n3. 在打开的 'Clone pipeline' 对话框中，选择克隆模式（如 Copy 或 Share）。\n4. 点击 'OK' 按钮完成克隆操作。\n5. 确认在可视化场景中同时显示原始通道和克隆通道的输出。"
+  }
+}

evaluation_examples/examples/ovito/code_generation_task1.json

@@ -0,0 +1,44 @@
+{
+  "id": "code_generation_task1",
+  "snapshot": "ovito",
+  "instruction": "在 OVITO 中，通过 File → Generate Python Script 打开代码生成器窗口。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OVITO Basic\\ovito.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "ovito"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 启动 OVITO 软件。\n2. 点击顶部菜单栏中的 File。\n3. 在下拉菜单中选择 Generate Python Script。\n4. 确保代码生成器窗口正常打开（可见 Python 代码编辑界面）。"
+  }
+}

evaluation_examples/examples/ovito/customize_init_state_task1.json

@@ -0,0 +1,44 @@
+{
+  "id": "customize_init_state_task1",
+  "snapshot": "ovito",
+  "instruction": "在 OVITO 中创建一个名为 defaults.ovito 的文件，以保存空的初始会话状态。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OVITO Basic\\ovito.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "ovito"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 打开 OVITO。\n2. 点击菜单栏中的 File → Save Session State As。\n3. 在弹出的文件保存对话框中，将文件命名为 defaults.ovito。\n4. 确保会话为空（即不包含数据集和管道）。\n5. 点击保存按钮。"
+  }
+}

evaluation_examples/examples/ovito/data_model_task1.json

@@ -0,0 +1,44 @@
+{
+  "id": "data_model_task1",
+  "snapshot": "ovito",
+  "instruction": "在 OVITO 中，通过 Data Inspector 检查导入的粒子属性表，包括 Position 和 Potential Energy 列。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OVITO Basic\\ovito.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "ovito"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 打开 OVITO 软件。\n2. 导入一个包含粒子属性的模拟文件（如 .xyz 格式）。\n3. 点击顶部工具栏中的 Data Inspector 按钮。\n4. 在 Data Inspector 面板查看粒子属性表，包括 Position 和 Potential Energy 列。"
+  }
+}

evaluation_examples/examples/ovito/export_task1.json

@@ -0,0 +1,44 @@
+{
+  "id": "export_task1",
+  "snapshot": "ovito",
+  "instruction": "在 OVITO 中，将当前数据管道导出为粒子及其属性的数据表，保存为文件 particle_data.csv。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OVITO Basic\\ovito.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "ovito"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 在菜单栏中，点击 File → Export File。\n2. 在弹出的对话框中，选择导出格式为 'Table of Particles'.\n3. 指定文件名为 particle_data.csv，并选择保存位置。\n4. 点击 'Save' 按钮以完成导出。"
+  }
+}

evaluation_examples/examples/ovito/marker_particles_task2.json

@@ -0,0 +1,44 @@
+{
+  "id": "marker_particles_task2",
+  "snapshot": "ovito",
+  "instruction": "在 OVITO 中，调整动画播放速度为每秒 15 帧。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OVITO Basic\\ovito.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "ovito"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 在 OVITO 界面上，点击 Animation Settings 按钮（小钟图标）。\n2. 在 Animation Settings 窗口中，找到 'Frames per second'（帧率）选项。\n3. 将 Frames per second 的值设置为 15。\n4. 点击 'OK' 确认设置。"
+  }
+}

evaluation_examples/examples/ovito/miscellaneous_task1.json

@@ -0,0 +1,44 @@
+{
+  "id": "miscellaneous_task1",
+  "snapshot": "ovito",
+  "instruction": "在 OVITO 中，通过 File → Save Session State 保存当前会话为 'session.ovitostate'。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OVITO Basic\\ovito.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "ovito"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 点击菜单栏中的 File。\n2. 选择 Save Session State。\n3. 在弹出的保存对话框中选择目标路径并输入文件名 'session.ovitostate'。\n4. 点击 Save 保存文件。"
+  }
+}

evaluation_examples/examples/ovito/python_extensions_task1.json

@@ -0,0 +1,44 @@
+{
+  "id": "python_extensions_task1",
+  "snapshot": "ovito",
+  "instruction": "在 OVITO Pro 中，通过 Edit → Python Extensions 打开扩展目录。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OVITO Basic\\ovito.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "ovito"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 打开 OVITO Pro 软件。\n2. 点击顶部菜单栏中的 Edit 菜单。\n3. 从下拉菜单中选择 Python Extensions。\n4. 查看扩展目录窗口，确认已打开。"
+  }
+}

evaluation_examples/examples/ovito/remote_file_access_task1.json

@@ -0,0 +1,44 @@
+{
+  "id": "remote_file_access_task1",
+  "snapshot": "ovito",
+  "instruction": "在 OVITO 中，通过 File → Load Remote File 打开远程 SSH 文件 sftp://user@hostname/path/file",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OVITO Basic\\ovito.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "ovito"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 打开 OVITO 软件。\n2. 点击菜单 File → Load Remote File。\n3. 在弹出的对话框中填写 Remote URL 字段，例如：sftp://user@hostname/path/file。\n4. 在 File type 下选择 Auto-detect file format。\n5. 在 SSH connection method 下选择 Integrated client (default)。\n6. 点击 Open 完成连接并加载文件。"
+  }
+}

evaluation_examples/examples/ovito/remote_rendering_task1.json

@@ -0,0 +1,44 @@
+{
+  "id": "remote_rendering_task1",
+  "snapshot": "ovito",
+  "instruction": "在 OVITO Pro 中设置远程渲染任务的导出目录并配置 CPU 核心数。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OVITO Basic\\ovito.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "ovito"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 打开 OVITO Pro 软件。\n2. 点击顶部菜单中的 Utilities 标签。\n3. 选择 Render On Remote Computer 工具。\n4. 在弹出的对话框中，点击 'Choose' 按钮为 Bundle Export Directory 设置一个本地导出目录。\n5. 在 CPU cores per task 选项框中输入渲染任务所需的 CPU 核心数量（可为空，默认使用所有核心）。"
+  }
+}

evaluation_examples/examples/ovito/rendering_task1.json

@@ -0,0 +1,44 @@
+{
+  "id": "rendering_task1",
+  "snapshot": "ovito",
+  "instruction": "在 OVITO 中，通过 Render Settings 面板渲染主动观察窗口为分辨率 1024x768 的图像，背景为透明色。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OVITO Basic\\ovito.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "ovito"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 打开 OVITO 软件。\n2. 确保观察窗口激活（黄色边框）。\n3. 点击右侧命令面板上的 Render 图标。\n4. 在弹出的 Render Settings 面板中，选择 'Single frame'。\n5. 设置输出图像大小为 Width: 1024 和 Height: 768。\n6. 选择背景为 'Transparent'。\n7. 点击 'Render active viewport' 按钮完成渲染。"
+  }
+}

evaluation_examples/examples/ovito/transparent_particles_task1.json

@@ -0,0 +1,44 @@
+{
+  "id": "transparent_particles_task1",
+  "snapshot": "ovito",
+  "instruction": "在软件中，将所有粒子的 Transparency 属性设置为 0.5，使粒子半透明。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\OVITO Basic\\ovito.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "ovito"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 打开软件并加载需要的粒子数据。\n2. 插入 Compute 属性修正器到数据管道。\n3. 在 Compute 属性修正器中找到 Transparency 属性。\n4. 在表达式字段中输入透明度值 0.5。\n5. 应用设置，确保 Transparency 属性被分配给所有粒子。"
+  }
+}

evaluation_examples/examples/pymol/MovieSchool_1_task1.json

@@ -0,0 +1,44 @@
+{
+  "id": "MovieSchool_1_task1",
+  "snapshot": "pymol",
+  "instruction": "在 PyMOL 中，通过命令行制作一个简单动画，播放 NMR ensemble。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\PYMOL\\PyMOLWin.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "pymol"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 在 PyMOL 命令行中输入 `fetch 1nmr` 来加载 NMR ensemble。\n2. 输入 `mplay` 命令开始播放动画。\n3. 如果需要停止播放动画，输入 `mstop` 。"
+  }
+}

evaluation_examples/examples/pymol/MovieSchool_1_task2.json

@@ -0,0 +1,44 @@
+{
+  "id": "MovieSchool_1_task2",
+  "snapshot": "pymol",
+  "instruction": "在 PyMOL 中制作一个场景绕 Y 轴 360 度旋转的动画。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\PYMOL\\PyMOLWin.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "pymol"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 从菜单中选择适当选项，用于创建场景绕 Y 轴旋转的动画。\n2. 按下“Pressplay”开始播放动画。"
+  }
+}

evaluation_examples/examples/pymol/MovieSchool_1_task3.json

@@ -0,0 +1,44 @@
+{
+  "id": "MovieSchool_1_task3",
+  "snapshot": "pymol",
+  "instruction": "在 PyMOL 中制作场景摇摆动画，可选择 30, 60, 90, 120 或 180 度摇摆角度。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\PYMOL\\PyMOLWin.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "pymol"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 从菜单中选择用于设置场景摇摆动画的选项。\n2. 选择摇摆角度（例如 30、60、90、120 或 180 度）。\n3. 按下“Pressplay”启动动画。"
+  }
+}

evaluation_examples/examples/pymol/MovieSchool_1_task4.json

@@ -0,0 +1,44 @@
+{
+  "id": "MovieSchool_1_task4",
+  "snapshot": "pymol",
+  "instruction": "在 PyMOL 中制作一个简单的场景“摇摆（Nutate）”动画。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\PYMOL\\PyMOLWin.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "pymol"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 从菜单中选择制作摇摆动画的选项。\n2. 设置摇摆效果参数。\n3. 按下“Pressplay”启动摇摆动画。"
+  }
+}

evaluation_examples/examples/pymol/MovieSchool_1_task5.json

@@ -0,0 +1,44 @@
+{
+  "id": "MovieSchool_1_task5",
+  "snapshot": "pymol",
+  "instruction": "在 PyMOL 中使用 Scene Loop 制作一个从原子缩放并返回的动画。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\PYMOL\\PyMOLWin.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "pymol"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 创建一个场景并设置为缩放到特定原子。\n2. 保存该场景。\n3. 使用 PyMOL 中的 Scene Loop 功能连接多个保存的场景。\n4. 播放动画以观察缩放效果。"
+  }
+}

evaluation_examples/examples/pymol/MovieSchool_3_task1.json

@@ -0,0 +1,44 @@
+{
+  "id": "MovieSchool_3_task1",
+  "snapshot": "pymol",
+  "instruction": "在 PyMOL 中，通过 Movie 菜单添加 2 秒到当前视频的尾部。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\PYMOL\\PyMOLWin.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "pymol"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 点击菜单栏中的 Movie。\n2. 从下拉菜单中选择 Append。\n3. 在 Append 子菜单中选择 2 seconds。"
+  }
+}

evaluation_examples/examples/pymol/MovieSchool_3_task10.json

@@ -0,0 +1,44 @@
+{
+  "id": "MovieSchool_3_task10",
+  "snapshot": "pymol",
+  "instruction": "在 PyMOL 中，通过 ALA Motions 菜单查看 ALA fragment 的运动选项。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\PYMOL\\PyMOLWin.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "pymol"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 在 PyMOL 的底部工具栏上点击 All。\n2. 从下拉菜单中选择 ALA。\n3. 在 ALA 菜单中选择 Motions。\n4. 浏览显示的运动/位置选项。"
+  }
+}

evaluation_examples/examples/pymol/MovieSchool_3_task2.json

@@ -0,0 +1,44 @@
+{
+  "id": "MovieSchool_3_task2",
+  "snapshot": "pymol",
+  "instruction": "在 PyMOL 中，通过 Movie 菜单设置视频帧率为 15 FPS。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\PYMOL\\PyMOLWin.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "pymol"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 点击菜单栏中的 Movie。\n2. 从下拉菜单中选择 Frame Rate。\n3. 在 Frame Rate 子菜单中选择 15 FPS。"
+  }
+}

evaluation_examples/examples/pymol/MovieSchool_3_task3.json

@@ -0,0 +1,44 @@
+{
+  "id": "MovieSchool_3_task3",
+  "snapshot": "pymol",
+  "instruction": "在 PyMOL 中，通过 Scene 菜单将当前场景存储为名称 'my_scene'。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\PYMOL\\PyMOLWin.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "pymol"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 点击菜单栏中的 Scene。\n2. 从下拉菜单中选择 Store。\n3. 在弹出的窗口中输入 'my_scene' 作为场景名称。\n4. 点击确认按钮存储场景。"
+  }
+}

evaluation_examples/examples/pymol/MovieSchool_3_task4.json

@@ -0,0 +1,44 @@
+{
+  "id": "MovieSchool_3_task4",
+  "snapshot": "pymol",
+  "instruction": "在 PyMOL 中，通过 Scene 菜单清除所有存储的场景。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\PYMOL\\PyMOLWin.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "pymol"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 点击菜单栏中的 Scene。\n2. 从下拉菜单中选择 Clear。\n3. 在确认对话框中点击是以清除所有场景。"
+  }
+}

evaluation_examples/examples/pymol/MovieSchool_3_task5.json

@@ -0,0 +1,44 @@
+{
+  "id": "MovieSchool_3_task5",
+  "snapshot": "pymol",
+  "instruction": "在 PyMOL 中，通过 Mouse 菜单将鼠标模式设置为 3 Button Motions。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\PYMOL\\PyMOLWin.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "pymol"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 点击菜单栏中的 Mouse。\n2. 从下拉菜单中选择 Edit。\n3. 在 Edit 菜单中选择 Motions。\n4. 在弹出的子菜单中选择 3 Button Motions。"
+  }
+}

evaluation_examples/examples/pymol/Mutagenesis_task4.json

@@ -0,0 +1,44 @@
+{
+  "id": "Mutagenesis_task4",
+  "snapshot": "pymol",
+  "instruction": "解释 PyMOL Mutagenesis 工具中的颜色代码，以理解范德瓦尔斯半径重叠情况。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\PYMOL\\PyMOLWin.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "pymol"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 打开 PyMOL 软件并加载任意结构。\n2. 通过 Wizard → Mutagenesis 打开 Mutagenesis 工具。\n3. 查看 Mutagenesis 工具中指定区域的颜色提示。\n4. 确认颜色解释：绿色表示轻微重叠，红色表示显著重叠。\n5. 使用颜色信息选择适当的操作来优化结构。"
+  }
+}

evaluation_examples/examples/pymol/Practical_Pymol_for_Beginners_task6.json

@@ -0,0 +1,44 @@
+{
+  "id": "Practical_Pymol_for_Beginners_task6",
+  "snapshot": "pymol",
+  "instruction": "在 PyMOL 中，通过 File → Save Session 保存当前会话为 .pse 文件。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\PYMOL\\PyMOLWin.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "pymol"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 确保所有需要的对象和场景已设置好。\n2. 点击菜单栏 File → Save Session。\n3. 在弹出的窗口中命名文件并保存为 .pse 格式。\n4. 确认会话被成功保存。"
+  }
+}

evaluation_examples/examples/vesta/VESTA_Manual_task1.json

@@ -0,0 +1,57 @@
+{
+  "id": "VESTA_Manual_task1",
+  "snapshot": "vesta",
+  "instruction": "在 VESTA 中启动软件并加载结构文件 example_structure.cif。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "upload_file",
+      "parameters": {
+        "files": [
+          {
+            "local_path": "evaluation_examples/data/vesta/example_structure.cif",
+            "path": "C:\\Users\\user\\Desktop\\example_structure.cif"
+          }
+        ]
+      }
+    },
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\VESTA-win64\\VESTA.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "vesta"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [
+      "example_structure.cif"
+    ],
+    "steps": "1. 启动 VESTA 软件。\n2. 点击 File → Open。\n3. 在文件浏览窗口中选择 example_structure.cif 文件。\n4. 点击 Open 按钮加载文件。\n5. 确认结构已显示在视图窗口中。"
+  }
+}

evaluation_examples/examples/vesta/VESTA_Manual_task10.json

@@ -0,0 +1,57 @@
+{
+  "id": "VESTA_Manual_task10",
+  "snapshot": "vesta",
+  "instruction": "在 VESTA 中导入 Crystallographic Information File (CIF) 并查看其对称性。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "upload_file",
+      "parameters": {
+        "files": [
+          {
+            "local_path": "evaluation_examples/data/vesta/sample.cif",
+            "path": "C:\\Users\\user\\Desktop\\sample.cif"
+          }
+        ]
+      }
+    },
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\VESTA-win64\\VESTA.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "vesta"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [
+      "sample.cif"
+    ],
+    "steps": "1. 启动 VESTA 软件。\n2. 点击 File → Open，打开文件浏览器。\n3. 选择 sample.cif 文件并点击 Open。\n4. 加载文件后，点击 Edit → Data。\n5. 选择 Unit Cell 标签。\n6. 查看 Symmetry 选项卡中显示的对称性信息。"
+  }
+}

evaluation_examples/examples/vesta/VESTA_Manual_task11.json

@@ -0,0 +1,57 @@
+{
+  "id": "VESTA_Manual_task11",
+  "snapshot": "vesta",
+  "instruction": "在 VESTA 中生成 polyhedra 并调整其透明度。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "upload_file",
+      "parameters": {
+        "files": [
+          {
+            "local_path": "evaluation_examples/data/vesta/example_structure.cif",
+            "path": "C:\\Users\\user\\Desktop\\example_structure.cif"
+          }
+        ]
+      }
+    },
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\VESTA-win64\\VESTA.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "vesta"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [
+      "example_structure.cif"
+    ],
+    "steps": "1. 打开 VESTA 软件并加载 example_structure.vesta 文件。\n2. 点击 Edit → Properties。\n3. 在 Properties 对话框中选择 Polyhedra 标签。\n4. 勾选 Enable Polyhedra 绘图。\n5. 调整 Transparency 滑块到所需透明度值，例如 50%。\n6. 点击 OK 按钮保存设置。\n7. 验证主视图窗口中 Polyhedra 的更新显示。"
+  }
+}

evaluation_examples/examples/vesta/VESTA_Manual_task2.json

@@ -0,0 +1,57 @@
+{
+  "id": "VESTA_Manual_task2",
+  "snapshot": "vesta",
+  "instruction": "在 VESTA 中设置显示模式为 Ball-and-Stick，用于 example_structure.cif 文件。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "upload_file",
+      "parameters": {
+        "files": [
+          {
+            "local_path": "evaluation_examples/data/vesta/example_structure.cif",
+            "path": "C:\\Users\\user\\Desktop\\example_structure.cif"
+          }
+        ]
+      }
+    },
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\VESTA-win64\\VESTA.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "vesta"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [
+      "example_structure.cif"
+    ],
+    "steps": "1. 打开 VESTA 软件并加载文件 loaded_structure.vesta。\n2. 在顶部菜单中选择 View → Display Style。\n3. 在弹出的对话框中选择 Ball-and-Stick 模式。\n4. 点击 OK 按钮应用设置。\n5. 查看主视图窗口以确认显示模式已改变。"
+  }
+}

evaluation_examples/examples/vesta/VESTA_Manual_task3.json

@@ -0,0 +1,44 @@
+{
+  "id": "VESTA_Manual_task3",
+  "snapshot": "vesta",
+  "instruction": "在 VESTA 中测量当前晶体中两个原子之间的距离。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\VESTA-win64\\VESTA.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "vesta"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [],
+    "steps": "1. 在 VESTA 软件中打开任何结构文件。\n2. 点击垂直工具栏中的 Measure Distance 工具。\n3. 在主视图窗口中选择两个要测量距离的原子。\n4. 在 Measure Distance 工具下确认显示两个原子之间的距离。\n5. 验证输出的距离值是否正确显示。"
+  }
+}

evaluation_examples/examples/vesta/VESTA_Manual_task4.json

@@ -0,0 +1,57 @@
+{
+  "id": "VESTA_Manual_task4",
+  "snapshot": "vesta",
+  "instruction": "在 VESTA 中定义自定义绘图边界。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "upload_file",
+      "parameters": {
+        "files": [
+          {
+            "local_path": "evaluation_examples/data/vesta/MgB2.cif",
+            "path": "C:\\Users\\user\\Desktop\\MgB2.cif"
+          }
+        ]
+      }
+    },
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\VESTA-win64\\VESTA.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "vesta"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [
+      "MgB2.cif"
+    ],
+    "steps": "1. 打开 VESTA 软件并加载文件 MgB2.cif。\n2. 点击左侧侧边栏的 Objects → Boundary 按钮，打开 Boundary 对话框。\n3. 在对话框中调整范围 (x[min], x[max], y[min], y[max], z[min], z[max]) 为自定义值，例如 0 到 1。\n4. 点击 OK 或 Apply 按钮。\n5. 查看修改后的晶体绘图边界显示在主视图中。"
+  }
+}

evaluation_examples/examples/vesta/VESTA_Manual_task5.json

@@ -0,0 +1,57 @@
+{
+  "id": "VESTA_Manual_task5",
+  "snapshot": "vesta",
+  "instruction": "通过 VESTA 的 Properties 对话框调整晶体键的颜色和半径。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "upload_file",
+      "parameters": {
+        "files": [
+          {
+            "local_path": "evaluation_examples/data/vesta/xTiO2.cif",
+            "path": "C:\\Users\\user\\Desktop\\xTiO2.cif"
+          }
+        ]
+      }
+    },
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\VESTA-win64\\VESTA.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "vesta"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [
+      "xTiO2.cif"
+    ],
+    "steps": "1. 打开 VESTA 软件并加载 xTiO2.vesta 文件。\n2. 点击 Edit → Properties。\n3. 在对话框中导航到 Bonds 页面。\n4. 调整 Radius (cylinder) 输入框值，例如更改为 0.3。\n5. 修改颜色设置为 RGB 值 (100, 150, 200)。\n6. 点击 OK 按钮保存更改并关闭对话框。\n7. 确保更改在主视图中可见。"
+  }
+}

evaluation_examples/examples/vesta/VESTA_Manual_task6.json

@@ -0,0 +1,57 @@
+{
+  "id": "VESTA_Manual_task6",
+  "snapshot": "vesta",
+  "instruction": "在 VESTA 中切换晶体投影为 [110] 方向。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "upload_file",
+      "parameters": {
+        "files": [
+          {
+            "local_path": "evaluation_examples/data/vesta/Si.cif",
+            "path": "C:\\Users\\user\\Desktop\\Si.cif"
+          }
+        ]
+      }
+    },
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\VESTA-win64\\VESTA.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "vesta"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [
+      "Si.cif"
+    ],
+    "steps": "1. 打开 VESTA 软件并加载文件 Si.cif。\n2. 在顶部菜单中选择 View → Lattice Planes。\n3. 在对话框中选择 [110] 方向作为投影。\n4. 点击 OK 按钮应用更改。\n5. 确认主视图窗口中显示的是 [110] 方向的晶体投影。"
+  }
+}

evaluation_examples/examples/vesta/VESTA_Manual_task7.json

@@ -0,0 +1,57 @@
+{
+  "id": "VESTA_Manual_task7",
+  "snapshot": "vesta",
+  "instruction": "在 VESTA 中生成晶体的二维 (2D) 投影视图。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "upload_file",
+      "parameters": {
+        "files": [
+          {
+            "local_path": "evaluation_examples/data/vesta/rutile_TiO2.cif",
+            "path": "C:\\Users\\user\\Desktop\\rutile_TiO2.cif"
+          }
+        ]
+      }
+    },
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\VESTA-win64\\VESTA.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "vesta"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [
+      "rutile_TiO2.cif"
+    ],
+    "steps": "1. 打开 VESTA 软件并加载 rutile_TiO2.cif 文件。\n2. 在顶部菜单中选择 File → Export → 2D Image。\n3. 在弹出的对话框中设置输出格式为 PNG，并选择合适的分辨率 (例如 300 dpi)。\n4. 设置保存路径为桌面并命名文件为 projection.png。\n5. 点击 Save 以导出图像。\n6. 验证桌面的 PNG 文件是否正确生成。"
+  }
+}

evaluation_examples/examples/vesta/VESTA_Manual_task8.json

@@ -0,0 +1,57 @@
+{
+  "id": "VESTA_Manual_task8",
+  "snapshot": "vesta",
+  "instruction": "在 VESTA 中查看倒易晶格的详细几何参数。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "upload_file",
+      "parameters": {
+        "files": [
+          {
+            "local_path": "evaluation_examples/data/vesta/YBa2Cu3O7.cif",
+            "path": "C:\\Users\\user\\Desktop\\YBa2Cu3O7.cif"
+          }
+        ]
+      }
+    },
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\VESTA-win64\\VESTA.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "vesta"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [
+      "YBa2Cu3O7.cif"
+    ],
+    "steps": "1. 打开 VESTA 软件并加载文件 YBa2Cu3O7.vesta。\n2. 在顶部菜单中选择 Edit → Data → Reciprocal Lattice Parameters。\n3. 查看弹出的对话框中的倒易晶格详细数据。\n4. 点击 OK 关闭对话框。\n5. 验证数据是否已在 Text Area 中正确显示。"
+  }
+}

evaluation_examples/examples/vesta/VESTA_Manual_task9.json

@@ -0,0 +1,57 @@
+{
+  "id": "VESTA_Manual_task9",
+  "snapshot": "vesta",
+  "instruction": "在 VESTA 中使用 Fourier Synthesis 生成电子密度图。",
+  "source": "custom",
+  "config": [
+    {
+      "type": "upload_file",
+      "parameters": {
+        "files": [
+          {
+            "local_path": "evaluation_examples/data/vesta/monazite.cif",
+            "path": "C:\\Users\\user\\Desktop\\monazite.cif"
+          }
+        ]
+      }
+    },
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "C:\\VESTA-win64\\VESTA.exe"
+        ]
+      }
+    },
+    {
+      "type": "sleep",
+      "parameters": {
+        "seconds": 5
+      }
+    }
+  ],
+  "trajectory": "trajectories/",
+  "related_apps": [
+    "vesta"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 3
+        }
+      }
+    ],
+    "func": "vllm_eval"
+  },
+  "proxy": false,
+  "fixed_ip": false,
+  "possibility_of_env_change": "low",
+  "metadata": {
+    "input_files": [
+      "monazite.cif"
+    ],
+    "steps": "1. 打开 VESTA 软件并加载文件 monazite.vesta。\n2. 在顶部菜单中选择 Utilities → Fourier Synthesis。\n3. 在弹出的对话框中设置分辨率值为 0.05。\n4. 点击 Calculate 按钮开始生成电子密度图。\n5. 验证生成的图形是否出现在主视图中。"
+  }
+}

evaluation_examples/extract_instructions.py

@@ -0,0 +1,604 @@
+import os
+import sys
+import asyncio
+import aiohttp
+import base64
+import logging
+from pathlib import Path
+from typing import List, Optional
+import tempfile
+import shutil
+from dataclasses import dataclass
+from datetime import datetime
+import json
+
+# Configuration
+SCRIPT_DIR = Path(__file__).parent
+PROJECT_ROOT = SCRIPT_DIR.parent
+
+API_BASE_URL = os.getenv("OPENAI_BASE_URL")
+API_URL = f"{API_BASE_URL}/chat/completions" if API_BASE_URL else None
+API_KEY = os.getenv("OPENAI_API_KEY")
+MODEL_NAME = "gemini-2.5-pro"
+MAX_CONCURRENT_REQUESTS = 5
+INPUT_FOLDER = "/Users/cuihang/Downloads/test_files"
+EXAMPLES_FOLDER = PROJECT_ROOT / "evaluation_examples" / "examples"
+TEST_ALL_JSON = PROJECT_ROOT / "evaluation_examples" / "test_all.json"
+
+# Retry configuration
+MAX_RETRY_ATTEMPTS = 3
+RETRY_DELAY = 5
+RETRY_BACKOFF = 2
+
+# Image limit
+MAX_IMAGES_PER_REQUEST = 50
+
+# Supported file extensions
+SUPPORTED_EXTENSIONS = {'.docx', '.doc', '.ppt', '.pptx', '.pdf', '.mp4', '.avi', '.mov', '.mkv'}
+
+SYSTEM_PROMPT = """You are an AI assistant that generates precise, executable step-by-step instructions for desktop software operations.
+
+Your task:
+Convert the provided document information into precise operation instructions that can be executed step-by-step by an AI agent in a software GUI.
+
+Output requirements (no additional explanatory text):
+------------------------------------------------
+
+[Task Goal]
+Describe in one sentence the final task result to be achieved in the software.
+
+[Input Files]
+Specify the file names, types, and locations involved in this operation.
+- If the document provides complete paths, record them as is
+- If only file names are mentioned (e.g., data.xlsx), record the filename and note "complete path not specified in document"
+- If no input files are mentioned, write "no input files required"
+
+[Detailed Operation Steps (GUI Level)]
+Break down the task into atomic GUI operation steps.
+Each step must meet the following conditions:
+- Contains only one explicit, indivisible GUI atomic action
+- Must specify the menus, panels, buttons, or controls involved
+- Must specify parameter names and option values involved
+- Arranged in the actual operation order of the software
+- Must include software launch steps (e.g., double-click desktop icon, launch from start menu, etc.)
+
+Step format example:
+1. Double-click the [Software Name] icon on the desktop to launch the software.
+2. Click "File → Open" in the main menu bar.
+3. In the file selection dialog, navigate to the specified directory and select file [filename].
+4. Click the "Open" button to confirm.
+5. ... (and so on)
+
+------------------------------------------------
+
+[Handling Uncertain Information]
+- Strictly generate operation steps based on document content, do not add features or menus not mentioned
+- If operation steps are unclear or ambiguous, infer based on common software operation flows
+- If parameter values in the document are unclear, note "[set according to actual needs]" in the step
+
+[Output Format]
+Output in JSON format with the following fields:
+{
+    "input_files": ["file1", "file2", "..."],
+    "task_goal": "...",
+    "steps": "A string containing all operation steps, arranged in order, with numbered prefix for each step, separated by newlines"
+}
+Note: Output must be strict JSON format, with no extra text or explanations."""
+
+
+# Logging configuration
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(message)s',
+    handlers=[
+        logging.StreamHandler(sys.stdout)
+    ]
+)
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class ProcessingStats:
+    """Processing statistics tracker"""
+    total_files: int = 0
+    completed_files: int = 0
+    failed_files: int = 0
+    retried_files: int = 0
+    start_time: datetime = None
+    failed_list: List[tuple] = None
+
+    def __post_init__(self):
+        if self.start_time is None:
+            self.start_time = datetime.now()
+        if self.failed_list is None:
+            self.failed_list = []
+
+    def add_completed(self):
+        self.completed_files += 1
+        self._log_progress()
+
+    def add_failed(self, file_path: str, error: str):
+        self.failed_files += 1
+        self.failed_list.append((file_path, error))
+        self._log_progress()
+
+    def add_retry(self):
+        self.retried_files += 1
+
+    def _log_progress(self):
+        processed = self.completed_files + self.failed_files
+        percentage = (processed / self.total_files * 100) if self.total_files > 0 else 0
+        elapsed = (datetime.now() - self.start_time).total_seconds()
+
+        if processed > 0:
+            avg_time = elapsed / processed
+            remaining = (self.total_files - processed) * avg_time
+            eta = f"{int(remaining // 60)}m{int(remaining % 60)}s"
+        else:
+            eta = "calculating..."
+
+        logger.info(f"Progress: {processed}/{self.total_files} ({percentage:.1f}%) | "
+                   f"Success: {self.completed_files} | Failed: {self.failed_files} | "
+                   f"Retried: {self.retried_files} | ETA: {eta}")
+
+    def print_summary(self):
+        elapsed = (datetime.now() - self.start_time).total_seconds()
+        logger.info("=" * 60)
+        logger.info("Processing Complete")
+        logger.info("=" * 60)
+        logger.info(f"Total files: {self.total_files}")
+        logger.info(f"Success: {self.completed_files}")
+        logger.info(f"Failed: {self.failed_files}")
+        logger.info(f"Total retries: {self.retried_files}")
+        logger.info(f"Total time: {int(elapsed // 60)}m{int(elapsed % 60)}s")
+
+        if self.failed_list:
+            logger.info("\nFailed files:")
+            for file_path, error in self.failed_list:
+                logger.info(f"  - {file_path}")
+                logger.info(f"    Error: {error}")
+
+        self._save_report()
+
+    def _save_report(self):
+        report = {
+            "total_files": self.total_files,
+            "completed": self.completed_files,
+            "failed": self.failed_files,
+            "retries": self.retried_files,
+            "start_time": self.start_time.isoformat(),
+            "end_time": datetime.now().isoformat(),
+            "elapsed_seconds": (datetime.now() - self.start_time).total_seconds(),
+            "failed_files": [{"file": f, "error": e} for f, e in self.failed_list]
+        }
+
+        report_file = Path(EXAMPLES_FOLDER) / "processing_report.json"
+        with open(report_file, 'w', encoding='utf-8') as f:
+            json.dump(report, f, ensure_ascii=False, indent=2)
+
+        logger.info(f"\nDetailed report saved to: {report_file}")
+
+
+stats = ProcessingStats()
+software_tests = {}
+
+
+def check_dependencies():
+    """Check and prompt for missing dependencies"""
+    missing = []
+
+    try:
+        import pdf2image
+    except ImportError:
+        missing.append("pdf2image")
+
+    try:
+        import PIL
+    except ImportError:
+        missing.append("Pillow")
+
+    try:
+        import cv2
+    except ImportError:
+        missing.append("opencv-python or opencv-python-headless")
+
+    if not shutil.which("soffice") and not shutil.which("libreoffice"):
+        logger.warning("LibreOffice not detected, cannot convert .doc and .ppt files")
+        logger.info("Install: sudo apt-get install libreoffice (Linux) or download from https://www.libreoffice.org/")
+
+    if missing:
+        logger.error(f"Missing dependencies: {', '.join(missing)}")
+        logger.info(f"Install with: pip install {' '.join(missing)}")
+        logger.info("Note: pdf2image also requires poppler")
+        logger.info("  - Ubuntu/Debian: sudo apt-get install poppler-utils")
+        logger.info("  - macOS: brew install poppler")
+        logger.info("  - Windows: download from https://github.com/oschwartz10612/poppler-windows/releases/")
+        return False
+    return True
+
+
+def convert_pdf_to_images(pdf_path: str) -> List[str]:
+    """Convert PDF to base64-encoded images"""
+    try:
+        from pdf2image import convert_from_path
+        from PIL import Image
+        import io
+
+        images = convert_from_path(pdf_path, dpi=150, fmt='jpeg')
+        base64_images = []
+
+        for img in images:
+            buffer = io.BytesIO()
+            img.save(buffer, format='JPEG', quality=100)
+            img_base64 = base64.b64encode(buffer.getvalue()).decode('utf-8')
+            base64_images.append(img_base64)
+
+        return base64_images
+    except Exception as e:
+        logger.error(f"PDF conversion failed for {pdf_path}: {str(e)}")
+        return []
+
+
+def convert_office_to_pdf(input_path: str) -> Optional[str]:
+    """Convert Office documents to PDF using LibreOffice"""
+    try:
+        import subprocess
+
+        temp_dir = tempfile.mkdtemp()
+        soffice_cmd = "soffice" if shutil.which("soffice") else "libreoffice"
+
+        cmd = [
+            soffice_cmd,
+            "--headless",
+            "--convert-to", "pdf",
+            "--outdir", temp_dir,
+            input_path
+        ]
+
+        result = subprocess.run(cmd, capture_output=True, text=True, timeout=60)
+
+        if result.returncode == 0:
+            pdf_name = Path(input_path).stem + ".pdf"
+            pdf_path = os.path.join(temp_dir, pdf_name)
+
+            if os.path.exists(pdf_path):
+                return pdf_path
+
+        logger.error(f"LibreOffice conversion failed: {result.stderr}")
+        return None
+
+    except Exception as e:
+        logger.error(f"Office conversion failed for {input_path}: {str(e)}")
+        return None
+
+
+def convert_document_to_images(file_path: str) -> List[str]:
+    """Convert any supported document to base64-encoded images"""
+    file_ext = Path(file_path).suffix.lower()
+
+    if file_ext == '.pdf':
+        return convert_pdf_to_images(file_path)
+
+    elif file_ext in ['.docx', '.doc', '.ppt', '.pptx']:
+        pdf_path = convert_office_to_pdf(file_path)
+        if pdf_path:
+            images = convert_pdf_to_images(pdf_path)
+            try:
+                os.remove(pdf_path)
+                os.rmdir(os.path.dirname(pdf_path))
+            except:
+                pass
+            return images
+        return []
+
+    elif file_ext in ['.mp4', '.avi', '.mov', '.mkv']:
+        return extract_video_frames(file_path)
+
+    return []
+
+
+def extract_video_frames(video_path: str, num_frames: int = 10) -> List[str]:
+    """Extract key frames from video"""
+    try:
+        import cv2
+
+        cap = cv2.VideoCapture(video_path)
+        total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
+
+        if total_frames == 0:
+            return []
+
+        frame_indices = [int(total_frames * i / (num_frames + 1)) for i in range(1, num_frames + 1)]
+        base64_frames = []
+
+        for idx in frame_indices:
+            cap.set(cv2.CAP_PROP_POS_FRAMES, idx)
+            ret, frame = cap.read()
+
+            if ret:
+                height, width = frame.shape[:2]
+                if width > 1280:
+                    scale = 1280 / width
+                    frame = cv2.resize(frame, (1280, int(height * scale)))
+
+                _, buffer = cv2.imencode('.jpg', frame, [cv2.IMWRITE_JPEG_QUALITY, 85])
+                frame_base64 = base64.b64encode(buffer).decode('utf-8')
+                base64_frames.append(frame_base64)
+
+        cap.release()
+        return base64_frames
+
+    except Exception as e:
+        logger.error(f"Video frame extraction failed for {video_path}: {str(e)}")
+        return []
+
+
+async def call_api_single_batch(images_batch: List[str], file_type: str,
+                                session: aiohttp.ClientSession, batch_num: int = 0) -> tuple[str, bool, int]:
+    """
+    Call API to process a single batch of images
+    Returns: (content, success, status_code)
+    """
+    messages = [{"role": "system", "content": SYSTEM_PROMPT}]
+
+    batch_info = f" (batch {batch_num})" if batch_num > 0 else ""
+    content = [
+        {"type": "text", "text": f"Please analyze the following {file_type} pages/frames{batch_info} and extract the operation workflow:"}
+    ]
+
+    for img_b64 in images_batch:
+        content.append({
+            "type": "image_url",
+            "image_url": {"url": f"data:image/jpeg;base64,{img_b64}"}
+        })
+
+    messages.append({"role": "user", "content": content})
+
+    try:
+        headers = {
+            "Authorization": f"Bearer {API_KEY}",
+            "Content-Type": "application/json"
+        }
+
+        payload = {
+            "model": MODEL_NAME,
+            "messages": messages,
+            "max_tokens": 8192
+        }
+
+        async with session.post(API_URL, headers=headers, json=payload, timeout=180) as response:
+            status_code = response.status
+            if status_code == 200:
+                result = await response.json()
+                return result['choices'][0]['message']['content'], True, status_code
+            else:
+                error_text = await response.text()
+                return f"[API call failed: {status_code}]\n{error_text}", False, status_code
+
+    except asyncio.TimeoutError:
+        return "[API call timeout]", False, 0
+    except Exception as e:
+        return f"[API call error: {str(e)}]", False, 0
+
+
+async def call_multimodal_api_with_retry(file_path: str, session: aiohttp.ClientSession) -> tuple[str, bool]:
+    """
+    Call multimodal API to analyze document images with retry mechanism
+    Returns: (content, success)
+    """
+    images_base64 = convert_document_to_images(file_path)
+
+    if not images_base64:
+        error_msg = f"[Document conversion failed: unable to convert {Path(file_path).name} to images]"
+        return error_msg, False
+
+    file_type = "video" if Path(file_path).suffix.lower() in ['.mp4', '.avi', '.mov', '.mkv'] else "document"
+    total_images = len(images_base64)
+
+    if total_images > MAX_IMAGES_PER_REQUEST:
+        images_base64 = images_base64[:MAX_IMAGES_PER_REQUEST]
+        total_images = MAX_IMAGES_PER_REQUEST
+
+    for attempt in range(1, MAX_RETRY_ATTEMPTS + 1):
+        try:
+            content, success, status_code = await call_api_single_batch(images_base64, file_type, session)
+
+            if success:
+                return content, True
+
+            if status_code == 413:
+                return f"[File too large: server refused to process the file]", False
+
+            if attempt < MAX_RETRY_ATTEMPTS:
+                delay = RETRY_DELAY * (RETRY_BACKOFF ** (attempt - 1))
+                logger.info(f"\nRetry {attempt}/{MAX_RETRY_ATTEMPTS}: {Path(file_path).name} (waiting {delay}s)")
+                stats.add_retry()
+                await asyncio.sleep(delay)
+                continue
+
+            return content, False
+
+        except asyncio.TimeoutError:
+            if attempt < MAX_RETRY_ATTEMPTS:
+                delay = RETRY_DELAY * (RETRY_BACKOFF ** (attempt - 1))
+                logger.info(f"\nRetry {attempt}/{MAX_RETRY_ATTEMPTS}: {Path(file_path).name} (timeout, waiting {delay}s)")
+                stats.add_retry()
+                await asyncio.sleep(delay)
+                continue
+            return "[API call timeout]", False
+
+        except Exception as e:
+            if attempt < MAX_RETRY_ATTEMPTS:
+                delay = RETRY_DELAY * (RETRY_BACKOFF ** (attempt - 1))
+                logger.info(f"\nRetry {attempt}/{MAX_RETRY_ATTEMPTS}: {Path(file_path).name} (error, waiting {delay}s)")
+                stats.add_retry()
+                await asyncio.sleep(delay)
+                continue
+            return f"[API call error: {str(e)}]", False
+
+    return "[Max retry attempts reached]", False
+
+
+async def process_file(file_path: str, session: aiohttp.ClientSession,
+                       semaphore: asyncio.Semaphore):
+    """Process a single file"""
+    async with semaphore:
+        try:
+            content, success = await call_multimodal_api_with_retry(file_path, session)
+
+            file_path_obj = Path(file_path).resolve()
+            input_folder_obj = Path(INPUT_FOLDER).resolve()
+
+            try:
+                rel_path = file_path_obj.relative_to(input_folder_obj)
+                software_name = rel_path.parts[0] if len(rel_path.parts) > 1 else "unknown"
+            except ValueError:
+                software_name = "unknown"
+
+            file_stem = file_path_obj.stem
+            test_id = file_stem
+            output_file = Path(EXAMPLES_FOLDER) / software_name / f"{file_stem}.json"
+            output_file.parent.mkdir(parents=True, exist_ok=True)
+
+            import re
+            match = re.search(r'```json\s*([\s\S]*?)\s*```', content)
+            content = match.group(1) if match else content
+
+            if success:
+                api_result = json.loads(content)
+
+                data = {
+                    "id": test_id,
+                    "snapshot": "snapshot",
+                    "instruction": api_result.get("steps", ""),
+                    "source": "custom",
+                    "config": [],
+                    "trajectory": "trajectories/",
+                    "related_apps": [software_name],
+                    "evaluator": {
+                        "postconfig": [
+                            {
+                                "type": "sleep",
+                                "parameters": {
+                                    "seconds": 3
+                                }
+                            }
+                        ],
+                        "func": "vllm_eval"
+                    },
+                    "proxy": False,
+                    "fixed_ip": False,
+                    "possibility_of_env_change": "low",
+                    "metadata": {
+                        "input_files": api_result.get("input_files", []),
+                        "task_goal": api_result.get("task_goal", "")
+                    }
+                }
+
+                if software_name not in software_tests:
+                    software_tests[software_name] = []
+                software_tests[software_name].append(test_id)
+
+            else:
+                data = {
+                    "id": test_id,
+                    "error": content,
+                    "status": "failed"
+                }
+
+            with open(output_file, 'w', encoding='utf-8') as f:
+                json.dump(data, f, ensure_ascii=False, indent=2)
+
+            if success:
+                stats.add_completed()
+            else:
+                stats.add_failed(file_path, content)
+
+        except Exception as e:
+            error_msg = str(e)
+            stats.add_failed(file_path, error_msg)
+            logger.error(f"\nError processing {file_path}: {error_msg}")
+
+
+def find_all_files(input_folder: str) -> List[str]:
+    """Recursively find all supported files"""
+    all_files = []
+
+    for root, dirs, files in os.walk(input_folder):
+        for file in files:
+            file_path = os.path.join(root, file)
+            if Path(file_path).suffix.lower() in SUPPORTED_EXTENSIONS:
+                all_files.append(file_path)
+
+    return all_files
+
+
+def save_test_all_json():
+    """Save aggregated test_all.json"""
+    test_all_path = Path(TEST_ALL_JSON)
+    if test_all_path.exists():
+        with open(test_all_path, 'r', encoding='utf-8') as f:
+            existing_data = json.load(f)
+    else:
+        existing_data = {}
+
+    for software, test_ids in software_tests.items():
+        if software in existing_data:
+            existing_data[software] = list(set(existing_data[software] + test_ids))
+        else:
+            existing_data[software] = test_ids
+
+    test_all_path.parent.mkdir(parents=True, exist_ok=True)
+    with open(test_all_path, 'w', encoding='utf-8') as f:
+        json.dump(existing_data, f, ensure_ascii=False, indent=2)
+
+    logger.info(f"\nTest index updated: {test_all_path}")
+    logger.info(f"Software included: {list(existing_data.keys())}")
+
+
+async def main():
+    """Main function"""
+    if not check_dependencies():
+        return
+
+    if not Path(INPUT_FOLDER).exists():
+        logger.error(f"Input directory does not exist: {INPUT_FOLDER}")
+        return
+
+    Path(EXAMPLES_FOLDER).mkdir(parents=True, exist_ok=True)
+
+    logger.info("Scanning files...")
+    logger.info(f"Input directory: {INPUT_FOLDER}")
+    logger.info(f"Output directory: {EXAMPLES_FOLDER}")
+    logger.info(f"Test index file: {TEST_ALL_JSON}\n")
+
+    files = find_all_files(INPUT_FOLDER)
+    stats.total_files = len(files)
+
+    logger.info(f"Found {len(files)} files")
+    logger.info(f"Configuration: max retries={MAX_RETRY_ATTEMPTS}, concurrency={MAX_CONCURRENT_REQUESTS}")
+    logger.info("=" * 60 + "\n")
+
+    if not files:
+        logger.warning("No supported files found")
+        return
+
+    semaphore = asyncio.Semaphore(MAX_CONCURRENT_REQUESTS)
+
+    async with aiohttp.ClientSession() as session:
+        tasks = [
+            process_file(file, session, semaphore)
+            for file in files
+        ]
+        await asyncio.gather(*tasks, return_exceptions=True)
+
+    save_test_all_json()
+    stats.print_summary()
+
+    logger.info("\nCompleted!")
+    logger.info(f"  - Test cases saved to: {EXAMPLES_FOLDER}")
+    logger.info(f"  - Test index updated: {TEST_ALL_JSON}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

evaluation_examples/extract_instructions_v2.py

@@ -0,0 +1,565 @@
+import os
+import sys
+import asyncio
+import aiohttp
+import base64
+import logging
+from pathlib import Path
+from typing import List, Optional
+import tempfile
+import shutil
+from dataclasses import dataclass
+from datetime import datetime
+import json
+import re
+
+# Configuration
+SCRIPT_DIR = Path(__file__).parent
+PROJECT_ROOT = SCRIPT_DIR.parent
+
+API_BASE_URL = os.getenv("OPENAI_BASE_URL", "https://api.openai.com/v1")
+API_URL = f"{API_BASE_URL}/chat/completions"
+API_KEY = os.getenv("OPENAI_API_KEY")
+MODEL_NAME = os.getenv("EXTRACT_MODEL", "gpt-4o")  # Configurable via env var
+MAX_CONCURRENT_REQUESTS = 5
+
+# Input folder where PDFs/Docs are stored, organized by software name
+# e.g. evaluation_examples/inputs/vesta/tutorial.pdf
+INPUT_FOLDER = PROJECT_ROOT / "evaluation_examples" / "inputs"
+EXAMPLES_FOLDER = PROJECT_ROOT / "evaluation_examples" / "examples"
+TEST_ALL_JSON = PROJECT_ROOT / "evaluation_examples" / "test_all.json"
+
+# Retry configuration
+MAX_RETRY_ATTEMPTS = 3
+RETRY_DELAY = 5
+RETRY_BACKOFF = 2
+
+# Image limit - keep low to avoid 413 payload too large errors
+MAX_IMAGES_PER_REQUEST = 20
+
+# Supported file extensions
+SUPPORTED_EXTENSIONS = {'.docx', '.doc', '.ppt', '.pptx', '.pdf', '.mp4', '.avi', '.mov', '.mkv'}
+
+# Software-specific launch config and snapshot mapping
+# Maps software folder name -> {"snapshot": ..., "config": [...]}
+SOFTWARE_CONFIG = {
+    "avogadro": {
+        "snapshot": "avogadro",
+        "config": [
+            {"type": "launch", "parameters": {"command": ["C:\\Avogadro2\\bin\\avogadro2.exe"]}},
+            {"type": "sleep", "parameters": {"seconds": 5}}
+        ]
+    },
+    "imagej": {
+        "snapshot": "imagej",
+        "config": [
+            {"type": "launch", "parameters": {"command": ["C:\\ImageJ\\ImageJ.exe"]}},
+            {"type": "sleep", "parameters": {"seconds": 5}}
+        ]
+    },
+    "origin": {
+        "snapshot": "origin",
+        "config": [
+            {"type": "launch", "parameters": {"command": ["C:\\OriginLab\\Origin2025b\\Origin64.exe"]}},
+            {"type": "sleep", "parameters": {"seconds": 5}}
+        ]
+    },
+    "ovito": {
+        "snapshot": "ovito",
+        "config": [
+            {"type": "launch", "parameters": {"command": ["C:\\OVITO Basic\\ovito.exe"]}},
+            {"type": "sleep", "parameters": {"seconds": 5}}
+        ]
+    },
+    "pymol": {
+        "snapshot": "pymol",
+        "config": [
+            {"type": "launch", "parameters": {"command": ["C:\\PYMOL\\PyMOLWin.exe"]}},
+            {"type": "sleep", "parameters": {"seconds": 5}}
+        ]
+    },
+    "vesta": {
+        "snapshot": "vesta",
+        "config": [
+            {"type": "launch", "parameters": {"command": ["C:\\VESTA-win64\\VESTA.exe"]}},
+            {"type": "sleep", "parameters": {"seconds": 5}}
+        ]
+    },
+}
+
+# Default config for unknown software
+DEFAULT_SOFTWARE_CONFIG = {
+    "snapshot": "snapshot",
+    "config": []
+}
+
+SYSTEM_PROMPT = """你是一个科研软件 GUI 自动化测试专家。你的任务是从教程文档中提取出多个**具体的、可执行的、可验证的** GUI 操作任务。
+
+## 核心要求
+这些任务将被用于测试 AI Agent 操控桌面软件的能力。每个任务必须足够具体，让 Agent 明确知道要做什么，做完后能通过截图判断是否成功。
+
+## 任务粒度要求（非常重要）
+- **每个任务应该是 3-8 步 GUI 操作就能完成的小任务**
+- **task_goal 必须包含具体的参数值、文件名、菜单路径等细节**
+- **绝对不要写模糊的指令**
+
+### ❌ 错误示例（太模糊）：
+- "Perform phase identification" — Agent 不知道用哪个文件、选什么参数
+- "Export data" — 导出什么格式？保存到哪里？
+- "Calculate crystallite size" — 选哪个峰？什么参数？
+
+### ✅ 正确示例（具体可执行）：
+- "在 ImageJ 中，通过 File → Open 打开桌面上的 cell_image.tif 文件"
+- "在 ImageJ 中，使用 Image → Adjust → Threshold 对当前图像进行阈值分割，选择 Default 方法并点击 Apply"
+- "在 ImageJ 中，通过 Analyze → Measure 测量当前选区的面积和平均灰度值"
+- "在 ImageJ 中，使用 Process → Filters → Gaussian Blur 对图像施加半径为 2.0 像素的高斯模糊"
+- "在 Avogadro 2 中，通过 Build → Insert → Molecule 搜索并插入一个 benzene 分子"
+- "在 VESTA 中通过 File → Open 打开桌面上的 Si.cif 文件，然后将视角旋转到 [110] 方向"
+
+## 输出格式
+返回严格的 JSON 对象：
+{
+    "tasks": [
+        {
+            "task_goal": "一句话具体描述要做什么（包含软件名、菜单路径、文件名、参数值等具体信息）。用中文。",
+            "input_files": ["涉及的文件名列表，如 'sample.raw'。如果不需要输入文件则为空列表 []"],
+            "steps": "详细的 GUI 操作步骤，带编号，用换行分隔"
+        }
+    ]
+}
+
+## 任务提取规则
+1. **独立性**：每个任务都能独立完成（假设软件已打开或从头启动）
+2. **具体性**：task_goal 中必须包含教程中提到的具体文件名、参数值、菜单名称
+3. **可验证性**：完成后应该能从屏幕截图看出任务是否成功（例如：文件已打开、图表已显示、对话框已出现等）
+4. **忠实性**：只描述教程中实际出现的操作，不要编造功能
+5. **数量**：从一份教程中提取 10-15 个不同的任务，覆盖教程的各个章节。优先选择最常用、最有代表性的操作
+6. **软件名称**：task_goal 必须以「在 XXX 中，」开头，明确指出软件名称
+7. **难度分布**：包含简单（2-3步）、中等（4-5步）、较难（6-8步）的任务各占三分之一
+
+"""
+
+# Logging configuration
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(message)s',
+    handlers=[
+        logging.StreamHandler(sys.stdout)
+    ]
+)
+logger = logging.getLogger(__name__)
+
+stats = None # Will be initialized in main
+
+@dataclass
+class ProcessingStats:
+    """Processing statistics tracker"""
+    total_files: int = 0
+    completed_files: int = 0
+    failed_files: int = 0
+    retried_files: int = 0
+    generated_tasks: int = 0
+    start_time: datetime = None
+    failed_list: List[tuple] = None
+
+    def __post_init__(self):
+        if self.start_time is None:
+            self.start_time = datetime.now()
+        if self.failed_list is None:
+            self.failed_list = []
+
+    def add_completed(self, num_tasks=1):
+        self.completed_files += 1
+        self.generated_tasks += num_tasks
+        self._log_progress()
+
+    def add_failed(self, file_path: str, error: str):
+        self.failed_files += 1
+        self.failed_list.append((file_path, error))
+        self._log_progress()
+
+    def add_retry(self):
+        self.retried_files += 1
+
+    def _log_progress(self):
+        processed = self.completed_files + self.failed_files
+        percentage = (processed / self.total_files * 100) if self.total_files > 0 else 0
+        elapsed = (datetime.now() - self.start_time).total_seconds()
+
+        logger.info(f"Progress: {processed}/{self.total_files} ({percentage:.1f}%) | "
+                   f"Tasks Gen: {self.generated_tasks} | Failed: {self.failed_files}")
+
+# -----------------------------------------------------------------------------
+# Dependency Checks & File Conversion (Copied & Adapted from original script)
+# -----------------------------------------------------------------------------
+
+def check_dependencies():
+    """Check and prompt for missing dependencies"""
+    missing = []
+    try: import pdf2image
+    except ImportError: missing.append("pdf2image")
+    try: import PIL
+    except ImportError: missing.append("Pillow")
+    try: import cv2
+    except ImportError: missing.append("opencv-python")
+
+    if not shutil.which("soffice") and not shutil.which("libreoffice"):
+        logger.warning("LibreOffice not detected (needed for .doc/.ppt)")
+
+    if missing:
+        logger.error(f"Missing dependencies: {', '.join(missing)}")
+        return False
+    return True
+
+def convert_pdf_to_images(pdf_path: str) -> List[str]:
+    try:
+        from pdf2image import convert_from_path
+        import io
+
+        # First, get total page count at very low DPI
+        quick_check = convert_from_path(pdf_path, dpi=36, fmt='jpeg')
+        total_pages = len(quick_check)
+        del quick_check
+
+        # For large PDFs: lower DPI + sample pages evenly
+        if total_pages > MAX_IMAGES_PER_REQUEST:
+            dpi = 100  # lower DPI for large docs
+            quality = 80
+            # Sample pages evenly across the document
+            step = total_pages / MAX_IMAGES_PER_REQUEST
+            selected_pages = [int(step * i) + 1 for i in range(MAX_IMAGES_PER_REQUEST)]
+            logger.info(f"Large PDF ({total_pages} pages): sampling {len(selected_pages)} pages at {dpi} DPI")
+            base64_images = []
+            for page_num in selected_pages:
+                imgs = convert_from_path(pdf_path, dpi=dpi, fmt='jpeg',
+                                        first_page=page_num, last_page=page_num)
+                if imgs:
+                    buffer = io.BytesIO()
+                    imgs[0].save(buffer, format='JPEG', quality=quality)
+                    base64_images.append(base64.b64encode(buffer.getvalue()).decode('utf-8'))
+            return base64_images
+        else:
+            # Small PDF: convert all pages at normal quality
+            dpi = 150
+            quality = 90
+            logger.info(f"PDF ({total_pages} pages) at {dpi} DPI")
+            images = convert_from_path(pdf_path, dpi=dpi, fmt='jpeg')
+            base64_images = []
+            for img in images:
+                buffer = io.BytesIO()
+                img.save(buffer, format='JPEG', quality=quality)
+                base64_images.append(base64.b64encode(buffer.getvalue()).decode('utf-8'))
+            return base64_images
+    except Exception as e:
+        logger.error(f"PDF conversion failed: {e}")
+        return []
+
+def convert_office_to_pdf(input_path: str) -> Optional[str]:
+    try:
+        import subprocess
+        temp_dir = tempfile.mkdtemp()
+        soffice_cmd = "soffice" if shutil.which("soffice") else "libreoffice"
+        if not soffice_cmd: return None
+
+        cmd = [soffice_cmd, "--headless", "--convert-to", "pdf", "--outdir", temp_dir, input_path]
+        subprocess.run(cmd, capture_output=True, timeout=60)
+
+        pdf_name = Path(input_path).stem + ".pdf"
+        pdf_path = os.path.join(temp_dir, pdf_name)
+        return pdf_path if os.path.exists(pdf_path) else None
+    except Exception:
+        return None
+
+def extract_video_frames(video_path: str, num_frames=10) -> List[str]:
+    try:
+        import cv2
+        cap = cv2.VideoCapture(video_path)
+        total = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
+        if total == 0: return []
+        indices = [int(total * i / (num_frames + 1)) for i in range(1, num_frames + 1)]
+        frames = []
+        for idx in indices:
+            cap.set(cv2.CAP_PROP_POS_FRAMES, idx)
+            ret, frame = cap.read()
+            if ret:
+                h, w = frame.shape[:2]
+                if w > 1280:
+                    scale = 1280/w
+                    frame = cv2.resize(frame, (1280, int(h*scale)))
+                _, buf = cv2.imencode('.jpg', frame)
+                frames.append(base64.b64encode(buf).decode('utf-8'))
+        cap.release()
+        return frames
+    except Exception:
+        return []
+
+def convert_document_to_images(file_path: str) -> List[str]:
+    path = Path(file_path)
+    ext = path.suffix.lower()
+    if ext == '.pdf':
+        return convert_pdf_to_images(file_path)
+    elif ext in ['.docx', '.doc', '.ppt', '.pptx']:
+        pdf = convert_office_to_pdf(file_path)
+        if pdf:
+            imgs = convert_pdf_to_images(pdf)
+            shutil.rmtree(os.path.dirname(pdf), ignore_errors=True)
+            return imgs
+    elif ext in ['.mp4', '.avi', '.mov', '.mkv']:
+        return extract_video_frames(file_path)
+    return []
+
+# -----------------------------------------------------------------------------
+# API Interaction
+# -----------------------------------------------------------------------------
+
+async def call_multimodal_api(images_b64: List[str], session: aiohttp.ClientSession) -> tuple[str, bool]:
+    messages = [{"role": "system", "content": SYSTEM_PROMPT}]
+
+    content = [{"type": "text", "text": "Analyze these tutorial pages and extract benchmark tasks as JSON."}]
+
+    # Cap images to avoid huge payloads
+    subset_images = images_b64[:MAX_IMAGES_PER_REQUEST]
+    for img in subset_images:
+        content.append({
+            "type": "image_url",
+            "image_url": {"url": f"data:image/jpeg;base64,{img}"}
+        })
+
+    messages.append({"role": "user", "content": content})
+
+    for attempt in range(1, MAX_RETRY_ATTEMPTS + 1):
+        try:
+            headers = {
+                "Authorization": f"Bearer {API_KEY}",
+                "Content-Type": "application/json"
+            }
+            # Add site specific headers if using openrouter or others if needed
+
+            payload = {
+                "model": MODEL_NAME,
+                "messages": messages,
+                "max_tokens": 4096,
+            }
+
+            async with session.post(API_URL, headers=headers, json=payload, timeout=180) as response:
+                if response.status == 200:
+                    res_json = await response.json()
+                    return res_json['choices'][0]['message']['content'], True
+                else:
+                    err = await response.text()
+                    logger.warning(f"API Error ({response.status}): {err}")
+                    if attempt < MAX_RETRY_ATTEMPTS:
+                        await asyncio.sleep(RETRY_DELAY)
+                    else:
+                        return f"API Error: {err}", False
+        except Exception as e:
+            logger.warning(f"Exception: {e}")
+            if attempt < MAX_RETRY_ATTEMPTS:
+                await asyncio.sleep(RETRY_DELAY)
+            else:
+                return str(e), False
+    return "Max retries", False
+
+# -----------------------------------------------------------------------------
+# Main Logic
+# -----------------------------------------------------------------------------
+
+software_tests = {}  # Global dict to track software -> [test_ids]
+FORCE_REGENERATE = False  # Set via --force flag
+
+async def process_file(file_path: str, session: aiohttp.ClientSession, semaphore: asyncio.Semaphore):
+    async with semaphore:
+        file_path_obj = Path(file_path)
+        file_stem = file_path_obj.stem
+
+        # Infer software name from folder structure
+        try:
+            rel_path = file_path_obj.relative_to(INPUT_FOLDER)
+            software_name = rel_path.parts[0] if len(rel_path.parts) > 1 else "unknown"
+        except ValueError:
+            software_name = "unknown"
+
+        # Skip if already processed (check if task1 json exists)
+        existing_task1 = EXAMPLES_FOLDER / software_name / f"{file_stem}_task1.json"
+        if existing_task1.exists() and not FORCE_REGENERATE:
+            logger.info(f"Skipping (already processed): {file_path_obj.name} → use --force to regenerate")
+            # Still register existing tasks in software_tests for test_all.json
+            import glob as g
+            existing_tasks = g.glob(str(EXAMPLES_FOLDER / software_name / f"{file_stem}_task*.json"))
+            for t in existing_tasks:
+                tid = Path(t).stem
+                if software_name not in software_tests:
+                    software_tests[software_name] = []
+                software_tests[software_name].append(tid)
+            stats.add_completed(num_tasks=len(existing_tasks))
+            return
+
+        logger.info(f"Processing: {file_path_obj.name}")
+
+        # 1. Convert to images
+        images = convert_document_to_images(file_path)
+        if not images:
+            stats.add_failed(file_path, "No images extracted")
+            return
+
+        # 2. Call API
+        content, success = await call_multimodal_api(images, session)
+        if not success:
+            stats.add_failed(file_path, content)
+            return
+
+        # 3. Parse JSON
+        try:
+            # Try to find JSON block if mixed with text
+            json_match = re.search(r'\{.*\}', content, re.DOTALL)
+            if json_match:
+                json_str = json_match.group(0)
+            else:
+                json_str = content
+
+            api_result = json.loads(json_str)
+            tasks = api_result.get("tasks", [])
+            if not tasks:
+                logger.warning(f"No tasks found in JSON for {file_path}")
+                return
+
+        except json.JSONDecodeError as e:
+            stats.add_failed(file_path, f"JSON Parse Error: {e}")
+            logger.error(f"Raw content: {content[:200]}...")
+            return
+
+        # 4. Generate Output Files
+        for i, task in enumerate(tasks, 1):
+            test_id = f"{file_stem}_task{i}"
+            output_file = EXAMPLES_FOLDER / software_name / f"{test_id}.json"
+            output_file.parent.mkdir(parents=True, exist_ok=True)
+
+            # Get software-specific config
+            sw_cfg = SOFTWARE_CONFIG.get(software_name, DEFAULT_SOFTWARE_CONFIG)
+
+            # Construct the OSWorld/Jade Benchmark Standard JSON
+            task_json = {
+                "id": test_id,
+                "snapshot": sw_cfg["snapshot"],
+                "instruction": task.get("task_goal", ""),
+                "source": "custom",
+                "config": sw_cfg["config"],
+                "trajectory": "trajectories/",
+                "related_apps": [software_name],
+                "evaluator": {
+                    "postconfig": [
+                        {
+                            "type": "sleep",
+                            "parameters": {
+                                "seconds": 3
+                            }
+                        }
+                    ],
+                    "func": "vllm_eval"
+                    # "result" field is NOT needed for vllm_eval
+                },
+                "proxy": False,
+                "fixed_ip": False,
+                "possibility_of_env_change": "low",
+                "metadata": {
+                    "input_files": task.get("input_files", []),
+                    "steps": task.get("steps", "")
+                }
+            }
+
+            with open(output_file, 'w', encoding='utf-8') as f:
+                json.dump(task_json, f, ensure_ascii=False, indent=2)
+
+            # Register to global index
+            if software_name not in software_tests:
+                software_tests[software_name] = []
+            software_tests[software_name].append(test_id)
+
+        stats.add_completed(num_tasks=len(tasks))
+        logger.info(f"Generated {len(tasks)} tasks for {file_path_obj.name}")
+
+def save_test_all_json():
+    """Update test_all.json with new tests"""
+    test_all_meta_path = Path(TEST_ALL_JSON)
+    existing_data = {}
+    if test_all_meta_path.exists():
+        try:
+            with open(test_all_meta_path, 'r', encoding='utf-8') as f:
+                existing_data = json.load(f)
+        except: pass
+
+    # Merge new tests
+    for software, test_ids in software_tests.items():
+        current_list = existing_data.get(software, [])
+        # Append unique
+        updated_list = sorted(list(set(current_list + test_ids)))
+        existing_data[software] = updated_list
+
+    with open(test_all_meta_path, 'w', encoding='utf-8') as f:
+        json.dump(existing_data, f, ensure_ascii=False, indent=2)
+
+    # Also save a 'test_custom.json' that ONLY contains the softwares we just processed/have in our inputs
+    # This is useful for running ONLY your custom benchmarks without OSWorld defaults
+    custom_data = {}
+
+    # We scan the INPUT_FOLDER to see which softwares are "ours"
+    custom_softwares = set()
+    if INPUT_FOLDER.exists():
+        for item in os.listdir(INPUT_FOLDER):
+            if (INPUT_FOLDER / item).is_dir():
+                custom_softwares.add(item)
+
+    for software in custom_softwares:
+        if software in existing_data:
+            custom_data[software] = existing_data[software]
+
+    test_custom_path = PROJECT_ROOT / "evaluation_examples" / "test_custom.json"
+    with open(test_custom_path, 'w', encoding='utf-8') as f:
+        json.dump(custom_data, f, ensure_ascii=False, indent=2)
+    logger.info(f"Custom test index saved to: {test_custom_path}")
+
+async def main():
+    global stats, FORCE_REGENERATE
+    stats = ProcessingStats()
+
+    # Parse --force flag
+    FORCE_REGENERATE = "--force" in sys.argv
+
+    if not API_KEY:
+        logger.error("OPENAI_API_KEY environment variable not set.")
+        return
+
+    # Check/Create Input Folder
+    if not INPUT_FOLDER.exists():
+        logger.warning(f"Input folder {INPUT_FOLDER} does not exist. Creating it.")
+        INPUT_FOLDER.mkdir(parents=True, exist_ok=True)
+        logger.info(f"Please put software PDF tutorials into subfolders in: {INPUT_FOLDER}")
+        return
+
+    # Find files
+    files = []
+    for root, _, filenames in os.walk(INPUT_FOLDER):
+        for f in filenames:
+            if Path(f).suffix.lower() in SUPPORTED_EXTENSIONS:
+                files.append(os.path.join(root, f))
+
+    stats.total_files = len(files)
+    logger.info(f"Found {len(files)} files in {INPUT_FOLDER}")
+
+    if not files:
+        logger.info("No files to process.")
+        return
+
+    # Process
+    semaphore = asyncio.Semaphore(MAX_CONCURRENT_REQUESTS)
+    async with aiohttp.ClientSession() as session:
+        tasks = [process_file(f, session, semaphore) for f in files]
+        await asyncio.gather(*tasks)
+
+    # Save Index
+    save_test_all_json()
+
+    logger.info("Done.")
+
+if __name__ == "__main__":
+    asyncio.run(main())

evaluation_examples/test_all.json

@@ -389,6 +389,314 @@
     "971cbb5b-3cbf-4ff7-9e24-b5c84fcebfa6"
   ],
   "jade": [
+    "MDIJade6.5使用手册_task1",
+    "MDIJade6.5使用手册_task10",
+    "MDIJade6.5使用手册_task2",
+    "MDIJade6.5使用手册_task3",
+    "MDIJade6.5使用手册_task4",
+    "MDIJade6.5使用手册_task5",
+    "MDIJade6.5使用手册_task6",
+    "MDIJade6.5使用手册_task7",
+    "MDIJade6.5使用手册_task8",
+    "MDIJade6.5使用手册_task9",
     "jade_test"
+  ],
+  "avogadro": [
+    "building-metal-complexes_task1",
+    "building-metal-complexes_task2",
+    "building-metal-complexes_task3",
+    "building-metal-complexes_task4",
+    "building-metal-complexes_task5",
+    "building-metal-complexes_task6",
+    "building-metal-complexes_task7",
+    "building-organic-molecules_task1",
+    "building-organic-molecules_task10",
+    "building-organic-molecules_task2",
+    "building-organic-molecules_task3",
+    "building-organic-molecules_task4",
+    "building-organic-molecules_task5",
+    "building-organic-molecules_task6",
+    "building-organic-molecules_task7",
+    "building-organic-molecules_task8",
+    "building-organic-molecules_task9",
+    "learning-avogadro_task1",
+    "learning-avogadro_task2",
+    "learning-avogadro_task3",
+    "learning-avogadro_task4",
+    "learning-avogadro_task5",
+    "learning-avogadro_task6",
+    "learning-avogadro_task7",
+    "learning-avogadro_task8",
+    "learning-avogadro_task9",
+    "naming-a-molecule_task1",
+    "naming-a-molecule_task2",
+    "using-qtaim-and-wfn_task1",
+    "using-qtaim-and-wfn_task2",
+    "using-qtaim-and-wfn_task3",
+    "viewing-electrostatic-potential_task1",
+    "viewing-electrostatic-potential_task2",
+    "viewing-molecular-orbitals_task1",
+    "viewing-molecular-orbitals_task2",
+    "viewing-molecular-orbitals_task3",
+    "viewing-vibrations_task1",
+    "viewing-vibrations_task2",
+    "viewing-vibrations_task3",
+    "viewing-vibrations_task4",
+    "viewing-vibrations_task5"
+  ],
+  "imagej": [
+    "user-guide_task1",
+    "user-guide_task10",
+    "user-guide_task2",
+    "user-guide_task3",
+    "user-guide_task4",
+    "user-guide_task5",
+    "user-guide_task6",
+    "user-guide_task7",
+    "user-guide_task8",
+    "user-guide_task9"
+  ],
+  "origin": [
+    "Origin_User_Guide_2025b_E_task1",
+    "Origin_User_Guide_2025b_E_task10",
+    "Origin_User_Guide_2025b_E_task11",
+    "Origin_User_Guide_2025b_E_task12",
+    "Origin_User_Guide_2025b_E_task2",
+    "Origin_User_Guide_2025b_E_task3",
+    "Origin_User_Guide_2025b_E_task4",
+    "Origin_User_Guide_2025b_E_task5",
+    "Origin_User_Guide_2025b_E_task6",
+    "Origin_User_Guide_2025b_E_task7",
+    "Origin_User_Guide_2025b_E_task8",
+    "Origin_User_Guide_2025b_E_task9"
+  ],
+  "ovito": [
+    "animation_task1",
+    "animation_task10",
+    "animation_task2",
+    "animation_task3",
+    "animation_task4",
+    "animation_task5",
+    "animation_task6",
+    "animation_task7",
+    "animation_task8",
+    "animation_task9",
+    "aspherical_particles_task1",
+    "aspherical_particles_task10",
+    "aspherical_particles_task2",
+    "aspherical_particles_task3",
+    "aspherical_particles_task4",
+    "aspherical_particles_task5",
+    "aspherical_particles_task6",
+    "aspherical_particles_task7",
+    "aspherical_particles_task8",
+    "aspherical_particles_task9",
+    "clone_pipeline_task1",
+    "clone_pipeline_task2",
+    "clone_pipeline_task3",
+    "clone_pipeline_task4",
+    "clone_pipeline_task5",
+    "clone_pipeline_task6",
+    "clone_pipeline_task7",
+    "clone_pipeline_task8",
+    "code_generation_task1",
+    "code_generation_task2",
+    "code_generation_task3",
+    "code_generation_task4",
+    "code_generation_task5",
+    "code_generation_task6",
+    "code_generation_task7",
+    "code_generation_task8",
+    "customize_init_state_task1",
+    "customize_init_state_task2",
+    "customize_init_state_task3",
+    "customize_init_state_task4",
+    "customize_init_state_task5",
+    "data_model_task1",
+    "data_model_task10",
+    "data_model_task2",
+    "data_model_task3",
+    "data_model_task4",
+    "data_model_task5",
+    "data_model_task6",
+    "data_model_task7",
+    "data_model_task8",
+    "data_model_task9",
+    "export_task1",
+    "export_task2",
+    "export_task3",
+    "export_task4",
+    "export_task5",
+    "import_task1",
+    "import_task10",
+    "import_task2",
+    "import_task3",
+    "import_task4",
+    "import_task5",
+    "import_task6",
+    "import_task7",
+    "import_task8",
+    "import_task9",
+    "marker_particles_task1",
+    "marker_particles_task2",
+    "marker_particles_task3",
+    "marker_particles_task4",
+    "marker_particles_task5",
+    "marker_particles_task6",
+    "marker_particles_task7",
+    "marker_particles_task8",
+    "marker_particles_task9",
+    "miscellaneous_task1",
+    "miscellaneous_task10",
+    "miscellaneous_task2",
+    "miscellaneous_task3",
+    "miscellaneous_task4",
+    "miscellaneous_task5",
+    "miscellaneous_task6",
+    "miscellaneous_task7",
+    "miscellaneous_task8",
+    "miscellaneous_task9",
+    "pipeline_task1",
+    "pipeline_task2",
+    "pipeline_task3",
+    "pipeline_task4",
+    "pipeline_task5",
+    "pipeline_task6",
+    "pipeline_task7",
+    "pipeline_task8",
+    "pipeline_task9",
+    "python_extensions_task1",
+    "python_extensions_task10",
+    "python_extensions_task2",
+    "python_extensions_task3",
+    "python_extensions_task4",
+    "python_extensions_task5",
+    "python_extensions_task6",
+    "python_extensions_task7",
+    "python_extensions_task8",
+    "python_extensions_task9",
+    "remote_file_access_task1",
+    "remote_file_access_task10",
+    "remote_file_access_task2",
+    "remote_file_access_task3",
+    "remote_file_access_task4",
+    "remote_file_access_task5",
+    "remote_file_access_task6",
+    "remote_file_access_task7",
+    "remote_file_access_task8",
+    "remote_file_access_task9",
+    "remote_rendering_task1",
+    "remote_rendering_task2",
+    "remote_rendering_task3",
+    "remote_rendering_task4",
+    "remote_rendering_task5",
+    "remote_rendering_task6",
+    "remote_rendering_task7",
+    "remote_rendering_task8",
+    "remote_rendering_task9",
+    "rendering_task1",
+    "rendering_task2",
+    "rendering_task3",
+    "rendering_task4",
+    "rendering_task5",
+    "rendering_task6",
+    "rendering_task7",
+    "rendering_task8",
+    "rendering_task9",
+    "transparent_particles_task1",
+    "transparent_particles_task2",
+    "turntable_animation_task1",
+    "turntable_animation_task2",
+    "turntable_animation_task3",
+    "turntable_animation_task4",
+    "turntable_animation_task5",
+    "turntable_animation_task6",
+    "viewport_layouts_task1",
+    "viewport_layouts_task10",
+    "viewport_layouts_task2",
+    "viewport_layouts_task3",
+    "viewport_layouts_task4",
+    "viewport_layouts_task5",
+    "viewport_layouts_task6",
+    "viewport_layouts_task7",
+    "viewport_layouts_task8",
+    "viewport_layouts_task9",
+    "viewports_task1",
+    "viewports_task10",
+    "viewports_task11",
+    "viewports_task2",
+    "viewports_task3",
+    "viewports_task4",
+    "viewports_task5",
+    "viewports_task6",
+    "viewports_task7",
+    "viewports_task8",
+    "viewports_task9"
+  ],
+  "pymol": [
+    "Biochemistry_student_intro_task1",
+    "Biochemistry_student_intro_task10",
+    "Biochemistry_student_intro_task2",
+    "Biochemistry_student_intro_task3",
+    "Biochemistry_student_intro_task4",
+    "Biochemistry_student_intro_task5",
+    "Biochemistry_student_intro_task6",
+    "Biochemistry_student_intro_task7",
+    "Biochemistry_student_intro_task8",
+    "Biochemistry_student_intro_task9",
+    "MovieSchool_1_task1",
+    "MovieSchool_1_task2",
+    "MovieSchool_1_task3",
+    "MovieSchool_1_task4",
+    "MovieSchool_1_task5",
+    "MovieSchool_1_task6",
+    "MovieSchool_1_task7",
+    "MovieSchool_3_task1",
+    "MovieSchool_3_task10",
+    "MovieSchool_3_task2",
+    "MovieSchool_3_task3",
+    "MovieSchool_3_task4",
+    "MovieSchool_3_task5",
+    "MovieSchool_3_task6",
+    "MovieSchool_3_task7",
+    "MovieSchool_3_task8",
+    "MovieSchool_3_task9",
+    "Mutagenesis_task1",
+    "Mutagenesis_task2",
+    "Mutagenesis_task3",
+    "Mutagenesis_task4",
+    "Mutagenesis_task5",
+    "Mutagenesis_task6",
+    "Mutagenesis_task7",
+    "Practical_Pymol_for_Beginners_task1",
+    "Practical_Pymol_for_Beginners_task10",
+    "Practical_Pymol_for_Beginners_task11",
+    "Practical_Pymol_for_Beginners_task12",
+    "Practical_Pymol_for_Beginners_task13",
+    "Practical_Pymol_for_Beginners_task2",
+    "Practical_Pymol_for_Beginners_task3",
+    "Practical_Pymol_for_Beginners_task4",
+    "Practical_Pymol_for_Beginners_task5",
+    "Practical_Pymol_for_Beginners_task6",
+    "Practical_Pymol_for_Beginners_task7",
+    "Practical_Pymol_for_Beginners_task8",
+    "Practical_Pymol_for_Beginners_task9",
+    "Visualizing_a_computed_structure_-_a_commented_example_task1",
+    "Visualizing_a_computed_structure_-_a_commented_example_task2",
+    "Visualizing_a_computed_structure_-_a_commented_example_task3",
+    "Visualizing_a_computed_structure_-_a_commented_example_task4"
+  ],
+  "vesta": [
+    "VESTA_Manual_task1",
+    "VESTA_Manual_task10",
+    "VESTA_Manual_task11",
+    "VESTA_Manual_task2",
+    "VESTA_Manual_task3",
+    "VESTA_Manual_task4",
+    "VESTA_Manual_task5",
+    "VESTA_Manual_task6",
+    "VESTA_Manual_task7",
+    "VESTA_Manual_task8",
+    "VESTA_Manual_task9"
   ]
 }

evaluation_examples/test_curated.json

@@ -0,0 +1,93 @@
+{
+  "avogadro": [
+    "building-metal-complexes_task1",
+    "building-metal-complexes_task3",
+    "building-metal-complexes_task7",
+    "building-organic-molecules_task1",
+    "building-organic-molecules_task3",
+    "building-organic-molecules_task4",
+    "building-organic-molecules_task5",
+    "building-organic-molecules_task9",
+    "naming-a-molecule_task1",
+    "viewing-electrostatic-potential_task1"
+  ],
+  "imagej": [
+    "user-guide_task1",
+    "user-guide_task10",
+    "user-guide_task2",
+    "user-guide_task3",
+    "user-guide_task4",
+    "user-guide_task5",
+    "user-guide_task6",
+    "user-guide_task7",
+    "user-guide_task8",
+    "user-guide_task9"
+  ],
+  "jade": [
+    "MDIJade6.5使用手册_task1",
+    "MDIJade6.5使用手册_task10",
+    "MDIJade6.5使用手册_task2",
+    "MDIJade6.5使用手册_task3",
+    "MDIJade6.5使用手册_task4",
+    "MDIJade6.5使用手册_task5",
+    "MDIJade6.5使用手册_task6",
+    "MDIJade6.5使用手册_task7",
+    "MDIJade6.5使用手册_task8"
+
+  ],
+  "origin": [
+    "Origin_User_Guide_2025b_E_task1",
+    "Origin_User_Guide_2025b_E_task11",
+    "Origin_User_Guide_2025b_E_task12",
+    "Origin_User_Guide_2025b_E_task2",
+    "Origin_User_Guide_2025b_E_task3",
+    "Origin_User_Guide_2025b_E_task4",
+    "Origin_User_Guide_2025b_E_task5",
+    "Origin_User_Guide_2025b_E_task8",
+    "Origin_User_Guide_2025b_E_task9"
+  ],
+  "ovito": [
+    "animation_task3",
+    "aspherical_particles_task1",
+    "clone_pipeline_task1",
+    "code_generation_task1",
+    "customize_init_state_task1",
+    "data_model_task1",
+    "export_task1",
+    "marker_particles_task2",
+    "miscellaneous_task1",
+    "python_extensions_task1",
+    "remote_file_access_task1",
+    "remote_rendering_task1",
+    "rendering_task1",
+    "transparent_particles_task1"
+  ],
+  "pymol": [
+    "MovieSchool_1_task1",
+    "MovieSchool_1_task2",
+    "MovieSchool_1_task3",
+    "MovieSchool_1_task4",
+    "MovieSchool_1_task5",
+    "MovieSchool_3_task1",
+    "MovieSchool_3_task10",
+    "MovieSchool_3_task2",
+    "MovieSchool_3_task3",
+    "MovieSchool_3_task4",
+    "MovieSchool_3_task5",
+    "Mutagenesis_task4",
+    "Practical_Pymol_for_Beginners_task6"
+  ],
+  "vesta": [
+    "VESTA_Manual_task1",
+    "VESTA_Manual_task10",
+    "VESTA_Manual_task11",
+    "VESTA_Manual_task2",
+    "VESTA_Manual_task3",
+    "VESTA_Manual_task4",
+    "VESTA_Manual_task5",
+    "VESTA_Manual_task6",
+    "VESTA_Manual_task7",
+    "VESTA_Manual_task8",
+    "VESTA_Manual_task9"
+  ]
+}

evaluation_examples/test_custom.json

@@ -0,0 +1,93 @@
+{
+  "avogadro": [
+    "building-metal-complexes_task1",
+    "building-metal-complexes_task3",
+    "building-metal-complexes_task7",
+    "building-organic-molecules_task1",
+    "building-organic-molecules_task3",
+    "building-organic-molecules_task4",
+    "building-organic-molecules_task5",
+    "building-organic-molecules_task9",
+    "naming-a-molecule_task1",
+    "viewing-electrostatic-potential_task1"
+  ],
+  "imagej": [
+    "user-guide_task1",
+    "user-guide_task10",
+    "user-guide_task2",
+    "user-guide_task3",
+    "user-guide_task4",
+    "user-guide_task5",
+    "user-guide_task6",
+    "user-guide_task7",
+    "user-guide_task8",
+    "user-guide_task9"
+  ],
+  "jade": [
+    "MDIJade6.5使用手册_task1",
+    "MDIJade6.5使用手册_task10",
+    "MDIJade6.5使用手册_task2",
+    "MDIJade6.5使用手册_task3",
+    "MDIJade6.5使用手册_task4",
+    "MDIJade6.5使用手册_task5",
+    "MDIJade6.5使用手册_task6",
+    "MDIJade6.5使用手册_task7",
+    "MDIJade6.5使用手册_task8",
+    "jade_test"
+  ],
+  "origin": [
+    "Origin_User_Guide_2025b_E_task1",
+    "Origin_User_Guide_2025b_E_task11",
+    "Origin_User_Guide_2025b_E_task12",
+    "Origin_User_Guide_2025b_E_task2",
+    "Origin_User_Guide_2025b_E_task3",
+    "Origin_User_Guide_2025b_E_task4",
+    "Origin_User_Guide_2025b_E_task5",
+    "Origin_User_Guide_2025b_E_task8",
+    "Origin_User_Guide_2025b_E_task9"
+  ],
+  "ovito": [
+    "animation_task3",
+    "aspherical_particles_task1",
+    "clone_pipeline_task1",
+    "code_generation_task1",
+    "customize_init_state_task1",
+    "data_model_task1",
+    "export_task1",
+    "marker_particles_task2",
+    "miscellaneous_task1",
+    "python_extensions_task1",
+    "remote_file_access_task1",
+    "remote_rendering_task1",
+    "rendering_task1",
+    "transparent_particles_task1"
+  ],
+  "pymol": [
+    "MovieSchool_1_task1",
+    "MovieSchool_1_task2",
+    "MovieSchool_1_task3",
+    "MovieSchool_1_task4",
+    "MovieSchool_1_task5",
+    "MovieSchool_3_task1",
+    "MovieSchool_3_task10",
+    "MovieSchool_3_task2",
+    "MovieSchool_3_task3",
+    "MovieSchool_3_task4",
+    "MovieSchool_3_task5",
+    "Mutagenesis_task4",
+    "Practical_Pymol_for_Beginners_task6"
+  ],
+  "vesta": [
+    "VESTA_Manual_task1",
+    "VESTA_Manual_task10",
+    "VESTA_Manual_task11",
+    "VESTA_Manual_task2",
+    "VESTA_Manual_task3",
+    "VESTA_Manual_task4",
+    "VESTA_Manual_task5",
+    "VESTA_Manual_task6",
+    "VESTA_Manual_task7",
+    "VESTA_Manual_task8",
+    "VESTA_Manual_task9"
+  ]
+}

evaluation_examples/test_each_domain_a11y_tree.json

@@ -0,0 +1,9 @@
+{
+    "avogadro": ["building-organic-molecules_task1"],
+    "imagej": ["user-guide_task1"],
+    "jade": ["MDIJade6.5使用手册_task1"],
+    "origin": ["Origin_User_Guide_2025b_E_task1"],
+    "ovito": ["animation_task3"],
+    "pymol": ["MovieSchool_1_task1"],
+    "vesta": ["VESTA_Manual_task1"]
+}

evaluation_examples/test_single.json

@@ -0,0 +1 @@
+{"avogadro": ["building-organic-molecules_task1"]}

lib_run_single.py

@@ -9,21 +9,21 @@ from lib_results_logger import log_task_completion
 logger = logging.getLogger("desktopenv.experiment")
 
 
-def run_single_example(agent, env, example, max_steps, instruction, args, example_result_dir, scores):
+def run_single_example(agent, env, example, max_steps, instruction, args, example_result_dir, scores, metadata_steps=""):
     runtime_logger = setup_logger(example, example_result_dir)
 
     # Reset environment first to get fresh VM IP
-    # env.reset(task_config=example)
-    # logger.info("=======Environment reset completed=======")
+    env.reset(task_config=example)
+    logger.info("=======Environment reset completed=======")
 
     # # Reset agent with fresh VM IP (for snapshot reverts)
     # try:
     #     agent.reset(runtime_logger, vm_ip=env.vm_ip)
     # except Exception as e:
     #     agent.reset(vm_ip=env.vm_ip)
-    
-    # time.sleep(10) # Wait for the environment to be ready
-    
+
+    time.sleep(15) # Wait for the environment to be ready (apps like Avogadro need time to fully load)
+
     # get initial observation
     logger.info("Getting initial observation...")
     obs = env._get_obs() # Get the initial observation
@@ -36,21 +36,22 @@ def run_single_example(agent, env, example, max_steps, instruction, args, exampl
         logger.info(f"Step {step_idx + 1} prediction...")
         response, actions = agent.predict(
             instruction,
-            obs
+            obs,
+            metadata_steps=metadata_steps,
         )
         logger.info(f"Response: {response}")
         logger.info(f"Actions: {actions}")
-        
+
         logger.info(f"Executing actions...")
         for action in actions:
             # Capture the timestamp before executing the action
             action_timestamp = datetime.datetime.now().strftime("%Y%m%d@%H%M%S%f")
             logger.info("Step %d: %s", step_idx + 1, action)
-            
+
             logger.info("执行动作中...")
             obs, reward, done, info = env.step(action, args.sleep_after_execution)
             logger.info("动作执行完成。")
-            
+
             logger.info("Reward: %.2f", reward)
             logger.info("Done: %s", done)
             # Save screenshot and trajectory information
@@ -74,15 +75,14 @@ def run_single_example(agent, env, example, max_steps, instruction, args, exampl
                 break
         step_idx += 1
     time.sleep(20) # Wait for the environment to settle
-    result = env.evaluate()
-    logger.info("Result: %.2f", result)
+    result = env.evaluate(result_dir=example_result_dir)
     scores.append(result)
     with open(os.path.join(example_result_dir, "result.txt"), "w", encoding="utf-8") as f:
         f.write(f"{result}\n")
-    
+
     # Log task completion to results.json
     log_task_completion(example, result, example_result_dir, args)
-    
+
     if getattr(args, 'enable_recording', False):
         env.controller.end_recording(os.path.join(example_result_dir, "recording.mp4"))
 
@@ -98,11 +98,11 @@ def run_single_example_human(env, example, max_steps, instruction, args, example
     env.reset(task_config=example)
     time.sleep(60) # Wait for the environment to be ready
     obs = env._get_obs() # Get the initial observation
-    
+
     # Save initial screenshot
     with open(os.path.join(example_result_dir, "initial_state.png"), "wb") as _f:
         _f.write(obs['screenshot'])
-    
+
     # Save trajectory information
     with open(os.path.join(example_result_dir, "traj.jsonl"), "a") as f:
         f.write(json.dumps({
@@ -110,9 +110,9 @@ def run_single_example_human(env, example, max_steps, instruction, args, example
             "initial_state": "initial_state.png"
         }))
         f.write("\n")
-    
+
     # Evaluate the result
-    result = env.evaluate()
+    result = env.evaluate(result_dir=example_result_dir)
     logger.info("Result: %.2f", result)
     scores.append(result)
     with open(os.path.join(example_result_dir, "result.txt"), "w", encoding="utf-8") as f:
@@ -255,14 +255,14 @@ def run_single_example_opencua(agent, env, example, max_steps, instruction, args
 
         logger.info(f"Got Action: {actions}")
         # Breack if no actions
-        if not actions or len(actions)==0 or actions[0]=="" or actions[0].lower().startswith("error"): 
+        if not actions or len(actions)==0 or actions[0]=="" or actions[0].lower().startswith("error"):
             break
 
         for action in actions:
             # Capture the timestamp before executing the action
             action_timestamp = datetime.datetime.now().strftime("%Y%m%d@%H%M%S")
             logger.info("Step %d: %s", step_idx + 1, action)
-            
+
             obs, reward, done, info = env.step(action, args.sleep_after_execution)
 
             logger.info(f"Action {action} executed, reward: {reward}, done: {done}")
@@ -305,7 +305,7 @@ def run_single_example_autoglm(agent, env, example, max_steps, instruction, args
         agent.reset()
 
     env.reset(task_config=example)
-    
+
     time.sleep(60) # Wait for the environment to be ready
     obs = env._get_obs() # Get the initial observation
     done = False
@@ -340,20 +340,20 @@ def run_single_example_autoglm(agent, env, example, max_steps, instruction, args
                     "screenshot_file": f"step_{step_idx + 1}_{action_timestamp}.png"
                 }))
                 f.write("\n")
-                
+
             if done:
                 logger.info("The episode is done.")
                 break
-        
+
         # Invalid Action
         if not actions:
             obs = env._get_obs() # update observation
-            
+
         step_idx += 1
-    
+
     if not done: # not completed the task yet
         env.action_history.append('FAIL')
-    
+
     result = env.evaluate()
     logger.info("Result: %.2f", result)
     scores.append(result)
@@ -370,7 +370,7 @@ def run_single_example_mano(agent, env, example, max_steps, instruction, args, e
     done = False
     step_idx = 0
     env.controller.start_recording()
-    
+
     with open(os.path.join(example_result_dir, f"step_0.png"),
       "wb") as _f:
         _f.write(obs['screenshot'])
@@ -380,12 +380,12 @@ def run_single_example_mano(agent, env, example, max_steps, instruction, args, e
             obs
         )
         if len(actions) > 1:
-            if (("pyautogui.hotkey('shift')" in actions[0] or "pyautogui.hotkey('ctrl')" in actions[0]) 
+            if (("pyautogui.hotkey('shift')" in actions[0] or "pyautogui.hotkey('ctrl')" in actions[0])
                 and "pyautogui.click" in actions[1]):
                 hotkey_type = 'shift' if "shift" in actions[0] else 'ctrl'
                 action = f"pyautogui.keyDown('{hotkey_type}')\n{actions[1]}\npyautogui.keyUp('{hotkey_type}')"
-                actions = [action]  
-                
+                actions = [action]
+
         for action in actions:
             # Capture the timestamp before executing the action
             action_timestamp = datetime.datetime.now().strftime("%Y%m%d@%H%M%S")
@@ -420,7 +420,7 @@ def run_single_example_mano(agent, env, example, max_steps, instruction, args, e
     with open(os.path.join(example_result_dir, "result.txt"), "w", encoding="utf-8") as f:
         f.write(f"{result}\n")
     env.controller.end_recording(os.path.join(example_result_dir, "recording.mp4"))
-    
+
 def run_single_example_uipath(agent, env, example, max_steps, instruction, args, example_result_dir, scores):
     runtime_logger = setup_logger(example, example_result_dir)
     try:
@@ -486,7 +486,7 @@ logger = logging.getLogger("desktopenv.experiment")
 
 def run_single_example_os_symphony(agent, env, example, max_steps, instruction, args, example_result_dir, scores):
     set_current_result_dir(example_result_dir)
-    
+
     agent.reset(result_dir=example_result_dir)
     env.reset(task_config=example)
     time.sleep(30) # Wait for the environment to be ready
@@ -508,14 +508,14 @@ def run_single_example_os_symphony(agent, env, example, max_steps, instruction,
                 img_name = f"step_{step_idx + 1}_milestone.png"
             else:
                 img_name = f"step_{step_idx + 1}.png"
-                
+
             with open(os.path.join(example_result_dir, img_name),
                       "wb") as _f:
                 _f.write(obs['screenshot'])
             if "coordinates" in response and response["coordinates"]:
                 draw_coordinates(
-                    image_bytes=obs['screenshot'], 
-                    coordinates=response["coordinates"], 
+                    image_bytes=obs['screenshot'],
+                    coordinates=response["coordinates"],
                     save_path=os.path.join(example_result_dir, img_name[:-4] + "_draw.png")
                 )
 
@@ -549,7 +549,7 @@ def run_single_example_os_symphony(agent, env, example, max_steps, instruction,
                 break
         step_idx += 1
     end_time = time.time()
-    result = float(env.evaluate())
+    result = float(env.evaluate(result_dir=example_result_dir))
     logger.info("Result: %.2f", result)
     scores.append(result)
     with open(os.path.join(example_result_dir, "result.txt"), "w", encoding="utf-8") as f:
@@ -564,10 +564,10 @@ def run_single_example_evocua(agent, env, example, max_steps, instruction, args,
     Unified run function for EvoCUAAgent (supporting both S1 and S2 modes).
     """
     runtime_logger = setup_logger(example, example_result_dir)
-    
+
     # Reset Environment
     env.reset(task_config=example)
-    
+
     # Reset Agent
     # Handle agent reset signature differences if any
     try:
@@ -588,7 +588,7 @@ def run_single_example_evocua(agent, env, example, max_steps, instruction, args,
         # EvoCUAAgent.predict unified signature: returns (response, actions)
         # It handles both modes internally.
         predict_res = agent.predict(instruction, obs)
-        
+
         # Check return signature logic
         if len(predict_res) == 3:
             # Compatibility with S1 original signature if agent was updated to match
@@ -598,7 +598,7 @@ def run_single_example_evocua(agent, env, example, max_steps, instruction, args,
             info_dict = {}
 
         logger.info(f"Step {step_idx + 1} Actions: {actions}")
-        
+
         # Break if no actions (fail-safe)
         if not actions or (len(actions) == 1 and (actions[0] == "" or "error" in actions[0].lower())):
              # Allow "FAIL" or "DONE" to process through execution loop if agent outputs them as actions
@@ -609,18 +609,18 @@ def run_single_example_evocua(agent, env, example, max_steps, instruction, args,
         for action in actions:
             action_timestamp = datetime.datetime.now().strftime("%Y%m%d@%H%M%S%f")
             logger.info("Executing action: %s", action)
-            
+
             # Execute
             obs, reward, done, info = env.step(action, args.sleep_after_execution)
-            
+
             logger.info("Reward: %.2f", reward)
             logger.info("Done: %s", done)
-            
+
             # Save screenshot
             screenshot_file = f"step_{step_idx + 1}_{action_timestamp}.png"
             with open(os.path.join(example_result_dir, screenshot_file), "wb") as _f:
                 _f.write(obs['screenshot'])
-            
+
             # Log Trajectory
             log_entry = {
                 "step_num": step_idx + 1,
@@ -635,25 +635,25 @@ def run_single_example_evocua(agent, env, example, max_steps, instruction, args,
             # Add natural language info if available (S1 style)
             if info_dict:
                 log_entry["natural_language_action"] = info_dict.get("action")
-            
+
             with open(os.path.join(example_result_dir, "traj.jsonl"), "a", encoding="utf-8") as f:
                 f.write(json.dumps(log_entry, ensure_ascii=False))
                 f.write("\n")
-                
+
             if done:
                 logger.info("The episode is done.")
                 break
-        
+
         step_idx += 1
-        
+
     time.sleep(20) # Wait for environment to settle
-    result = env.evaluate()
+    result = env.evaluate(result_dir=example_result_dir)
     logger.info("Result: %.2f", result)
     scores.append(result)
-    
+
     with open(os.path.join(example_result_dir, "result.txt"), "w", encoding="utf-8") as f:
         f.write(f"{result}\n")
-    
+
     log_task_completion(example, result, example_result_dir, args)
 
     env.controller.end_recording(os.path.join(example_result_dir, "recording.mp4"))

mm_agents/accessibility_tree_wrap/heuristic_retrieve.py

@@ -46,6 +46,15 @@ def judge_node(node: ET, platform="ubuntu", check_image=False) -> bool:
         raise ValueError("Invalid platform, must be 'ubuntu' or 'windows'")
 
     keeps: bool = node.tag.startswith("document") \
+                  or node.tag.startswith("sunawt") \
+                  or node.tag.startswith("qt5q") \
+                  or node.tag.startswith("qt6q") \
+                  or node.tag.startswith("ovito") \
+                  or node.tag.startswith("pymol") \
+                  or node.tag.startswith("contentspanel") \
+                  or node.tag.startswith("wx") \
+                  or node.tag.startswith("afx") \
+                  or node.tag.startswith("thunderrt") \
                   or node.tag.endswith("item") \
                   or node.tag.endswith("button") \
                   or node.tag.endswith("heading") \
@@ -58,6 +67,18 @@ def judge_node(node: ET, platform="ubuntu", check_image=False) -> bool:
                   or node.tag.endswith("textfield") \
                   or node.tag.endswith("textarea") \
                   or node.tag.endswith("menu") \
+                  or node.tag.endswith("menuitem") \
+                  or node.tag.endswith("menubar") \
+                  or node.tag.endswith("toolbar") \
+                  or node.tag.endswith("tabitem") \
+                  or node.tag.endswith("treeitem") \
+                  or node.tag.endswith("window") \
+                  or node.tag.endswith("edit") \
+                  or node.tag.endswith("widget") \
+                  or node.tag.endswith("box") \
+                  or node.tag.endswith("dialog") \
+                  or node.tag.endswith("view") \
+                  or node.tag.endswith("frame") \
                   or node.tag in {"alert", "canvas", "check-box"
                       , "combo-box", "entry", "icon"
                       , "image", "paragraph", "scroll-bar"
@@ -66,6 +87,16 @@ def judge_node(node: ET, platform="ubuntu", check_image=False) -> bool:
                       , "netuiribbontab", "start", "trayclockwclass"
                       , "traydummysearchcontrol", "uiimage", "uiproperty"
                       , "uiribboncommandbar"
+                      , "qt5qwindowicon", "textblock", "listview"
+                      , "chrome_widgetwin_1", "chrome_renderwidgethosthwnd"
+                      , "unknown", "pane", "tree", "tab"
+                      , "datagrid", "dataitem", "group"
+                      , "statusbar", "titlebar", "tooltip"
+                      , "toolbarwindow32", "richedit50w"
+                      , "msctls_statusbar32", "qaction"
+                      , "qsplitter", "qsplitterhandle"
+                      , "qtoolbarseparator", "qtextbrowser"
+                      , "qtabbar", "qopenglwidget"
                                   }
     keeps = keeps and (
             platform == "ubuntu"
@@ -83,6 +114,12 @@ def judge_node(node: ET, platform="ubuntu", check_image=False) -> bool:
             and (
                     node.get("name", "") != "" or node.text is not None and len(node.text) > 0 \
                     or check_image and node.get("image", "false") == "true"
+                    # Keep empty input fields (edit/textfield) - they are important interactive elements
+                    # even without name/text (e.g., search boxes, filter inputs)
+                    or node.tag.endswith("edit") or node.tag.endswith("textfield")
+                    or node.tag.endswith("textarea") or node.tag.endswith("textbox")
+                    or node.tag.endswith("searchbox") or node.tag.endswith("combobox")
+                    or node.tag in {"entry", "combo-box", "check-box", "slider"}
             )
 
     coordinates: Tuple[int, int] = eval(node.get("{{{:}}}screencoord".format(_component_ns), "(-1, -1)"))

mm_agents/agent.py

@@ -126,7 +126,7 @@ def linearize_accessibility_tree(accessibility_tree, platform="ubuntu"):
         raise ValueError("Invalid platform, must be 'ubuntu' or 'windows'")
 
     filtered_nodes = filter_nodes(ET.fromstring(accessibility_tree), platform)
-    linearized_accessibility_tree = ["tag\tname\ttext\tclass\tdescription\tposition (top-left x&y)\tsize (w&h)"]
+    linearized_accessibility_tree = ["tag\tname\ttext\tposition (center x&y)\tsize (w&h)\tstates"]
 
     # Linearize the accessibility tree nodes into a table format
     for node in filtered_nodes:
@@ -145,14 +145,36 @@ def linearize_accessibility_tree(accessibility_tree, platform="ubuntu"):
         else:
             text = '""'
 
+        # Compute center coordinates from top-left + size/2
+        coords_str = node.get('{{{:}}}screencoord'.format(_component_ns), "")
+        size_str = node.get('{{{:}}}size'.format(_component_ns), "")
+        if coords_str and size_str:
+            try:
+                cx, cy = coords_str.strip('()').split(', ')
+                sw, sh = size_str.strip('()').split(', ')
+                center_x = int(cx) + int(sw) // 2
+                center_y = int(cy) + int(sh) // 2
+                center_str = "({:d}, {:d})".format(center_x, center_y)
+            except (ValueError, IndexError):
+                center_str = coords_str
+        else:
+            center_str = coords_str
+
+        # Extract useful UI states (expanded/collapsed/checked/selected/focused)
+        state_flags = []
+        for state_name in ["expanded", "collapsed", "checked", "selected", "focused", "pressed"]:
+            val = node.get("{{{:}}}{:}".format(_state_ns, state_name), "")
+            if val == "true":
+                state_flags.append(state_name)
+        states_str = ",".join(state_flags) if state_flags else ""
+
         linearized_accessibility_tree.append(
-            "{:}\t{:}\t{:}\t{:}\t{:}\t{:}\t{:}".format(
+            "{:}\t{:}\t{:}\t{:}\t{:}\t{:}".format(
                 node.tag, node.get("name", ""),
                 text,
-                node.get("{{{:}}}class".format(_attributes_ns), "") if platform == "ubuntu" else node.get("{{{:}}}class".format(class_ns_windows), ""),
-                node.get("{{{:}}}description".format(_attributes_ns), ""),
-                node.get('{{{:}}}screencoord'.format(_component_ns), ""),
-                node.get('{{{:}}}size'.format(_component_ns), "")
+                center_str,
+                size_str,
+                states_str
             )
         )
 
@@ -329,14 +351,16 @@ class PromptAgent:
                 raise ValueError("Invalid action space: " + action_space)
         else:
             raise ValueError("Invalid experiment type: " + observation_type)
-        
+
         self.system_message = self.system_message.format(CLIENT_PASSWORD=self.client_password, SCREEN_WIDTH=self.screen_width, SCREEN_HEIGHT=self.screen_height)
 
-    def predict(self, instruction: str, obs: Dict) -> List:
+    def predict(self, instruction: str, obs: Dict, metadata_steps: str = "") -> List:
         """
         Predict the next action(s) based on the current observation.
         """
         system_message = self.system_message + "\nYou are asked to complete the following task: {}".format(instruction)
+        if metadata_steps:
+            system_message += "\n\nHere are the reference steps from the software tutorial, which may help you complete the task:\n{}".format(metadata_steps)
 
         # Prepare the payload for the API call
         messages = []
@@ -502,9 +526,30 @@ class PromptAgent:
                 ]
             })
         elif self.observation_type == "a11y_tree":
+            # Debug: log raw a11y tree XML to help diagnose missing elements
+            raw_tree = obs["accessibility_tree"]
+            if raw_tree:
+                # Log first 2000 chars of raw XML and count total nodes
+                root = ET.fromstring(raw_tree)
+                all_tags = set()
+                total_nodes = 0
+                for node in root.iter():
+                    all_tags.add(node.tag)
+                    total_nodes += 1
+                logger.info("Raw a11y tree: %d total nodes, unique tags: %s", total_nodes, all_tags)
+                logger.debug("Raw a11y tree XML (first 2000 chars): %s", raw_tree[:2000])
+                # Also log nodes containing 'avogadro' or 'qt5' in their attributes
+                for node in root.iter():
+                    node_str = ET.tostring(node, encoding="unicode")
+                    if 'avogadro' in node_str.lower() or 'qt5' in node_str.lower():
+                        logger.info("Avogadro/Qt5 node: tag=%s, name=%s, visible=%s, enabled=%s",
+                                    node.tag, node.get("name", ""),
+                                    node.get("{https://accessibility.windows.example.org/ns/state}visible", "?"),
+                                    node.get("{https://accessibility.windows.example.org/ns/state}enabled", "?"))
             linearized_accessibility_tree = linearize_accessibility_tree(accessibility_tree=obs["accessibility_tree"],
                                                                          platform=self.platform)
             logger.debug("LINEAR AT: %s", linearized_accessibility_tree)
+            logger.info("Linearized a11y tree lines: %d", len(linearized_accessibility_tree.split('\n')) if linearized_accessibility_tree else 0)
 
             if linearized_accessibility_tree:
                 linearized_accessibility_tree = trim_accessibility_tree(linearized_accessibility_tree,
@@ -670,7 +715,7 @@ class PromptAgent:
                 return response.json()['choices'][0]['message']['content']
         elif self.model.startswith("gpt"):
             # Support custom OpenAI base URL via environment variable
-            base_url = os.environ.get('OPENAI_BASE_URL', 'https://api.openai.com')
+            base_url = os.environ.get('OPENAI_BASE_URL', os.environ.get('OPENAI_API_BASE', 'https://api.openai.com'))
             # Smart handling: avoid duplicate /v1 if base_url already ends with /v1
             api_url = f"{base_url}/chat/completions" if base_url.endswith('/v1') else f"{base_url}/v1/chat/completions"
             headers = {
@@ -1153,7 +1198,7 @@ class PromptAgent:
             except Exception as e:
                 print("Failed to call LLM: " + str(e))
                 return ""
-        
+
         else:
             raise ValueError("Invalid model: " + self.model)
 
@@ -1192,4 +1237,4 @@ class PromptAgent:
 
         self.thoughts = []
         self.actions = []
-        self.observations = []
+        self.observations = []

mm_agents/prompts.py

@@ -15,6 +15,8 @@ When you think you have to wait for some time, return ```WAIT```;
 When you think the task can not be done, return ```FAIL```, don't easily say ```FAIL```, try your best to do the task;
 When you think the task is done, return ```DONE```.
 
+IMPORTANT: You must return EITHER a python code block OR a special code (WAIT/DONE/FAIL) in each response, NEVER BOTH. Do NOT include ```DONE``` in the same response as a ```python``` code block. After executing an action, wait for the next observation to verify the result before returning ```DONE```.
+
 My computer's password is '{{CLIENT_PASSWORD}}', feel free to use it when you need sudo rights.
 First give the current screenshot and previous things we did a short reflection, then RETURN ME THE CODE OR SPECIAL CODE I ASKED FOR. NEVER EVER RETURN ME ANYTHING ELSE.
 """.strip()
@@ -36,6 +38,8 @@ When you think you have to wait for some time, return ```WAIT```;
 When you think the task can not be done, return ```FAIL```, don't easily say ```FAIL```, try your best to do the task;
 When you think the task is done, return ```DONE```.
 
+IMPORTANT: You must return EITHER a python code block OR a special code (WAIT/DONE/FAIL) in each response, NEVER BOTH. Do NOT include ```DONE``` in the same response as a ```python``` code block. After executing an action, wait for the next observation to verify the result before returning ```DONE```.
+
 My computer's password is '{{CLIENT_PASSWORD}}', feel free to use it when you need sudo rights.
 Our past communication is great, and what you have done is very helpful. I will now give you another task to complete.
 First take a deep breath, think step by step, give the current screenshot a thinking, then RETURN ME THE CODE OR SPECIAL CODE I ASKED FOR. NEVER EVER RETURN ME ANYTHING ELSE.
@@ -550,6 +554,8 @@ When you think you have to wait for some time, return ```WAIT```;
 When you think the task can not be done, return ```FAIL```, don't easily say ```FAIL```, try your best to do the task;
 When you think the task is done, return ```DONE```.
 
+IMPORTANT: You must return EITHER a python code block OR a special code (WAIT/DONE/FAIL) in each response, NEVER BOTH. Do NOT include ```DONE``` in the same response as a ```python``` code block. After executing an action, wait for the next observation to verify the result before returning ```DONE```.
+
 My computer's password is '{{CLIENT_PASSWORD}}', feel free to use it when you need sudo rights.
 First give the current screenshot and previous things we did a short reflection, then RETURN ME THE CODE OR SPECIAL CODE I ASKED FOR. NEVER EVER RETURN ME ANYTHING ELSE.
 """.strip()
@@ -817,6 +823,8 @@ When you think you have to wait for some time, return ```WAIT```;
 When you think the task can not be done, return ```FAIL```, don't easily say ```FAIL```, try your best to do the task;
 When you think the task is done, return ```DONE```.
 
+IMPORTANT: You must return EITHER a python code block OR a special code (WAIT/DONE/FAIL) in each response, NEVER BOTH. Do NOT include ```DONE``` in the same response as a ```python``` code block. After executing an action, wait for the next observation to verify the result before returning ```DONE```.
+
 My computer's password is '{{CLIENT_PASSWORD}}', feel free to use it when you need sudo rights.
 First give the current screenshot and previous things we did a short reflection, then RETURN ME THE CODE OR SPECIAL CODE I ASKED FOR. NEVER EVER RETURN ME ANYTHING ELSE.
 """.strip()

run.py

@@ -85,7 +85,7 @@ def config() -> argparse.Namespace:
     parser.add_argument("--screen_width", type=int, default=1920)
     parser.add_argument("--screen_height", type=int, default=1080)
     parser.add_argument("--sleep_after_execution", type=float, default=0.0)
-    parser.add_argument("--max_steps", type=int, default=15)
+    parser.add_argument("--max_steps", type=int, default=8)
     parser.add_argument("--enable_recording", action="store_true", help="Enable video recording (disabled by default)")
 
     # agent config
@@ -100,6 +100,7 @@ def config() -> argparse.Namespace:
     parser.add_argument("--top_p", type=float, default=0.9)
     parser.add_argument("--max_tokens", type=int, default=16384)
     parser.add_argument("--stop_token", type=str, default=None)
+    parser.add_argument("--eval_model", type=str, default="gpt-5.2-chat-latest")
 
     # example config
     parser.add_argument("--domain", type=str, default="all")
@@ -141,6 +142,7 @@ def test(args: argparse.Namespace, test_all_meta: dict) -> None:
     }
 
     agent = PromptAgent(
+        platform="windows",
         model=args.model,
         max_tokens=args.max_tokens,
         top_p=args.top_p,
@@ -161,6 +163,7 @@ def test(args: argparse.Namespace, test_all_meta: dict) -> None:
         os_type = "Windows",
         require_a11y_tree=args.observation_type
         in ["a11y_tree", "screenshot_a11y_tree", "som"],
+        eval_model=args.eval_model
     )
 
     # get actual VM screen size after environment initialization
@@ -195,8 +198,11 @@ def test(args: argparse.Namespace, test_all_meta: dict) -> None:
             logger.info(f"[Example ID]: {example_id}")
 
             instruction = example["instruction"]
+            metadata_steps = example.get("metadata", {}).get("steps", "")
 
             logger.info(f"[Instruction]: {instruction}")
+            if metadata_steps:
+                logger.info(f"[Metadata Steps]: {metadata_steps}")
             # wandb each example config settings
             cfg_args["instruction"] = instruction
             cfg_args["start_time"] = datetime.datetime.now().strftime(
@@ -224,6 +230,7 @@ def test(args: argparse.Namespace, test_all_meta: dict) -> None:
                     args,
                     example_result_dir,
                     scores,
+                    metadata_steps=metadata_steps,
                 )
             except Exception as e:
                 logger.error(f"Exception in {domain}/{example_id}: {e}")
@@ -319,8 +326,19 @@ def get_result(action_space, use_model, observation_type, result_dir, total_file
 if __name__ == "__main__":
     ####### The complete version of the list of examples #######
     os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+    # Bypass proxy for local/VM IPs so Clash global mode doesn't break VM communication
+    no_proxy = os.environ.get("NO_PROXY", os.environ.get("no_proxy", ""))
+    extra = "localhost,127.0.0.1,192.168.*,10.*,172.16.*,172.17.*,172.18.*,172.19.*,172.20.*,172.21.*,172.22.*,172.23.*,172.24.*,172.25.*,172.26.*,172.27.*,172.28.*,172.29.*,172.30.*,172.31.*"
+    if no_proxy:
+        no_proxy = f"{no_proxy},{extra}"
+    else:
+        no_proxy = extra
+    os.environ["NO_PROXY"] = no_proxy
+    os.environ["no_proxy"] = no_proxy
+
     args = config()
-    
+
     # save args to json in result_dir/action_space/observation_type/model/args.json
     path_to_args = os.path.join(
         args.result_dir,