CoACT initialize (#292)

mm_agents/coact/autogen/coding/__init__.py

@@ -0,0 +1,22 @@
+# Copyright (c) 2023 - 2025, AG2ai, Inc., AG2ai open-source projects maintainers and core contributors
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Original portions of this file are derived from https://github.com/microsoft/autogen under the MIT License.
+# SPDX-License-Identifier: MIT
+from .base import CodeBlock, CodeExecutor, CodeExtractor, CodeResult
+from .docker_commandline_code_executor import DockerCommandLineCodeExecutor
+from .factory import CodeExecutorFactory
+from .local_commandline_code_executor import LocalCommandLineCodeExecutor
+from .markdown_code_extractor import MarkdownCodeExtractor
+
+__all__ = (
+    "CodeBlock",
+    "CodeExecutor",
+    "CodeExecutorFactory",
+    "CodeExtractor",
+    "CodeResult",
+    "DockerCommandLineCodeExecutor",
+    "LocalCommandLineCodeExecutor",
+    "MarkdownCodeExtractor",
+)

mm_agents/coact/autogen/coding/base.py

@@ -0,0 +1,119 @@
+# Copyright (c) 2023 - 2025, AG2ai, Inc., AG2ai open-source projects maintainers and core contributors
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Portions derived from https://github.com/microsoft/autogen are under the MIT License.
+# SPDX-License-Identifier: MIT
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any, Literal, Optional, Protocol, TypedDict, Union, runtime_checkable
+
+from pydantic import BaseModel, Field
+
+from ..doc_utils import export_module
+from ..types import UserMessageImageContentPart, UserMessageTextContentPart
+
+__all__ = ("CodeBlock", "CodeExecutionConfig", "CodeExecutor", "CodeExtractor", "CodeResult")
+
+
+@export_module("autogen.coding")
+class CodeBlock(BaseModel):
+    """(Experimental) A class that represents a code block."""
+
+    code: str = Field(description="The code to execute.")
+
+    language: str = Field(description="The language of the code.")
+
+
+@export_module("autogen.coding")
+class CodeResult(BaseModel):
+    """(Experimental) A class that represents the result of a code execution."""
+
+    exit_code: int = Field(description="The exit code of the code execution.")
+
+    output: str = Field(description="The output of the code execution.")
+
+
+@export_module("autogen.coding")
+class CodeExtractor(Protocol):
+    """(Experimental) A code extractor class that extracts code blocks from a message."""
+
+    def extract_code_blocks(
+        self, message: Optional[Union[str, list[Union[UserMessageTextContentPart, UserMessageImageContentPart]]]]
+    ) -> list[CodeBlock]:
+        """(Experimental) Extract code blocks from a message.
+
+        Args:
+            message (str): The message to extract code blocks from.
+
+        Returns:
+            List[CodeBlock]: The extracted code blocks.
+        """
+        ...  # pragma: no cover
+
+
+@runtime_checkable
+@export_module("autogen.coding")
+class CodeExecutor(Protocol):
+    """(Experimental) A code executor class that executes code blocks and returns the result."""
+
+    @property
+    def code_extractor(self) -> CodeExtractor:
+        """(Experimental) The code extractor used by this code executor."""
+        ...  # pragma: no cover
+
+    def execute_code_blocks(self, code_blocks: list[CodeBlock]) -> CodeResult:
+        """(Experimental) Execute code blocks and return the result.
+
+        This method should be implemented by the code executor.
+
+        Args:
+            code_blocks (List[CodeBlock]): The code blocks to execute.
+
+        Returns:
+            CodeResult: The result of the code execution.
+        """
+        ...  # pragma: no cover
+
+    def restart(self) -> None:
+        """(Experimental) Restart the code executor.
+
+        This method should be implemented by the code executor.
+
+        This method is called when the agent is reset.
+        """
+        ...  # pragma: no cover
+
+
+class IPythonCodeResult(CodeResult):
+    """(Experimental) A code result class for IPython code executor."""
+
+    output_files: list[str] = Field(
+        default_factory=list,
+        description="The list of files that the executed code blocks generated.",
+    )
+
+
+CodeExecutionConfig = TypedDict(
+    "CodeExecutionConfig",
+    {
+        "executor": Union[Literal["ipython-embedded", "commandline-local"], CodeExecutor],
+        "last_n_messages": Union[int, Literal["auto"]],
+        "timeout": int,
+        "use_docker": Union[bool, str, list[str]],
+        "work_dir": str,
+        "ipython-embedded": Mapping[str, Any],
+        "commandline-local": Mapping[str, Any],
+    },
+    total=False,
+)
+
+
+class CommandLineCodeResult(CodeResult):
+    """(Experimental) A code result class for command line code executor."""
+
+    code_file: Optional[str] = Field(
+        default=None,
+        description="The file that the executed code block was saved to.",
+    )

mm_agents/coact/autogen/coding/docker_commandline_code_executor.py

@@ -0,0 +1,268 @@
+# Copyright (c) 2023 - 2025, AG2ai, Inc., AG2ai open-source projects maintainers and core contributors
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Portions derived from https://github.com/microsoft/autogen are under the MIT License.
+# SPDX-License-Identifier: MIT
+from __future__ import annotations
+
+import atexit
+import logging
+import sys
+import uuid
+from hashlib import md5
+from pathlib import Path
+from time import sleep
+from types import TracebackType
+from typing import Any, ClassVar, Optional, Union
+
+import docker
+from docker.errors import ImageNotFound
+
+from ..code_utils import TIMEOUT_MSG, _cmd
+from ..doc_utils import export_module
+from .base import CodeBlock, CodeExecutor, CodeExtractor, CommandLineCodeResult
+from .markdown_code_extractor import MarkdownCodeExtractor
+from .utils import _get_file_name_from_content, silence_pip
+
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
+
+def _wait_for_ready(container: Any, timeout: int = 60, stop_time: float = 0.1) -> None:
+    elapsed_time = 0.0
+    while container.status != "running" and elapsed_time < timeout:
+        sleep(stop_time)
+        elapsed_time += stop_time
+        container.reload()
+        continue
+    if container.status != "running":
+        raise ValueError("Container failed to start")
+
+
+__all__ = ("DockerCommandLineCodeExecutor",)
+
+
+@export_module("autogen.coding")
+class DockerCommandLineCodeExecutor(CodeExecutor):
+    DEFAULT_EXECUTION_POLICY: ClassVar[dict[str, bool]] = {
+        "bash": True,
+        "shell": True,
+        "sh": True,
+        "pwsh": True,
+        "powershell": True,
+        "ps1": True,
+        "python": True,
+        "javascript": False,
+        "html": False,
+        "css": False,
+    }
+    LANGUAGE_ALIASES: ClassVar[dict[str, str]] = {"py": "python", "js": "javascript"}
+
+    def __init__(
+        self,
+        image: str = "python:3-slim",
+        container_name: Optional[str] = None,
+        timeout: int = 60,
+        work_dir: Optional[Union[Path, str]] = None,
+        bind_dir: Optional[Union[Path, str]] = None,
+        auto_remove: bool = True,
+        stop_container: bool = True,
+        execution_policies: Optional[dict[str, bool]] = None,
+    ):
+        """(Experimental) A code executor class that executes code through
+        a command line environment in a Docker container.
+
+        The executor first saves each code block in a file in the working
+        directory, and then executes the code file in the container.
+        The executor executes the code blocks in the order they are received.
+        Currently, the executor only supports Python and shell scripts.
+        For Python code, use the language "python" for the code block.
+        For shell scripts, use the language "bash", "shell", or "sh" for the code
+        block.
+
+        Args:
+            image: Docker image to use for code execution. Defaults to "python:3-slim".
+            container_name: Name of the Docker container which is created. If None, will autogenerate a name. Defaults to None.
+            timeout: The timeout for code execution. Defaults to 60.
+            work_dir: The working directory for the code execution. Defaults to Path(".").
+            bind_dir: The directory that will be bound to the code executor container. Useful for cases where you want to spawn
+                the container from within a container. Defaults to work_dir.
+            auto_remove: If true, will automatically remove the Docker container when it is stopped. Defaults to True.
+            stop_container: If true, will automatically stop the
+                container when stop is called, when the context manager exits or when
+                the Python process exits with atext. Defaults to True.
+            execution_policies: A dictionary mapping language names to boolean values that determine
+                whether code in that language should be executed. True means code in that language
+                will be executed, False means it will only be saved to a file. This overrides the
+                default execution policies. Defaults to None.
+
+        Raises:
+            ValueError: On argument error, or if the container fails to start.
+        """
+        work_dir = work_dir if work_dir is not None else Path()
+
+        if timeout < 1:
+            raise ValueError("Timeout must be greater than or equal to 1.")
+
+        if isinstance(work_dir, str):
+            work_dir = Path(work_dir)
+        work_dir.mkdir(exist_ok=True)
+
+        if bind_dir is None:
+            bind_dir = work_dir
+        elif isinstance(bind_dir, str):
+            bind_dir = Path(bind_dir)
+
+        client = docker.from_env()
+        # Check if the image exists
+        try:
+            client.images.get(image)
+        except ImageNotFound:
+            logging.info(f"Pulling image {image}...")
+            # Let the docker exception escape if this fails.
+            client.images.pull(image)
+
+        if container_name is None:
+            container_name = f"autogen-code-exec-{uuid.uuid4()}"
+
+        # Start a container from the image, read to exec commands later
+        self._container = client.containers.create(
+            image,
+            name=container_name,
+            entrypoint="/bin/sh",
+            tty=True,
+            auto_remove=auto_remove,
+            volumes={str(bind_dir.resolve()): {"bind": "/workspace", "mode": "rw"}},
+            working_dir="/workspace",
+        )
+        self._container.start()
+
+        _wait_for_ready(self._container)
+
+        def cleanup() -> None:
+            try:
+                container = client.containers.get(container_name)
+                container.stop()
+            except docker.errors.NotFound:
+                pass
+            atexit.unregister(cleanup)
+
+        if stop_container:
+            atexit.register(cleanup)
+
+        self._cleanup = cleanup
+
+        # Check if the container is running
+        if self._container.status != "running":
+            raise ValueError(f"Failed to start container from image {image}. Logs: {self._container.logs()}")
+
+        self._timeout = timeout
+        self._work_dir: Path = work_dir
+        self._bind_dir: Path = bind_dir
+        self.execution_policies = self.DEFAULT_EXECUTION_POLICY.copy()
+        if execution_policies is not None:
+            self.execution_policies.update(execution_policies)
+
+    @property
+    def timeout(self) -> int:
+        """(Experimental) The timeout for code execution."""
+        return self._timeout
+
+    @property
+    def work_dir(self) -> Path:
+        """(Experimental) The working directory for the code execution."""
+        return self._work_dir
+
+    @property
+    def bind_dir(self) -> Path:
+        """(Experimental) The binding directory for the code execution container."""
+        return self._bind_dir
+
+    @property
+    def code_extractor(self) -> CodeExtractor:
+        """(Experimental) Export a code extractor that can be used by an agent."""
+        return MarkdownCodeExtractor()
+
+    def execute_code_blocks(self, code_blocks: list[CodeBlock]) -> CommandLineCodeResult:
+        """(Experimental) Execute the code blocks and return the result.
+
+        Args:
+            code_blocks (List[CodeBlock]): The code blocks to execute.
+
+        Returns:
+            CommandlineCodeResult: The result of the code execution.
+        """
+        if len(code_blocks) == 0:
+            raise ValueError("No code blocks to execute.")
+
+        outputs = []
+        files = []
+        last_exit_code = 0
+        for code_block in code_blocks:
+            lang = self.LANGUAGE_ALIASES.get(code_block.language.lower(), code_block.language.lower())
+            if lang not in self.DEFAULT_EXECUTION_POLICY:
+                outputs.append(f"Unsupported language {lang}\n")
+                last_exit_code = 1
+                break
+
+            execute_code = self.execution_policies.get(lang, False)
+            code = silence_pip(code_block.code, lang)
+
+            # Check if there is a filename comment
+            try:
+                filename = _get_file_name_from_content(code, self._work_dir)
+            except ValueError:
+                outputs.append("Filename is not in the workspace")
+                last_exit_code = 1
+                break
+
+            if not filename:
+                filename = f"tmp_code_{md5(code.encode()).hexdigest()}.{lang}"
+
+            code_path = self._work_dir / filename
+            with code_path.open("w", encoding="utf-8") as fout:
+                fout.write(code)
+            files.append(code_path)
+
+            if not execute_code:
+                outputs.append(f"Code saved to {code_path!s}\n")
+                continue
+
+            command = ["timeout", str(self._timeout), _cmd(lang), filename]
+            result = self._container.exec_run(command)
+            exit_code = result.exit_code
+            output = result.output.decode("utf-8")
+            if exit_code == 124:
+                output += "\n" + TIMEOUT_MSG
+            outputs.append(output)
+
+            last_exit_code = exit_code
+            if exit_code != 0:
+                break
+
+        code_file = str(files[0]) if files else None
+        return CommandLineCodeResult(exit_code=last_exit_code, output="".join(outputs), code_file=code_file)
+
+    def restart(self) -> None:
+        """(Experimental) Restart the code executor."""
+        self._container.restart()
+        if self._container.status != "running":
+            raise ValueError(f"Failed to restart container. Logs: {self._container.logs()}")
+
+    def stop(self) -> None:
+        """(Experimental) Stop the code executor."""
+        self._cleanup()
+
+    def __enter__(self) -> Self:
+        return self
+
+    def __exit__(
+        self,
+        exc_type: Optional[type[BaseException]],
+        exc_val: Optional[BaseException],
+        exc_tb: Optional[TracebackType],
+    ) -> None:
+        self.stop()

mm_agents/coact/autogen/coding/factory.py

@@ -0,0 +1,47 @@
+# Copyright (c) 2023 - 2025, AG2ai, Inc., AG2ai open-source projects maintainers and core contributors
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Portions derived from https://github.com/microsoft/autogen are under the MIT License.
+# SPDX-License-Identifier: MIT
+from ..doc_utils import export_module
+from .base import CodeExecutionConfig, CodeExecutor
+
+__all__ = ("CodeExecutorFactory",)
+
+
+@export_module("autogen.coding")
+class CodeExecutorFactory:
+    """(Experimental) A factory class for creating code executors."""
+
+    @staticmethod
+    def create(code_execution_config: CodeExecutionConfig) -> CodeExecutor:
+        """(Experimental) Get a code executor based on the code execution config.
+
+        Args:
+            code_execution_config (Dict): The code execution config,
+                which is a dictionary that must contain the key "executor".
+                The value of the key "executor" can be either a string
+                or an instance of CodeExecutor, in which case the code
+                executor is returned directly.
+
+        Returns:
+            CodeExecutor: The code executor.
+
+        Raises:
+            ValueError: If the code executor is unknown or not specified.
+        """
+        executor = code_execution_config.get("executor")
+        if isinstance(executor, CodeExecutor):
+            # If the executor is already an instance of CodeExecutor, return it.
+            return executor
+        if executor == "ipython-embedded":
+            from .jupyter.embedded_ipython_code_executor import EmbeddedIPythonCodeExecutor
+
+            return EmbeddedIPythonCodeExecutor(**code_execution_config.get("ipython-embedded", {}))
+        elif executor == "commandline-local":
+            from .local_commandline_code_executor import LocalCommandLineCodeExecutor
+
+            return LocalCommandLineCodeExecutor(**code_execution_config.get("commandline-local", {}))
+        else:
+            raise ValueError(f"Unknown code executor {executor}")

mm_agents/coact/autogen/coding/func_with_reqs.py

@@ -0,0 +1,202 @@
+# Copyright (c) 2023 - 2025, AG2ai, Inc., AG2ai open-source projects maintainers and core contributors
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Portions derived from https://github.com/microsoft/autogen are under the MIT License.
+# SPDX-License-Identifier: MIT
+
+import functools
+import importlib
+import inspect
+from dataclasses import dataclass, field
+from importlib.abc import SourceLoader
+from textwrap import dedent, indent
+from typing import Any, Callable, Generic, TypeVar, Union
+
+from typing_extensions import ParamSpec
+
+T = TypeVar("T")
+P = ParamSpec("P")
+
+
+def _to_code(func: Union["FunctionWithRequirements[T, P]", Callable[P, T], "FunctionWithRequirementsStr"]) -> str:
+    if isinstance(func, FunctionWithRequirementsStr):
+        return func.func
+
+    code = inspect.getsource(func)
+    # Strip the decorator
+    if code.startswith("@"):
+        code = code[code.index("\n") + 1 :]
+    return code
+
+
+@dataclass
+class Alias:
+    name: str
+    alias: str
+
+
+@dataclass
+class ImportFromModule:
+    module: str
+    imports: list[Union[str, Alias]]
+
+
+Import = Union[str, ImportFromModule, Alias]
+
+
+def _import_to_str(im: Import) -> str:
+    if isinstance(im, str):
+        return f"import {im}"
+    elif isinstance(im, Alias):
+        return f"import {im.name} as {im.alias}"
+    else:
+
+        def to_str(i: Union[str, Alias]) -> str:
+            if isinstance(i, str):
+                return i
+            else:
+                return f"{i.name} as {i.alias}"
+
+        imports = ", ".join(map(to_str, im.imports))
+        return f"from {im.module} import {imports}"
+
+
+class _StringLoader(SourceLoader):
+    def __init__(self, data: str):
+        self.data = data
+
+    def get_source(self, fullname: str) -> str:
+        return self.data
+
+    def get_data(self, path: str) -> bytes:
+        return self.data.encode("utf-8")
+
+    def get_filename(self, fullname: str) -> str:
+        return "<not a real path>/" + fullname + ".py"
+
+
+@dataclass
+class FunctionWithRequirementsStr:
+    func: str
+    _compiled_func: Callable[..., Any]
+    _func_name: str
+    python_packages: list[str] = field(default_factory=list)
+    global_imports: list[Import] = field(default_factory=list)
+
+    def __init__(self, func: str, python_packages: list[str] = [], global_imports: list[Import] = []):
+        self.func = func
+        self.python_packages = python_packages
+        self.global_imports = global_imports
+
+        module_name = "func_module"
+        loader = _StringLoader(func)
+        spec = importlib.util.spec_from_loader(module_name, loader)
+        if spec is None:
+            raise ValueError("Could not create spec")
+        module = importlib.util.module_from_spec(spec)
+        if spec.loader is None:
+            raise ValueError("Could not create loader")
+
+        try:
+            spec.loader.exec_module(module)
+        except Exception as e:
+            raise ValueError(f"Could not compile function: {e}") from e
+
+        functions = inspect.getmembers(module, inspect.isfunction)
+        if len(functions) != 1:
+            raise ValueError("The string must contain exactly one function")
+
+        self._func_name, self._compiled_func = functions[0]
+
+    def __call__(self, *args: Any, **kwargs: Any) -> None:
+        raise NotImplementedError("String based function with requirement objects are not directly callable")
+
+
+@dataclass
+class FunctionWithRequirements(Generic[T, P]):
+    func: Callable[P, T]
+    python_packages: list[str] = field(default_factory=list)
+    global_imports: list[Import] = field(default_factory=list)
+
+    @classmethod
+    def from_callable(
+        cls, func: Callable[P, T], python_packages: list[str] = [], global_imports: list[Import] = []
+    ) -> "FunctionWithRequirements[T, P]":
+        return cls(python_packages=python_packages, global_imports=global_imports, func=func)
+
+    @staticmethod
+    def from_str(
+        func: str, python_packages: list[str] = [], global_imports: list[Import] = []
+    ) -> FunctionWithRequirementsStr:
+        return FunctionWithRequirementsStr(func=func, python_packages=python_packages, global_imports=global_imports)
+
+    # Type this based on F
+    def __call__(self, *args: P.args, **kwargs: P.kwargs) -> T:
+        return self.func(*args, **kwargs)
+
+
+def with_requirements(
+    python_packages: list[str] = [], global_imports: list[Import] = []
+) -> Callable[[Callable[P, T]], FunctionWithRequirements[T, P]]:
+    """Decorate a function with package and import requirements
+
+    Args:
+        python_packages (List[str], optional): Packages required to function. Can include version info.. Defaults to [].
+        global_imports (List[Import], optional): Required imports. Defaults to [].
+
+    Returns:
+        Callable[[Callable[P, T]], FunctionWithRequirements[T, P]]: The decorated function
+    """
+
+    def wrapper(func: Callable[P, T]) -> FunctionWithRequirements[T, P]:
+        func_with_reqs = FunctionWithRequirements(
+            python_packages=python_packages, global_imports=global_imports, func=func
+        )
+
+        functools.update_wrapper(func_with_reqs, func)
+        return func_with_reqs
+
+    return wrapper
+
+
+def _build_python_functions_file(
+    funcs: list[Union[FunctionWithRequirements[Any, P], Callable[..., Any], FunctionWithRequirementsStr]],
+) -> str:
+    # First collect all global imports
+    global_imports: set[str] = set()
+    for func in funcs:
+        if isinstance(func, (FunctionWithRequirements, FunctionWithRequirementsStr)):
+            global_imports.update(map(_import_to_str, func.global_imports))
+
+    content = "\n".join(global_imports) + "\n\n"
+
+    for func in funcs:
+        content += _to_code(func) + "\n\n"
+
+    return content
+
+
+def to_stub(func: Union[Callable[..., Any], FunctionWithRequirementsStr]) -> str:
+    """Generate a stub for a function as a string
+
+    Args:
+        func (Callable[..., Any]): The function to generate a stub for
+
+    Returns:
+        str: The stub for the function
+    """
+    if isinstance(func, FunctionWithRequirementsStr):
+        return to_stub(func._compiled_func)
+
+    content = f"def {func.__name__}{inspect.signature(func)}:\n"
+    docstring = func.__doc__
+
+    if docstring:
+        docstring = dedent(docstring)
+        docstring = '"""' + docstring + '"""'
+        docstring = indent(docstring, "    ")
+        content += docstring + "\n"
+
+    content += "    ..."
+    return content

mm_agents/coact/autogen/coding/jupyter/__init__.py

@@ -0,0 +1,23 @@
+# Copyright (c) 2023 - 2025, AG2ai, Inc., AG2ai open-source projects maintainers and core contributors
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Original portions of this file are derived from https://github.com/microsoft/autogen under the MIT License.
+# SPDX-License-Identifier: MIT
+
+from .base import JupyterConnectable, JupyterConnectionInfo
+from .docker_jupyter_server import DockerJupyterServer
+from .embedded_ipython_code_executor import EmbeddedIPythonCodeExecutor
+from .jupyter_client import JupyterClient
+from .jupyter_code_executor import JupyterCodeExecutor
+from .local_jupyter_server import LocalJupyterServer
+
+__all__ = [
+    "DockerJupyterServer",
+    "EmbeddedIPythonCodeExecutor",
+    "JupyterClient",
+    "JupyterCodeExecutor",
+    "JupyterConnectable",
+    "JupyterConnectionInfo",
+    "LocalJupyterServer",
+]

mm_agents/coact/autogen/coding/jupyter/base.py

@@ -0,0 +1,36 @@
+# Copyright (c) 2023 - 2025, AG2ai, Inc., AG2ai open-source projects maintainers and core contributors
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Portions derived from https://github.com/microsoft/autogen are under the MIT License.
+# SPDX-License-Identifier: MIT
+from dataclasses import dataclass
+from typing import Optional, Protocol, runtime_checkable
+
+from ...doc_utils import export_module
+
+
+@dataclass
+@export_module("autogen.coding.jupyter")
+class JupyterConnectionInfo:
+    """(Experimental)"""
+
+    host: str
+    """`str` - Host of the Jupyter gateway server"""
+    use_https: bool
+    """`bool` - Whether to use HTTPS"""
+    port: Optional[int] = None
+    """`Optional[int]` - Port of the Jupyter gateway server. If None, the default port is used"""
+    token: Optional[str] = None
+    """`Optional[str]` - Token for authentication. If None, no token is used"""
+
+
+@runtime_checkable
+@export_module("autogen.coding.jupyter")
+class JupyterConnectable(Protocol):
+    """(Experimental)"""
+
+    @property
+    def connection_info(self) -> JupyterConnectionInfo:
+        """Return the connection information for this connectable."""
+        pass

mm_agents/coact/autogen/coding/jupyter/docker_jupyter_server.py

@@ -0,0 +1,167 @@
+# Copyright (c) 2023 - 2025, AG2ai, Inc., AG2ai open-source projects maintainers and core contributors
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Portions derived from https://github.com/microsoft/autogen are under the MIT License.
+# SPDX-License-Identifier: MIT
+from __future__ import annotations
+
+import atexit
+import io
+import logging
+import secrets
+import sys
+import uuid
+from pathlib import Path
+from types import TracebackType
+from typing import Optional
+
+import docker
+
+from ...doc_utils import export_module
+
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
+from ..docker_commandline_code_executor import _wait_for_ready
+from .base import JupyterConnectable, JupyterConnectionInfo
+from .import_utils import require_jupyter_kernel_gateway_installed
+from .jupyter_client import JupyterClient
+
+
+@require_jupyter_kernel_gateway_installed()
+@export_module("autogen.coding.jupyter")
+class DockerJupyterServer(JupyterConnectable):
+    DEFAULT_DOCKERFILE = """FROM quay.io/jupyter/docker-stacks-foundation
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+USER ${NB_UID}
+RUN mamba install --yes jupyter_kernel_gateway ipykernel && \
+    mamba clean --all -f -y && \
+    fix-permissions "${CONDA_DIR}" && \
+    fix-permissions "/home/${NB_USER}"
+
+ENV TOKEN="UNSET"
+CMD python -m jupyter kernelgateway --KernelGatewayApp.ip=0.0.0.0 \
+    --KernelGatewayApp.port=8888 \
+    --KernelGatewayApp.auth_token="${TOKEN}" \
+    --JupyterApp.answer_yes=true \
+    --JupyterWebsocketPersonality.list_kernels=true
+
+EXPOSE 8888
+
+WORKDIR "${HOME}"
+"""
+
+    class GenerateToken:
+        pass
+
+    def __init__(
+        self,
+        *,
+        custom_image_name: Optional[str] = None,
+        container_name: Optional[str] = None,
+        auto_remove: bool = True,
+        stop_container: bool = True,
+        docker_env: dict[str, str] = {},
+        token: str | GenerateToken = GenerateToken(),
+    ):
+        """Start a Jupyter kernel gateway server in a Docker container.
+
+        Args:
+            custom_image_name (Optional[str], optional): Custom image to use. If this is None,
+                then the bundled image will be built and used. The default image is based on
+                quay.io/jupyter/docker-stacks-foundation and extended to include jupyter_kernel_gateway
+            container_name (Optional[str], optional): Name of the container to start.
+                A name will be generated if None.
+            auto_remove (bool, optional): If true the Docker container will be deleted
+                when it is stopped.
+            stop_container (bool, optional): If true the container will be stopped,
+                either by program exit or using the context manager
+            docker_env (Dict[str, str], optional): Extra environment variables to pass
+                to the running Docker container.
+            token (Union[str, GenerateToken], optional): Token to use for authentication.
+                If GenerateToken is used, a random token will be generated. Empty string
+                will be unauthenticated.
+        """
+        if container_name is None:
+            container_name = f"autogen-jupyterkernelgateway-{uuid.uuid4()}"
+
+        client = docker.from_env()
+        if custom_image_name is None:
+            image_name = "autogen-jupyterkernelgateway"
+            # Make sure the image exists
+            try:
+                client.images.get(image_name)
+            except docker.errors.ImageNotFound:
+                # Build the image
+                # Get this script directory
+                here = Path(__file__).parent
+                dockerfile = io.BytesIO(self.DEFAULT_DOCKERFILE.encode("utf-8"))
+                logging.info(f"Image {image_name} not found. Building it now.")
+                client.images.build(path=here, fileobj=dockerfile, tag=image_name)
+                logging.info(f"Image {image_name} built successfully.")
+        else:
+            image_name = custom_image_name
+            # Check if the image exists
+            try:
+                client.images.get(image_name)
+            except docker.errors.ImageNotFound:
+                raise ValueError(f"Custom image {image_name} does not exist")
+
+        if isinstance(token, DockerJupyterServer.GenerateToken):
+            self._token = secrets.token_hex(32)
+        else:
+            self._token = token
+
+        # Run the container
+        env = {"TOKEN": self._token}
+        env.update(docker_env)
+        container = client.containers.run(
+            image_name,
+            detach=True,
+            auto_remove=auto_remove,
+            environment=env,
+            publish_all_ports=True,
+            name=container_name,
+        )
+        _wait_for_ready(container)
+        container_ports = container.ports
+        self._port = int(container_ports["8888/tcp"][0]["HostPort"])
+        self._container_id = container.id
+
+        def cleanup() -> None:
+            try:
+                inner_container = client.containers.get(container.id)
+                inner_container.stop()
+            except docker.errors.NotFound:
+                pass
+
+            atexit.unregister(cleanup)
+
+        if stop_container:
+            atexit.register(cleanup)
+
+        self._cleanup_func = cleanup
+        self._stop_container = stop_container
+
+    @property
+    def connection_info(self) -> JupyterConnectionInfo:
+        return JupyterConnectionInfo(host="127.0.0.1", use_https=False, port=self._port, token=self._token)
+
+    def stop(self) -> None:
+        self._cleanup_func()
+
+    def get_client(self) -> JupyterClient:
+        return JupyterClient(self.connection_info)
+
+    def __enter__(self) -> Self:
+        return self
+
+    def __exit__(
+        self, exc_type: Optional[type[BaseException]], exc_val: Optional[BaseException], exc_tb: Optional[TracebackType]
+    ) -> None:
+        self.stop()

mm_agents/coact/autogen/coding/jupyter/embedded_ipython_code_executor.py

@@ -0,0 +1,182 @@
+# Copyright (c) 2023 - 2025, AG2ai, Inc., AG2ai open-source projects maintainers and core contributors
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Portions derived from https://github.com/microsoft/autogen are under the MIT License.
+# SPDX-License-Identifier: MIT
+import base64
+import json
+import os
+import re
+import uuid
+from pathlib import Path
+from queue import Empty
+from typing import Any
+
+from pydantic import BaseModel, Field, field_validator
+
+from ...doc_utils import export_module
+from ...import_utils import optional_import_block, require_optional_import
+from ..base import CodeBlock, CodeExtractor, IPythonCodeResult
+from ..markdown_code_extractor import MarkdownCodeExtractor
+from .import_utils import require_jupyter_kernel_gateway_installed
+
+with optional_import_block():
+    from jupyter_client import KernelManager  # type: ignore[attr-defined]
+    from jupyter_client.kernelspec import KernelSpecManager
+
+__all__ = ["EmbeddedIPythonCodeExecutor"]
+
+
+@require_optional_import("jupyter_client", "jupyter-executor")
+@require_jupyter_kernel_gateway_installed()
+@export_module("autogen.coding.jupyter")
+class EmbeddedIPythonCodeExecutor(BaseModel):
+    """(Experimental) A code executor class that executes code statefully using an embedded
+    IPython kernel managed by this class.
+
+    **This will execute LLM generated code on the local machine.**
+
+    Each execution is stateful and can access variables created from previous
+    executions in the same session. The kernel must be installed before using
+    this class. The kernel can be installed using the following command:
+    `python -m ipykernel install --user --name {kernel_name}`
+    where `kernel_name` is the name of the kernel to install.
+    """
+
+    timeout: int = Field(default=60, ge=1, description="The timeout for code execution.")
+    kernel_name: str = Field(default="python3", description="The kernel name to use. Make sure it is installed.")
+    output_dir: str = Field(default=".", description="The directory to save output files.")
+
+    @field_validator("output_dir")
+    @classmethod
+    def _output_dir_must_exist(cls, value: str) -> str:
+        if not os.path.exists(value):
+            raise ValueError(f"Output directory {value} does not exist.")
+        return value
+
+    def __init__(self, **kwargs: Any):
+        super().__init__(**kwargs)
+        # Check if the kernel is installed.
+        if self.kernel_name not in KernelSpecManager().find_kernel_specs():
+            raise ValueError(
+                f"Kernel {self.kernel_name} is not installed. "
+                "Please first install it with "
+                f"`python -m ipykernel install --user --name {self.kernel_name}`."
+            )
+        self._kernel_manager = KernelManager(kernel_name=self.kernel_name)
+        self._kernel_manager.start_kernel()
+        self._kernel_client = self._kernel_manager.client()
+        self._kernel_client.start_channels()
+        self._timeout = self.timeout
+        self._kernel_name = self.kernel_name
+        self._output_dir = Path(self.output_dir)
+
+    @property
+    def code_extractor(self) -> CodeExtractor:
+        """(Experimental) Export a code extractor that can be used by an agent."""
+        return MarkdownCodeExtractor()
+
+    def execute_code_blocks(self, code_blocks: list[CodeBlock]) -> IPythonCodeResult:
+        """(Experimental) Execute a list of code blocks and return the result.
+
+        This method executes a list of code blocks as cells in an IPython kernel
+        managed by this class.
+        See: https://jupyter-client.readthedocs.io/en/stable/messaging.html
+        for the message protocol.
+
+        Args:
+            code_blocks (List[CodeBlock]): A list of code blocks to execute.
+
+        Returns:
+            IPythonCodeResult: The result of the code execution.
+        """
+        self._kernel_client.wait_for_ready()
+        outputs = []
+        output_files = []
+        for code_block in code_blocks:
+            code = self._process_code(code_block.code)
+            self._kernel_client.execute(code, store_history=True)
+            while True:
+                try:
+                    msg = self._kernel_client.get_iopub_msg(timeout=self._timeout)
+                    msg_type = msg["msg_type"]
+                    content = msg["content"]
+                    if msg_type in ["execute_result", "display_data"]:
+                        for data_type, data in content["data"].items():
+                            if data_type == "text/plain":
+                                # Output is a text.
+                                outputs.append(data)
+                            elif data_type.startswith("image/"):
+                                # Output is an image.
+                                path = self._save_image(data)
+                                outputs.append(f"Image data saved to {path}")
+                                output_files.append(path)
+                            elif data_type == "text/html":
+                                # Output is an html.
+                                path = self._save_html(data)
+                                outputs.append(f"HTML data saved to {path}")
+                                output_files.append(path)
+                            else:
+                                # Output raw data.
+                                outputs.append(json.dumps(data))
+                    elif msg_type == "stream":
+                        # Output is a text.
+                        outputs.append(content["text"])
+                    elif msg_type == "error":
+                        # Output is an error.
+                        return IPythonCodeResult(
+                            exit_code=1,
+                            output=f"ERROR: {content['ename']}: {content['evalue']}\n{content['traceback']}",
+                        )
+                    if msg_type == "status" and content["execution_state"] == "idle":
+                        break
+                # handle time outs.
+                except Empty:
+                    return IPythonCodeResult(
+                        exit_code=1,
+                        output=f"ERROR: Timeout waiting for output from code block: {code_block.code}",
+                    )
+        # We return the full output.
+        return IPythonCodeResult(
+            exit_code=0, output="\n".join([str(output) for output in outputs]), output_files=output_files
+        )
+
+    def restart(self) -> None:
+        """(Experimental) Restart a new session."""
+        self._kernel_client.stop_channels()
+        self._kernel_manager.shutdown_kernel()
+        self._kernel_manager = KernelManager(kernel_name=self.kernel_name)
+        self._kernel_manager.start_kernel()
+        self._kernel_client = self._kernel_manager.client()
+        self._kernel_client.start_channels()
+
+    def _save_image(self, image_data_base64: str) -> str:
+        """Save image data to a file."""
+        image_data = base64.b64decode(image_data_base64)
+        # Randomly generate a filename.
+        filename = f"{uuid.uuid4().hex}.png"
+        path = os.path.join(self.output_dir, filename)
+        with open(path, "wb") as f:
+            f.write(image_data)
+        return os.path.abspath(path)
+
+    def _save_html(self, html_data: str) -> str:
+        """Save html data to a file."""
+        # Randomly generate a filename.
+        filename = f"{uuid.uuid4().hex}.html"
+        path = os.path.join(self.output_dir, filename)
+        with open(path, "w") as f:
+            f.write(html_data)
+        return os.path.abspath(path)
+
+    def _process_code(self, code: str) -> str:
+        """Process code before execution."""
+        # Find lines that start with `! pip install` and make sure "-qqq" flag is added.
+        lines = code.split("\n")
+        for i, line in enumerate(lines):
+            # use regex to find lines that start with `! pip install` or `!pip install`.
+            match = re.search(r"^! ?pip install", line)
+            if match is not None and "-qqq" not in line:
+                lines[i] = line.replace(match.group(0), match.group(0) + " -qqq")
+        return "\n".join(lines)

mm_agents/coact/autogen/coding/jupyter/import_utils.py

@@ -0,0 +1,82 @@
+# Copyright (c) 2023 - 2025, AG2ai, Inc., AG2ai open-source projects maintainers and core contributors
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import subprocess
+from functools import lru_cache
+from logging import getLogger
+from typing import Callable, TypeVar
+
+from ...import_utils import patch_object
+
+logger = getLogger(__name__)
+
+__all__ = ["require_jupyter_kernel_gateway_installed", "skip_on_missing_jupyter_kernel_gateway"]
+
+
+@lru_cache
+def is_jupyter_kernel_gateway_installed() -> bool:
+    """Check if jupyter-kernel-gateway is installed."""
+    try:
+        subprocess.run(
+            ["jupyter", "kernelgateway", "--version"],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            check=True,
+        )
+        return True
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        logger.warning(
+            "jupyter-kernel-gateway is required for JupyterCodeExecutor, please install it with `pip install ag2[jupyter-executor]`"
+        )
+        return False
+
+
+T = TypeVar("T")
+
+
+def require_jupyter_kernel_gateway_installed() -> Callable[[T], T]:
+    """Decorator that checks if jupyter-kernel-gateway is installed before function execution.
+
+    Returns:
+        Callable[[T], T]: A decorator function that either:
+            - Returns the original function unchanged if jupyter-kernel-gateway is installed
+            - Returns a patched version of the function that will raise a helpful error indicating the missing dependency when called
+    """
+    if is_jupyter_kernel_gateway_installed():
+
+        def decorator(o: T) -> T:
+            return o
+
+    else:
+
+        def decorator(o: T) -> T:
+            return patch_object(o, missing_modules={}, dep_target="jupyter-executor")
+
+    return decorator
+
+
+def skip_on_missing_jupyter_kernel_gateway() -> Callable[[T], T]:
+    """Decorator to skip a test if an optional module is missing"""
+    # Add pytest.mark.jupyter_executor decorator
+    mark_name = "jupyter_executor"
+
+    if is_jupyter_kernel_gateway_installed():
+
+        def decorator(o: T) -> T:
+            import pytest
+
+            pytest_mark_o = getattr(pytest.mark, mark_name)(o)
+            return pytest_mark_o  # type: ignore[no-any-return]
+
+    else:
+
+        def decorator(o: T) -> T:
+            import pytest
+
+            pytest_mark_o = getattr(pytest.mark, mark_name)(o)
+            return pytest.mark.skip(  # type: ignore[return-value,no-any-return]
+                reason="jupyter-kernel-gateway is required for JupyterCodeExecutor, please install it with `pip install ag2[jupyter-executor]`"
+            )(pytest_mark_o)
+
+    return decorator

mm_agents/coact/autogen/coding/jupyter/jupyter_client.py

@@ -0,0 +1,231 @@
+# Copyright (c) 2023 - 2025, AG2ai, Inc., AG2ai open-source projects maintainers and core contributors
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Portions derived from https://github.com/microsoft/autogen are under the MIT License.
+# SPDX-License-Identifier: MIT
+from __future__ import annotations
+
+import datetime
+import json
+import sys
+import uuid
+from dataclasses import dataclass
+from types import TracebackType
+from typing import Any, Optional, cast
+
+from ...doc_utils import export_module
+
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
+
+import requests
+from requests.adapters import HTTPAdapter, Retry
+
+from ...import_utils import optional_import_block, require_optional_import
+from .base import JupyterConnectionInfo
+
+with optional_import_block():
+    import websocket
+    from websocket import WebSocket
+
+
+@export_module("autogen.coding.jupyter")
+class JupyterClient:
+    def __init__(self, connection_info: JupyterConnectionInfo):
+        """(Experimental) A client for communicating with a Jupyter gateway server.
+
+        Args:
+            connection_info (JupyterConnectionInfo): Connection information
+        """
+        self._connection_info = connection_info
+        self._session = requests.Session()
+        retries = Retry(total=5, backoff_factor=0.1)
+        self._session.mount("http://", HTTPAdapter(max_retries=retries))
+
+    def _get_headers(self) -> dict[str, str]:
+        if self._connection_info.token is None:
+            return {}
+        return {"Authorization": f"token {self._connection_info.token}"}
+
+    def _get_api_base_url(self) -> str:
+        protocol = "https" if self._connection_info.use_https else "http"
+        port = f":{self._connection_info.port}" if self._connection_info.port else ""
+        return f"{protocol}://{self._connection_info.host}{port}"
+
+    def _get_ws_base_url(self) -> str:
+        port = f":{self._connection_info.port}" if self._connection_info.port else ""
+        return f"ws://{self._connection_info.host}{port}"
+
+    def list_kernel_specs(self) -> dict[str, dict[str, str]]:
+        response = self._session.get(f"{self._get_api_base_url()}/api/kernelspecs", headers=self._get_headers())
+        return cast(dict[str, dict[str, str]], response.json())
+
+    def list_kernels(self) -> list[dict[str, str]]:
+        response = self._session.get(f"{self._get_api_base_url()}/api/kernels", headers=self._get_headers())
+        return cast(list[dict[str, str]], response.json())
+
+    def start_kernel(self, kernel_spec_name: str) -> str:
+        """Start a new kernel.
+
+        Args:
+            kernel_spec_name (str): Name of the kernel spec to start
+
+        Returns:
+            str: ID of the started kernel
+        """
+        response = self._session.post(
+            f"{self._get_api_base_url()}/api/kernels",
+            headers=self._get_headers(),
+            json={"name": kernel_spec_name},
+        )
+        return cast(str, response.json()["id"])
+
+    def delete_kernel(self, kernel_id: str) -> None:
+        response = self._session.delete(
+            f"{self._get_api_base_url()}/api/kernels/{kernel_id}", headers=self._get_headers()
+        )
+        response.raise_for_status()
+
+    def restart_kernel(self, kernel_id: str) -> None:
+        response = self._session.post(
+            f"{self._get_api_base_url()}/api/kernels/{kernel_id}/restart", headers=self._get_headers()
+        )
+        response.raise_for_status()
+
+    @require_optional_import("websocket", "jupyter-executor")
+    def get_kernel_client(self, kernel_id: str) -> JupyterKernelClient:
+        ws_url = f"{self._get_ws_base_url()}/api/kernels/{kernel_id}/channels"
+        ws = websocket.create_connection(ws_url, header=self._get_headers())
+        return JupyterKernelClient(ws)
+
+
+@require_optional_import("websocket", "jupyter-executor")
+class JupyterKernelClient:
+    """(Experimental) A client for communicating with a Jupyter kernel."""
+
+    @dataclass
+    class ExecutionResult:
+        @dataclass
+        class DataItem:
+            mime_type: str
+            data: str
+
+        is_ok: bool
+        output: str
+        data_items: list[DataItem]
+
+    def __init__(self, websocket: WebSocket):  # type: ignore[no-any-unimported]
+        self._session_id: str = uuid.uuid4().hex
+        self._websocket: WebSocket = websocket  # type: ignore[no-any-unimported]
+
+    def __enter__(self) -> Self:
+        return self
+
+    def __exit__(
+        self, exc_type: Optional[type[BaseException]], exc_val: Optional[BaseException], exc_tb: Optional[TracebackType]
+    ) -> None:
+        self.stop()
+
+    def stop(self) -> None:
+        self._websocket.close()
+
+    def _send_message(self, *, content: dict[str, Any], channel: str, message_type: str) -> str:
+        timestamp = datetime.datetime.now().isoformat()
+        message_id = uuid.uuid4().hex
+        message = {
+            "header": {
+                "username": "autogen",
+                "version": "5.0",
+                "session": self._session_id,
+                "msg_id": message_id,
+                "msg_type": message_type,
+                "date": timestamp,
+            },
+            "parent_header": {},
+            "channel": channel,
+            "content": content,
+            "metadata": {},
+            "buffers": {},
+        }
+        self._websocket.send_text(json.dumps(message))
+        return message_id
+
+    def _receive_message(self, timeout_seconds: Optional[float]) -> Optional[dict[str, Any]]:
+        self._websocket.settimeout(timeout_seconds)
+        try:
+            data = self._websocket.recv()
+            if isinstance(data, bytes):
+                data = data.decode("utf-8")
+            return cast(dict[str, Any], json.loads(data))
+        except websocket.WebSocketTimeoutException:
+            return None
+
+    def wait_for_ready(self, timeout_seconds: Optional[float] = None) -> bool:
+        message_id = self._send_message(content={}, channel="shell", message_type="kernel_info_request")
+        while True:
+            message = self._receive_message(timeout_seconds)
+            # This means we timed out with no new messages.
+            if message is None:
+                return False
+            if (
+                message.get("parent_header", {}).get("msg_id") == message_id
+                and message["msg_type"] == "kernel_info_reply"
+            ):
+                return True
+
+    def execute(self, code: str, timeout_seconds: Optional[float] = None) -> ExecutionResult:
+        message_id = self._send_message(
+            content={
+                "code": code,
+                "silent": False,
+                "store_history": True,
+                "user_expressions": {},
+                "allow_stdin": False,
+                "stop_on_error": True,
+            },
+            channel="shell",
+            message_type="execute_request",
+        )
+
+        text_output = []
+        data_output = []
+        while True:
+            message = self._receive_message(timeout_seconds)
+            if message is None:
+                return JupyterKernelClient.ExecutionResult(
+                    is_ok=False, output="ERROR: Timeout waiting for output from code block.", data_items=[]
+                )
+
+            # Ignore messages that are not for this execution.
+            if message.get("parent_header", {}).get("msg_id") != message_id:
+                continue
+
+            msg_type = message["msg_type"]
+            content = message["content"]
+            if msg_type in ["execute_result", "display_data"]:
+                for data_type, data in content["data"].items():
+                    if data_type == "text/plain":
+                        text_output.append(data)
+                    elif data_type.startswith("image/") or data_type == "text/html":
+                        data_output.append(self.ExecutionResult.DataItem(mime_type=data_type, data=data))
+                    else:
+                        text_output.append(json.dumps(data))
+            elif msg_type == "stream":
+                text_output.append(content["text"])
+            elif msg_type == "error":
+                # Output is an error.
+                return JupyterKernelClient.ExecutionResult(
+                    is_ok=False,
+                    output=f"ERROR: {content['ename']}: {content['evalue']}\n{content['traceback']}",
+                    data_items=[],
+                )
+            if msg_type == "status" and content["execution_state"] == "idle":
+                break
+
+        return JupyterKernelClient.ExecutionResult(
+            is_ok=True, output="\n".join([str(output) for output in text_output]), data_items=data_output
+        )

mm_agents/coact/autogen/coding/jupyter/jupyter_code_executor.py

@@ -0,0 +1,160 @@
+# Copyright (c) 2023 - 2025, AG2ai, Inc., AG2ai open-source projects maintainers and core contributors
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Portions derived from https://github.com/microsoft/autogen are under the MIT License.
+# SPDX-License-Identifier: MIT
+import base64
+import json
+import os
+import sys
+import uuid
+from pathlib import Path
+from types import TracebackType
+from typing import Optional, Union
+
+from ...doc_utils import export_module
+
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
+from ..base import CodeBlock, CodeExecutor, CodeExtractor, IPythonCodeResult
+from ..markdown_code_extractor import MarkdownCodeExtractor
+from ..utils import silence_pip
+from .base import JupyterConnectable, JupyterConnectionInfo
+from .jupyter_client import JupyterClient
+
+
+@export_module("autogen.coding.jupyter")
+class JupyterCodeExecutor(CodeExecutor):
+    def __init__(
+        self,
+        jupyter_server: Union[JupyterConnectable, JupyterConnectionInfo],
+        kernel_name: str = "python3",
+        timeout: int = 60,
+        output_dir: Union[Path, str] = Path(),
+    ):
+        """(Experimental) A code executor class that executes code statefully using
+        a Jupyter server supplied to this class.
+
+        Each execution is stateful and can access variables created from previous
+        executions in the same session.
+
+        Args:
+            jupyter_server (Union[JupyterConnectable, JupyterConnectionInfo]): The Jupyter server to use.
+            timeout (int): The timeout for code execution, by default 60.
+            kernel_name (str): The kernel name to use. Make sure it is installed.
+                By default, it is "python3".
+            output_dir (str): The directory to save output files, by default ".".
+        """
+        if timeout < 1:
+            raise ValueError("Timeout must be greater than or equal to 1.")
+
+        if isinstance(output_dir, str):
+            output_dir = Path(output_dir)
+
+        if not output_dir.exists():
+            raise ValueError(f"Output directory {output_dir} does not exist.")
+
+        if isinstance(jupyter_server, JupyterConnectable):
+            self._connection_info = jupyter_server.connection_info
+        elif isinstance(jupyter_server, JupyterConnectionInfo):
+            self._connection_info = jupyter_server
+        else:
+            raise ValueError("jupyter_server must be a JupyterConnectable or JupyterConnectionInfo.")
+
+        self._jupyter_client = JupyterClient(self._connection_info)
+        available_kernels = self._jupyter_client.list_kernel_specs()
+        if kernel_name not in available_kernels["kernelspecs"]:
+            raise ValueError(f"Kernel {kernel_name} is not installed.")
+
+        self._kernel_id = self._jupyter_client.start_kernel(kernel_name)
+        self._kernel_name = kernel_name
+        self._jupyter_kernel_client = self._jupyter_client.get_kernel_client(self._kernel_id)
+        self._timeout = timeout
+        self._output_dir = output_dir
+
+    @property
+    def code_extractor(self) -> CodeExtractor:
+        """(Experimental) Export a code extractor that can be used by an agent."""
+        return MarkdownCodeExtractor()
+
+    def execute_code_blocks(self, code_blocks: list[CodeBlock]) -> IPythonCodeResult:
+        """(Experimental) Execute a list of code blocks and return the result.
+
+        This method executes a list of code blocks as cells in the Jupyter kernel.
+        See: https://jupyter-client.readthedocs.io/en/stable/messaging.html
+        for the message protocol.
+
+        Args:
+            code_blocks (List[CodeBlock]): A list of code blocks to execute.
+
+        Returns:
+            IPythonCodeResult: The result of the code execution.
+        """
+        self._jupyter_kernel_client.wait_for_ready()
+        outputs = []
+        output_files = []
+        for code_block in code_blocks:
+            code = silence_pip(code_block.code, code_block.language)
+            result = self._jupyter_kernel_client.execute(code, timeout_seconds=self._timeout)
+            if result.is_ok:
+                outputs.append(result.output)
+                for data in result.data_items:
+                    if data.mime_type == "image/png":
+                        path = self._save_image(data.data)
+                        outputs.append(f"Image data saved to {path}")
+                        output_files.append(path)
+                    elif data.mime_type == "text/html":
+                        path = self._save_html(data.data)
+                        outputs.append(f"HTML data saved to {path}")
+                        output_files.append(path)
+                    else:
+                        outputs.append(json.dumps(data.data))
+            else:
+                return IPythonCodeResult(
+                    exit_code=1,
+                    output=f"ERROR: {result.output}",
+                )
+
+        return IPythonCodeResult(
+            exit_code=0, output="\n".join([str(output) for output in outputs]), output_files=output_files
+        )
+
+    def restart(self) -> None:
+        """(Experimental) Restart a new session."""
+        self._jupyter_client.restart_kernel(self._kernel_id)
+        self._jupyter_kernel_client = self._jupyter_client.get_kernel_client(self._kernel_id)
+
+    def _save_image(self, image_data_base64: str) -> str:
+        """Save image data to a file."""
+        image_data = base64.b64decode(image_data_base64)
+        # Randomly generate a filename.
+        filename = f"{uuid.uuid4().hex}.png"
+        path = os.path.join(self._output_dir, filename)
+        with open(path, "wb") as f:
+            f.write(image_data)
+        return os.path.abspath(path)
+
+    def _save_html(self, html_data: str) -> str:
+        """Save html data to a file."""
+        # Randomly generate a filename.
+        filename = f"{uuid.uuid4().hex}.html"
+        path = os.path.join(self._output_dir, filename)
+        with open(path, "w") as f:
+            f.write(html_data)
+        return os.path.abspath(path)
+
+    def stop(self) -> None:
+        """Stop the kernel."""
+        self._jupyter_client.delete_kernel(self._kernel_id)
+
+    def __enter__(self) -> Self:
+        return self
+
+    def __exit__(
+        self, exc_type: Optional[type[BaseException]], exc_val: Optional[BaseException], exc_tb: Optional[TracebackType]
+    ) -> None:
+        self.stop()

mm_agents/coact/autogen/coding/jupyter/local_jupyter_server.py

@@ -0,0 +1,172 @@
+# Copyright (c) 2023 - 2025, AG2ai, Inc., AG2ai open-source projects maintainers and core contributors
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Portions derived from https://github.com/microsoft/autogen are under the MIT License.
+# SPDX-License-Identifier: MIT
+from __future__ import annotations
+
+import atexit
+import json
+import secrets
+import signal
+import subprocess
+import sys
+from types import TracebackType
+from typing import Optional
+
+from ...doc_utils import export_module
+
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
+from .base import JupyterConnectable, JupyterConnectionInfo
+from .import_utils import require_jupyter_kernel_gateway_installed
+from .jupyter_client import JupyterClient
+
+
+@require_jupyter_kernel_gateway_installed()
+@export_module("autogen.coding.jupyter")
+class LocalJupyterServer(JupyterConnectable):
+    class GenerateToken:
+        pass
+
+    def __init__(
+        self,
+        ip: str = "127.0.0.1",
+        port: Optional[int] = None,
+        token: str | GenerateToken = GenerateToken(),
+        log_file: str = "jupyter_gateway.log",
+        log_level: str = "INFO",
+        log_max_bytes: int = 1048576,
+        log_backup_count: int = 3,
+    ):
+        """Runs a Jupyter Kernel Gateway server locally.
+
+        Args:
+            ip (str, optional): IP address to bind to. Defaults to "127.0.0.1".
+            port (Optional[int], optional): Port to use, if None it automatically selects a port. Defaults to None.
+            token (Union[str, GenerateToken], optional): Token to use for Jupyter server. By default will generate a token. Using None will use no token for authentication. Defaults to GenerateToken().
+            log_file (str, optional): File for Jupyter Kernel Gateway logs. Defaults to "jupyter_gateway.log".
+            log_level (str, optional): Level for Jupyter Kernel Gateway logs. Defaults to "INFO".
+            log_max_bytes (int, optional): Max logfile size. Defaults to 1048576.
+            log_backup_count (int, optional): Number of backups for rotating log. Defaults to 3.
+        """
+        # Remove as soon as https://github.com/jupyter-server/kernel_gateway/issues/398 is fixed
+        if sys.platform == "win32":
+            raise ValueError("LocalJupyterServer is not supported on Windows due to kernelgateway bug.")
+
+        # Check Jupyter gateway server is installed
+        try:
+            subprocess.run(
+                [sys.executable, "-m", "jupyter", "kernelgateway", "--version"],
+                check=True,
+                capture_output=True,
+                text=True,
+            )
+        except subprocess.CalledProcessError:
+            raise ValueError(
+                "Jupyter gateway server is not installed. Please install it with `pip install jupyter_kernel_gateway`."
+            )
+
+        self.ip: str = ip
+
+        if isinstance(token, LocalJupyterServer.GenerateToken):
+            token = secrets.token_hex(32)
+
+        self.token: str = token
+        self._subprocess: subprocess.Popen[str]
+        logging_config = {
+            "handlers": {
+                "file": {
+                    "class": "logging.handlers.RotatingFileHandler",
+                    "level": log_level,
+                    "maxBytes": log_max_bytes,
+                    "backupCount": log_backup_count,
+                    "filename": log_file,
+                }
+            },
+            "loggers": {"KernelGatewayApp": {"level": log_level, "handlers": ["file", "console"]}},
+        }
+
+        # Run Jupyter gateway server with detached subprocess
+        args = [
+            sys.executable,
+            "-m",
+            "jupyter",
+            "kernelgateway",
+            "--KernelGatewayApp.ip",
+            ip,
+            "--KernelGatewayApp.auth_token",
+            token,
+            "--JupyterApp.answer_yes",
+            "true",
+            "--JupyterApp.logging_config",
+            json.dumps(logging_config),
+            "--JupyterWebsocketPersonality.list_kernels",
+            "true",
+        ]
+        if port is not None:
+            args.extend(["--KernelGatewayApp.port", str(port)])
+            args.extend(["--KernelGatewayApp.port_retries", "0"])
+        self._subprocess = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+
+        # Satisfy mypy, we know this is not None because we passed PIPE
+        assert self._subprocess.stderr is not None
+        # Read stderr until we see "is available at" or the process has exited with an error
+        stderr = ""
+        while True:
+            result = self._subprocess.poll()
+            if result is not None:
+                stderr += self._subprocess.stderr.read()
+                raise ValueError(f"Jupyter gateway server failed to start with exit code: {result}. stderr:\n{stderr}")
+            line = self._subprocess.stderr.readline()
+            stderr += line
+
+            if "ERROR:" in line:
+                error_info = line.split("ERROR:")[1]
+                raise ValueError(f"Jupyter gateway server failed to start. {error_info}")
+
+            if "is available at" in line:
+                # We need to extract what port it settled on
+                # Example output:
+                #   Jupyter Kernel Gateway 3.0.0 is available at http://127.0.0.1:8890
+                if port is None:
+                    port = int(line.split(":")[-1])
+                self.port: int = port
+
+                break
+
+        # Poll the subprocess to check if it is still running
+        result = self._subprocess.poll()
+        if result is not None:
+            raise ValueError(
+                f"Jupyter gateway server failed to start. Please check the logs ({log_file}) for more information."
+            )
+
+        atexit.register(self.stop)
+
+    def stop(self) -> None:
+        if self._subprocess.poll() is None:
+            if sys.platform == "win32":
+                self._subprocess.send_signal(signal.CTRL_C_EVENT)
+            else:
+                self._subprocess.send_signal(signal.SIGINT)
+            self._subprocess.wait()
+
+    @property
+    def connection_info(self) -> JupyterConnectionInfo:
+        return JupyterConnectionInfo(host=self.ip, use_https=False, port=self.port, token=self.token)
+
+    def get_client(self) -> JupyterClient:
+        return JupyterClient(self.connection_info)
+
+    def __enter__(self) -> Self:
+        return self
+
+    def __exit__(
+        self, exc_type: Optional[type[BaseException]], exc_val: Optional[BaseException], exc_tb: Optional[TracebackType]
+    ) -> None:
+        self.stop()

mm_agents/coact/autogen/coding/local_commandline_code_executor.py

@@ -0,0 +1,405 @@
+# Copyright (c) 2023 - 2025, AG2ai, Inc., AG2ai open-source projects maintainers and core contributors
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Portions derived from https://github.com/microsoft/autogen are under the MIT License.
+# SPDX-License-Identifier: MIT
+import logging
+import os
+import re
+import subprocess
+import sys
+import warnings
+from hashlib import md5
+from pathlib import Path
+from string import Template
+from types import SimpleNamespace
+from typing import Any, Callable, ClassVar, Optional, Union
+
+from typing_extensions import ParamSpec
+
+from ..code_utils import PYTHON_VARIANTS, TIMEOUT_MSG, WIN32, _cmd
+from ..doc_utils import export_module
+from .base import CodeBlock, CodeExecutor, CodeExtractor, CommandLineCodeResult
+from .func_with_reqs import (
+    FunctionWithRequirements,
+    FunctionWithRequirementsStr,
+    _build_python_functions_file,
+    to_stub,
+)
+from .markdown_code_extractor import MarkdownCodeExtractor
+from .utils import _get_file_name_from_content, silence_pip
+
+__all__ = ("LocalCommandLineCodeExecutor",)
+
+A = ParamSpec("A")
+
+
+@export_module("autogen.coding")
+class LocalCommandLineCodeExecutor(CodeExecutor):
+    SUPPORTED_LANGUAGES: ClassVar[list[str]] = [
+        "bash",
+        "shell",
+        "sh",
+        "pwsh",
+        "powershell",
+        "ps1",
+        "python",
+        "javascript",
+        "html",
+        "css",
+    ]
+    DEFAULT_EXECUTION_POLICY: ClassVar[dict[str, bool]] = {
+        "bash": True,
+        "shell": True,
+        "sh": True,
+        "pwsh": True,
+        "powershell": True,
+        "ps1": True,
+        "python": True,
+        "javascript": False,
+        "html": False,
+        "css": False,
+    }
+
+    FUNCTION_PROMPT_TEMPLATE: ClassVar[
+        str
+    ] = """You have access to the following user defined functions. They can be accessed from the module called `$module_name` by their function names.
+
+For example, if there was a function called `foo` you could import it by writing `from $module_name import foo`
+
+$functions"""
+
+    def __init__(
+        self,
+        timeout: int = 60,
+        virtual_env_context: Optional[SimpleNamespace] = None,
+        work_dir: Union[Path, str] = Path(),
+        functions: list[Union[FunctionWithRequirements[Any, A], Callable[..., Any], FunctionWithRequirementsStr]] = [],
+        functions_module: str = "functions",
+        execution_policies: Optional[dict[str, bool]] = None,
+    ):
+        """(Experimental) A code executor class that executes or saves LLM generated code a local command line
+        environment.
+
+        **This will execute or save LLM generated code on the local machine.**
+
+        Each code block is saved as a file in the working directory. Depending on the execution policy,
+        the code may be executed in a separate process.
+        The code blocks are executed or save in the order they are received.
+        Command line code is sanitized against a list of dangerous commands to prevent self-destructive commands from being executed,
+        which could potentially affect the user's environment. Supported languages include Python, shell scripts (bash, shell, sh),
+        PowerShell (pwsh, powershell, ps1), HTML, CSS, and JavaScript.
+        Execution policies determine whether each language's code blocks are executed or saved only.
+
+        ## Execution with a Python virtual environment
+        A python virtual env can be used to execute code and install dependencies. This has the added benefit of not polluting the
+        base environment with unwanted modules.
+        ```python
+        from autogen.code_utils import create_virtual_env
+        from autogen.coding import LocalCommandLineCodeExecutor
+
+        venv_dir = ".venv"
+        venv_context = create_virtual_env(venv_dir)
+
+        executor = LocalCommandLineCodeExecutor(virtual_env_context=venv_context)
+        ```
+
+        Args:
+            timeout (int): The timeout for code execution, default is 60 seconds.
+            virtual_env_context (Optional[SimpleNamespace]): The virtual environment context to use.
+            work_dir (Union[Path, str]): The working directory for code execution, defaults to the current directory.
+            functions (List[Union[FunctionWithRequirements[Any, A], Callable[..., Any], FunctionWithRequirementsStr]]): A list of callable functions available to the executor.
+            functions_module (str): The module name under which functions are accessible.
+            execution_policies (Optional[Dict[str, bool]]): A dictionary mapping languages to execution policies (True for execution, False for saving only). Defaults to class-wide DEFAULT_EXECUTION_POLICY.
+        """
+        if timeout < 1:
+            raise ValueError("Timeout must be greater than or equal to 1.")
+
+        if isinstance(work_dir, str):
+            work_dir = Path(work_dir)
+
+        if not functions_module.isidentifier():
+            raise ValueError("Module name must be a valid Python identifier")
+
+        self._functions_module = functions_module
+
+        work_dir.mkdir(exist_ok=True)
+
+        self._timeout = timeout
+        self._work_dir: Path = work_dir
+        self._virtual_env_context: Optional[SimpleNamespace] = virtual_env_context
+
+        self._functions = functions
+        # Setup could take some time so we intentionally wait for the first code block to do it.
+        if len(functions) > 0:
+            self._setup_functions_complete = False
+        else:
+            self._setup_functions_complete = True
+
+        self.execution_policies = self.DEFAULT_EXECUTION_POLICY.copy()
+        if execution_policies is not None:
+            self.execution_policies.update(execution_policies)
+
+    def format_functions_for_prompt(self, prompt_template: str = FUNCTION_PROMPT_TEMPLATE) -> str:
+        """(Experimental) Format the functions for a prompt.
+
+        The template includes two variables:
+        - `$module_name`: The module name.
+        - `$functions`: The functions formatted as stubs with two newlines between each function.
+
+        Args:
+            prompt_template (str): The prompt template. Default is the class default.
+
+        Returns:
+            str: The formatted prompt.
+        """
+        template = Template(prompt_template)
+        return template.substitute(
+            module_name=self._functions_module,
+            functions="\n\n".join([to_stub(func) for func in self._functions]),
+        )
+
+    @property
+    def functions_module(self) -> str:
+        """(Experimental) The module name for the functions."""
+        return self._functions_module
+
+    @property
+    def functions(
+        self,
+    ) -> list[Union[FunctionWithRequirements[Any, A], Callable[..., Any], FunctionWithRequirementsStr]]:
+        """(Experimental) The functions that are available to the code executor."""
+        return self._functions
+
+    @property
+    def timeout(self) -> int:
+        """(Experimental) The timeout for code execution."""
+        return self._timeout
+
+    @property
+    def work_dir(self) -> Path:
+        """(Experimental) The working directory for the code execution."""
+        return self._work_dir
+
+    @property
+    def code_extractor(self) -> CodeExtractor:
+        """(Experimental) Export a code extractor that can be used by an agent."""
+        return MarkdownCodeExtractor()
+
+    @staticmethod
+    def sanitize_command(lang: str, code: str) -> None:
+        """Sanitize the code block to prevent dangerous commands.
+        This approach acknowledges that while Docker or similar
+        containerization/sandboxing technologies provide a robust layer of security,
+        not all users may have Docker installed or may choose not to use it.
+        Therefore, having a baseline level of protection helps mitigate risks for users who,
+        either out of choice or necessity, run code outside of a sandboxed environment.
+        """
+        dangerous_patterns = [
+            (r"\brm\s+-rf\b", "Use of 'rm -rf' command is not allowed."),
+            (r"\bmv\b.*?\s+/dev/null", "Moving files to /dev/null is not allowed."),
+            (r"\bdd\b", "Use of 'dd' command is not allowed."),
+            (r">\s*/dev/sd[a-z][1-9]?", "Overwriting disk blocks directly is not allowed."),
+            (r":\(\)\{\s*:\|\:&\s*\};:", "Fork bombs are not allowed."),
+        ]
+        if lang in ["bash", "shell", "sh"]:
+            for pattern, message in dangerous_patterns:
+                if re.search(pattern, code):
+                    raise ValueError(f"Potentially dangerous command detected: {message}")
+
+    def _setup_functions(self) -> None:
+        func_file_content = _build_python_functions_file(self._functions)
+        func_file = self._work_dir / f"{self._functions_module}.py"
+        func_file.write_text(func_file_content)
+
+        # Collect requirements
+        lists_of_packages = [x.python_packages for x in self._functions if isinstance(x, FunctionWithRequirements)]
+        flattened_packages = [item for sublist in lists_of_packages for item in sublist]
+        required_packages = list(set(flattened_packages))
+        if len(required_packages) > 0:
+            logging.info("Ensuring packages are installed in executor.")
+            py_executable = self._virtual_env_context.env_exe if self._virtual_env_context else sys.executable
+            cmd = [py_executable, "-m", "pip", "install"] + required_packages
+            try:
+                result = subprocess.run(
+                    cmd,
+                    cwd=self._work_dir,
+                    capture_output=True,
+                    text=True,
+                    timeout=float(self._timeout),
+                    encoding="utf-8",
+                )
+            except subprocess.TimeoutExpired as e:
+                raise ValueError("Pip install timed out") from e
+            if result.returncode != 0:
+                raise ValueError(f"Pip install failed. {result.stdout}, {result.stderr}")
+        # Attempt to load the function file to check for syntax errors, imports etc.
+        exec_result = self._execute_code_dont_check_setup([CodeBlock(code=func_file_content, language="python")])
+        if exec_result.exit_code != 0:
+            raise ValueError(f"Functions failed to load: {exec_result.output}")
+        self._setup_functions_complete = True
+
+    def execute_code_blocks(self, code_blocks: list[CodeBlock]) -> CommandLineCodeResult:
+        """(Experimental) Execute the code blocks and return the result.
+
+        Args:
+            code_blocks (List[CodeBlock]): The code blocks to execute.
+
+        Returns:
+            CommandLineCodeResult: The result of the code execution.
+        """
+        if not self._setup_functions_complete:
+            self._setup_functions()
+        return self._execute_code_dont_check_setup(code_blocks)
+
+    def _execute_code_dont_check_setup(self, code_blocks: list[CodeBlock]) -> CommandLineCodeResult:
+        logs_all = ""
+        file_names = []
+        for code_block in code_blocks:
+            lang, code = code_block.language, code_block.code
+            lang = lang.lower()
+
+            LocalCommandLineCodeExecutor.sanitize_command(lang, code)
+            code = silence_pip(code, lang)
+
+            if lang in PYTHON_VARIANTS:
+                lang = "python"
+
+            if WIN32 and lang in ["sh", "shell"]:
+                lang = "ps1"
+
+            if lang not in self.SUPPORTED_LANGUAGES:
+                # In case the language is not supported, we return an error message.
+                exitcode = 1
+                logs_all += "\n" + f"unknown language {lang}"
+                break
+
+            execute_code = self.execution_policies.get(lang, False)
+            try:
+                # Check if there is a filename comment
+                filename = _get_file_name_from_content(code, self._work_dir)
+            except ValueError:
+                return CommandLineCodeResult(exit_code=1, output="Filename is not in the workspace")
+
+            if filename is None:
+                # create a file with an automatically generated name
+                code_hash = md5(code.encode()).hexdigest()
+                filename = f"tmp_code_{code_hash}.{'py' if lang.startswith('python') else lang}"
+            written_file = (self._work_dir / filename).resolve()
+            with written_file.open("w", encoding="utf-8") as f:
+                f.write(code)
+            file_names.append(written_file)
+
+            if not execute_code:
+                # Just return a message that the file is saved.
+                logs_all += f"Code saved to {written_file!s}\n"
+                exitcode = 0
+                continue
+
+            program = _cmd(lang)
+            cmd = [program, str(written_file.absolute())]
+            env = os.environ.copy()
+
+            if self._virtual_env_context:
+                virtual_env_abs_path = os.path.abspath(self._virtual_env_context.bin_path)
+                path_with_virtualenv = rf"{virtual_env_abs_path}{os.pathsep}{env['PATH']}"
+                env["PATH"] = path_with_virtualenv
+                if WIN32:
+                    activation_script = os.path.join(virtual_env_abs_path, "activate.bat")
+                    cmd = [activation_script, "&&", *cmd]
+
+            try:
+                result = subprocess.run(
+                    cmd,
+                    cwd=self._work_dir,
+                    capture_output=True,
+                    text=True,
+                    timeout=float(self._timeout),
+                    env=env,
+                    encoding="utf-8",
+                )
+            except subprocess.TimeoutExpired:
+                logs_all += "\n" + TIMEOUT_MSG
+                # Same exit code as the timeout command on linux.
+                exitcode = 124
+                break
+
+            logs_all += result.stderr
+            logs_all += result.stdout
+            exitcode = result.returncode
+
+            if exitcode != 0:
+                break
+
+        code_file = str(file_names[0]) if len(file_names) > 0 else None
+        return CommandLineCodeResult(exit_code=exitcode, output=logs_all, code_file=code_file)
+
+    def restart(self) -> None:
+        """(Experimental) Restart the code executor."""
+        warnings.warn("Restarting local command line code executor is not supported. No action is taken.")
+
+
+# From stack overflow: https://stackoverflow.com/a/52087847/2214524
+class _DeprecatedClassMeta(type):
+    def __new__(cls, name, bases, classdict, *args, **kwargs):  # type: ignore[no-untyped-def]
+        alias = classdict.get("_DeprecatedClassMeta__alias")
+
+        if alias is not None:
+
+            def new(cls, *args, **kwargs):  # type: ignore[no-untyped-def]
+                alias = cls._DeprecatedClassMeta__alias
+
+                if alias is not None:
+                    warnings.warn(
+                        f"{cls.__name__} has been renamed to {alias.__name__}, the alias will be removed in the future",
+                        DeprecationWarning,
+                        stacklevel=2,
+                    )
+
+                return alias(*args, **kwargs)
+
+            classdict["__new__"] = new
+            classdict["_DeprecatedClassMeta__alias"] = alias
+
+        fixed_bases = []
+
+        for b in bases:
+            alias = getattr(b, "_DeprecatedClassMeta__alias", None)
+
+            if alias is not None:
+                warnings.warn(
+                    f"{b.__name__} has been renamed to {alias.__name__}, the alias will be removed in the future",
+                    DeprecationWarning,
+                    stacklevel=2,
+                )
+
+            # Avoid duplicate base classes.
+            b = alias or b
+            if b not in fixed_bases:
+                fixed_bases.append(b)
+
+        fixed_bases = tuple(fixed_bases)  # type: ignore[assignment]
+
+        return super().__new__(cls, name, fixed_bases, classdict, *args, **kwargs)  # type: ignore[call-overload]
+
+    def __instancecheck__(cls, instance):  # type: ignore[no-untyped-def]
+        return any(cls.__subclasscheck__(c) for c in {type(instance), instance.__class__})  # type: ignore[no-untyped-call]
+
+    def __subclasscheck__(cls, subclass):  # type: ignore[no-untyped-def]
+        if subclass is cls:
+            return True
+        else:
+            return issubclass(subclass, cls._DeprecatedClassMeta__alias)  # type: ignore[attr-defined]
+
+
+class LocalCommandlineCodeExecutor(metaclass=_DeprecatedClassMeta):
+    """LocalCommandlineCodeExecutor renamed to LocalCommandLineCodeExecutor"""
+
+    _DeprecatedClassMeta__alias = LocalCommandLineCodeExecutor
+
+
+class CommandlineCodeResult(metaclass=_DeprecatedClassMeta):
+    """CommandlineCodeResult renamed to CommandLineCodeResult"""
+
+    _DeprecatedClassMeta__alias = CommandLineCodeResult

mm_agents/coact/autogen/coding/markdown_code_extractor.py

@@ -0,0 +1,45 @@
+# Copyright (c) 2023 - 2025, AG2ai, Inc., AG2ai open-source projects maintainers and core contributors
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Portions derived from https://github.com/microsoft/autogen are under the MIT License.
+# SPDX-License-Identifier: MIT
+import re
+from typing import Union
+
+from ..code_utils import CODE_BLOCK_PATTERN, UNKNOWN, content_str, infer_lang
+from ..doc_utils import export_module
+from ..types import UserMessageImageContentPart, UserMessageTextContentPart
+from .base import CodeBlock, CodeExtractor
+
+__all__ = ("MarkdownCodeExtractor",)
+
+
+@export_module("autogen.coding")
+class MarkdownCodeExtractor(CodeExtractor):
+    """(Experimental) A class that extracts code blocks from a message using Markdown syntax."""
+
+    def extract_code_blocks(
+        self, message: Union[str, list[Union[UserMessageTextContentPart, UserMessageImageContentPart]], None]
+    ) -> list[CodeBlock]:
+        """(Experimental) Extract code blocks from a message. If no code blocks are found,
+        return an empty list.
+
+        Args:
+            message (str): The message to extract code blocks from.
+
+        Returns:
+            List[CodeBlock]: The extracted code blocks or an empty list.
+        """
+        text = content_str(message)
+        match = re.findall(CODE_BLOCK_PATTERN, text, flags=re.DOTALL)
+        if not match:
+            return []
+        code_blocks = []
+        for lang, code in match:
+            if lang == "":
+                lang = infer_lang(code)
+            if lang == UNKNOWN:
+                lang = ""
+            code_blocks.append(CodeBlock(code=code, language=lang))
+        return code_blocks

mm_agents/coact/autogen/coding/utils.py

@@ -0,0 +1,56 @@
+# Copyright (c) 2023 - 2025, AG2ai, Inc., AG2ai open-source projects maintainers and core contributors
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Portions derived from https://github.com/ag2ai/ag2 are under the MIT License.
+# SPDX-License-Identifier: MIT
+# Will return the filename relative to the workspace path
+import re
+from pathlib import Path
+from typing import Optional
+
+filename_patterns = [
+    re.compile(r"^<!-- (filename:)?(.+?) -->", re.DOTALL),
+    re.compile(r"^/\* (filename:)?(.+?) \*/", re.DOTALL),
+    re.compile(r"^// (filename:)?(.+?)$", re.DOTALL),
+    re.compile(r"^# (filename:)?(.+?)$", re.DOTALL),
+]
+
+
+# Raises ValueError if the file is not in the workspace
+def _get_file_name_from_content(code: str, workspace_path: Path) -> Optional[str]:
+    first_line = code.split("\n")[0].strip()
+    # TODO - support other languages
+    for pattern in filename_patterns:
+        matches = pattern.match(first_line)
+        if matches is not None:
+            filename = matches.group(2).strip()
+
+            # Handle relative paths in the filename
+            path = Path(filename)
+            if not path.is_absolute():
+                path = workspace_path / path
+            path = path.resolve()
+            # Throws an error if the file is not in the workspace
+            relative = path.relative_to(workspace_path.resolve())
+            return str(relative)
+    return None
+
+
+def silence_pip(code: str, lang: str) -> str:
+    """Apply -qqq flag to pip install commands."""
+    if lang == "python":
+        regex = r"^! ?pip install"
+    elif lang in ["bash", "shell", "sh", "pwsh", "powershell", "ps1"]:
+        regex = r"^pip install"
+    else:
+        return code
+
+    # Find lines that start with pip install and make sure "-qqq" flag is added.
+    lines = code.split("\n")
+    for i, line in enumerate(lines):
+        # use regex to find lines that start with pip install.
+        match = re.search(regex, line)
+        if match is not None and "-qqq" not in line:
+            lines[i] = line.replace(match.group(0), match.group(0) + " -qqq")
+    return "\n".join(lines)