libs/core/langchain_core/language_models/chat_models.py PYTHON 2,795 lines View on github.com → Search inside
File is large — showing lines 1–2,000 of 2,795.
1"""Chat models for conversational AI."""23from __future__ import annotations45import asyncio6import builtins  # noqa: TC003  # runtime-evaluated; subclass `dict()` shadows the builtin7import contextlib8import json9from abc import ABC, abstractmethod10from collections.abc import AsyncIterator, Callable, Iterator, Sequence11from functools import cached_property12from operator import itemgetter13from typing import TYPE_CHECKING, Any, Literal, Self, cast, overload1415from langchain_protocol.protocol import MessageFinishData16from pydantic import BaseModel, ConfigDict, Field, model_validator17from typing_extensions import override1819from langchain_core._api import beta, deprecated, suppress_langchain_deprecation_warning20from langchain_core.caches import BaseCache21from langchain_core.callbacks import (22    AsyncCallbackManager,23    AsyncCallbackManagerForLLMRun,24    CallbackManager,25    CallbackManagerForLLMRun,26    Callbacks,27)28from langchain_core.globals import get_llm_cache29from langchain_core.language_models._compat_bridge import (30    achunks_to_events,31    amessage_to_events,32    chunks_to_events,33    message_to_events,34)35from langchain_core.language_models._utils import (36    _filter_invocation_params_for_tracing,37    _normalize_messages,38    _update_message_content_to_blocks,39)40from langchain_core.language_models.base import (41    BaseLanguageModel,42    LangSmithParams,43    LanguageModelInput,44)45from langchain_core.language_models.chat_model_stream import (46    AsyncChatModelStream,47    ChatModelStream,48)49from langchain_core.language_models.model_profile import (50    ModelProfile,51    _warn_unknown_profile_keys,52)53from langchain_core.load import dumpd, dumps54from langchain_core.messages import (55    AIMessage,56    AIMessageChunk,57    AnyMessage,58    BaseMessage,59    convert_to_messages,60    is_data_content_block,61    message_chunk_to_message,62)63from langchain_core.messages import content as types64from langchain_core.messages.block_translators.openai import (65    convert_to_openai_image_block,66)67from langchain_core.output_parsers.openai_tools import (68    JsonOutputKeyToolsParser,69    JsonOutputToolsParser,70    PydanticToolsParser,71)72from langchain_core.outputs import (73    ChatGeneration,74    ChatGenerationChunk,75    ChatResult,76    Generation,77    LLMResult,78    RunInfo,79)80from langchain_core.outputs.chat_generation import merge_chat_generation_chunks81from langchain_core.prompt_values import ChatPromptValue, PromptValue, StringPromptValue82from langchain_core.rate_limiters import BaseRateLimiter83from langchain_core.runnables import RunnableBinding, RunnableMap, RunnablePassthrough84from langchain_core.runnables.config import ensure_config, run_in_executor85from langchain_core.tracers._streaming import (86    _StreamingCallbackHandler,87    _V2StreamingCallbackHandler,88)89from langchain_core.utils._gateway import (90    GATEWAY_METADATA_RESPONSE_KEY,91    _parse_gateway_metadata,92)93from langchain_core.utils._signature import _signature_parameters94from langchain_core.utils.function_calling import (95    convert_to_json_schema,96    convert_to_openai_tool,97)98from langchain_core.utils.pydantic import is_basemodel_subclass99from langchain_core.utils.utils import LC_ID_PREFIX, from_env100101if TYPE_CHECKING:102    import uuid103    from collections.abc import Awaitable104105    from langchain_protocol.protocol import MessagesData106107    from langchain_core.runnables import Runnable, RunnableConfig108    from langchain_core.runnables.schema import StreamEvent109    from langchain_core.tools import BaseTool110111112def _generate_response_from_error(error: BaseException) -> list[ChatGeneration]:113    if hasattr(error, "response"):114        response = error.response115        metadata: dict[str, Any] = {}116        generation_info: dict[str, Any] = {}117        if hasattr(response, "json"):118            try:119                metadata["body"] = response.json()120            except Exception:121                try:122                    metadata["body"] = getattr(response, "text", None)123                except Exception:124                    metadata["body"] = None125        if hasattr(response, "headers"):126            try:127                headers = response.headers128                metadata["headers"] = dict(headers)129                gateway_metadata = _parse_gateway_metadata(headers)130                if gateway_metadata is not None:131                    generation_info[GATEWAY_METADATA_RESPONSE_KEY] = gateway_metadata132            except Exception:133                metadata["headers"] = None134        if hasattr(response, "status_code"):135            metadata["status_code"] = response.status_code136        if hasattr(error, "request_id"):137            metadata["request_id"] = error.request_id138        generations = [139            ChatGeneration(140                message=AIMessage(content="", response_metadata=metadata),141                generation_info=generation_info or None,142            )143        ]144    else:145        generations = []146147    return generations148149150def _v3_generation(151    message: AIMessage, stream: ChatModelStream | AsyncChatModelStream152) -> ChatGeneration:153    """Build the final generation for a v3 stream.154155    Attaches tracing-only metadata (e.g. gateway metadata) collected while156    bridging chunks, which is kept off the public message.157    """158    return ChatGeneration(159        message=message,160        generation_info=stream._generation_info or None,  # noqa: SLF001161    )162163164def _v3_error_response(165    error: BaseException, stream: ChatModelStream | AsyncChatModelStream166) -> LLMResult:167    """Build the `on_llm_error` response for a v3 stream.168169    Mirrors v1 `stream()` so gateway metadata reaches the trace on failure:170    parsed from the error's HTTP response, or collected from chunks received171    before the failure.172    """173    generations = _generate_response_from_error(error)174    generation_info = stream._generation_info  # noqa: SLF001175    if generation_info:176        if generations:177            generations[0].generation_info = {178                **generation_info,179                **(generations[0].generation_info or {}),180            }181        else:182            generations = [183                ChatGeneration(184                    message=AIMessage(content=""),185                    generation_info=dict(generation_info),186                )187            ]188    return LLMResult(generations=[generations] if generations else [])189190191def _format_for_tracing(messages: list[BaseMessage]) -> list[BaseMessage]:192    """Format messages for tracing in `on_chat_model_start`.193194    - Update image content blocks to OpenAI Chat Completions format (backward195    compatibility).196    - Add `type` key to content blocks that have a single key.197198    Args:199        messages: List of messages to format.200201    Returns:202        List of messages formatted for tracing.203204    """205    messages_to_trace = []206    for message in messages:207        message_to_trace = message208        if isinstance(message.content, list):209            for idx, block in enumerate(message.content):210                if isinstance(block, dict):211                    # Update image content blocks to OpenAI # Chat Completions format.212                    if (213                        block.get("type") == "image"214                        and is_data_content_block(block)215                        and not ("file_id" in block or block.get("source_type") == "id")216                    ):217                        if message_to_trace is message:218                            # Shallow copy219                            message_to_trace = message.model_copy()220                            message_to_trace.content = list(message_to_trace.content)221222                        message_to_trace.content[idx] = (  # type: ignore[index]  # mypy confused by .model_copy223                            convert_to_openai_image_block(block)224                        )225                    elif (226                        block.get("type") == "file"227                        and is_data_content_block(block)  # v0 (image/audio/file) or v1228                        and "base64" in block229                        # Backward compat: convert v1 base64 blocks to v0230                    ):231                        if message_to_trace is message:232                            # Shallow copy233                            message_to_trace = message.model_copy()234                            message_to_trace.content = list(message_to_trace.content)235236                        message_to_trace.content[idx] = {  # type: ignore[index]237                            **{k: v for k, v in block.items() if k != "base64"},238                            "data": block["base64"],239                            "source_type": "base64",240                        }241                    elif len(block) == 1 and "type" not in block:242                        # Tracing assumes all content blocks have a "type" key. Here243                        # we add this key if it is missing, and there's an obvious244                        # choice for the type (e.g., a single key in the block).245                        if message_to_trace is message:246                            # Shallow copy247                            message_to_trace = message.model_copy()248                            message_to_trace.content = list(message_to_trace.content)249                        key = next(iter(block))250                        message_to_trace.content[idx] = {  # type: ignore[index]251                            "type": key,252                            key: block[key],253                        }254        messages_to_trace.append(message_to_trace)255256    return messages_to_trace257258259def generate_from_stream(stream: Iterator[ChatGenerationChunk]) -> ChatResult:260    """Generate from a stream.261262    Args:263        stream: Iterator of `ChatGenerationChunk`.264265    Raises:266        ValueError: If no generations are found in the stream.267268    Returns:269        Chat result.270271    """272    generation = next(stream, None)273    if generation:274        generation += list(stream)275    if generation is None:276        msg = "No generations found in stream."277        raise ValueError(msg)278    return ChatResult(279        generations=[280            ChatGeneration(281                message=message_chunk_to_message(generation.message),282                generation_info=generation.generation_info,283            )284        ]285    )286287288async def agenerate_from_stream(289    stream: AsyncIterator[ChatGenerationChunk],290) -> ChatResult:291    """Async generate from a stream.292293    Args:294        stream: AsyncIterator of `ChatGenerationChunk`.295296    Returns:297        Chat result.298299    """300    chunks = [chunk async for chunk in stream]301    return await run_in_executor(None, generate_from_stream, iter(chunks))302303304def _format_ls_structured_output(305    ls_structured_output_format: dict[str, Any] | None,306) -> dict[str, Any]:307    if ls_structured_output_format:308        try:309            ls_structured_output_format_dict = {310                "ls_structured_output_format": {311                    "kwargs": ls_structured_output_format.get("kwargs", {}),312                    "schema": convert_to_json_schema(313                        ls_structured_output_format["schema"]314                    ),315                }316            }317        except ValueError:318            ls_structured_output_format_dict = {}319    else:320        ls_structured_output_format_dict = {}321322    return ls_structured_output_format_dict323324325class BaseChatModel(BaseLanguageModel[AIMessage], ABC):326    r"""Base class for chat models.327328    Key imperative methods:329        Methods that actually call the underlying model.330331        This table provides a brief overview of the main imperative methods. Please see the base `Runnable` reference for full documentation.332333        | Method                 | Input                                                        | Output                                                     | Description                                                                      |334        | ---------------------- | ------------------------------------------------------------ | ---------------------------------------------------------- | -------------------------------------------------------------------------------- |335        | `invoke`               | `str` \| `list[dict | tuple | BaseMessage]` \| `PromptValue` | `BaseMessage`                                              | A single chat model call.                                                        |336        | `ainvoke`              | `'''`                                                        | `BaseMessage`                                              | Defaults to running `invoke` in an async executor.                               |337        | `stream`               | `'''`                                                        | `Iterator[BaseMessageChunk]`                               | Defaults to yielding output of `invoke`.                                         |338        | `astream`              | `'''`                                                        | `AsyncIterator[BaseMessageChunk]`                          | Defaults to yielding output of `ainvoke`.                                        |339        | `astream_events`       | `'''`                                                        | `AsyncIterator[StreamEvent]`                               | Event types: `on_chat_model_start`, `on_chat_model_stream`, `on_chat_model_end`. |340        | `batch`                | `list[''']`                                                  | `list[BaseMessage]`                                        | Defaults to running `invoke` in concurrent threads.                              |341        | `abatch`               | `list[''']`                                                  | `list[BaseMessage]`                                        | Defaults to running `ainvoke` in concurrent threads.                             |342        | `batch_as_completed`   | `list[''']`                                                  | `Iterator[tuple[int, Union[BaseMessage, Exception]]]`      | Defaults to running `invoke` in concurrent threads.                              |343        | `abatch_as_completed`  | `list[''']`                                                  | `AsyncIterator[tuple[int, Union[BaseMessage, Exception]]]` | Defaults to running `ainvoke` in concurrent threads.                             |344345    Key declarative methods:346        Methods for creating another `Runnable` using the chat model.347348        This table provides a brief overview of the main declarative methods. Please see the reference for each method for full documentation.349350        | Method                       | Description                                                                                |351        | ---------------------------- | ------------------------------------------------------------------------------------------ |352        | `bind_tools`                 | Create chat model that can call tools.                                                     |353        | `with_structured_output`     | Create wrapper that structures model output using schema.                                  |354        | `with_retry`                 | Create wrapper that retries model calls on failure.                                        |355        | `with_fallbacks`             | Create wrapper that falls back to other models on failure.                                 |356        | `configurable_fields`        | Specify init args of the model that can be configured at runtime via the `RunnableConfig`. |357        | `configurable_alternatives`  | Specify alternative models which can be swapped in at runtime via the `RunnableConfig`.    |358359    Creating custom chat model:360        Custom chat model implementations should inherit from this class.361        Please reference the table below for information about which362        methods and properties are required or optional for implementations.363364        | Method/Property                  | Description                                                        | Required          |365        | -------------------------------- | ------------------------------------------------------------------ | ----------------- |366        | `_generate`                      | Use to generate a chat result from a prompt                        | Required          |367        | `_llm_type` (property)           | Used to uniquely identify the type of the model. Used for logging. | Required          |368        | `_identifying_params` (property) | Represent model parameterization for tracing purposes.             | Optional          |369        | `_stream`                        | Use to implement streaming                                         | Optional          |370        | `_agenerate`                     | Use to implement a native async method                             | Optional          |371        | `_astream`                       | Use to implement async version of `_stream`                        | Optional          |372373    """  # noqa: E501374375    rate_limiter: BaseRateLimiter | None = Field(default=None, exclude=True)376    "An optional rate limiter to use for limiting the number of requests."377378    disable_streaming: bool | Literal["tool_calling"] = False379    """Whether to disable streaming for this model.380381    If streaming is bypassed, then `stream`/`astream`/`astream_events` will382    defer to `invoke`/`ainvoke`.383384    - If `True`, will always bypass streaming case.385    - If `'tool_calling'`, will bypass streaming case only when the model is called386        with a `tools` keyword argument. In other words, LangChain will automatically387        switch to non-streaming behavior (`invoke`) only when the tools argument is388        provided. This offers the best of both worlds.389    - If `False` (Default), will always use streaming case if available.390391    The main reason for this flag is that code might be written using `stream` and392    a user may want to swap out a given model for another model whose implementation393    does not properly support streaming.394    """395396    output_version: str | None = Field(397        default_factory=from_env("LC_OUTPUT_VERSION", default=None)398    )399    """Version of `AIMessage` output format to store in message content.400401    `AIMessage.content_blocks` will lazily parse the contents of `content` into a402    standard format. This flag can be used to additionally store the standard format403    in message content, e.g., for serialization purposes.404405    Supported values:406407    - `'v0'`: provider-specific format in content (can lazily-parse with408        `content_blocks`)409    - `'v1'`: standardized format in content (consistent with `content_blocks`)410411    Partner packages (e.g.,412    [`langchain-openai`](https://pypi.org/project/langchain-openai)) can also use this413    field to roll out new content formats in a backward-compatible way.414415    !!! version-added "Added in `langchain-core` 1.0.0"416417    """418419    profile: ModelProfile | None = Field(default=None, exclude=True)420    """Profile detailing model capabilities.421422    !!! warning "Beta feature"423424        This is a beta feature. The format of model profiles is subject to change.425426    If not specified, automatically loaded from the provider package on initialization427    if data is available.428429    Example profile data includes context window sizes, supported modalities, or support430    for tool calling, structured output, and other features.431432    !!! version-added "Added in `langchain-core` 1.1.0"433    """434435    model_config = ConfigDict(436        arbitrary_types_allowed=True,437    )438439    def _resolve_model_profile(self) -> ModelProfile | None:440        """Return the default model profile, or `None` if unavailable.441442        Override this in subclasses instead of `_set_model_profile`. The base443        validator calls it automatically and handles assignment. This avoids444        coupling partner code to Pydantic validator mechanics.445446        Each partner needs its own override because things can vary per-partner,447        such as the attribute that identifies the model (e.g., `model`,448        `model_name`, `model_id`, `deployment_name`) and the partner-local449        `_get_default_model_profile` function that reads from each partner's own450        profile data.451        """452        # TODO: consider adding a `_model_identifier` property on BaseChatModel453        # to standardize how partners identify their model, which could allow a454        # default implementation here that calls a shared455        # profile-loading mechanism.456        return None457458    @model_validator(mode="after")459    def _set_model_profile(self) -> Self:460        """Populate `profile` from `_resolve_model_profile` if not provided.461462        Partners should override `_resolve_model_profile` rather than this463        validator. Overriding this with a new `@model_validator` replaces the464        base validator (Pydantic v2 behavior), bypassing the standard resolution465        path. A plain method override does not prevent the base validator from466        running.467        """468        if self.profile is None:469            # Suppress errors from partner overrides (e.g., missing profile470            # files, broken imports) so model construction never fails over an471            # optional field.472            with contextlib.suppress(Exception):473                self.profile = self._resolve_model_profile()474        return self475476    # NOTE: _check_profile_keys must be defined AFTER _set_model_profile.477    # Pydantic v2 runs mode="after" validators in definition order.478    @model_validator(mode="after")479    def _check_profile_keys(self) -> Self:480        """Warn on unrecognized profile keys."""481        # isinstance guard: ModelProfile is a TypedDict (always a dict), but482        # protects against unexpected types from partner overrides.483        if self.profile and isinstance(self.profile, dict):484            _warn_unknown_profile_keys(self.profile)485        return self486487    @cached_property488    def _serialized(self) -> builtins.dict[str, Any]:489        # self is always a Serializable object in this case, thus the result is490        # guaranteed to be a dict since dumpd uses the default callback, which uses491        # obj.to_json which always returns TypedDict subclasses492        return cast("builtins.dict[str, Any]", dumpd(self))493494    # --- Runnable methods ---495496    @property497    @override498    def OutputType(self) -> Any:499        """Get the output type for this `Runnable`."""500        return AnyMessage501502    def _convert_input(self, model_input: LanguageModelInput) -> PromptValue:503        if isinstance(model_input, PromptValue):504            return model_input505        if isinstance(model_input, str):506            return StringPromptValue(text=model_input)507        if isinstance(model_input, Sequence):508            return ChatPromptValue(messages=convert_to_messages(model_input))509        msg = (  # type: ignore[unreachable]510            f"Invalid input type {type(model_input)}. "511            "Must be a PromptValue, str, or list of BaseMessages."512        )513        raise ValueError(msg)514515    @override516    def invoke(517        self,518        input: LanguageModelInput,519        config: RunnableConfig | None = None,520        *,521        stop: list[str] | None = None,522        **kwargs: Any,523    ) -> AIMessage:524        config = ensure_config(config)525        return cast(526            "AIMessage",527            cast(528                "ChatGeneration",529                self.generate_prompt(530                    [self._convert_input(input)],531                    stop=stop,532                    callbacks=config.get("callbacks"),533                    tags=config.get("tags"),534                    metadata=config.get("metadata"),535                    run_name=config.get("run_name"),536                    run_id=config.pop("run_id", None),537                    **kwargs,538                ).generations[0][0],539            ).message,540        )541542    @override543    async def ainvoke(544        self,545        input: LanguageModelInput,546        config: RunnableConfig | None = None,547        *,548        stop: list[str] | None = None,549        **kwargs: Any,550    ) -> AIMessage:551        config = ensure_config(config)552        llm_result = await self.agenerate_prompt(553            [self._convert_input(input)],554            stop=stop,555            callbacks=config.get("callbacks"),556            tags=config.get("tags"),557            metadata=config.get("metadata"),558            run_name=config.get("run_name"),559            run_id=config.pop("run_id", None),560            **kwargs,561        )562        return cast(563            "AIMessage", cast("ChatGeneration", llm_result.generations[0][0]).message564        )565566    def _streaming_disabled(self, **kwargs: Any) -> bool:567        """Return whether streaming is hard-disabled for this call.568569        Shared opt-outs honored by both `_should_stream` and570        `_should_use_protocol_streaming` — these override any affirmative trigger571        (attached handler, `stream=True`, etc.):572573        - `self.disable_streaming is True`574        - `self.disable_streaming == "tool_calling"` with `tools` passed575        - `stream=<falsy>` in call kwargs576        - `self.streaming is False` on the instance577        """578        if self.disable_streaming is True:579            return True580        # We assume tools are passed in via "tools" kwarg in all models.581        if self.disable_streaming == "tool_calling" and kwargs.get("tools"):582            return True583        if "stream" in kwargs and not kwargs["stream"]:584            return True585        return (586            "streaming" in self.model_fields_set587            and getattr(self, "streaming", None) is False588        )589590    def _should_stream(591        self,592        *,593        async_api: bool,594        run_manager: CallbackManagerForLLMRun595        | AsyncCallbackManagerForLLMRun596        | None = None,597        **kwargs: Any,598    ) -> bool:599        """Determine if a given model call should hit the streaming API."""600        sync_not_implemented = type(self)._stream == BaseChatModel._stream  # noqa: SLF001601        async_not_implemented = type(self)._astream == BaseChatModel._astream  # noqa: SLF001602603        # Check if streaming is implemented.604        if (not async_api) and sync_not_implemented:605            return False606        # Note, since async falls back to sync we check both here.607        if async_api and async_not_implemented and sync_not_implemented:608            return False609610        if self._streaming_disabled(**kwargs):611            return False612613        # Affirmative: explicit `stream=<truthy>` kwarg.614        if kwargs.get("stream"):615            return True616617        # Affirmative: instance-level `streaming=True` attribute.618        if (619            "streaming" in self.model_fields_set620            and getattr(self, "streaming", None) is True621        ):622            return True623624        # Affirmative: a v1 streaming callback handler is attached.625        handlers = run_manager.handlers if run_manager else []626        return any(isinstance(h, _StreamingCallbackHandler) for h in handlers)627628    def _should_use_protocol_streaming(629        self,630        *,631        async_api: bool,632        run_manager: CallbackManagerForLLMRun633        | AsyncCallbackManagerForLLMRun634        | None = None,635        **kwargs: Any,636    ) -> bool:637        """Determine whether an invoke should route through the v2 event path.638639        Runs alongside `_should_stream` inside `_generate_with_cache` /640        `_agenerate_with_cache` — after the run manager is open — and641        wins over the v1 streaming branch when a handler has declared642        itself a `_V2StreamingCallbackHandler`. Parallel to643        `_should_stream` rather than a delegation — v1 and v2 have644        disjoint affirmative triggers.645646        Args:647            async_api: Whether the caller is on the async path.648            run_manager: The active LLM run manager.649            **kwargs: Call kwargs; inspected for `disable_streaming`650                semantics and an explicit `stream=False` override.651652        Returns:653            `True` if any attached handler inherits654            `_V2StreamingCallbackHandler` and the model can drive the v2655            event generator (natively or via the `_stream` compat656            bridge).657        """658        # Opt-in: only route through v2 when a v2 handler is attached.659        handlers = run_manager.handlers if run_manager else []660        if not any(isinstance(h, _V2StreamingCallbackHandler) for h in handlers):661            return False662663        # Need a source of v2 events on the requested flavor. A native664        # `_(a)stream_chat_model_events` hook bypasses the bridge;665        # otherwise the bridge wraps `_stream` / `_astream`. Async can666        # fall back to sync.667        #668        # `cls._stream is not BaseChatModel._stream` is an identity669        # check for "subclass overrode `_stream`" — same pattern as670        # `_should_stream`.671        cls = type(self)672        has_native_sync = getattr(cls, "_stream_chat_model_events", None) is not None673        has_native_async = getattr(cls, "_astream_chat_model_events", None) is not None674        overrides_sync = cls._stream is not BaseChatModel._stream675        overrides_async = cls._astream is not BaseChatModel._astream676        has_sync_source = has_native_sync or overrides_sync677        has_async_source = has_native_async or overrides_async678        has_source = (679            (has_sync_source or has_async_source) if async_api else has_sync_source680        )681        if not has_source:682            return False683684        return not self._streaming_disabled(**kwargs)685686    def _iter_v2_events(687        self,688        messages: list[BaseMessage],689        *,690        run_manager: CallbackManagerForLLMRun,691        stream: ChatModelStream,692        stop: list[str] | None = None,693        **kwargs: Any,694    ) -> Iterator[MessagesData]:695        """Drive the v2 event generator with per-event dispatch.696697        Shared between the `stream_events(version="v3")` pump and the698        invoke-time v2 branch in `_generate_with_cache`. Picks the native699        `_stream_chat_model_events` hook when the subclass provides one,700        else bridges `_stream` chunks via `chunks_to_events`. Each event701        is dispatched into `stream` and fired as `on_stream_event` on702        the run manager. Run-lifecycle callbacks703        (`on_chat_model_start` / `on_llm_end` / `on_llm_error`) and704        rate-limiter acquisition are the caller's responsibility.705706        Args:707            messages: Normalized input messages.708            run_manager: Active LLM run manager; receives709                `on_stream_event` per event.710            stream: Accumulator owned by the caller; receives each711                event via `stream.dispatch`.712            stop: Optional stop sequences.713            **kwargs: Forwarded to the event producer.714715        Yields:716            Each protocol event produced by the model.717        """718        native = cast(719            "Callable[..., Iterator[MessagesData]] | None",720            getattr(self, "_stream_chat_model_events", None),721        )722        if native is not None:723            event_iter: Iterator[MessagesData] = native(724                messages, stop=stop, run_manager=run_manager, **kwargs725            )726        else:727            event_iter = chunks_to_events(728                self._stream(messages, stop=stop, run_manager=run_manager, **kwargs),729                message_id=stream.message_id,730                generation_info=stream._generation_info,  # noqa: SLF001731            )732        for event in event_iter:733            stream.dispatch(event)734            run_manager.on_stream_event(event)735            yield event736737    async def _aiter_v2_events(738        self,739        messages: list[BaseMessage],740        *,741        run_manager: AsyncCallbackManagerForLLMRun,742        stream: AsyncChatModelStream,743        stop: list[str] | None = None,744        **kwargs: Any,745    ) -> AsyncIterator[MessagesData]:746        """Async counterpart to `_iter_v2_events`.747748        See `_iter_v2_events` for the shared contract.749        """750        native = cast(751            "Callable[..., AsyncIterator[MessagesData]] | None",752            getattr(self, "_astream_chat_model_events", None),753        )754        if native is not None:755            event_iter: AsyncIterator[MessagesData] = native(756                messages, stop=stop, run_manager=run_manager, **kwargs757            )758        else:759            event_iter = achunks_to_events(760                self._astream(messages, stop=stop, run_manager=run_manager, **kwargs),761                message_id=stream.message_id,762                generation_info=stream._generation_info,  # noqa: SLF001763            )764        async for event in event_iter:765            stream.dispatch(event)766            await run_manager.on_stream_event(event)767            yield event768769    @override770    def stream(771        self,772        input: LanguageModelInput,773        config: RunnableConfig | None = None,774        *,775        stop: list[str] | None = None,776        **kwargs: Any,777    ) -> Iterator[AIMessageChunk]:778        if not self._should_stream(async_api=False, **{**kwargs, "stream": True}):779            # Model doesn't implement streaming, so use default implementation780            yield cast(781                "AIMessageChunk",782                self.invoke(input, config=config, stop=stop, **kwargs),783            )784        else:785            config = ensure_config(config)786            messages = self._convert_input(input).to_messages()787            ls_structured_output_format = kwargs.pop(788                "ls_structured_output_format", None789            ) or kwargs.pop("structured_output_format", None)790            ls_structured_output_format_dict = _format_ls_structured_output(791                ls_structured_output_format792            )793794            params = self._get_invocation_params(stop=stop, **kwargs)795            options = {796                "stop": stop,797                **{key: params[key] for key in kwargs if key in params},798                **ls_structured_output_format_dict,799            }800            inheritable_metadata = {801                **(config.get("metadata") or {}),802                **self._get_ls_params_with_defaults(stop=stop, **kwargs),803            }804            callback_manager = CallbackManager.configure(805                config.get("callbacks"),806                self.callbacks,807                self.verbose,808                config.get("tags"),809                self.tags,810                inheritable_metadata,811                self.metadata,812                langsmith_inheritable_metadata=_filter_invocation_params_for_tracing(813                    params814                ),815            )816            (run_manager,) = callback_manager.on_chat_model_start(817                self._serialized,818                [_format_for_tracing(messages)],819                invocation_params=params,820                options=options,821                name=config.get("run_name"),822                run_id=config.pop("run_id", None),823                batch_size=1,824            )825826            chunks: list[ChatGenerationChunk] = []827828            if self.rate_limiter:829                self.rate_limiter.acquire(blocking=True)830831            try:832                input_messages = _normalize_messages(messages)833                run_id = "-".join((LC_ID_PREFIX, str(run_manager.run_id)))834                yielded = False835                index = -1836                index_type = ""837                for chunk in self._stream(input_messages, stop=stop, **kwargs):838                    if chunk.message.id is None:839                        chunk.message.id = run_id840                    chunk.message.response_metadata = _gen_info_and_msg_metadata(chunk)841                    if self.output_version == "v1":842                        # Overwrite .content with .content_blocks843                        chunk.message = _update_message_content_to_blocks(844                            chunk.message, "v1"845                        )846                        for block in cast(847                            "list[types.ContentBlock]", chunk.message.content848                        ):849                            if block["type"] != index_type:850                                index_type = block["type"]851                                index += 1852                            if "index" not in block:853                                block["index"] = index854                    run_manager.on_llm_new_token(chunk.message.content, chunk=chunk)855                    chunks.append(chunk)856                    yield cast("AIMessageChunk", chunk.message)857                    yielded = True858859                # Yield a final empty chunk with chunk_position="last" if not yet860                # yielded861                if (862                    yielded863                    and isinstance(chunk.message, AIMessageChunk)864                    and not chunk.message.chunk_position865                ):866                    empty_content: str | list[str | dict[str, Any]] = (867                        "" if isinstance(chunk.message.content, str) else []868                    )869                    msg_chunk = AIMessageChunk(870                        content=empty_content, chunk_position="last", id=run_id871                    )872                    run_manager.on_llm_new_token(873                        "", chunk=ChatGenerationChunk(message=msg_chunk)874                    )875                    yield msg_chunk876            except BaseException as e:877                generations_with_error_metadata = _generate_response_from_error(e)878                chat_generation_chunk = merge_chat_generation_chunks(chunks)879                if chat_generation_chunk:880                    generations = [881                        [chat_generation_chunk],882                        generations_with_error_metadata,883                    ]884                else:885                    generations = [generations_with_error_metadata]886                run_manager.on_llm_error(887                    e,888                    response=LLMResult(generations=generations),889                )890                raise891892            generation = merge_chat_generation_chunks(chunks)893            if generation is None:894                err = ValueError("No generation chunks were returned")895                run_manager.on_llm_error(err, response=LLMResult(generations=[]))896                raise err897898            run_manager.on_llm_end(LLMResult(generations=[[generation]]))899900    @override901    async def astream(902        self,903        input: LanguageModelInput,904        config: RunnableConfig | None = None,905        *,906        stop: list[str] | None = None,907        **kwargs: Any,908    ) -> AsyncIterator[AIMessageChunk]:909        if not self._should_stream(async_api=True, **{**kwargs, "stream": True}):910            # No async or sync stream is implemented, so fall back to ainvoke911            yield cast(912                "AIMessageChunk",913                await self.ainvoke(input, config=config, stop=stop, **kwargs),914            )915            return916917        config = ensure_config(config)918        messages = self._convert_input(input).to_messages()919920        ls_structured_output_format = kwargs.pop(921            "ls_structured_output_format", None922        ) or kwargs.pop("structured_output_format", None)923        ls_structured_output_format_dict = _format_ls_structured_output(924            ls_structured_output_format925        )926927        params = self._get_invocation_params(stop=stop, **kwargs)928        options = {929            "stop": stop,930            **{key: params[key] for key in kwargs if key in params},931            **ls_structured_output_format_dict,932        }933        inheritable_metadata = {934            **(config.get("metadata") or {}),935            **self._get_ls_params_with_defaults(stop=stop, **kwargs),936        }937        callback_manager = AsyncCallbackManager.configure(938            config.get("callbacks"),939            self.callbacks,940            self.verbose,941            config.get("tags"),942            self.tags,943            inheritable_metadata,944            self.metadata,945            langsmith_inheritable_metadata=_filter_invocation_params_for_tracing(946                params947            ),948        )949        (run_manager,) = await callback_manager.on_chat_model_start(950            self._serialized,951            [_format_for_tracing(messages)],952            invocation_params=params,953            options=options,954            name=config.get("run_name"),955            run_id=config.pop("run_id", None),956            batch_size=1,957        )958959        if self.rate_limiter:960            await self.rate_limiter.aacquire(blocking=True)961962        chunks: list[ChatGenerationChunk] = []963964        try:965            input_messages = _normalize_messages(messages)966            run_id = "-".join((LC_ID_PREFIX, str(run_manager.run_id)))967            yielded = False968            index = -1969            index_type = ""970            async for chunk in self._astream(971                input_messages,972                stop=stop,973                **kwargs,974            ):975                if chunk.message.id is None:976                    chunk.message.id = run_id977                chunk.message.response_metadata = _gen_info_and_msg_metadata(chunk)978                if self.output_version == "v1":979                    # Overwrite .content with .content_blocks980                    chunk.message = _update_message_content_to_blocks(981                        chunk.message, "v1"982                    )983                    for block in cast(984                        "list[types.ContentBlock]", chunk.message.content985                    ):986                        if block["type"] != index_type:987                            index_type = block["type"]988                            index += 1989                        if "index" not in block:990                            block["index"] = index991                await run_manager.on_llm_new_token(chunk.message.content, chunk=chunk)992                chunks.append(chunk)993                yield cast("AIMessageChunk", chunk.message)994                yielded = True995996            # Yield a final empty chunk with chunk_position="last" if not yet yielded997            if (998                yielded999                and isinstance(chunk.message, AIMessageChunk)1000                and not chunk.message.chunk_position1001            ):1002                empty_content: str | list[str | dict[str, Any]] = (1003                    "" if isinstance(chunk.message.content, str) else []1004                )1005                msg_chunk = AIMessageChunk(1006                    content=empty_content, chunk_position="last", id=run_id1007                )1008                await run_manager.on_llm_new_token(1009                    "", chunk=ChatGenerationChunk(message=msg_chunk)1010                )1011                yield msg_chunk1012        except BaseException as e:1013            generations_with_error_metadata = _generate_response_from_error(e)1014            chat_generation_chunk = merge_chat_generation_chunks(chunks)1015            if chat_generation_chunk:1016                generations = [[chat_generation_chunk], generations_with_error_metadata]1017            else:1018                generations = [generations_with_error_metadata]1019            await run_manager.on_llm_error(1020                e,1021                response=LLMResult(generations=generations),1022            )1023            raise10241025        generation = merge_chat_generation_chunks(chunks)1026        if not generation:1027            err = ValueError("No generation chunks were returned")1028            await run_manager.on_llm_error(err, response=LLMResult(generations=[]))1029            raise err10301031        await run_manager.on_llm_end(1032            LLMResult(generations=[[generation]]),1033        )10341035    # --- stream_events v3 ---10361037    @beta()1038    def _chat_model_stream_v3(1039        self,1040        input: LanguageModelInput,1041        config: RunnableConfig | None = None,1042        *,1043        stop: list[str] | None = None,1044        **kwargs: Any,1045    ) -> ChatModelStream:1046        """Internal v3 sync streaming implementation.10471048        Public entry point: `stream_events(version='v3')`.1049        """1050        config = ensure_config(config)1051        messages = self._convert_input(input).to_messages()1052        input_messages = _normalize_messages(messages)10531054        # Strip tracing-only kwargs before forwarding to `_stream` — matches1055        # `stream()` / `astream()`. Provider clients reject unknown kwargs,1056        # so `.with_structured_output().stream_events(version="v3", ...)`1057        # and any other binding that carries `ls_structured_output_format`1058        # / `structured_output_format` would raise without this pop.1059        ls_structured_output_format = kwargs.pop(1060            "ls_structured_output_format", None1061        ) or kwargs.pop("structured_output_format", None)1062        ls_structured_output_format_dict = _format_ls_structured_output(1063            ls_structured_output_format1064        )10651066        params = self._get_invocation_params(stop=stop, **kwargs)1067        options = {1068            "stop": stop,1069            **{key: params[key] for key in kwargs if key in params},1070            **ls_structured_output_format_dict,1071        }1072        inheritable_metadata = {1073            **(config.get("metadata") or {}),1074            **self._get_ls_params_with_defaults(stop=stop, **kwargs),1075        }1076        callback_manager = CallbackManager.configure(1077            config.get("callbacks"),1078            self.callbacks,1079            self.verbose,1080            config.get("tags"),1081            self.tags,1082            inheritable_metadata,1083            self.metadata,1084            langsmith_inheritable_metadata=_filter_invocation_params_for_tracing(1085                params1086            ),1087        )1088        stream = ChatModelStream()1089        run_manager: CallbackManagerForLLMRun | None = None1090        event_iter_ref: Iterator[MessagesData] | None = None1091        rate_limiter_acquired = self.rate_limiter is None1092        run_name = config.get("run_name")1093        run_id = config.pop("run_id", None)10941095        def ensure_started() -> None:1096            nonlocal event_iter_ref, run_manager1097            if event_iter_ref is not None:1098                return10991100            (run_manager,) = callback_manager.on_chat_model_start(1101                self._serialized,1102                [_format_for_tracing(messages)],1103                invocation_params=params,1104                options=options,1105                name=run_name,1106                run_id=run_id,1107                batch_size=1,1108            )1109            stream.set_message_id("-".join((LC_ID_PREFIX, str(run_manager.run_id))))1110            event_iter_ref = iter(1111                self._iter_v2_events(1112                    input_messages,1113                    run_manager=run_manager,1114                    stream=stream,1115                    stop=stop,1116                    **kwargs,1117                )1118            )11191120        def pump_one() -> bool:1121            nonlocal rate_limiter_acquired1122            ensure_started()1123            if not rate_limiter_acquired:1124                assert self.rate_limiter is not None  # noqa: S1011125                self.rate_limiter.acquire(blocking=True)1126                rate_limiter_acquired = True1127            assert event_iter_ref is not None  # noqa: S1011128            assert run_manager is not None  # noqa: S1011129            try:1130                next(event_iter_ref)1131            except StopIteration:1132                if not stream.done:1133                    if stream.has_events:1134                        # Native event producers may omit the terminal1135                        # `message-finish`. Close the lifecycle here so1136                        # `on_llm_end` still observes the assembled1137                        # message. A truly empty stream remains an error1138                        # for parity with `stream()`.1139                        stream.dispatch(MessageFinishData(event="message-finish"))1140                    else:1141                        err = ValueError("No generation chunks were returned")1142                        stream.fail(err)1143                        run_manager.on_llm_error(1144                            err,1145                            response=_v3_error_response(err, stream),1146                        )1147                        return False1148                if stream.done and stream.output_message is not None:1149                    run_manager.on_llm_end(1150                        LLMResult(1151                            generations=[1152                                [_v3_generation(stream.output_message, stream)],1153                            ],1154                        ),1155                    )1156                return False1157            except BaseException as exc:1158                stream.fail(exc)1159                run_manager.on_llm_error(1160                    exc,1161                    response=_v3_error_response(exc, stream),1162                )1163                return False1164            if stream.done and stream.output_message is not None:1165                run_manager.on_llm_end(1166                    LLMResult(1167                        generations=[1168                            [_v3_generation(stream.output_message, stream)],1169                        ],1170                    ),1171                )1172            return True11731174        stream.set_start(ensure_started)1175        stream.bind_pump(pump_one)1176        return stream11771178    @beta()1179    async def _achat_model_stream_v3(1180        self,1181        input: LanguageModelInput,1182        config: RunnableConfig | None = None,1183        *,1184        stop: list[str] | None = None,1185        **kwargs: Any,1186    ) -> AsyncChatModelStream:1187        """Internal v3 async streaming implementation.11881189        Public entry point: `astream_events(version='v3')`.1190        """1191        config = ensure_config(config)1192        messages = self._convert_input(input).to_messages()1193        input_messages = _normalize_messages(messages)11941195        # Strip tracing-only kwargs before forwarding — see the sync v31196        # implementation for the full rationale.1197        ls_structured_output_format = kwargs.pop(1198            "ls_structured_output_format", None1199        ) or kwargs.pop("structured_output_format", None)1200        ls_structured_output_format_dict = _format_ls_structured_output(1201            ls_structured_output_format1202        )12031204        params = self._get_invocation_params(stop=stop, **kwargs)1205        options = {1206            "stop": stop,1207            **{key: params[key] for key in kwargs if key in params},1208            **ls_structured_output_format_dict,1209        }1210        inheritable_metadata = {1211            **(config.get("metadata") or {}),1212            **self._get_ls_params_with_defaults(stop=stop, **kwargs),1213        }1214        callback_manager = AsyncCallbackManager.configure(1215            config.get("callbacks"),1216            self.callbacks,1217            self.verbose,1218            config.get("tags"),1219            self.tags,1220            inheritable_metadata,1221            self.metadata,1222            langsmith_inheritable_metadata=_filter_invocation_params_for_tracing(1223                params1224            ),1225        )1226        stream = AsyncChatModelStream()1227        run_manager: AsyncCallbackManagerForLLMRun | None = None1228        run_name = config.get("run_name")1229        run_id = config.pop("run_id", None)1230        start_lock = asyncio.Lock()12311232        async def _produce() -> None:1233            assert run_manager is not None  # noqa: S1011234            try:1235                if self.rate_limiter:1236                    await self.rate_limiter.aacquire(blocking=True)12371238                async for _event in self._aiter_v2_events(1239                    input_messages,1240                    run_manager=run_manager,1241                    stream=stream,1242                    stop=stop,1243                    **kwargs,1244                ):1245                    pass1246                if not stream.done:1247                    if stream.has_events:1248                        # Native event producers may omit the terminal1249                        # `message-finish`. Close the lifecycle here so1250                        # `on_llm_end` sees the finalized message. A1251                        # truly empty stream remains an error for parity1252                        # with `astream()`.1253                        stream.dispatch(MessageFinishData(event="message-finish"))1254                    else:1255                        err = ValueError("No generation chunks were returned")1256                        stream.fail(err)1257                        await run_manager.on_llm_error(1258                            err,1259                            response=_v3_error_response(err, stream),1260                        )1261                        return1262                if stream.done and stream.output_message is not None:1263                    await run_manager.on_llm_end(1264                        LLMResult(1265                            generations=[1266                                [_v3_generation(stream.output_message, stream)],1267                            ],1268                        ),1269                    )1270            except asyncio.CancelledError as exc:1271                stream.fail(exc)1272                # Close the callback lifecycle so tracing observes a1273                # matching end event for the earlier `on_chat_model_start`.1274                # `on_llm_error` is `@shielded`, so the callback runs to1275                # completion in the background even though the `await`1276                # here re-raises our cancellation.1277                with contextlib.suppress(Exception):1278                    await run_manager.on_llm_error(1279                        exc,1280                        response=_v3_error_response(exc, stream),1281                    )1282                raise1283            except BaseException as exc:1284                stream.fail(exc)1285                await run_manager.on_llm_error(1286                    exc,1287                    response=_v3_error_response(exc, stream),1288                )12891290        async def ensure_started() -> None:1291            nonlocal run_manager1292            if stream._producer_task is not None:  # noqa: SLF0011293                return12941295            async with start_lock:1296                if stream._producer_task is not None:  # noqa: SLF0011297                    return  # type: ignore[unreachable]12981299                (run_manager,) = await callback_manager.on_chat_model_start(1300                    self._serialized,1301                    [_format_for_tracing(messages)],1302                    invocation_params=params,1303                    options=options,1304                    name=run_name,1305                    run_id=run_id,1306                    batch_size=1,1307                )1308                stream.set_message_id("-".join((LC_ID_PREFIX, str(run_manager.run_id))))1309                stream._producer_task = asyncio.get_running_loop().create_task(  # noqa: SLF0011310                    _produce()1311                )13121313        async def _on_aclose_fail(exc: BaseException) -> None:1314            assert run_manager is not None  # noqa: S1011315            # Invoked by `stream.aclose()` only when the producer was1316            # cancelled before `_produce` ran — so `on_llm_error` from1317            # the CancelledError handler never fired. Shielded by the1318            # callback manager; runs to completion even if our caller1319            # is being cancelled.1320            await run_manager.on_llm_error(1321                exc,1322                response=_v3_error_response(exc, stream),1323            )13241325        stream.set_start(ensure_started)1326        stream._on_aclose_fail = _on_aclose_fail  # noqa: SLF0011327        return stream13281329    @overload  # type: ignore[override]1330    def stream_events(1331        self,1332        input: LanguageModelInput,1333        config: RunnableConfig | None = None,1334        *,1335        version: Literal["v1", "v2"] = "v2",1336        **kwargs: Any,1337    ) -> Iterator[StreamEvent]: ...13381339    @overload1340    def stream_events(1341        self,1342        input: LanguageModelInput,1343        config: RunnableConfig | None = None,1344        *,1345        version: Literal["v3"],1346        stop: list[str] | None = None,1347        **kwargs: Any,1348    ) -> ChatModelStream: ...13491350    def stream_events(1351        self,1352        input: LanguageModelInput,1353        config: RunnableConfig | None = None,1354        *,1355        version: Literal["v1", "v2", "v3"] = "v2",1356        stop: list[str] | None = None,1357        **kwargs: Any,1358    ) -> Iterator[StreamEvent] | ChatModelStream:1359        """Stream events from this chat model.13601361        For `version="v1"` / `"v2"`, yields `StreamEvent` dicts (see1362        `Runnable.stream_events`). For `version="v3"`, returns a1363        `ChatModelStream` exposing typed projections (`.text`,1364        `.reasoning`, `.tool_calls`, `.output`).13651366        !!! warning "Beta"13671368            `version="v3"` is in beta. The protocol shape, return type,1369            and surface area may change in future releases. Calling it1370            emits a `LangChainBetaWarning` at runtime.13711372        !!! note "v3 always produces v1-shaped content"13731374            `ChatModelStream.output.content` is always a list of v11375            content blocks (text / reasoning / tool_call / image / …),1376            regardless of the model's `output_version` attribute. The1377            setting only affects the legacy `stream()` / `astream()` /1378            `invoke()` paths. If you're mixing1379            `stream_events(version="v3")` with those paths in the same1380            pipeline and need a consistent output shape across them,1381            set `output_version="v1"` on the model.13821383        Args:1384            input: The model input.1385            config: Optional runnable config.1386            version: Streaming-event schema version. `"v3"` selects the1387                content-block-centric streaming protocol.1388            stop: Optional stop sequences. Only used for `version="v3"`;1389                ignored otherwise.1390            **kwargs: Additional keyword arguments. For `version="v3"`,1391                forwarded to the model.13921393        Returns:1394            For `version="v3"`, a `ChatModelStream` with typed1395            projections. Otherwise an `Iterator[StreamEvent]`.1396        """1397        if version == "v3":1398            return self._chat_model_stream_v3(input, config, stop=stop, **kwargs)1399        return super().stream_events(1400            input, config, version=version, stop=stop, **kwargs1401        )14021403    @overload1404    def astream_events(1405        self,1406        input: LanguageModelInput,1407        config: RunnableConfig | None = None,1408        *,1409        version: Literal["v1", "v2"] = "v2",1410        **kwargs: Any,1411    ) -> AsyncIterator[StreamEvent]: ...14121413    @overload1414    def astream_events(1415        self,1416        input: LanguageModelInput,1417        config: RunnableConfig | None = None,1418        *,1419        version: Literal["v3"],1420        stop: list[str] | None = None,1421        **kwargs: Any,1422    ) -> Awaitable[AsyncChatModelStream]: ...14231424    def astream_events(1425        self,1426        input: LanguageModelInput,1427        config: RunnableConfig | None = None,1428        *,1429        version: Literal["v1", "v2", "v3"] = "v2",1430        stop: list[str] | None = None,1431        **kwargs: Any,1432    ) -> AsyncIterator[StreamEvent] | Awaitable[AsyncChatModelStream]:1433        """Async variant of `stream_events`. See `stream_events` for full docs."""1434        if version == "v3":1435            return self._achat_model_stream_v3(input, config, stop=stop, **kwargs)1436        # v1/v2: forward to Runnable.astream_events (async generator).1437        return super().astream_events(1438            input, config, version=version, stop=stop, **kwargs1439        )14401441    # --- Custom methods ---14421443    def _combine_llm_outputs(1444        self, _llm_outputs: list[builtins.dict[str, Any] | None], /1445    ) -> builtins.dict[str, Any]:1446        return {}14471448    def _convert_cached_generations(1449        self, cache_val: list[Generation]1450    ) -> list[ChatGeneration]:1451        """Convert cached Generation objects to ChatGeneration objects.14521453        Handle case where cache contains Generation objects instead of1454        ChatGeneration objects. This can happen due to serialization/deserialization1455        issues or legacy cache data (see #22389).14561457        Args:1458            cache_val: List of cached generation objects.14591460        Returns:1461            List of ChatGeneration objects.14621463        """1464        converted_generations = []1465        for gen in cache_val:1466            if isinstance(gen, Generation) and not isinstance(gen, ChatGeneration):1467                # Convert Generation to ChatGeneration by creating AIMessage1468                # from the text content1469                chat_gen = ChatGeneration(1470                    message=AIMessage(content=gen.text),1471                    generation_info=gen.generation_info,1472                )1473                converted_generations.append(chat_gen)1474            else:1475                # Already a ChatGeneration or other expected type1476                if hasattr(gen, "message") and isinstance(gen.message, AIMessage):1477                    # We zero out cost on cache hits1478                    gen.message = gen.message.model_copy(1479                        update={1480                            "usage_metadata": {1481                                **(gen.message.usage_metadata or {}),1482                                "total_cost": 0,1483                            }1484                        }1485                    )1486                converted_generations.append(gen)1487        return converted_generations14881489    def _replay_v2_events_for_cache_hit(1490        self,1491        generations: list[ChatGeneration],1492        *,1493        run_manager: CallbackManagerForLLMRun | None,1494        **kwargs: Any,1495    ) -> None:1496        """Replay cached messages as v2 events when a v2 handler is attached.14971498        A warm cache must produce the same `on_stream_event` stream as a1499        cold call so LangGraph-style consumers do not observe behavior1500        that depends on cache state. Gated by1501        `_should_use_protocol_streaming` so a `disable_streaming` config1502        that suppresses v2 on cold calls also suppresses it here.1503        """1504        if run_manager is None or not self._should_use_protocol_streaming(1505            async_api=False, run_manager=run_manager, **kwargs1506        ):1507            return1508        message_id = f"{LC_ID_PREFIX}-{run_manager.run_id}"1509        for gen in generations:1510            msg = getattr(gen, "message", None)1511            if not isinstance(msg, AIMessage):1512                continue1513            for event in message_to_events(msg, message_id=message_id):1514                run_manager.on_stream_event(event)15151516    async def _areplay_v2_events_for_cache_hit(1517        self,1518        generations: list[ChatGeneration],1519        *,1520        run_manager: AsyncCallbackManagerForLLMRun | None,1521        **kwargs: Any,1522    ) -> None:1523        """Async counterpart to `_replay_v2_events_for_cache_hit`."""1524        if run_manager is None or not self._should_use_protocol_streaming(1525            async_api=True, run_manager=run_manager, **kwargs1526        ):1527            return1528        message_id = f"{LC_ID_PREFIX}-{run_manager.run_id}"1529        for gen in generations:1530            msg = getattr(gen, "message", None)1531            if not isinstance(msg, AIMessage):1532                continue1533            async for event in amessage_to_events(msg, message_id=message_id):1534                await run_manager.on_stream_event(event)15351536    def _get_invocation_params(1537        self,1538        stop: list[str] | None = None,1539        **kwargs: Any,1540    ) -> builtins.dict[str, Any]:1541        params = self._dict_for_compat()1542        params["stop"] = stop1543        return {**params, **kwargs}15441545    def _get_ls_params(1546        self,1547        stop: list[str] | None = None,1548        **kwargs: Any,1549    ) -> LangSmithParams:1550        """Get standard params for LangSmith tracing.15511552        Subclasses **should override** this method to populate `ls_provider`1553        and `ls_model_name` from provider-specific attributes (e.g. `self.model`,1554        `self.model_name`, `self.model_id`) and to honor per-call overrides1555        passed via `kwargs["model"]` so that runtime `bind`/`invoke` model1556        changes are reflected in traces.15571558        The implementation here is a best-effort fallback for subclasses that1559        do not override it. It is not part of a stable contract and the1560        derivation rules may change:15611562        - `ls_provider` is derived from the class name by stripping a leading1563            or trailing `"Chat"` and lowercasing the remainder. This produces1564            ugly values for multi-word providers (e.g. `ChatGoogleGenerativeAI`1565            would become `"googlegenerativeai"`).15661567            Override to set a stable, conventional value1568            such as `"google_genai"`.1569        - `ls_model_name` is resolved from `kwargs["model"]`, then1570            `self.model`, then `self.model_name`.15711572            Subclasses whose model attribute has a different name1573            (`model_id`, `deployment_name`, ...) must override.1574        """1575        # get default provider from class name1576        default_provider = self.__class__.__name__1577        if default_provider.startswith("Chat"):1578            default_provider = default_provider[4:].lower()1579        elif default_provider.endswith("Chat"):1580            default_provider = default_provider[:-4]1581        default_provider = default_provider.lower()15821583        ls_params = LangSmithParams(ls_provider=default_provider, ls_model_type="chat")1584        if stop:1585            ls_params["ls_stop"] = stop15861587        # model1588        if "model" in kwargs and isinstance(kwargs["model"], str):1589            ls_params["ls_model_name"] = kwargs["model"]1590        elif hasattr(self, "model") and isinstance(self.model, str):1591            ls_params["ls_model_name"] = self.model1592        elif hasattr(self, "model_name") and isinstance(self.model_name, str):1593            ls_params["ls_model_name"] = self.model_name15941595        # temperature1596        if "temperature" in kwargs and isinstance(kwargs["temperature"], (int, float)):1597            ls_params["ls_temperature"] = kwargs["temperature"]1598        elif hasattr(self, "temperature") and isinstance(1599            self.temperature, (int, float)1600        ):1601            ls_params["ls_temperature"] = self.temperature16021603        # max_tokens1604        if "max_tokens" in kwargs and isinstance(kwargs["max_tokens"], int):1605            ls_params["ls_max_tokens"] = kwargs["max_tokens"]1606        elif hasattr(self, "max_tokens") and isinstance(self.max_tokens, int):1607            ls_params["ls_max_tokens"] = self.max_tokens16081609        return ls_params16101611    def _get_ls_params_with_defaults(1612        self,1613        stop: list[str] | None = None,1614        **kwargs: Any,1615    ) -> LangSmithParams:1616        """Wrap _get_ls_params to always include ls_integration."""1617        ls_params = self._get_ls_params(stop=stop, **kwargs)1618        ls_params["ls_integration"] = "langchain_chat_model"1619        return ls_params16201621    def _get_llm_string(self, stop: list[str] | None = None, **kwargs: Any) -> str:1622        if self.is_lc_serializable():1623            params = {**kwargs, "stop": stop}1624            param_string = str(sorted(params.items()))1625            # This code is not super efficient as it goes back and forth between1626            # json and dict.1627            serialized_repr = self._serialized1628            _cleanup_llm_representation(serialized_repr, 1)1629            llm_string = json.dumps(serialized_repr, sort_keys=True)1630            return llm_string + "---" + param_string1631        params = self._get_invocation_params(stop=stop, **kwargs)1632        params = {**params, **kwargs}1633        return str(sorted(params.items()))16341635    def generate(1636        self,1637        messages: list[list[BaseMessage]],1638        stop: list[str] | None = None,1639        callbacks: Callbacks = None,1640        *,1641        tags: list[str] | None = None,1642        metadata: builtins.dict[str, Any] | None = None,1643        run_name: str | None = None,1644        run_id: uuid.UUID | None = None,1645        **kwargs: Any,1646    ) -> LLMResult:1647        """Pass a sequence of prompts to the model and return model generations.16481649        This method should make use of batched calls for models that expose a batched1650        API.16511652        Use this method when you want to:16531654        1. Take advantage of batched calls,1655        2. Need more output from the model than just the top generated value,1656        3. Are building chains that are agnostic to the underlying language model1657            type (e.g., pure text completion models vs chat models).16581659        Args:1660            messages: List of list of messages.1661            stop: Stop words to use when generating.16621663                Model output is cut off at the first occurrence of any of these1664                substrings.1665            callbacks: `Callbacks` to pass through.16661667                Used for executing additional functionality, such as logging or1668                streaming, throughout generation.1669            tags: The tags to apply.1670            metadata: The metadata to apply.1671            run_name: The name of the run.1672            run_id: The ID of the run.1673            **kwargs: Arbitrary additional keyword arguments.16741675                These are usually passed to the model provider API call.16761677        Returns:1678            An `LLMResult`, which contains a list of candidate `Generations` for each1679                input prompt and additional model provider-specific output.16801681        """1682        ls_structured_output_format = kwargs.pop(1683            "ls_structured_output_format", None1684        ) or kwargs.pop("structured_output_format", None)1685        ls_structured_output_format_dict = _format_ls_structured_output(1686            ls_structured_output_format1687        )16881689        params = self._get_invocation_params(stop=stop, **kwargs)1690        options = {"stop": stop, **ls_structured_output_format_dict}1691        inheritable_metadata = {1692            **(metadata or {}),1693            **self._get_ls_params_with_defaults(stop=stop, **kwargs),1694        }16951696        callback_manager = CallbackManager.configure(1697            callbacks,1698            self.callbacks,1699            self.verbose,1700            tags,1701            self.tags,1702            inheritable_metadata,1703            self.metadata,1704            langsmith_inheritable_metadata=_filter_invocation_params_for_tracing(1705                params1706            ),1707        )1708        messages_to_trace = [1709            _format_for_tracing(message_list) for message_list in messages1710        ]1711        run_managers = callback_manager.on_chat_model_start(1712            self._serialized,1713            messages_to_trace,1714            invocation_params=params,1715            options=options,1716            name=run_name,1717            run_id=run_id,1718            batch_size=len(messages),1719        )1720        results = []1721        input_messages = [1722            _normalize_messages(message_list) for message_list in messages1723        ]1724        for i, m in enumerate(input_messages):1725            try:1726                results.append(1727                    self._generate_with_cache(1728                        m,1729                        stop=stop,1730                        run_manager=run_managers[i] if run_managers else None,1731                        **kwargs,1732                    )1733                )1734            except BaseException as e:1735                if run_managers:1736                    generations_with_error_metadata = _generate_response_from_error(e)1737                    run_managers[i].on_llm_error(1738                        e,1739                        response=LLMResult(1740                            generations=[generations_with_error_metadata]1741                        ),1742                    )1743                raise1744        flattened_outputs = [1745            LLMResult(generations=[res.generations], llm_output=res.llm_output)1746            for res in results1747        ]1748        llm_output = self._combine_llm_outputs([res.llm_output for res in results])1749        generations = [res.generations for res in results]1750        output = LLMResult(generations=generations, llm_output=llm_output)1751        if run_managers:1752            run_infos = []1753            for manager, flattened_output in zip(1754                run_managers, flattened_outputs, strict=False1755            ):1756                manager.on_llm_end(flattened_output)1757                run_infos.append(RunInfo(run_id=manager.run_id))1758            output.run = run_infos1759        return output17601761    async def agenerate(1762        self,1763        messages: list[list[BaseMessage]],1764        stop: list[str] | None = None,1765        callbacks: Callbacks = None,1766        *,1767        tags: list[str] | None = None,1768        metadata: builtins.dict[str, Any] | None = None,1769        run_name: str | None = None,1770        run_id: uuid.UUID | None = None,1771        **kwargs: Any,1772    ) -> LLMResult:1773        """Asynchronously pass a sequence of prompts to a model and return generations.17741775        This method should make use of batched calls for models that expose a batched1776        API.17771778        Use this method when you want to:17791780        1. Take advantage of batched calls,1781        2. Need more output from the model than just the top generated value,1782        3. Are building chains that are agnostic to the underlying language model1783            type (e.g., pure text completion models vs chat models).17841785        Args:1786            messages: List of list of messages.1787            stop: Stop words to use when generating.17881789                Model output is cut off at the first occurrence of any of these1790                substrings.1791            callbacks: `Callbacks` to pass through.17921793                Used for executing additional functionality, such as logging or1794                streaming, throughout generation.1795            tags: The tags to apply.1796            metadata: The metadata to apply.1797            run_name: The name of the run.1798            run_id: The ID of the run.1799            **kwargs: Arbitrary additional keyword arguments.18001801                These are usually passed to the model provider API call.18021803        Returns:1804            An `LLMResult`, which contains a list of candidate `Generations` for each1805                input prompt and additional model provider-specific output.18061807        """1808        ls_structured_output_format = kwargs.pop(1809            "ls_structured_output_format", None1810        ) or kwargs.pop("structured_output_format", None)1811        ls_structured_output_format_dict = _format_ls_structured_output(1812            ls_structured_output_format1813        )18141815        params = self._get_invocation_params(stop=stop, **kwargs)1816        options = {"stop": stop, **ls_structured_output_format_dict}1817        inheritable_metadata = {1818            **(metadata or {}),1819            **self._get_ls_params_with_defaults(stop=stop, **kwargs),1820        }18211822        callback_manager = AsyncCallbackManager.configure(1823            callbacks,1824            self.callbacks,1825            self.verbose,1826            tags,1827            self.tags,1828            inheritable_metadata,1829            self.metadata,1830            langsmith_inheritable_metadata=_filter_invocation_params_for_tracing(1831                params1832            ),1833        )18341835        messages_to_trace = [1836            _format_for_tracing(message_list) for message_list in messages1837        ]1838        run_managers = await callback_manager.on_chat_model_start(1839            self._serialized,1840            messages_to_trace,1841            invocation_params=params,1842            options=options,1843            name=run_name,1844            batch_size=len(messages),1845            run_id=run_id,1846        )18471848        input_messages = [1849            _normalize_messages(message_list) for message_list in messages1850        ]1851        results = await asyncio.gather(1852            *[1853                self._agenerate_with_cache(1854                    m,1855                    stop=stop,1856                    run_manager=run_managers[i] if run_managers else None,1857                    **kwargs,1858                )1859                for i, m in enumerate(input_messages)1860            ],1861            return_exceptions=True,1862        )1863        exceptions = []1864        for i, res in enumerate(results):1865            if isinstance(res, BaseException):1866                if run_managers:1867                    generations_with_error_metadata = _generate_response_from_error(res)1868                    await run_managers[i].on_llm_error(1869                        res,1870                        response=LLMResult(1871                            generations=[generations_with_error_metadata]1872                        ),1873                    )1874                exceptions.append(res)1875        if exceptions:1876            if run_managers:1877                await asyncio.gather(1878                    *[1879                        run_manager.on_llm_end(1880                            LLMResult(1881                                generations=[res.generations],  # type: ignore[union-attr]1882                                llm_output=res.llm_output,  # type: ignore[union-attr]1883                            )1884                        )1885                        for run_manager, res in zip(run_managers, results, strict=False)1886                        if not isinstance(res, Exception)1887                    ]1888                )1889            raise exceptions[0]1890        flattened_outputs = [1891            LLMResult(generations=[res.generations], llm_output=res.llm_output)  # type: ignore[union-attr]1892            for res in results1893        ]1894        llm_output = self._combine_llm_outputs([res.llm_output for res in results])  # type: ignore[union-attr]1895        generations = [res.generations for res in results]  # type: ignore[union-attr]1896        output = LLMResult(generations=generations, llm_output=llm_output)1897        await asyncio.gather(1898            *[1899                run_manager.on_llm_end(flattened_output)1900                for run_manager, flattened_output in zip(1901                    run_managers, flattened_outputs, strict=False1902                )1903            ]1904        )1905        if run_managers:1906            output.run = [1907                RunInfo(run_id=run_manager.run_id) for run_manager in run_managers1908            ]1909        return output19101911    @override1912    def generate_prompt(1913        self,1914        prompts: list[PromptValue],1915        stop: list[str] | None = None,1916        callbacks: Callbacks = None,1917        **kwargs: Any,1918    ) -> LLMResult:1919        prompt_messages = [p.to_messages() for p in prompts]1920        return self.generate(prompt_messages, stop=stop, callbacks=callbacks, **kwargs)19211922    @override1923    async def agenerate_prompt(1924        self,1925        prompts: list[PromptValue],1926        stop: list[str] | None = None,1927        callbacks: Callbacks = None,1928        **kwargs: Any,1929    ) -> LLMResult:1930        prompt_messages = [p.to_messages() for p in prompts]1931        return await self.agenerate(1932            prompt_messages, stop=stop, callbacks=callbacks, **kwargs1933        )19341935    def _generate_with_cache(1936        self,1937        messages: list[BaseMessage],1938        stop: list[str] | None = None,1939        run_manager: CallbackManagerForLLMRun | None = None,1940        **kwargs: Any,1941    ) -> ChatResult:1942        llm_cache = self.cache if isinstance(self.cache, BaseCache) else get_llm_cache()1943        # We should check the cache unless it's explicitly set to False1944        # A None cache means we should use the default global cache1945        # if it's configured.1946        check_cache = self.cache is not False1947        if check_cache:1948            if llm_cache is not None:1949                llm_string = self._get_llm_string(stop=stop, **kwargs)1950                normalized_messages = [1951                    (1952                        msg.model_copy(update={"id": None})1953                        if getattr(msg, "id", None) is not None1954                        else msg1955                    )1956                    for msg in messages1957                ]1958                prompt = dumps(normalized_messages)1959                cache_val = llm_cache.lookup(prompt, llm_string)1960                if isinstance(cache_val, list):1961                    converted_generations = self._convert_cached_generations(cache_val)1962                    self._replay_v2_events_for_cache_hit(1963                        converted_generations,1964                        run_manager=run_manager,1965                        **kwargs,1966                    )1967                    return ChatResult(generations=converted_generations)1968            elif self.cache is None:1969                pass1970            else:1971                msg = "Asked to cache, but no cache found at `langchain.cache`."1972                raise ValueError(msg)19731974        # Apply the rate limiter after checking the cache, since1975        # we usually don't want to rate limit cache lookups, but1976        # we do want to rate limit API requests.1977        if self.rate_limiter:1978            self.rate_limiter.acquire(blocking=True)19791980        # v2 streaming: preferred over v1 when any attached handler opts in via1981        # `_V2StreamingCallbackHandler`. Drives the protocol event generator1982        # (native or `_stream` compat bridge) through the shared helper so1983        # `on_stream_event` fires per event, then returns a normal `ChatResult`1984        # so caching / `on_llm_end` stay on the existing generate path.1985        if self._should_use_protocol_streaming(1986            async_api=False,1987            run_manager=run_manager,1988            **kwargs,1989        ):1990            stream_accum = ChatModelStream(1991                message_id=(1992                    f"{LC_ID_PREFIX}-{run_manager.run_id}" if run_manager else None1993                )1994            )1995            assert run_manager is not None  # noqa: S1011996            for _event in self._iter_v2_events(1997                messages,1998                run_manager=run_manager,1999                stream=stream_accum,2000                stop=stop,

Findings

✓ No findings reported for this file.

Get this view in your editor

Same data, no extra tab — call code_get_file + code_get_findings over MCP from Claude/Cursor/Copilot.

Generated by searchcode.com · powered by code_get_file + code_get_findings