1"""Chat models for conversational AI."""23from __future__ import annotations45import asyncio6import builtins # noqa: TC003 # runtime-evaluated; subclass `dict()` shadows the builtin7import contextlib8import json9from abc import ABC, abstractmethod10from collections.abc import AsyncIterator, Callable, Iterator, Sequence11from functools import cached_property12from operator import itemgetter13from typing import TYPE_CHECKING, Any, Literal, Self, cast, overload1415from langchain_protocol.protocol import MessageFinishData16from pydantic import BaseModel, ConfigDict, Field, model_validator17from typing_extensions import override1819from langchain_core._api import beta, deprecated, suppress_langchain_deprecation_warning20from langchain_core.caches import BaseCache21from langchain_core.callbacks import (22 AsyncCallbackManager,23 AsyncCallbackManagerForLLMRun,24 CallbackManager,25 CallbackManagerForLLMRun,26 Callbacks,27)28from langchain_core.globals import get_llm_cache29from langchain_core.language_models._compat_bridge import (30 achunks_to_events,31 amessage_to_events,32 chunks_to_events,33 message_to_events,34)35from langchain_core.language_models._utils import (36 _filter_invocation_params_for_tracing,37 _normalize_messages,38 _update_message_content_to_blocks,39)40from langchain_core.language_models.base import (41 BaseLanguageModel,42 LangSmithParams,43 LanguageModelInput,44)45from langchain_core.language_models.chat_model_stream import (46 AsyncChatModelStream,47 ChatModelStream,48)49from langchain_core.language_models.model_profile import (50 ModelProfile,51 _warn_unknown_profile_keys,52)53from langchain_core.load import dumpd, dumps54from langchain_core.messages import (55 AIMessage,56 AIMessageChunk,57 AnyMessage,58 BaseMessage,59 convert_to_messages,60 is_data_content_block,61 message_chunk_to_message,62)63from langchain_core.messages import content as types64from langchain_core.messages.block_translators.openai import (65 convert_to_openai_image_block,66)67from langchain_core.output_parsers.openai_tools import (68 JsonOutputKeyToolsParser,69 JsonOutputToolsParser,70 PydanticToolsParser,71)72from langchain_core.outputs import (73 ChatGeneration,74 ChatGenerationChunk,75 ChatResult,76 Generation,77 LLMResult,78 RunInfo,79)80from langchain_core.outputs.chat_generation import merge_chat_generation_chunks81from langchain_core.prompt_values import ChatPromptValue, PromptValue, StringPromptValue82from langchain_core.rate_limiters import BaseRateLimiter83from langchain_core.runnables import RunnableBinding, RunnableMap, RunnablePassthrough84from langchain_core.runnables.config import ensure_config, run_in_executor85from langchain_core.tracers._streaming import (86 _StreamingCallbackHandler,87 _V2StreamingCallbackHandler,88)89from langchain_core.utils._gateway import (90 GATEWAY_METADATA_RESPONSE_KEY,91 _parse_gateway_metadata,92)93from langchain_core.utils._signature import _signature_parameters94from langchain_core.utils.function_calling import (95 convert_to_json_schema,96 convert_to_openai_tool,97)98from langchain_core.utils.pydantic import is_basemodel_subclass99from langchain_core.utils.utils import LC_ID_PREFIX, from_env100101if TYPE_CHECKING:102 import uuid103 from collections.abc import Awaitable104105 from langchain_protocol.protocol import MessagesData106107 from langchain_core.runnables import Runnable, RunnableConfig108 from langchain_core.runnables.schema import StreamEvent109 from langchain_core.tools import BaseTool110111112def _generate_response_from_error(error: BaseException) -> list[ChatGeneration]:113 if hasattr(error, "response"):114 response = error.response115 metadata: dict[str, Any] = {}116 generation_info: dict[str, Any] = {}117 if hasattr(response, "json"):118 try:119 metadata["body"] = response.json()120 except Exception:121 try:122 metadata["body"] = getattr(response, "text", None)123 except Exception:124 metadata["body"] = None125 if hasattr(response, "headers"):126 try:127 headers = response.headers128 metadata["headers"] = dict(headers)129 gateway_metadata = _parse_gateway_metadata(headers)130 if gateway_metadata is not None:131 generation_info[GATEWAY_METADATA_RESPONSE_KEY] = gateway_metadata132 except Exception:133 metadata["headers"] = None134 if hasattr(response, "status_code"):135 metadata["status_code"] = response.status_code136 if hasattr(error, "request_id"):137 metadata["request_id"] = error.request_id138 generations = [139 ChatGeneration(140 message=AIMessage(content="", response_metadata=metadata),141 generation_info=generation_info or None,142 )143 ]144 else:145 generations = []146147 return generations148149150def _v3_generation(151 message: AIMessage, stream: ChatModelStream | AsyncChatModelStream152) -> ChatGeneration:153 """Build the final generation for a v3 stream.154155 Attaches tracing-only metadata (e.g. gateway metadata) collected while156 bridging chunks, which is kept off the public message.157 """158 return ChatGeneration(159 message=message,160 generation_info=stream._generation_info or None, # noqa: SLF001161 )162163164def _v3_error_response(165 error: BaseException, stream: ChatModelStream | AsyncChatModelStream166) -> LLMResult:167 """Build the `on_llm_error` response for a v3 stream.168169 Mirrors v1 `stream()` so gateway metadata reaches the trace on failure:170 parsed from the error's HTTP response, or collected from chunks received171 before the failure.172 """173 generations = _generate_response_from_error(error)174 generation_info = stream._generation_info # noqa: SLF001175 if generation_info:176 if generations:177 generations[0].generation_info = {178 **generation_info,179 **(generations[0].generation_info or {}),180 }181 else:182 generations = [183 ChatGeneration(184 message=AIMessage(content=""),185 generation_info=dict(generation_info),186 )187 ]188 return LLMResult(generations=[generations] if generations else [])189190191def _format_for_tracing(messages: list[BaseMessage]) -> list[BaseMessage]:192 """Format messages for tracing in `on_chat_model_start`.193194 - Update image content blocks to OpenAI Chat Completions format (backward195 compatibility).196 - Add `type` key to content blocks that have a single key.197198 Args:199 messages: List of messages to format.200201 Returns:202 List of messages formatted for tracing.203204 """205 messages_to_trace = []206 for message in messages:207 message_to_trace = message208 if isinstance(message.content, list):209 for idx, block in enumerate(message.content):210 if isinstance(block, dict):211 # Update image content blocks to OpenAI # Chat Completions format.212 if (213 block.get("type") == "image"214 and is_data_content_block(block)215 and not ("file_id" in block or block.get("source_type") == "id")216 ):217 if message_to_trace is message:218 # Shallow copy219 message_to_trace = message.model_copy()220 message_to_trace.content = list(message_to_trace.content)221222 message_to_trace.content[idx] = ( # type: ignore[index] # mypy confused by .model_copy223 convert_to_openai_image_block(block)224 )225 elif (226 block.get("type") == "file"227 and is_data_content_block(block) # v0 (image/audio/file) or v1228 and "base64" in block229 # Backward compat: convert v1 base64 blocks to v0230 ):231 if message_to_trace is message:232 # Shallow copy233 message_to_trace = message.model_copy()234 message_to_trace.content = list(message_to_trace.content)235236 message_to_trace.content[idx] = { # type: ignore[index]237 **{k: v for k, v in block.items() if k != "base64"},238 "data": block["base64"],239 "source_type": "base64",240 }241 elif len(block) == 1 and "type" not in block:242 # Tracing assumes all content blocks have a "type" key. Here243 # we add this key if it is missing, and there's an obvious244 # choice for the type (e.g., a single key in the block).245 if message_to_trace is message:246 # Shallow copy247 message_to_trace = message.model_copy()248 message_to_trace.content = list(message_to_trace.content)249 key = next(iter(block))250 message_to_trace.content[idx] = { # type: ignore[index]251 "type": key,252 key: block[key],253 }254 messages_to_trace.append(message_to_trace)255256 return messages_to_trace257258259def generate_from_stream(stream: Iterator[ChatGenerationChunk]) -> ChatResult:260 """Generate from a stream.261262 Args:263 stream: Iterator of `ChatGenerationChunk`.264265 Raises:266 ValueError: If no generations are found in the stream.267268 Returns:269 Chat result.270271 """272 generation = next(stream, None)273 if generation:274 generation += list(stream)275 if generation is None:276 msg = "No generations found in stream."277 raise ValueError(msg)278 return ChatResult(279 generations=[280 ChatGeneration(281 message=message_chunk_to_message(generation.message),282 generation_info=generation.generation_info,283 )284 ]285 )286287288async def agenerate_from_stream(289 stream: AsyncIterator[ChatGenerationChunk],290) -> ChatResult:291 """Async generate from a stream.292293 Args:294 stream: AsyncIterator of `ChatGenerationChunk`.295296 Returns:297 Chat result.298299 """300 chunks = [chunk async for chunk in stream]301 return await run_in_executor(None, generate_from_stream, iter(chunks))302303304def _format_ls_structured_output(305 ls_structured_output_format: dict[str, Any] | None,306) -> dict[str, Any]:307 if ls_structured_output_format:308 try:309 ls_structured_output_format_dict = {310 "ls_structured_output_format": {311 "kwargs": ls_structured_output_format.get("kwargs", {}),312 "schema": convert_to_json_schema(313 ls_structured_output_format["schema"]314 ),315 }316 }317 except ValueError:318 ls_structured_output_format_dict = {}319 else:320 ls_structured_output_format_dict = {}321322 return ls_structured_output_format_dict323324325class BaseChatModel(BaseLanguageModel[AIMessage], ABC):326 r"""Base class for chat models.327328 Key imperative methods:329 Methods that actually call the underlying model.330331 This table provides a brief overview of the main imperative methods. Please see the base `Runnable` reference for full documentation.332333 | Method | Input | Output | Description |334 | ---------------------- | ------------------------------------------------------------ | ---------------------------------------------------------- | -------------------------------------------------------------------------------- |335 | `invoke` | `str` \| `list[dict | tuple | BaseMessage]` \| `PromptValue` | `BaseMessage` | A single chat model call. |336 | `ainvoke` | `'''` | `BaseMessage` | Defaults to running `invoke` in an async executor. |337 | `stream` | `'''` | `Iterator[BaseMessageChunk]` | Defaults to yielding output of `invoke`. |338 | `astream` | `'''` | `AsyncIterator[BaseMessageChunk]` | Defaults to yielding output of `ainvoke`. |339 | `astream_events` | `'''` | `AsyncIterator[StreamEvent]` | Event types: `on_chat_model_start`, `on_chat_model_stream`, `on_chat_model_end`. |340 | `batch` | `list[''']` | `list[BaseMessage]` | Defaults to running `invoke` in concurrent threads. |341 | `abatch` | `list[''']` | `list[BaseMessage]` | Defaults to running `ainvoke` in concurrent threads. |342 | `batch_as_completed` | `list[''']` | `Iterator[tuple[int, Union[BaseMessage, Exception]]]` | Defaults to running `invoke` in concurrent threads. |343 | `abatch_as_completed` | `list[''']` | `AsyncIterator[tuple[int, Union[BaseMessage, Exception]]]` | Defaults to running `ainvoke` in concurrent threads. |344345 Key declarative methods:346 Methods for creating another `Runnable` using the chat model.347348 This table provides a brief overview of the main declarative methods. Please see the reference for each method for full documentation.349350 | Method | Description |351 | ---------------------------- | ------------------------------------------------------------------------------------------ |352 | `bind_tools` | Create chat model that can call tools. |353 | `with_structured_output` | Create wrapper that structures model output using schema. |354 | `with_retry` | Create wrapper that retries model calls on failure. |355 | `with_fallbacks` | Create wrapper that falls back to other models on failure. |356 | `configurable_fields` | Specify init args of the model that can be configured at runtime via the `RunnableConfig`. |357 | `configurable_alternatives` | Specify alternative models which can be swapped in at runtime via the `RunnableConfig`. |358359 Creating custom chat model:360 Custom chat model implementations should inherit from this class.361 Please reference the table below for information about which362 methods and properties are required or optional for implementations.363364 | Method/Property | Description | Required |365 | -------------------------------- | ------------------------------------------------------------------ | ----------------- |366 | `_generate` | Use to generate a chat result from a prompt | Required |367 | `_llm_type` (property) | Used to uniquely identify the type of the model. Used for logging. | Required |368 | `_identifying_params` (property) | Represent model parameterization for tracing purposes. | Optional |369 | `_stream` | Use to implement streaming | Optional |370 | `_agenerate` | Use to implement a native async method | Optional |371 | `_astream` | Use to implement async version of `_stream` | Optional |372373 """ # noqa: E501374375 rate_limiter: BaseRateLimiter | None = Field(default=None, exclude=True)376 "An optional rate limiter to use for limiting the number of requests."377378 disable_streaming: bool | Literal["tool_calling"] = False379 """Whether to disable streaming for this model.380381 If streaming is bypassed, then `stream`/`astream`/`astream_events` will382 defer to `invoke`/`ainvoke`.383384 - If `True`, will always bypass streaming case.385 - If `'tool_calling'`, will bypass streaming case only when the model is called386 with a `tools` keyword argument. In other words, LangChain will automatically387 switch to non-streaming behavior (`invoke`) only when the tools argument is388 provided. This offers the best of both worlds.389 - If `False` (Default), will always use streaming case if available.390391 The main reason for this flag is that code might be written using `stream` and392 a user may want to swap out a given model for another model whose implementation393 does not properly support streaming.394 """395396 output_version: str | None = Field(397 default_factory=from_env("LC_OUTPUT_VERSION", default=None)398 )399 """Version of `AIMessage` output format to store in message content.400401 `AIMessage.content_blocks` will lazily parse the contents of `content` into a402 standard format. This flag can be used to additionally store the standard format403 in message content, e.g., for serialization purposes.404405 Supported values:406407 - `'v0'`: provider-specific format in content (can lazily-parse with408 `content_blocks`)409 - `'v1'`: standardized format in content (consistent with `content_blocks`)410411 Partner packages (e.g.,412 [`langchain-openai`](https://pypi.org/project/langchain-openai)) can also use this413 field to roll out new content formats in a backward-compatible way.414415 !!! version-added "Added in `langchain-core` 1.0.0"416417 """418419 profile: ModelProfile | None = Field(default=None, exclude=True)420 """Profile detailing model capabilities.421422 !!! warning "Beta feature"423424 This is a beta feature. The format of model profiles is subject to change.425426 If not specified, automatically loaded from the provider package on initialization427 if data is available.428429 Example profile data includes context window sizes, supported modalities, or support430 for tool calling, structured output, and other features.431432 !!! version-added "Added in `langchain-core` 1.1.0"433 """434435 model_config = ConfigDict(436 arbitrary_types_allowed=True,437 )438439 def _resolve_model_profile(self) -> ModelProfile | None:440 """Return the default model profile, or `None` if unavailable.441442 Override this in subclasses instead of `_set_model_profile`. The base443 validator calls it automatically and handles assignment. This avoids444 coupling partner code to Pydantic validator mechanics.445446 Each partner needs its own override because things can vary per-partner,447 such as the attribute that identifies the model (e.g., `model`,448 `model_name`, `model_id`, `deployment_name`) and the partner-local449 `_get_default_model_profile` function that reads from each partner's own450 profile data.451 """452 # TODO: consider adding a `_model_identifier` property on BaseChatModel453 # to standardize how partners identify their model, which could allow a454 # default implementation here that calls a shared455 # profile-loading mechanism.456 return None457458 @model_validator(mode="after")459 def _set_model_profile(self) -> Self:460 """Populate `profile` from `_resolve_model_profile` if not provided.461462 Partners should override `_resolve_model_profile` rather than this463 validator. Overriding this with a new `@model_validator` replaces the464 base validator (Pydantic v2 behavior), bypassing the standard resolution465 path. A plain method override does not prevent the base validator from466 running.467 """468 if self.profile is None:469 # Suppress errors from partner overrides (e.g., missing profile470 # files, broken imports) so model construction never fails over an471 # optional field.472 with contextlib.suppress(Exception):473 self.profile = self._resolve_model_profile()474 return self475476 # NOTE: _check_profile_keys must be defined AFTER _set_model_profile.477 # Pydantic v2 runs mode="after" validators in definition order.478 @model_validator(mode="after")479 def _check_profile_keys(self) -> Self:480 """Warn on unrecognized profile keys."""481 # isinstance guard: ModelProfile is a TypedDict (always a dict), but482 # protects against unexpected types from partner overrides.483 if self.profile and isinstance(self.profile, dict):484 _warn_unknown_profile_keys(self.profile)485 return self486487 @cached_property488 def _serialized(self) -> builtins.dict[str, Any]:489 # self is always a Serializable object in this case, thus the result is490 # guaranteed to be a dict since dumpd uses the default callback, which uses491 # obj.to_json which always returns TypedDict subclasses492 return cast("builtins.dict[str, Any]", dumpd(self))493494 # --- Runnable methods ---495496 @property497 @override498 def OutputType(self) -> Any:499 """Get the output type for this `Runnable`."""500 return AnyMessage501502 def _convert_input(self, model_input: LanguageModelInput) -> PromptValue:503 if isinstance(model_input, PromptValue):504 return model_input505 if isinstance(model_input, str):506 return StringPromptValue(text=model_input)507 if isinstance(model_input, Sequence):508 return ChatPromptValue(messages=convert_to_messages(model_input))509 msg = ( # type: ignore[unreachable]510 f"Invalid input type {type(model_input)}. "511 "Must be a PromptValue, str, or list of BaseMessages."512 )513 raise ValueError(msg)514515 @override516 def invoke(517 self,518 input: LanguageModelInput,519 config: RunnableConfig | None = None,520 *,521 stop: list[str] | None = None,522 **kwargs: Any,523 ) -> AIMessage:524 config = ensure_config(config)525 return cast(526 "AIMessage",527 cast(528 "ChatGeneration",529 self.generate_prompt(530 [self._convert_input(input)],531 stop=stop,532 callbacks=config.get("callbacks"),533 tags=config.get("tags"),534 metadata=config.get("metadata"),535 run_name=config.get("run_name"),536 run_id=config.pop("run_id", None),537 **kwargs,538 ).generations[0][0],539 ).message,540 )541542 @override543 async def ainvoke(544 self,545 input: LanguageModelInput,546 config: RunnableConfig | None = None,547 *,548 stop: list[str] | None = None,549 **kwargs: Any,550 ) -> AIMessage:551 config = ensure_config(config)552 llm_result = await self.agenerate_prompt(553 [self._convert_input(input)],554 stop=stop,555 callbacks=config.get("callbacks"),556 tags=config.get("tags"),557 metadata=config.get("metadata"),558 run_name=config.get("run_name"),559 run_id=config.pop("run_id", None),560 **kwargs,561 )562 return cast(563 "AIMessage", cast("ChatGeneration", llm_result.generations[0][0]).message564 )565566 def _streaming_disabled(self, **kwargs: Any) -> bool:567 """Return whether streaming is hard-disabled for this call.568569 Shared opt-outs honored by both `_should_stream` and570 `_should_use_protocol_streaming` — these override any affirmative trigger571 (attached handler, `stream=True`, etc.):572573 - `self.disable_streaming is True`574 - `self.disable_streaming == "tool_calling"` with `tools` passed575 - `stream=<falsy>` in call kwargs576 - `self.streaming is False` on the instance577 """578 if self.disable_streaming is True:579 return True580 # We assume tools are passed in via "tools" kwarg in all models.581 if self.disable_streaming == "tool_calling" and kwargs.get("tools"):582 return True583 if "stream" in kwargs and not kwargs["stream"]:584 return True585 return (586 "streaming" in self.model_fields_set587 and getattr(self, "streaming", None) is False588 )589590 def _should_stream(591 self,592 *,593 async_api: bool,594 run_manager: CallbackManagerForLLMRun595 | AsyncCallbackManagerForLLMRun596 | None = None,597 **kwargs: Any,598 ) -> bool:599 """Determine if a given model call should hit the streaming API."""600 sync_not_implemented = type(self)._stream == BaseChatModel._stream # noqa: SLF001601 async_not_implemented = type(self)._astream == BaseChatModel._astream # noqa: SLF001602603 # Check if streaming is implemented.604 if (not async_api) and sync_not_implemented:605 return False606 # Note, since async falls back to sync we check both here.607 if async_api and async_not_implemented and sync_not_implemented:608 return False609610 if self._streaming_disabled(**kwargs):611 return False612613 # Affirmative: explicit `stream=<truthy>` kwarg.614 if kwargs.get("stream"):615 return True616617 # Affirmative: instance-level `streaming=True` attribute.618 if (619 "streaming" in self.model_fields_set620 and getattr(self, "streaming", None) is True621 ):622 return True623624 # Affirmative: a v1 streaming callback handler is attached.625 handlers = run_manager.handlers if run_manager else []626 return any(isinstance(h, _StreamingCallbackHandler) for h in handlers)627628 def _should_use_protocol_streaming(629 self,630 *,631 async_api: bool,632 run_manager: CallbackManagerForLLMRun633 | AsyncCallbackManagerForLLMRun634 | None = None,635 **kwargs: Any,636 ) -> bool:637 """Determine whether an invoke should route through the v2 event path.638639 Runs alongside `_should_stream` inside `_generate_with_cache` /640 `_agenerate_with_cache` — after the run manager is open — and641 wins over the v1 streaming branch when a handler has declared642 itself a `_V2StreamingCallbackHandler`. Parallel to643 `_should_stream` rather than a delegation — v1 and v2 have644 disjoint affirmative triggers.645646 Args:647 async_api: Whether the caller is on the async path.648 run_manager: The active LLM run manager.649 **kwargs: Call kwargs; inspected for `disable_streaming`650 semantics and an explicit `stream=False` override.651652 Returns:653 `True` if any attached handler inherits654 `_V2StreamingCallbackHandler` and the model can drive the v2655 event generator (natively or via the `_stream` compat656 bridge).657 """658 # Opt-in: only route through v2 when a v2 handler is attached.659 handlers = run_manager.handlers if run_manager else []660 if not any(isinstance(h, _V2StreamingCallbackHandler) for h in handlers):661 return False662663 # Need a source of v2 events on the requested flavor. A native664 # `_(a)stream_chat_model_events` hook bypasses the bridge;665 # otherwise the bridge wraps `_stream` / `_astream`. Async can666 # fall back to sync.667 #668 # `cls._stream is not BaseChatModel._stream` is an identity669 # check for "subclass overrode `_stream`" — same pattern as670 # `_should_stream`.671 cls = type(self)672 has_native_sync = getattr(cls, "_stream_chat_model_events", None) is not None673 has_native_async = getattr(cls, "_astream_chat_model_events", None) is not None674 overrides_sync = cls._stream is not BaseChatModel._stream675 overrides_async = cls._astream is not BaseChatModel._astream676 has_sync_source = has_native_sync or overrides_sync677 has_async_source = has_native_async or overrides_async678 has_source = (679 (has_sync_source or has_async_source) if async_api else has_sync_source680 )681 if not has_source:682 return False683684 return not self._streaming_disabled(**kwargs)685686 def _iter_v2_events(687 self,688 messages: list[BaseMessage],689 *,690 run_manager: CallbackManagerForLLMRun,691 stream: ChatModelStream,692 stop: list[str] | None = None,693 **kwargs: Any,694 ) -> Iterator[MessagesData]:695 """Drive the v2 event generator with per-event dispatch.696697 Shared between the `stream_events(version="v3")` pump and the698 invoke-time v2 branch in `_generate_with_cache`. Picks the native699 `_stream_chat_model_events` hook when the subclass provides one,700 else bridges `_stream` chunks via `chunks_to_events`. Each event701 is dispatched into `stream` and fired as `on_stream_event` on702 the run manager. Run-lifecycle callbacks703 (`on_chat_model_start` / `on_llm_end` / `on_llm_error`) and704 rate-limiter acquisition are the caller's responsibility.705706 Args:707 messages: Normalized input messages.708 run_manager: Active LLM run manager; receives709 `on_stream_event` per event.710 stream: Accumulator owned by the caller; receives each711 event via `stream.dispatch`.712 stop: Optional stop sequences.713 **kwargs: Forwarded to the event producer.714715 Yields:716 Each protocol event produced by the model.717 """718 native = cast(719 "Callable[..., Iterator[MessagesData]] | None",720 getattr(self, "_stream_chat_model_events", None),721 )722 if native is not None:723 event_iter: Iterator[MessagesData] = native(724 messages, stop=stop, run_manager=run_manager, **kwargs725 )726 else:727 event_iter = chunks_to_events(728 self._stream(messages, stop=stop, run_manager=run_manager, **kwargs),729 message_id=stream.message_id,730 generation_info=stream._generation_info, # noqa: SLF001731 )732 for event in event_iter:733 stream.dispatch(event)734 run_manager.on_stream_event(event)735 yield event736737 async def _aiter_v2_events(738 self,739 messages: list[BaseMessage],740 *,741 run_manager: AsyncCallbackManagerForLLMRun,742 stream: AsyncChatModelStream,743 stop: list[str] | None = None,744 **kwargs: Any,745 ) -> AsyncIterator[MessagesData]:746 """Async counterpart to `_iter_v2_events`.747748 See `_iter_v2_events` for the shared contract.749 """750 native = cast(751 "Callable[..., AsyncIterator[MessagesData]] | None",752 getattr(self, "_astream_chat_model_events", None),753 )754 if native is not None:755 event_iter: AsyncIterator[MessagesData] = native(756 messages, stop=stop, run_manager=run_manager, **kwargs757 )758 else:759 event_iter = achunks_to_events(760 self._astream(messages, stop=stop, run_manager=run_manager, **kwargs),761 message_id=stream.message_id,762 generation_info=stream._generation_info, # noqa: SLF001763 )764 async for event in event_iter:765 stream.dispatch(event)766 await run_manager.on_stream_event(event)767 yield event768769 @override770 def stream(771 self,772 input: LanguageModelInput,773 config: RunnableConfig | None = None,774 *,775 stop: list[str] | None = None,776 **kwargs: Any,777 ) -> Iterator[AIMessageChunk]:778 if not self._should_stream(async_api=False, **{**kwargs, "stream": True}):779 # Model doesn't implement streaming, so use default implementation780 yield cast(781 "AIMessageChunk",782 self.invoke(input, config=config, stop=stop, **kwargs),783 )784 else:785 config = ensure_config(config)786 messages = self._convert_input(input).to_messages()787 ls_structured_output_format = kwargs.pop(788 "ls_structured_output_format", None789 ) or kwargs.pop("structured_output_format", None)790 ls_structured_output_format_dict = _format_ls_structured_output(791 ls_structured_output_format792 )793794 params = self._get_invocation_params(stop=stop, **kwargs)795 options = {796 "stop": stop,797 **{key: params[key] for key in kwargs if key in params},798 **ls_structured_output_format_dict,799 }800 inheritable_metadata = {801 **(config.get("metadata") or {}),802 **self._get_ls_params_with_defaults(stop=stop, **kwargs),803 }804 callback_manager = CallbackManager.configure(805 config.get("callbacks"),806 self.callbacks,807 self.verbose,808 config.get("tags"),809 self.tags,810 inheritable_metadata,811 self.metadata,812 langsmith_inheritable_metadata=_filter_invocation_params_for_tracing(813 params814 ),815 )816 (run_manager,) = callback_manager.on_chat_model_start(817 self._serialized,818 [_format_for_tracing(messages)],819 invocation_params=params,820 options=options,821 name=config.get("run_name"),822 run_id=config.pop("run_id", None),823 batch_size=1,824 )825826 chunks: list[ChatGenerationChunk] = []827828 if self.rate_limiter:829 self.rate_limiter.acquire(blocking=True)830831 try:832 input_messages = _normalize_messages(messages)833 run_id = "-".join((LC_ID_PREFIX, str(run_manager.run_id)))834 yielded = False835 index = -1836 index_type = ""837 for chunk in self._stream(input_messages, stop=stop, **kwargs):838 if chunk.message.id is None:839 chunk.message.id = run_id840 chunk.message.response_metadata = _gen_info_and_msg_metadata(chunk)841 if self.output_version == "v1":842 # Overwrite .content with .content_blocks843 chunk.message = _update_message_content_to_blocks(844 chunk.message, "v1"845 )846 for block in cast(847 "list[types.ContentBlock]", chunk.message.content848 ):849 if block["type"] != index_type:850 index_type = block["type"]851 index += 1852 if "index" not in block:853 block["index"] = index854 run_manager.on_llm_new_token(chunk.message.content, chunk=chunk)855 chunks.append(chunk)856 yield cast("AIMessageChunk", chunk.message)857 yielded = True858859 # Yield a final empty chunk with chunk_position="last" if not yet860 # yielded861 if (862 yielded863 and isinstance(chunk.message, AIMessageChunk)864 and not chunk.message.chunk_position865 ):866 empty_content: str | list[str | dict[str, Any]] = (867 "" if isinstance(chunk.message.content, str) else []868 )869 msg_chunk = AIMessageChunk(870 content=empty_content, chunk_position="last", id=run_id871 )872 run_manager.on_llm_new_token(873 "", chunk=ChatGenerationChunk(message=msg_chunk)874 )875 yield msg_chunk876 except BaseException as e:877 generations_with_error_metadata = _generate_response_from_error(e)878 chat_generation_chunk = merge_chat_generation_chunks(chunks)879 if chat_generation_chunk:880 generations = [881 [chat_generation_chunk],882 generations_with_error_metadata,883 ]884 else:885 generations = [generations_with_error_metadata]886 run_manager.on_llm_error(887 e,888 response=LLMResult(generations=generations),889 )890 raise891892 generation = merge_chat_generation_chunks(chunks)893 if generation is None:894 err = ValueError("No generation chunks were returned")895 run_manager.on_llm_error(err, response=LLMResult(generations=[]))896 raise err897898 run_manager.on_llm_end(LLMResult(generations=[[generation]]))899900 @override901 async def astream(902 self,903 input: LanguageModelInput,904 config: RunnableConfig | None = None,905 *,906 stop: list[str] | None = None,907 **kwargs: Any,908 ) -> AsyncIterator[AIMessageChunk]:909 if not self._should_stream(async_api=True, **{**kwargs, "stream": True}):910 # No async or sync stream is implemented, so fall back to ainvoke911 yield cast(912 "AIMessageChunk",913 await self.ainvoke(input, config=config, stop=stop, **kwargs),914 )915 return916917 config = ensure_config(config)918 messages = self._convert_input(input).to_messages()919920 ls_structured_output_format = kwargs.pop(921 "ls_structured_output_format", None922 ) or kwargs.pop("structured_output_format", None)923 ls_structured_output_format_dict = _format_ls_structured_output(924 ls_structured_output_format925 )926927 params = self._get_invocation_params(stop=stop, **kwargs)928 options = {929 "stop": stop,930 **{key: params[key] for key in kwargs if key in params},931 **ls_structured_output_format_dict,932 }933 inheritable_metadata = {934 **(config.get("metadata") or {}),935 **self._get_ls_params_with_defaults(stop=stop, **kwargs),936 }937 callback_manager = AsyncCallbackManager.configure(938 config.get("callbacks"),939 self.callbacks,940 self.verbose,941 config.get("tags"),942 self.tags,943 inheritable_metadata,944 self.metadata,945 langsmith_inheritable_metadata=_filter_invocation_params_for_tracing(946 params947 ),948 )949 (run_manager,) = await callback_manager.on_chat_model_start(950 self._serialized,951 [_format_for_tracing(messages)],952 invocation_params=params,953 options=options,954 name=config.get("run_name"),955 run_id=config.pop("run_id", None),956 batch_size=1,957 )958959 if self.rate_limiter:960 await self.rate_limiter.aacquire(blocking=True)961962 chunks: list[ChatGenerationChunk] = []963964 try:965 input_messages = _normalize_messages(messages)966 run_id = "-".join((LC_ID_PREFIX, str(run_manager.run_id)))967 yielded = False968 index = -1969 index_type = ""970 async for chunk in self._astream(971 input_messages,972 stop=stop,973 **kwargs,974 ):975 if chunk.message.id is None:976 chunk.message.id = run_id977 chunk.message.response_metadata = _gen_info_and_msg_metadata(chunk)978 if self.output_version == "v1":979 # Overwrite .content with .content_blocks980 chunk.message = _update_message_content_to_blocks(981 chunk.message, "v1"982 )983 for block in cast(984 "list[types.ContentBlock]", chunk.message.content985 ):986 if block["type"] != index_type:987 index_type = block["type"]988 index += 1989 if "index" not in block:990 block["index"] = index991 await run_manager.on_llm_new_token(chunk.message.content, chunk=chunk)992 chunks.append(chunk)993 yield cast("AIMessageChunk", chunk.message)994 yielded = True995996 # Yield a final empty chunk with chunk_position="last" if not yet yielded997 if (998 yielded999 and isinstance(chunk.message, AIMessageChunk)1000 and not chunk.message.chunk_position1001 ):1002 empty_content: str | list[str | dict[str, Any]] = (1003 "" if isinstance(chunk.message.content, str) else []1004 )1005 msg_chunk = AIMessageChunk(1006 content=empty_content, chunk_position="last", id=run_id1007 )1008 await run_manager.on_llm_new_token(1009 "", chunk=ChatGenerationChunk(message=msg_chunk)1010 )1011 yield msg_chunk1012 except BaseException as e:1013 generations_with_error_metadata = _generate_response_from_error(e)1014 chat_generation_chunk = merge_chat_generation_chunks(chunks)1015 if chat_generation_chunk:1016 generations = [[chat_generation_chunk], generations_with_error_metadata]1017 else:1018 generations = [generations_with_error_metadata]1019 await run_manager.on_llm_error(1020 e,1021 response=LLMResult(generations=generations),1022 )1023 raise10241025 generation = merge_chat_generation_chunks(chunks)1026 if not generation:1027 err = ValueError("No generation chunks were returned")1028 await run_manager.on_llm_error(err, response=LLMResult(generations=[]))1029 raise err10301031 await run_manager.on_llm_end(1032 LLMResult(generations=[[generation]]),1033 )10341035 # --- stream_events v3 ---10361037 @beta()1038 def _chat_model_stream_v3(1039 self,1040 input: LanguageModelInput,1041 config: RunnableConfig | None = None,1042 *,1043 stop: list[str] | None = None,1044 **kwargs: Any,1045 ) -> ChatModelStream:1046 """Internal v3 sync streaming implementation.10471048 Public entry point: `stream_events(version='v3')`.1049 """1050 config = ensure_config(config)1051 messages = self._convert_input(input).to_messages()1052 input_messages = _normalize_messages(messages)10531054 # Strip tracing-only kwargs before forwarding to `_stream` — matches1055 # `stream()` / `astream()`. Provider clients reject unknown kwargs,1056 # so `.with_structured_output().stream_events(version="v3", ...)`1057 # and any other binding that carries `ls_structured_output_format`1058 # / `structured_output_format` would raise without this pop.1059 ls_structured_output_format = kwargs.pop(1060 "ls_structured_output_format", None1061 ) or kwargs.pop("structured_output_format", None)1062 ls_structured_output_format_dict = _format_ls_structured_output(1063 ls_structured_output_format1064 )10651066 params = self._get_invocation_params(stop=stop, **kwargs)1067 options = {1068 "stop": stop,1069 **{key: params[key] for key in kwargs if key in params},1070 **ls_structured_output_format_dict,1071 }1072 inheritable_metadata = {1073 **(config.get("metadata") or {}),1074 **self._get_ls_params_with_defaults(stop=stop, **kwargs),1075 }1076 callback_manager = CallbackManager.configure(1077 config.get("callbacks"),1078 self.callbacks,1079 self.verbose,1080 config.get("tags"),1081 self.tags,1082 inheritable_metadata,1083 self.metadata,1084 langsmith_inheritable_metadata=_filter_invocation_params_for_tracing(1085 params1086 ),1087 )1088 stream = ChatModelStream()1089 run_manager: CallbackManagerForLLMRun | None = None1090 event_iter_ref: Iterator[MessagesData] | None = None1091 rate_limiter_acquired = self.rate_limiter is None1092 run_name = config.get("run_name")1093 run_id = config.pop("run_id", None)10941095 def ensure_started() -> None:1096 nonlocal event_iter_ref, run_manager1097 if event_iter_ref is not None:1098 return10991100 (run_manager,) = callback_manager.on_chat_model_start(1101 self._serialized,1102 [_format_for_tracing(messages)],1103 invocation_params=params,1104 options=options,1105 name=run_name,1106 run_id=run_id,1107 batch_size=1,1108 )1109 stream.set_message_id("-".join((LC_ID_PREFIX, str(run_manager.run_id))))1110 event_iter_ref = iter(1111 self._iter_v2_events(1112 input_messages,1113 run_manager=run_manager,1114 stream=stream,1115 stop=stop,1116 **kwargs,1117 )1118 )11191120 def pump_one() -> bool:1121 nonlocal rate_limiter_acquired1122 ensure_started()1123 if not rate_limiter_acquired:1124 assert self.rate_limiter is not None # noqa: S1011125 self.rate_limiter.acquire(blocking=True)1126 rate_limiter_acquired = True1127 assert event_iter_ref is not None # noqa: S1011128 assert run_manager is not None # noqa: S1011129 try:1130 next(event_iter_ref)1131 except StopIteration:1132 if not stream.done:1133 if stream.has_events:1134 # Native event producers may omit the terminal1135 # `message-finish`. Close the lifecycle here so1136 # `on_llm_end` still observes the assembled1137 # message. A truly empty stream remains an error1138 # for parity with `stream()`.1139 stream.dispatch(MessageFinishData(event="message-finish"))1140 else:1141 err = ValueError("No generation chunks were returned")1142 stream.fail(err)1143 run_manager.on_llm_error(1144 err,1145 response=_v3_error_response(err, stream),1146 )1147 return False1148 if stream.done and stream.output_message is not None:1149 run_manager.on_llm_end(1150 LLMResult(1151 generations=[1152 [_v3_generation(stream.output_message, stream)],1153 ],1154 ),1155 )1156 return False1157 except BaseException as exc:1158 stream.fail(exc)1159 run_manager.on_llm_error(1160 exc,1161 response=_v3_error_response(exc, stream),1162 )1163 return False1164 if stream.done and stream.output_message is not None:1165 run_manager.on_llm_end(1166 LLMResult(1167 generations=[1168 [_v3_generation(stream.output_message, stream)],1169 ],1170 ),1171 )1172 return True11731174 stream.set_start(ensure_started)1175 stream.bind_pump(pump_one)1176 return stream11771178 @beta()1179 async def _achat_model_stream_v3(1180 self,1181 input: LanguageModelInput,1182 config: RunnableConfig | None = None,1183 *,1184 stop: list[str] | None = None,1185 **kwargs: Any,1186 ) -> AsyncChatModelStream:1187 """Internal v3 async streaming implementation.11881189 Public entry point: `astream_events(version='v3')`.1190 """1191 config = ensure_config(config)1192 messages = self._convert_input(input).to_messages()1193 input_messages = _normalize_messages(messages)11941195 # Strip tracing-only kwargs before forwarding — see the sync v31196 # implementation for the full rationale.1197 ls_structured_output_format = kwargs.pop(1198 "ls_structured_output_format", None1199 ) or kwargs.pop("structured_output_format", None)1200 ls_structured_output_format_dict = _format_ls_structured_output(1201 ls_structured_output_format1202 )12031204 params = self._get_invocation_params(stop=stop, **kwargs)1205 options = {1206 "stop": stop,1207 **{key: params[key] for key in kwargs if key in params},1208 **ls_structured_output_format_dict,1209 }1210 inheritable_metadata = {1211 **(config.get("metadata") or {}),1212 **self._get_ls_params_with_defaults(stop=stop, **kwargs),1213 }1214 callback_manager = AsyncCallbackManager.configure(1215 config.get("callbacks"),1216 self.callbacks,1217 self.verbose,1218 config.get("tags"),1219 self.tags,1220 inheritable_metadata,1221 self.metadata,1222 langsmith_inheritable_metadata=_filter_invocation_params_for_tracing(1223 params1224 ),1225 )1226 stream = AsyncChatModelStream()1227 run_manager: AsyncCallbackManagerForLLMRun | None = None1228 run_name = config.get("run_name")1229 run_id = config.pop("run_id", None)1230 start_lock = asyncio.Lock()12311232 async def _produce() -> None:1233 assert run_manager is not None # noqa: S1011234 try:1235 if self.rate_limiter:1236 await self.rate_limiter.aacquire(blocking=True)12371238 async for _event in self._aiter_v2_events(1239 input_messages,1240 run_manager=run_manager,1241 stream=stream,1242 stop=stop,1243 **kwargs,1244 ):1245 pass1246 if not stream.done:1247 if stream.has_events:1248 # Native event producers may omit the terminal1249 # `message-finish`. Close the lifecycle here so1250 # `on_llm_end` sees the finalized message. A1251 # truly empty stream remains an error for parity1252 # with `astream()`.1253 stream.dispatch(MessageFinishData(event="message-finish"))1254 else:1255 err = ValueError("No generation chunks were returned")1256 stream.fail(err)1257 await run_manager.on_llm_error(1258 err,1259 response=_v3_error_response(err, stream),1260 )1261 return1262 if stream.done and stream.output_message is not None:1263 await run_manager.on_llm_end(1264 LLMResult(1265 generations=[1266 [_v3_generation(stream.output_message, stream)],1267 ],1268 ),1269 )1270 except asyncio.CancelledError as exc:1271 stream.fail(exc)1272 # Close the callback lifecycle so tracing observes a1273 # matching end event for the earlier `on_chat_model_start`.1274 # `on_llm_error` is `@shielded`, so the callback runs to1275 # completion in the background even though the `await`1276 # here re-raises our cancellation.1277 with contextlib.suppress(Exception):1278 await run_manager.on_llm_error(1279 exc,1280 response=_v3_error_response(exc, stream),1281 )1282 raise1283 except BaseException as exc:1284 stream.fail(exc)1285 await run_manager.on_llm_error(1286 exc,1287 response=_v3_error_response(exc, stream),1288 )12891290 async def ensure_started() -> None:1291 nonlocal run_manager1292 if stream._producer_task is not None: # noqa: SLF0011293 return12941295 async with start_lock:1296 if stream._producer_task is not None: # noqa: SLF0011297 return # type: ignore[unreachable]12981299 (run_manager,) = await callback_manager.on_chat_model_start(1300 self._serialized,1301 [_format_for_tracing(messages)],1302 invocation_params=params,1303 options=options,1304 name=run_name,1305 run_id=run_id,1306 batch_size=1,1307 )1308 stream.set_message_id("-".join((LC_ID_PREFIX, str(run_manager.run_id))))1309 stream._producer_task = asyncio.get_running_loop().create_task( # noqa: SLF0011310 _produce()1311 )13121313 async def _on_aclose_fail(exc: BaseException) -> None:1314 assert run_manager is not None # noqa: S1011315 # Invoked by `stream.aclose()` only when the producer was1316 # cancelled before `_produce` ran — so `on_llm_error` from1317 # the CancelledError handler never fired. Shielded by the1318 # callback manager; runs to completion even if our caller1319 # is being cancelled.1320 await run_manager.on_llm_error(1321 exc,1322 response=_v3_error_response(exc, stream),1323 )13241325 stream.set_start(ensure_started)1326 stream._on_aclose_fail = _on_aclose_fail # noqa: SLF0011327 return stream13281329 @overload # type: ignore[override]1330 def stream_events(1331 self,1332 input: LanguageModelInput,1333 config: RunnableConfig | None = None,1334 *,1335 version: Literal["v1", "v2"] = "v2",1336 **kwargs: Any,1337 ) -> Iterator[StreamEvent]: ...13381339 @overload1340 def stream_events(1341 self,1342 input: LanguageModelInput,1343 config: RunnableConfig | None = None,1344 *,1345 version: Literal["v3"],1346 stop: list[str] | None = None,1347 **kwargs: Any,1348 ) -> ChatModelStream: ...13491350 def stream_events(1351 self,1352 input: LanguageModelInput,1353 config: RunnableConfig | None = None,1354 *,1355 version: Literal["v1", "v2", "v3"] = "v2",1356 stop: list[str] | None = None,1357 **kwargs: Any,1358 ) -> Iterator[StreamEvent] | ChatModelStream:1359 """Stream events from this chat model.13601361 For `version="v1"` / `"v2"`, yields `StreamEvent` dicts (see1362 `Runnable.stream_events`). For `version="v3"`, returns a1363 `ChatModelStream` exposing typed projections (`.text`,1364 `.reasoning`, `.tool_calls`, `.output`).13651366 !!! warning "Beta"13671368 `version="v3"` is in beta. The protocol shape, return type,1369 and surface area may change in future releases. Calling it1370 emits a `LangChainBetaWarning` at runtime.13711372 !!! note "v3 always produces v1-shaped content"13731374 `ChatModelStream.output.content` is always a list of v11375 content blocks (text / reasoning / tool_call / image / …),1376 regardless of the model's `output_version` attribute. The1377 setting only affects the legacy `stream()` / `astream()` /1378 `invoke()` paths. If you're mixing1379 `stream_events(version="v3")` with those paths in the same1380 pipeline and need a consistent output shape across them,1381 set `output_version="v1"` on the model.13821383 Args:1384 input: The model input.1385 config: Optional runnable config.1386 version: Streaming-event schema version. `"v3"` selects the1387 content-block-centric streaming protocol.1388 stop: Optional stop sequences. Only used for `version="v3"`;1389 ignored otherwise.1390 **kwargs: Additional keyword arguments. For `version="v3"`,1391 forwarded to the model.13921393 Returns:1394 For `version="v3"`, a `ChatModelStream` with typed1395 projections. Otherwise an `Iterator[StreamEvent]`.1396 """1397 if version == "v3":1398 return self._chat_model_stream_v3(input, config, stop=stop, **kwargs)1399 return super().stream_events(1400 input, config, version=version, stop=stop, **kwargs1401 )14021403 @overload1404 def astream_events(1405 self,1406 input: LanguageModelInput,1407 config: RunnableConfig | None = None,1408 *,1409 version: Literal["v1", "v2"] = "v2",1410 **kwargs: Any,1411 ) -> AsyncIterator[StreamEvent]: ...14121413 @overload1414 def astream_events(1415 self,1416 input: LanguageModelInput,1417 config: RunnableConfig | None = None,1418 *,1419 version: Literal["v3"],1420 stop: list[str] | None = None,1421 **kwargs: Any,1422 ) -> Awaitable[AsyncChatModelStream]: ...14231424 def astream_events(1425 self,1426 input: LanguageModelInput,1427 config: RunnableConfig | None = None,1428 *,1429 version: Literal["v1", "v2", "v3"] = "v2",1430 stop: list[str] | None = None,1431 **kwargs: Any,1432 ) -> AsyncIterator[StreamEvent] | Awaitable[AsyncChatModelStream]:1433 """Async variant of `stream_events`. See `stream_events` for full docs."""1434 if version == "v3":1435 return self._achat_model_stream_v3(input, config, stop=stop, **kwargs)1436 # v1/v2: forward to Runnable.astream_events (async generator).1437 return super().astream_events(1438 input, config, version=version, stop=stop, **kwargs1439 )14401441 # --- Custom methods ---14421443 def _combine_llm_outputs(1444 self, _llm_outputs: list[builtins.dict[str, Any] | None], /1445 ) -> builtins.dict[str, Any]:1446 return {}14471448 def _convert_cached_generations(1449 self, cache_val: list[Generation]1450 ) -> list[ChatGeneration]:1451 """Convert cached Generation objects to ChatGeneration objects.14521453 Handle case where cache contains Generation objects instead of1454 ChatGeneration objects. This can happen due to serialization/deserialization1455 issues or legacy cache data (see #22389).14561457 Args:1458 cache_val: List of cached generation objects.14591460 Returns:1461 List of ChatGeneration objects.14621463 """1464 converted_generations = []1465 for gen in cache_val:1466 if isinstance(gen, Generation) and not isinstance(gen, ChatGeneration):1467 # Convert Generation to ChatGeneration by creating AIMessage1468 # from the text content1469 chat_gen = ChatGeneration(1470 message=AIMessage(content=gen.text),1471 generation_info=gen.generation_info,1472 )1473 converted_generations.append(chat_gen)1474 else:1475 # Already a ChatGeneration or other expected type1476 if hasattr(gen, "message") and isinstance(gen.message, AIMessage):1477 # We zero out cost on cache hits1478 gen.message = gen.message.model_copy(1479 update={1480 "usage_metadata": {1481 **(gen.message.usage_metadata or {}),1482 "total_cost": 0,1483 }1484 }1485 )1486 converted_generations.append(gen)1487 return converted_generations14881489 def _replay_v2_events_for_cache_hit(1490 self,1491 generations: list[ChatGeneration],1492 *,1493 run_manager: CallbackManagerForLLMRun | None,1494 **kwargs: Any,1495 ) -> None:1496 """Replay cached messages as v2 events when a v2 handler is attached.14971498 A warm cache must produce the same `on_stream_event` stream as a1499 cold call so LangGraph-style consumers do not observe behavior1500 that depends on cache state. Gated by1501 `_should_use_protocol_streaming` so a `disable_streaming` config1502 that suppresses v2 on cold calls also suppresses it here.1503 """1504 if run_manager is None or not self._should_use_protocol_streaming(1505 async_api=False, run_manager=run_manager, **kwargs1506 ):1507 return1508 message_id = f"{LC_ID_PREFIX}-{run_manager.run_id}"1509 for gen in generations:1510 msg = getattr(gen, "message", None)1511 if not isinstance(msg, AIMessage):1512 continue1513 for event in message_to_events(msg, message_id=message_id):1514 run_manager.on_stream_event(event)15151516 async def _areplay_v2_events_for_cache_hit(1517 self,1518 generations: list[ChatGeneration],1519 *,1520 run_manager: AsyncCallbackManagerForLLMRun | None,1521 **kwargs: Any,1522 ) -> None:1523 """Async counterpart to `_replay_v2_events_for_cache_hit`."""1524 if run_manager is None or not self._should_use_protocol_streaming(1525 async_api=True, run_manager=run_manager, **kwargs1526 ):1527 return1528 message_id = f"{LC_ID_PREFIX}-{run_manager.run_id}"1529 for gen in generations:1530 msg = getattr(gen, "message", None)1531 if not isinstance(msg, AIMessage):1532 continue1533 async for event in amessage_to_events(msg, message_id=message_id):1534 await run_manager.on_stream_event(event)15351536 def _get_invocation_params(1537 self,1538 stop: list[str] | None = None,1539 **kwargs: Any,1540 ) -> builtins.dict[str, Any]:1541 params = self._dict_for_compat()1542 params["stop"] = stop1543 return {**params, **kwargs}15441545 def _get_ls_params(1546 self,1547 stop: list[str] | None = None,1548 **kwargs: Any,1549 ) -> LangSmithParams:1550 """Get standard params for LangSmith tracing.15511552 Subclasses **should override** this method to populate `ls_provider`1553 and `ls_model_name` from provider-specific attributes (e.g. `self.model`,1554 `self.model_name`, `self.model_id`) and to honor per-call overrides1555 passed via `kwargs["model"]` so that runtime `bind`/`invoke` model1556 changes are reflected in traces.15571558 The implementation here is a best-effort fallback for subclasses that1559 do not override it. It is not part of a stable contract and the1560 derivation rules may change:15611562 - `ls_provider` is derived from the class name by stripping a leading1563 or trailing `"Chat"` and lowercasing the remainder. This produces1564 ugly values for multi-word providers (e.g. `ChatGoogleGenerativeAI`1565 would become `"googlegenerativeai"`).15661567 Override to set a stable, conventional value1568 such as `"google_genai"`.1569 - `ls_model_name` is resolved from `kwargs["model"]`, then1570 `self.model`, then `self.model_name`.15711572 Subclasses whose model attribute has a different name1573 (`model_id`, `deployment_name`, ...) must override.1574 """1575 # get default provider from class name1576 default_provider = self.__class__.__name__1577 if default_provider.startswith("Chat"):1578 default_provider = default_provider[4:].lower()1579 elif default_provider.endswith("Chat"):1580 default_provider = default_provider[:-4]1581 default_provider = default_provider.lower()15821583 ls_params = LangSmithParams(ls_provider=default_provider, ls_model_type="chat")1584 if stop:1585 ls_params["ls_stop"] = stop15861587 # model1588 if "model" in kwargs and isinstance(kwargs["model"], str):1589 ls_params["ls_model_name"] = kwargs["model"]1590 elif hasattr(self, "model") and isinstance(self.model, str):1591 ls_params["ls_model_name"] = self.model1592 elif hasattr(self, "model_name") and isinstance(self.model_name, str):1593 ls_params["ls_model_name"] = self.model_name15941595 # temperature1596 if "temperature" in kwargs and isinstance(kwargs["temperature"], (int, float)):1597 ls_params["ls_temperature"] = kwargs["temperature"]1598 elif hasattr(self, "temperature") and isinstance(1599 self.temperature, (int, float)1600 ):1601 ls_params["ls_temperature"] = self.temperature16021603 # max_tokens1604 if "max_tokens" in kwargs and isinstance(kwargs["max_tokens"], int):1605 ls_params["ls_max_tokens"] = kwargs["max_tokens"]1606 elif hasattr(self, "max_tokens") and isinstance(self.max_tokens, int):1607 ls_params["ls_max_tokens"] = self.max_tokens16081609 return ls_params16101611 def _get_ls_params_with_defaults(1612 self,1613 stop: list[str] | None = None,1614 **kwargs: Any,1615 ) -> LangSmithParams:1616 """Wrap _get_ls_params to always include ls_integration."""1617 ls_params = self._get_ls_params(stop=stop, **kwargs)1618 ls_params["ls_integration"] = "langchain_chat_model"1619 return ls_params16201621 def _get_llm_string(self, stop: list[str] | None = None, **kwargs: Any) -> str:1622 if self.is_lc_serializable():1623 params = {**kwargs, "stop": stop}1624 param_string = str(sorted(params.items()))1625 # This code is not super efficient as it goes back and forth between1626 # json and dict.1627 serialized_repr = self._serialized1628 _cleanup_llm_representation(serialized_repr, 1)1629 llm_string = json.dumps(serialized_repr, sort_keys=True)1630 return llm_string + "---" + param_string1631 params = self._get_invocation_params(stop=stop, **kwargs)1632 params = {**params, **kwargs}1633 return str(sorted(params.items()))16341635 def generate(1636 self,1637 messages: list[list[BaseMessage]],1638 stop: list[str] | None = None,1639 callbacks: Callbacks = None,1640 *,1641 tags: list[str] | None = None,1642 metadata: builtins.dict[str, Any] | None = None,1643 run_name: str | None = None,1644 run_id: uuid.UUID | None = None,1645 **kwargs: Any,1646 ) -> LLMResult:1647 """Pass a sequence of prompts to the model and return model generations.16481649 This method should make use of batched calls for models that expose a batched1650 API.16511652 Use this method when you want to:16531654 1. Take advantage of batched calls,1655 2. Need more output from the model than just the top generated value,1656 3. Are building chains that are agnostic to the underlying language model1657 type (e.g., pure text completion models vs chat models).16581659 Args:1660 messages: List of list of messages.1661 stop: Stop words to use when generating.16621663 Model output is cut off at the first occurrence of any of these1664 substrings.1665 callbacks: `Callbacks` to pass through.16661667 Used for executing additional functionality, such as logging or1668 streaming, throughout generation.1669 tags: The tags to apply.1670 metadata: The metadata to apply.1671 run_name: The name of the run.1672 run_id: The ID of the run.1673 **kwargs: Arbitrary additional keyword arguments.16741675 These are usually passed to the model provider API call.16761677 Returns:1678 An `LLMResult`, which contains a list of candidate `Generations` for each1679 input prompt and additional model provider-specific output.16801681 """1682 ls_structured_output_format = kwargs.pop(1683 "ls_structured_output_format", None1684 ) or kwargs.pop("structured_output_format", None)1685 ls_structured_output_format_dict = _format_ls_structured_output(1686 ls_structured_output_format1687 )16881689 params = self._get_invocation_params(stop=stop, **kwargs)1690 options = {"stop": stop, **ls_structured_output_format_dict}1691 inheritable_metadata = {1692 **(metadata or {}),1693 **self._get_ls_params_with_defaults(stop=stop, **kwargs),1694 }16951696 callback_manager = CallbackManager.configure(1697 callbacks,1698 self.callbacks,1699 self.verbose,1700 tags,1701 self.tags,1702 inheritable_metadata,1703 self.metadata,1704 langsmith_inheritable_metadata=_filter_invocation_params_for_tracing(1705 params1706 ),1707 )1708 messages_to_trace = [1709 _format_for_tracing(message_list) for message_list in messages1710 ]1711 run_managers = callback_manager.on_chat_model_start(1712 self._serialized,1713 messages_to_trace,1714 invocation_params=params,1715 options=options,1716 name=run_name,1717 run_id=run_id,1718 batch_size=len(messages),1719 )1720 results = []1721 input_messages = [1722 _normalize_messages(message_list) for message_list in messages1723 ]1724 for i, m in enumerate(input_messages):1725 try:1726 results.append(1727 self._generate_with_cache(1728 m,1729 stop=stop,1730 run_manager=run_managers[i] if run_managers else None,1731 **kwargs,1732 )1733 )1734 except BaseException as e:1735 if run_managers:1736 generations_with_error_metadata = _generate_response_from_error(e)1737 run_managers[i].on_llm_error(1738 e,1739 response=LLMResult(1740 generations=[generations_with_error_metadata]1741 ),1742 )1743 raise1744 flattened_outputs = [1745 LLMResult(generations=[res.generations], llm_output=res.llm_output)1746 for res in results1747 ]1748 llm_output = self._combine_llm_outputs([res.llm_output for res in results])1749 generations = [res.generations for res in results]1750 output = LLMResult(generations=generations, llm_output=llm_output)1751 if run_managers:1752 run_infos = []1753 for manager, flattened_output in zip(1754 run_managers, flattened_outputs, strict=False1755 ):1756 manager.on_llm_end(flattened_output)1757 run_infos.append(RunInfo(run_id=manager.run_id))1758 output.run = run_infos1759 return output17601761 async def agenerate(1762 self,1763 messages: list[list[BaseMessage]],1764 stop: list[str] | None = None,1765 callbacks: Callbacks = None,1766 *,1767 tags: list[str] | None = None,1768 metadata: builtins.dict[str, Any] | None = None,1769 run_name: str | None = None,1770 run_id: uuid.UUID | None = None,1771 **kwargs: Any,1772 ) -> LLMResult:1773 """Asynchronously pass a sequence of prompts to a model and return generations.17741775 This method should make use of batched calls for models that expose a batched1776 API.17771778 Use this method when you want to:17791780 1. Take advantage of batched calls,1781 2. Need more output from the model than just the top generated value,1782 3. Are building chains that are agnostic to the underlying language model1783 type (e.g., pure text completion models vs chat models).17841785 Args:1786 messages: List of list of messages.1787 stop: Stop words to use when generating.17881789 Model output is cut off at the first occurrence of any of these1790 substrings.1791 callbacks: `Callbacks` to pass through.17921793 Used for executing additional functionality, such as logging or1794 streaming, throughout generation.1795 tags: The tags to apply.1796 metadata: The metadata to apply.1797 run_name: The name of the run.1798 run_id: The ID of the run.1799 **kwargs: Arbitrary additional keyword arguments.18001801 These are usually passed to the model provider API call.18021803 Returns:1804 An `LLMResult`, which contains a list of candidate `Generations` for each1805 input prompt and additional model provider-specific output.18061807 """1808 ls_structured_output_format = kwargs.pop(1809 "ls_structured_output_format", None1810 ) or kwargs.pop("structured_output_format", None)1811 ls_structured_output_format_dict = _format_ls_structured_output(1812 ls_structured_output_format1813 )18141815 params = self._get_invocation_params(stop=stop, **kwargs)1816 options = {"stop": stop, **ls_structured_output_format_dict}1817 inheritable_metadata = {1818 **(metadata or {}),1819 **self._get_ls_params_with_defaults(stop=stop, **kwargs),1820 }18211822 callback_manager = AsyncCallbackManager.configure(1823 callbacks,1824 self.callbacks,1825 self.verbose,1826 tags,1827 self.tags,1828 inheritable_metadata,1829 self.metadata,1830 langsmith_inheritable_metadata=_filter_invocation_params_for_tracing(1831 params1832 ),1833 )18341835 messages_to_trace = [1836 _format_for_tracing(message_list) for message_list in messages1837 ]1838 run_managers = await callback_manager.on_chat_model_start(1839 self._serialized,1840 messages_to_trace,1841 invocation_params=params,1842 options=options,1843 name=run_name,1844 batch_size=len(messages),1845 run_id=run_id,1846 )18471848 input_messages = [1849 _normalize_messages(message_list) for message_list in messages1850 ]1851 results = await asyncio.gather(1852 *[1853 self._agenerate_with_cache(1854 m,1855 stop=stop,1856 run_manager=run_managers[i] if run_managers else None,1857 **kwargs,1858 )1859 for i, m in enumerate(input_messages)1860 ],1861 return_exceptions=True,1862 )1863 exceptions = []1864 for i, res in enumerate(results):1865 if isinstance(res, BaseException):1866 if run_managers:1867 generations_with_error_metadata = _generate_response_from_error(res)1868 await run_managers[i].on_llm_error(1869 res,1870 response=LLMResult(1871 generations=[generations_with_error_metadata]1872 ),1873 )1874 exceptions.append(res)1875 if exceptions:1876 if run_managers:1877 await asyncio.gather(1878 *[1879 run_manager.on_llm_end(1880 LLMResult(1881 generations=[res.generations], # type: ignore[union-attr]1882 llm_output=res.llm_output, # type: ignore[union-attr]1883 )1884 )1885 for run_manager, res in zip(run_managers, results, strict=False)1886 if not isinstance(res, Exception)1887 ]1888 )1889 raise exceptions[0]1890 flattened_outputs = [1891 LLMResult(generations=[res.generations], llm_output=res.llm_output) # type: ignore[union-attr]1892 for res in results1893 ]1894 llm_output = self._combine_llm_outputs([res.llm_output for res in results]) # type: ignore[union-attr]1895 generations = [res.generations for res in results] # type: ignore[union-attr]1896 output = LLMResult(generations=generations, llm_output=llm_output)1897 await asyncio.gather(1898 *[1899 run_manager.on_llm_end(flattened_output)1900 for run_manager, flattened_output in zip(1901 run_managers, flattened_outputs, strict=False1902 )1903 ]1904 )1905 if run_managers:1906 output.run = [1907 RunInfo(run_id=run_manager.run_id) for run_manager in run_managers1908 ]1909 return output19101911 @override1912 def generate_prompt(1913 self,1914 prompts: list[PromptValue],1915 stop: list[str] | None = None,1916 callbacks: Callbacks = None,1917 **kwargs: Any,1918 ) -> LLMResult:1919 prompt_messages = [p.to_messages() for p in prompts]1920 return self.generate(prompt_messages, stop=stop, callbacks=callbacks, **kwargs)19211922 @override1923 async def agenerate_prompt(1924 self,1925 prompts: list[PromptValue],1926 stop: list[str] | None = None,1927 callbacks: Callbacks = None,1928 **kwargs: Any,1929 ) -> LLMResult:1930 prompt_messages = [p.to_messages() for p in prompts]1931 return await self.agenerate(1932 prompt_messages, stop=stop, callbacks=callbacks, **kwargs1933 )19341935 def _generate_with_cache(1936 self,1937 messages: list[BaseMessage],1938 stop: list[str] | None = None,1939 run_manager: CallbackManagerForLLMRun | None = None,1940 **kwargs: Any,1941 ) -> ChatResult:1942 llm_cache = self.cache if isinstance(self.cache, BaseCache) else get_llm_cache()1943 # We should check the cache unless it's explicitly set to False1944 # A None cache means we should use the default global cache1945 # if it's configured.1946 check_cache = self.cache is not False1947 if check_cache:1948 if llm_cache is not None:1949 llm_string = self._get_llm_string(stop=stop, **kwargs)1950 normalized_messages = [1951 (1952 msg.model_copy(update={"id": None})1953 if getattr(msg, "id", None) is not None1954 else msg1955 )1956 for msg in messages1957 ]1958 prompt = dumps(normalized_messages)1959 cache_val = llm_cache.lookup(prompt, llm_string)1960 if isinstance(cache_val, list):1961 converted_generations = self._convert_cached_generations(cache_val)1962 self._replay_v2_events_for_cache_hit(1963 converted_generations,1964 run_manager=run_manager,1965 **kwargs,1966 )1967 return ChatResult(generations=converted_generations)1968 elif self.cache is None:1969 pass1970 else:1971 msg = "Asked to cache, but no cache found at `langchain.cache`."1972 raise ValueError(msg)19731974 # Apply the rate limiter after checking the cache, since1975 # we usually don't want to rate limit cache lookups, but1976 # we do want to rate limit API requests.1977 if self.rate_limiter:1978 self.rate_limiter.acquire(blocking=True)19791980 # v2 streaming: preferred over v1 when any attached handler opts in via1981 # `_V2StreamingCallbackHandler`. Drives the protocol event generator1982 # (native or `_stream` compat bridge) through the shared helper so1983 # `on_stream_event` fires per event, then returns a normal `ChatResult`1984 # so caching / `on_llm_end` stay on the existing generate path.1985 if self._should_use_protocol_streaming(1986 async_api=False,1987 run_manager=run_manager,1988 **kwargs,1989 ):1990 stream_accum = ChatModelStream(1991 message_id=(1992 f"{LC_ID_PREFIX}-{run_manager.run_id}" if run_manager else None1993 )1994 )1995 assert run_manager is not None # noqa: S1011996 for _event in self._iter_v2_events(1997 messages,1998 run_manager=run_manager,1999 stream=stream_accum,2000 stop=stop,
Findings
✓ No findings reported for this file.