1"""Chat models for conversational AI."""23from __future__ import annotations45import asyncio6import builtins # noqa: TC003 # runtime-evaluated; subclass `dict()` shadows the builtin7import contextlib8import inspect9import json10from abc import ABC, abstractmethod11from collections.abc import AsyncIterator, Callable, Iterator, Sequence12from functools import cached_property13from operator import itemgetter14from typing import TYPE_CHECKING, Any, Literal, cast, overload1516from langchain_protocol.protocol import MessageFinishData17from pydantic import BaseModel, ConfigDict, Field, model_validator18from typing_extensions import Self, override1920from langchain_core._api import beta, deprecated, suppress_langchain_deprecation_warning21from langchain_core.caches import BaseCache22from langchain_core.callbacks import (23 AsyncCallbackManager,24 AsyncCallbackManagerForLLMRun,25 CallbackManager,26 CallbackManagerForLLMRun,27 Callbacks,28)29from langchain_core.globals import get_llm_cache30from langchain_core.language_models._compat_bridge import (31 achunks_to_events,32 amessage_to_events,33 chunks_to_events,34 message_to_events,35)36from langchain_core.language_models._utils import (37 _filter_invocation_params_for_tracing,38 _normalize_messages,39 _update_message_content_to_blocks,40)41from langchain_core.language_models.base import (42 BaseLanguageModel,43 LangSmithParams,44 LanguageModelInput,45)46from langchain_core.language_models.chat_model_stream import (47 AsyncChatModelStream,48 ChatModelStream,49)50from langchain_core.language_models.model_profile import (51 ModelProfile,52 _warn_unknown_profile_keys,53)54from langchain_core.load import dumpd, dumps55from langchain_core.messages import (56 AIMessage,57 AIMessageChunk,58 AnyMessage,59 BaseMessage,60 convert_to_messages,61 is_data_content_block,62 message_chunk_to_message,63)64from langchain_core.messages import content as types65from langchain_core.messages.block_translators.openai import (66 convert_to_openai_image_block,67)68from langchain_core.output_parsers.openai_tools import (69 JsonOutputKeyToolsParser,70 JsonOutputToolsParser,71 PydanticToolsParser,72)73from langchain_core.outputs import (74 ChatGeneration,75 ChatGenerationChunk,76 ChatResult,77 Generation,78 LLMResult,79 RunInfo,80)81from langchain_core.outputs.chat_generation import merge_chat_generation_chunks82from langchain_core.prompt_values import ChatPromptValue, PromptValue, StringPromptValue83from langchain_core.rate_limiters import BaseRateLimiter84from langchain_core.runnables import RunnableBinding, RunnableMap, RunnablePassthrough85from langchain_core.runnables.config import ensure_config, run_in_executor86from langchain_core.tracers._streaming import (87 _StreamingCallbackHandler,88 _V2StreamingCallbackHandler,89)90from langchain_core.utils._gateway import (91 GATEWAY_METADATA_RESPONSE_KEY,92 _parse_gateway_metadata,93)94from langchain_core.utils.function_calling import (95 convert_to_json_schema,96 convert_to_openai_tool,97)98from langchain_core.utils.pydantic import is_basemodel_subclass99from langchain_core.utils.utils import LC_ID_PREFIX, from_env100101if TYPE_CHECKING:102 import uuid103 from collections.abc import Awaitable104105 from langchain_protocol.protocol import MessagesData106107 from langchain_core.runnables import Runnable, RunnableConfig108 from langchain_core.runnables.schema import StreamEvent109 from langchain_core.tools import BaseTool110111112def _generate_response_from_error(error: BaseException) -> list[ChatGeneration]:113 if hasattr(error, "response"):114 response = error.response115 metadata: dict[str, Any] = {}116 generation_info: dict[str, Any] = {}117 if hasattr(response, "json"):118 try:119 metadata["body"] = response.json()120 except Exception:121 try:122 metadata["body"] = getattr(response, "text", None)123 except Exception:124 metadata["body"] = None125 if hasattr(response, "headers"):126 try:127 headers = response.headers128 metadata["headers"] = dict(headers)129 gateway_metadata = _parse_gateway_metadata(headers)130 if gateway_metadata is not None:131 generation_info[GATEWAY_METADATA_RESPONSE_KEY] = gateway_metadata132 except Exception:133 metadata["headers"] = None134 if hasattr(response, "status_code"):135 metadata["status_code"] = response.status_code136 if hasattr(error, "request_id"):137 metadata["request_id"] = error.request_id138 generations = [139 ChatGeneration(140 message=AIMessage(content="", response_metadata=metadata),141 generation_info=generation_info or None,142 )143 ]144 else:145 generations = []146147 return generations148149150def _format_for_tracing(messages: list[BaseMessage]) -> list[BaseMessage]:151 """Format messages for tracing in `on_chat_model_start`.152153 - Update image content blocks to OpenAI Chat Completions format (backward154 compatibility).155 - Add `type` key to content blocks that have a single key.156157 Args:158 messages: List of messages to format.159160 Returns:161 List of messages formatted for tracing.162163 """164 messages_to_trace = []165 for message in messages:166 message_to_trace = message167 if isinstance(message.content, list):168 for idx, block in enumerate(message.content):169 if isinstance(block, dict):170 # Update image content blocks to OpenAI # Chat Completions format.171 if (172 block.get("type") == "image"173 and is_data_content_block(block)174 and not ("file_id" in block or block.get("source_type") == "id")175 ):176 if message_to_trace is message:177 # Shallow copy178 message_to_trace = message.model_copy()179 message_to_trace.content = list(message_to_trace.content)180181 message_to_trace.content[idx] = ( # type: ignore[index] # mypy confused by .model_copy182 convert_to_openai_image_block(block)183 )184 elif (185 block.get("type") == "file"186 and is_data_content_block(block) # v0 (image/audio/file) or v1187 and "base64" in block188 # Backward compat: convert v1 base64 blocks to v0189 ):190 if message_to_trace is message:191 # Shallow copy192 message_to_trace = message.model_copy()193 message_to_trace.content = list(message_to_trace.content)194195 message_to_trace.content[idx] = { # type: ignore[index]196 **{k: v for k, v in block.items() if k != "base64"},197 "data": block["base64"],198 "source_type": "base64",199 }200 elif len(block) == 1 and "type" not in block:201 # Tracing assumes all content blocks have a "type" key. Here202 # we add this key if it is missing, and there's an obvious203 # choice for the type (e.g., a single key in the block).204 if message_to_trace is message:205 # Shallow copy206 message_to_trace = message.model_copy()207 message_to_trace.content = list(message_to_trace.content)208 key = next(iter(block))209 message_to_trace.content[idx] = { # type: ignore[index]210 "type": key,211 key: block[key],212 }213 messages_to_trace.append(message_to_trace)214215 return messages_to_trace216217218def generate_from_stream(stream: Iterator[ChatGenerationChunk]) -> ChatResult:219 """Generate from a stream.220221 Args:222 stream: Iterator of `ChatGenerationChunk`.223224 Raises:225 ValueError: If no generations are found in the stream.226227 Returns:228 Chat result.229230 """231 generation = next(stream, None)232 if generation:233 generation += list(stream)234 if generation is None:235 msg = "No generations found in stream."236 raise ValueError(msg)237 return ChatResult(238 generations=[239 ChatGeneration(240 message=message_chunk_to_message(generation.message),241 generation_info=generation.generation_info,242 )243 ]244 )245246247async def agenerate_from_stream(248 stream: AsyncIterator[ChatGenerationChunk],249) -> ChatResult:250 """Async generate from a stream.251252 Args:253 stream: AsyncIterator of `ChatGenerationChunk`.254255 Returns:256 Chat result.257258 """259 chunks = [chunk async for chunk in stream]260 return await run_in_executor(None, generate_from_stream, iter(chunks))261262263def _format_ls_structured_output(264 ls_structured_output_format: dict[str, Any] | None,265) -> dict[str, Any]:266 if ls_structured_output_format:267 try:268 ls_structured_output_format_dict = {269 "ls_structured_output_format": {270 "kwargs": ls_structured_output_format.get("kwargs", {}),271 "schema": convert_to_json_schema(272 ls_structured_output_format["schema"]273 ),274 }275 }276 except ValueError:277 ls_structured_output_format_dict = {}278 else:279 ls_structured_output_format_dict = {}280281 return ls_structured_output_format_dict282283284class BaseChatModel(BaseLanguageModel[AIMessage], ABC):285 r"""Base class for chat models.286287 Key imperative methods:288 Methods that actually call the underlying model.289290 This table provides a brief overview of the main imperative methods. Please see the base `Runnable` reference for full documentation.291292 | Method | Input | Output | Description |293 | ---------------------- | ------------------------------------------------------------ | ---------------------------------------------------------- | -------------------------------------------------------------------------------- |294 | `invoke` | `str` \| `list[dict | tuple | BaseMessage]` \| `PromptValue` | `BaseMessage` | A single chat model call. |295 | `ainvoke` | `'''` | `BaseMessage` | Defaults to running `invoke` in an async executor. |296 | `stream` | `'''` | `Iterator[BaseMessageChunk]` | Defaults to yielding output of `invoke`. |297 | `astream` | `'''` | `AsyncIterator[BaseMessageChunk]` | Defaults to yielding output of `ainvoke`. |298 | `astream_events` | `'''` | `AsyncIterator[StreamEvent]` | Event types: `on_chat_model_start`, `on_chat_model_stream`, `on_chat_model_end`. |299 | `batch` | `list[''']` | `list[BaseMessage]` | Defaults to running `invoke` in concurrent threads. |300 | `abatch` | `list[''']` | `list[BaseMessage]` | Defaults to running `ainvoke` in concurrent threads. |301 | `batch_as_completed` | `list[''']` | `Iterator[tuple[int, Union[BaseMessage, Exception]]]` | Defaults to running `invoke` in concurrent threads. |302 | `abatch_as_completed` | `list[''']` | `AsyncIterator[tuple[int, Union[BaseMessage, Exception]]]` | Defaults to running `ainvoke` in concurrent threads. |303304 Key declarative methods:305 Methods for creating another `Runnable` using the chat model.306307 This table provides a brief overview of the main declarative methods. Please see the reference for each method for full documentation.308309 | Method | Description |310 | ---------------------------- | ------------------------------------------------------------------------------------------ |311 | `bind_tools` | Create chat model that can call tools. |312 | `with_structured_output` | Create wrapper that structures model output using schema. |313 | `with_retry` | Create wrapper that retries model calls on failure. |314 | `with_fallbacks` | Create wrapper that falls back to other models on failure. |315 | `configurable_fields` | Specify init args of the model that can be configured at runtime via the `RunnableConfig`. |316 | `configurable_alternatives` | Specify alternative models which can be swapped in at runtime via the `RunnableConfig`. |317318 Creating custom chat model:319 Custom chat model implementations should inherit from this class.320 Please reference the table below for information about which321 methods and properties are required or optional for implementations.322323 | Method/Property | Description | Required |324 | -------------------------------- | ------------------------------------------------------------------ | ----------------- |325 | `_generate` | Use to generate a chat result from a prompt | Required |326 | `_llm_type` (property) | Used to uniquely identify the type of the model. Used for logging. | Required |327 | `_identifying_params` (property) | Represent model parameterization for tracing purposes. | Optional |328 | `_stream` | Use to implement streaming | Optional |329 | `_agenerate` | Use to implement a native async method | Optional |330 | `_astream` | Use to implement async version of `_stream` | Optional |331332 """ # noqa: E501333334 rate_limiter: BaseRateLimiter | None = Field(default=None, exclude=True)335 "An optional rate limiter to use for limiting the number of requests."336337 disable_streaming: bool | Literal["tool_calling"] = False338 """Whether to disable streaming for this model.339340 If streaming is bypassed, then `stream`/`astream`/`astream_events` will341 defer to `invoke`/`ainvoke`.342343 - If `True`, will always bypass streaming case.344 - If `'tool_calling'`, will bypass streaming case only when the model is called345 with a `tools` keyword argument. In other words, LangChain will automatically346 switch to non-streaming behavior (`invoke`) only when the tools argument is347 provided. This offers the best of both worlds.348 - If `False` (Default), will always use streaming case if available.349350 The main reason for this flag is that code might be written using `stream` and351 a user may want to swap out a given model for another model whose implementation352 does not properly support streaming.353 """354355 output_version: str | None = Field(356 default_factory=from_env("LC_OUTPUT_VERSION", default=None)357 )358 """Version of `AIMessage` output format to store in message content.359360 `AIMessage.content_blocks` will lazily parse the contents of `content` into a361 standard format. This flag can be used to additionally store the standard format362 in message content, e.g., for serialization purposes.363364 Supported values:365366 - `'v0'`: provider-specific format in content (can lazily-parse with367 `content_blocks`)368 - `'v1'`: standardized format in content (consistent with `content_blocks`)369370 Partner packages (e.g.,371 [`langchain-openai`](https://pypi.org/project/langchain-openai)) can also use this372 field to roll out new content formats in a backward-compatible way.373374 !!! version-added "Added in `langchain-core` 1.0.0"375376 """377378 profile: ModelProfile | None = Field(default=None, exclude=True)379 """Profile detailing model capabilities.380381 !!! warning "Beta feature"382383 This is a beta feature. The format of model profiles is subject to change.384385 If not specified, automatically loaded from the provider package on initialization386 if data is available.387388 Example profile data includes context window sizes, supported modalities, or support389 for tool calling, structured output, and other features.390391 !!! version-added "Added in `langchain-core` 1.1.0"392 """393394 model_config = ConfigDict(395 arbitrary_types_allowed=True,396 )397398 def _resolve_model_profile(self) -> ModelProfile | None:399 """Return the default model profile, or `None` if unavailable.400401 Override this in subclasses instead of `_set_model_profile`. The base402 validator calls it automatically and handles assignment. This avoids403 coupling partner code to Pydantic validator mechanics.404405 Each partner needs its own override because things can vary per-partner,406 such as the attribute that identifies the model (e.g., `model`,407 `model_name`, `model_id`, `deployment_name`) and the partner-local408 `_get_default_model_profile` function that reads from each partner's own409 profile data.410 """411 # TODO: consider adding a `_model_identifier` property on BaseChatModel412 # to standardize how partners identify their model, which could allow a413 # default implementation here that calls a shared414 # profile-loading mechanism.415 return None416417 @model_validator(mode="after")418 def _set_model_profile(self) -> Self:419 """Populate `profile` from `_resolve_model_profile` if not provided.420421 Partners should override `_resolve_model_profile` rather than this422 validator. Overriding this with a new `@model_validator` replaces the423 base validator (Pydantic v2 behavior), bypassing the standard resolution424 path. A plain method override does not prevent the base validator from425 running.426 """427 if self.profile is None:428 # Suppress errors from partner overrides (e.g., missing profile429 # files, broken imports) so model construction never fails over an430 # optional field.431 with contextlib.suppress(Exception):432 self.profile = self._resolve_model_profile()433 return self434435 # NOTE: _check_profile_keys must be defined AFTER _set_model_profile.436 # Pydantic v2 runs mode="after" validators in definition order.437 @model_validator(mode="after")438 def _check_profile_keys(self) -> Self:439 """Warn on unrecognized profile keys."""440 # isinstance guard: ModelProfile is a TypedDict (always a dict), but441 # protects against unexpected types from partner overrides.442 if self.profile and isinstance(self.profile, dict):443 _warn_unknown_profile_keys(self.profile)444 return self445446 @cached_property447 def _serialized(self) -> builtins.dict[str, Any]:448 # self is always a Serializable object in this case, thus the result is449 # guaranteed to be a dict since dumpd uses the default callback, which uses450 # obj.to_json which always returns TypedDict subclasses451 return cast("builtins.dict[str, Any]", dumpd(self))452453 # --- Runnable methods ---454455 @property456 @override457 def OutputType(self) -> Any:458 """Get the output type for this `Runnable`."""459 return AnyMessage460461 def _convert_input(self, model_input: LanguageModelInput) -> PromptValue:462 if isinstance(model_input, PromptValue):463 return model_input464 if isinstance(model_input, str):465 return StringPromptValue(text=model_input)466 if isinstance(model_input, Sequence):467 return ChatPromptValue(messages=convert_to_messages(model_input))468 msg = ( # type: ignore[unreachable]469 f"Invalid input type {type(model_input)}. "470 "Must be a PromptValue, str, or list of BaseMessages."471 )472 raise ValueError(msg)473474 @override475 def invoke(476 self,477 input: LanguageModelInput,478 config: RunnableConfig | None = None,479 *,480 stop: list[str] | None = None,481 **kwargs: Any,482 ) -> AIMessage:483 config = ensure_config(config)484 return cast(485 "AIMessage",486 cast(487 "ChatGeneration",488 self.generate_prompt(489 [self._convert_input(input)],490 stop=stop,491 callbacks=config.get("callbacks"),492 tags=config.get("tags"),493 metadata=config.get("metadata"),494 run_name=config.get("run_name"),495 run_id=config.pop("run_id", None),496 **kwargs,497 ).generations[0][0],498 ).message,499 )500501 @override502 async def ainvoke(503 self,504 input: LanguageModelInput,505 config: RunnableConfig | None = None,506 *,507 stop: list[str] | None = None,508 **kwargs: Any,509 ) -> AIMessage:510 config = ensure_config(config)511 llm_result = await self.agenerate_prompt(512 [self._convert_input(input)],513 stop=stop,514 callbacks=config.get("callbacks"),515 tags=config.get("tags"),516 metadata=config.get("metadata"),517 run_name=config.get("run_name"),518 run_id=config.pop("run_id", None),519 **kwargs,520 )521 return cast(522 "AIMessage", cast("ChatGeneration", llm_result.generations[0][0]).message523 )524525 def _streaming_disabled(self, **kwargs: Any) -> bool:526 """Return whether streaming is hard-disabled for this call.527528 Shared opt-outs honored by both `_should_stream` and529 `_should_use_protocol_streaming` — these override any affirmative trigger530 (attached handler, `stream=True`, etc.):531532 - `self.disable_streaming is True`533 - `self.disable_streaming == "tool_calling"` with `tools` passed534 - `stream=<falsy>` in call kwargs535 - `self.streaming is False` on the instance536 """537 if self.disable_streaming is True:538 return True539 # We assume tools are passed in via "tools" kwarg in all models.540 if self.disable_streaming == "tool_calling" and kwargs.get("tools"):541 return True542 if "stream" in kwargs and not kwargs["stream"]:543 return True544 return (545 "streaming" in self.model_fields_set546 and getattr(self, "streaming", None) is False547 )548549 def _should_stream(550 self,551 *,552 async_api: bool,553 run_manager: CallbackManagerForLLMRun554 | AsyncCallbackManagerForLLMRun555 | None = None,556 **kwargs: Any,557 ) -> bool:558 """Determine if a given model call should hit the streaming API."""559 sync_not_implemented = type(self)._stream == BaseChatModel._stream # noqa: SLF001560 async_not_implemented = type(self)._astream == BaseChatModel._astream # noqa: SLF001561562 # Check if streaming is implemented.563 if (not async_api) and sync_not_implemented:564 return False565 # Note, since async falls back to sync we check both here.566 if async_api and async_not_implemented and sync_not_implemented:567 return False568569 if self._streaming_disabled(**kwargs):570 return False571572 # Affirmative: explicit `stream=<truthy>` kwarg.573 if kwargs.get("stream"):574 return True575576 # Affirmative: instance-level `streaming=True` attribute.577 if (578 "streaming" in self.model_fields_set579 and getattr(self, "streaming", None) is True580 ):581 return True582583 # Affirmative: a v1 streaming callback handler is attached.584 handlers = run_manager.handlers if run_manager else []585 return any(isinstance(h, _StreamingCallbackHandler) for h in handlers)586587 def _should_use_protocol_streaming(588 self,589 *,590 async_api: bool,591 run_manager: CallbackManagerForLLMRun592 | AsyncCallbackManagerForLLMRun593 | None = None,594 **kwargs: Any,595 ) -> bool:596 """Determine whether an invoke should route through the v2 event path.597598 Runs alongside `_should_stream` inside `_generate_with_cache` /599 `_agenerate_with_cache` — after the run manager is open — and600 wins over the v1 streaming branch when a handler has declared601 itself a `_V2StreamingCallbackHandler`. Parallel to602 `_should_stream` rather than a delegation — v1 and v2 have603 disjoint affirmative triggers.604605 Args:606 async_api: Whether the caller is on the async path.607 run_manager: The active LLM run manager.608 **kwargs: Call kwargs; inspected for `disable_streaming`609 semantics and an explicit `stream=False` override.610611 Returns:612 `True` if any attached handler inherits613 `_V2StreamingCallbackHandler` and the model can drive the v2614 event generator (natively or via the `_stream` compat615 bridge).616 """617 # Opt-in: only route through v2 when a v2 handler is attached.618 handlers = run_manager.handlers if run_manager else []619 if not any(isinstance(h, _V2StreamingCallbackHandler) for h in handlers):620 return False621622 # Need a source of v2 events on the requested flavor. A native623 # `_(a)stream_chat_model_events` hook bypasses the bridge;624 # otherwise the bridge wraps `_stream` / `_astream`. Async can625 # fall back to sync.626 #627 # `cls._stream is not BaseChatModel._stream` is an identity628 # check for "subclass overrode `_stream`" — same pattern as629 # `_should_stream`.630 cls = type(self)631 has_native_sync = getattr(cls, "_stream_chat_model_events", None) is not None632 has_native_async = getattr(cls, "_astream_chat_model_events", None) is not None633 overrides_sync = cls._stream is not BaseChatModel._stream634 overrides_async = cls._astream is not BaseChatModel._astream635 has_sync_source = has_native_sync or overrides_sync636 has_async_source = has_native_async or overrides_async637 has_source = (638 (has_sync_source or has_async_source) if async_api else has_sync_source639 )640 if not has_source:641 return False642643 return not self._streaming_disabled(**kwargs)644645 def _iter_v2_events(646 self,647 messages: list[BaseMessage],648 *,649 run_manager: CallbackManagerForLLMRun,650 stream: ChatModelStream,651 stop: list[str] | None = None,652 **kwargs: Any,653 ) -> Iterator[MessagesData]:654 """Drive the v2 event generator with per-event dispatch.655656 Shared between the `stream_events(version="v3")` pump and the657 invoke-time v2 branch in `_generate_with_cache`. Picks the native658 `_stream_chat_model_events` hook when the subclass provides one,659 else bridges `_stream` chunks via `chunks_to_events`. Each event660 is dispatched into `stream` and fired as `on_stream_event` on661 the run manager. Run-lifecycle callbacks662 (`on_chat_model_start` / `on_llm_end` / `on_llm_error`) and663 rate-limiter acquisition are the caller's responsibility.664665 Args:666 messages: Normalized input messages.667 run_manager: Active LLM run manager; receives668 `on_stream_event` per event.669 stream: Accumulator owned by the caller; receives each670 event via `stream.dispatch`.671 stop: Optional stop sequences.672 **kwargs: Forwarded to the event producer.673674 Yields:675 Each protocol event produced by the model.676 """677 native = cast(678 "Callable[..., Iterator[MessagesData]] | None",679 getattr(self, "_stream_chat_model_events", None),680 )681 if native is not None:682 event_iter: Iterator[MessagesData] = native(683 messages, stop=stop, run_manager=run_manager, **kwargs684 )685 else:686 event_iter = chunks_to_events(687 self._stream(messages, stop=stop, run_manager=run_manager, **kwargs),688 message_id=stream.message_id,689 )690 for event in event_iter:691 stream.dispatch(event)692 run_manager.on_stream_event(event)693 yield event694695 async def _aiter_v2_events(696 self,697 messages: list[BaseMessage],698 *,699 run_manager: AsyncCallbackManagerForLLMRun,700 stream: AsyncChatModelStream,701 stop: list[str] | None = None,702 **kwargs: Any,703 ) -> AsyncIterator[MessagesData]:704 """Async counterpart to `_iter_v2_events`.705706 See `_iter_v2_events` for the shared contract.707 """708 native = cast(709 "Callable[..., AsyncIterator[MessagesData]] | None",710 getattr(self, "_astream_chat_model_events", None),711 )712 if native is not None:713 event_iter: AsyncIterator[MessagesData] = native(714 messages, stop=stop, run_manager=run_manager, **kwargs715 )716 else:717 event_iter = achunks_to_events(718 self._astream(messages, stop=stop, run_manager=run_manager, **kwargs),719 message_id=stream.message_id,720 )721 async for event in event_iter:722 stream.dispatch(event)723 await run_manager.on_stream_event(event)724 yield event725726 @override727 def stream(728 self,729 input: LanguageModelInput,730 config: RunnableConfig | None = None,731 *,732 stop: list[str] | None = None,733 **kwargs: Any,734 ) -> Iterator[AIMessageChunk]:735 if not self._should_stream(async_api=False, **{**kwargs, "stream": True}):736 # Model doesn't implement streaming, so use default implementation737 yield cast(738 "AIMessageChunk",739 self.invoke(input, config=config, stop=stop, **kwargs),740 )741 else:742 config = ensure_config(config)743 messages = self._convert_input(input).to_messages()744 ls_structured_output_format = kwargs.pop(745 "ls_structured_output_format", None746 ) or kwargs.pop("structured_output_format", None)747 ls_structured_output_format_dict = _format_ls_structured_output(748 ls_structured_output_format749 )750751 params = self._get_invocation_params(stop=stop, **kwargs)752 options = {753 "stop": stop,754 **{key: params[key] for key in kwargs if key in params},755 **ls_structured_output_format_dict,756 }757 inheritable_metadata = {758 **(config.get("metadata") or {}),759 **self._get_ls_params_with_defaults(stop=stop, **kwargs),760 }761 callback_manager = CallbackManager.configure(762 config.get("callbacks"),763 self.callbacks,764 self.verbose,765 config.get("tags"),766 self.tags,767 inheritable_metadata,768 self.metadata,769 langsmith_inheritable_metadata=_filter_invocation_params_for_tracing(770 params771 ),772 )773 (run_manager,) = callback_manager.on_chat_model_start(774 self._serialized,775 [_format_for_tracing(messages)],776 invocation_params=params,777 options=options,778 name=config.get("run_name"),779 run_id=config.pop("run_id", None),780 batch_size=1,781 )782783 chunks: list[ChatGenerationChunk] = []784785 if self.rate_limiter:786 self.rate_limiter.acquire(blocking=True)787788 try:789 input_messages = _normalize_messages(messages)790 run_id = "-".join((LC_ID_PREFIX, str(run_manager.run_id)))791 yielded = False792 index = -1793 index_type = ""794 for chunk in self._stream(input_messages, stop=stop, **kwargs):795 if chunk.message.id is None:796 chunk.message.id = run_id797 chunk.message.response_metadata = _gen_info_and_msg_metadata(chunk)798 if self.output_version == "v1":799 # Overwrite .content with .content_blocks800 chunk.message = _update_message_content_to_blocks(801 chunk.message, "v1"802 )803 for block in cast(804 "list[types.ContentBlock]", chunk.message.content805 ):806 if block["type"] != index_type:807 index_type = block["type"]808 index += 1809 if "index" not in block:810 block["index"] = index811 run_manager.on_llm_new_token(chunk.message.content, chunk=chunk)812 chunks.append(chunk)813 yield cast("AIMessageChunk", chunk.message)814 yielded = True815816 # Yield a final empty chunk with chunk_position="last" if not yet817 # yielded818 if (819 yielded820 and isinstance(chunk.message, AIMessageChunk)821 and not chunk.message.chunk_position822 ):823 empty_content: str | list[str | dict[str, Any]] = (824 "" if isinstance(chunk.message.content, str) else []825 )826 msg_chunk = AIMessageChunk(827 content=empty_content, chunk_position="last", id=run_id828 )829 run_manager.on_llm_new_token(830 "", chunk=ChatGenerationChunk(message=msg_chunk)831 )832 yield msg_chunk833 except BaseException as e:834 generations_with_error_metadata = _generate_response_from_error(e)835 chat_generation_chunk = merge_chat_generation_chunks(chunks)836 if chat_generation_chunk:837 generations = [838 [chat_generation_chunk],839 generations_with_error_metadata,840 ]841 else:842 generations = [generations_with_error_metadata]843 run_manager.on_llm_error(844 e,845 response=LLMResult(generations=generations),846 )847 raise848849 generation = merge_chat_generation_chunks(chunks)850 if generation is None:851 err = ValueError("No generation chunks were returned")852 run_manager.on_llm_error(err, response=LLMResult(generations=[]))853 raise err854855 run_manager.on_llm_end(LLMResult(generations=[[generation]]))856857 @override858 async def astream(859 self,860 input: LanguageModelInput,861 config: RunnableConfig | None = None,862 *,863 stop: list[str] | None = None,864 **kwargs: Any,865 ) -> AsyncIterator[AIMessageChunk]:866 if not self._should_stream(async_api=True, **{**kwargs, "stream": True}):867 # No async or sync stream is implemented, so fall back to ainvoke868 yield cast(869 "AIMessageChunk",870 await self.ainvoke(input, config=config, stop=stop, **kwargs),871 )872 return873874 config = ensure_config(config)875 messages = self._convert_input(input).to_messages()876877 ls_structured_output_format = kwargs.pop(878 "ls_structured_output_format", None879 ) or kwargs.pop("structured_output_format", None)880 ls_structured_output_format_dict = _format_ls_structured_output(881 ls_structured_output_format882 )883884 params = self._get_invocation_params(stop=stop, **kwargs)885 options = {886 "stop": stop,887 **{key: params[key] for key in kwargs if key in params},888 **ls_structured_output_format_dict,889 }890 inheritable_metadata = {891 **(config.get("metadata") or {}),892 **self._get_ls_params_with_defaults(stop=stop, **kwargs),893 }894 callback_manager = AsyncCallbackManager.configure(895 config.get("callbacks"),896 self.callbacks,897 self.verbose,898 config.get("tags"),899 self.tags,900 inheritable_metadata,901 self.metadata,902 langsmith_inheritable_metadata=_filter_invocation_params_for_tracing(903 params904 ),905 )906 (run_manager,) = await callback_manager.on_chat_model_start(907 self._serialized,908 [_format_for_tracing(messages)],909 invocation_params=params,910 options=options,911 name=config.get("run_name"),912 run_id=config.pop("run_id", None),913 batch_size=1,914 )915916 if self.rate_limiter:917 await self.rate_limiter.aacquire(blocking=True)918919 chunks: list[ChatGenerationChunk] = []920921 try:922 input_messages = _normalize_messages(messages)923 run_id = "-".join((LC_ID_PREFIX, str(run_manager.run_id)))924 yielded = False925 index = -1926 index_type = ""927 async for chunk in self._astream(928 input_messages,929 stop=stop,930 **kwargs,931 ):932 if chunk.message.id is None:933 chunk.message.id = run_id934 chunk.message.response_metadata = _gen_info_and_msg_metadata(chunk)935 if self.output_version == "v1":936 # Overwrite .content with .content_blocks937 chunk.message = _update_message_content_to_blocks(938 chunk.message, "v1"939 )940 for block in cast(941 "list[types.ContentBlock]", chunk.message.content942 ):943 if block["type"] != index_type:944 index_type = block["type"]945 index += 1946 if "index" not in block:947 block["index"] = index948 await run_manager.on_llm_new_token(chunk.message.content, chunk=chunk)949 chunks.append(chunk)950 yield cast("AIMessageChunk", chunk.message)951 yielded = True952953 # Yield a final empty chunk with chunk_position="last" if not yet yielded954 if (955 yielded956 and isinstance(chunk.message, AIMessageChunk)957 and not chunk.message.chunk_position958 ):959 empty_content: str | list[str | dict[str, Any]] = (960 "" if isinstance(chunk.message.content, str) else []961 )962 msg_chunk = AIMessageChunk(963 content=empty_content, chunk_position="last", id=run_id964 )965 await run_manager.on_llm_new_token(966 "", chunk=ChatGenerationChunk(message=msg_chunk)967 )968 yield msg_chunk969 except BaseException as e:970 generations_with_error_metadata = _generate_response_from_error(e)971 chat_generation_chunk = merge_chat_generation_chunks(chunks)972 if chat_generation_chunk:973 generations = [[chat_generation_chunk], generations_with_error_metadata]974 else:975 generations = [generations_with_error_metadata]976 await run_manager.on_llm_error(977 e,978 response=LLMResult(generations=generations),979 )980 raise981982 generation = merge_chat_generation_chunks(chunks)983 if not generation:984 err = ValueError("No generation chunks were returned")985 await run_manager.on_llm_error(err, response=LLMResult(generations=[]))986 raise err987988 await run_manager.on_llm_end(989 LLMResult(generations=[[generation]]),990 )991992 # --- stream_events v3 ---993994 @beta()995 def _chat_model_stream_v3(996 self,997 input: LanguageModelInput,998 config: RunnableConfig | None = None,999 *,1000 stop: list[str] | None = None,1001 **kwargs: Any,1002 ) -> ChatModelStream:1003 """Internal v3 sync streaming implementation.10041005 Public entry point: `stream_events(version='v3')`.1006 """1007 config = ensure_config(config)1008 messages = self._convert_input(input).to_messages()1009 input_messages = _normalize_messages(messages)10101011 # Strip tracing-only kwargs before forwarding to `_stream` — matches1012 # `stream()` / `astream()`. Provider clients reject unknown kwargs,1013 # so `.with_structured_output().stream_events(version="v3", ...)`1014 # and any other binding that carries `ls_structured_output_format`1015 # / `structured_output_format` would raise without this pop.1016 ls_structured_output_format = kwargs.pop(1017 "ls_structured_output_format", None1018 ) or kwargs.pop("structured_output_format", None)1019 ls_structured_output_format_dict = _format_ls_structured_output(1020 ls_structured_output_format1021 )10221023 params = self._get_invocation_params(stop=stop, **kwargs)1024 options = {1025 "stop": stop,1026 **{key: params[key] for key in kwargs if key in params},1027 **ls_structured_output_format_dict,1028 }1029 inheritable_metadata = {1030 **(config.get("metadata") or {}),1031 **self._get_ls_params_with_defaults(stop=stop, **kwargs),1032 }1033 callback_manager = CallbackManager.configure(1034 config.get("callbacks"),1035 self.callbacks,1036 self.verbose,1037 config.get("tags"),1038 self.tags,1039 inheritable_metadata,1040 self.metadata,1041 langsmith_inheritable_metadata=_filter_invocation_params_for_tracing(1042 params1043 ),1044 )1045 stream = ChatModelStream()1046 run_manager: CallbackManagerForLLMRun | None = None1047 event_iter_ref: Iterator[MessagesData] | None = None1048 rate_limiter_acquired = self.rate_limiter is None1049 run_name = config.get("run_name")1050 run_id = config.pop("run_id", None)10511052 def ensure_started() -> None:1053 nonlocal event_iter_ref, run_manager1054 if event_iter_ref is not None:1055 return10561057 (run_manager,) = callback_manager.on_chat_model_start(1058 self._serialized,1059 [_format_for_tracing(messages)],1060 invocation_params=params,1061 options=options,1062 name=run_name,1063 run_id=run_id,1064 batch_size=1,1065 )1066 stream.set_message_id("-".join((LC_ID_PREFIX, str(run_manager.run_id))))1067 event_iter_ref = iter(1068 self._iter_v2_events(1069 input_messages,1070 run_manager=run_manager,1071 stream=stream,1072 stop=stop,1073 **kwargs,1074 )1075 )10761077 def pump_one() -> bool:1078 nonlocal rate_limiter_acquired1079 ensure_started()1080 if not rate_limiter_acquired:1081 assert self.rate_limiter is not None # noqa: S1011082 self.rate_limiter.acquire(blocking=True)1083 rate_limiter_acquired = True1084 assert event_iter_ref is not None # noqa: S1011085 assert run_manager is not None # noqa: S1011086 try:1087 next(event_iter_ref)1088 except StopIteration:1089 if not stream.done:1090 if stream.has_events:1091 # Native event producers may omit the terminal1092 # `message-finish`. Close the lifecycle here so1093 # `on_llm_end` still observes the assembled1094 # message. A truly empty stream remains an error1095 # for parity with `stream()`.1096 stream.dispatch(MessageFinishData(event="message-finish"))1097 else:1098 err = ValueError("No generation chunks were returned")1099 stream.fail(err)1100 run_manager.on_llm_error(1101 err,1102 response=LLMResult(generations=[]),1103 )1104 return False1105 if stream.done and stream.output_message is not None:1106 run_manager.on_llm_end(1107 LLMResult(1108 generations=[1109 [ChatGeneration(message=stream.output_message)],1110 ],1111 ),1112 )1113 return False1114 except BaseException as exc:1115 stream.fail(exc)1116 run_manager.on_llm_error(1117 exc,1118 response=LLMResult(generations=[]),1119 )1120 return False1121 if stream.done and stream.output_message is not None:1122 run_manager.on_llm_end(1123 LLMResult(1124 generations=[1125 [ChatGeneration(message=stream.output_message)],1126 ],1127 ),1128 )1129 return True11301131 stream.set_start(ensure_started)1132 stream.bind_pump(pump_one)1133 return stream11341135 @beta()1136 async def _achat_model_stream_v3(1137 self,1138 input: LanguageModelInput,1139 config: RunnableConfig | None = None,1140 *,1141 stop: list[str] | None = None,1142 **kwargs: Any,1143 ) -> AsyncChatModelStream:1144 """Internal v3 async streaming implementation.11451146 Public entry point: `astream_events(version='v3')`.1147 """1148 config = ensure_config(config)1149 messages = self._convert_input(input).to_messages()1150 input_messages = _normalize_messages(messages)11511152 # Strip tracing-only kwargs before forwarding — see the sync v31153 # implementation for the full rationale.1154 ls_structured_output_format = kwargs.pop(1155 "ls_structured_output_format", None1156 ) or kwargs.pop("structured_output_format", None)1157 ls_structured_output_format_dict = _format_ls_structured_output(1158 ls_structured_output_format1159 )11601161 params = self._get_invocation_params(stop=stop, **kwargs)1162 options = {1163 "stop": stop,1164 **{key: params[key] for key in kwargs if key in params},1165 **ls_structured_output_format_dict,1166 }1167 inheritable_metadata = {1168 **(config.get("metadata") or {}),1169 **self._get_ls_params_with_defaults(stop=stop, **kwargs),1170 }1171 callback_manager = AsyncCallbackManager.configure(1172 config.get("callbacks"),1173 self.callbacks,1174 self.verbose,1175 config.get("tags"),1176 self.tags,1177 inheritable_metadata,1178 self.metadata,1179 langsmith_inheritable_metadata=_filter_invocation_params_for_tracing(1180 params1181 ),1182 )1183 stream = AsyncChatModelStream()1184 run_manager: AsyncCallbackManagerForLLMRun | None = None1185 run_name = config.get("run_name")1186 run_id = config.pop("run_id", None)1187 start_lock = asyncio.Lock()11881189 async def _produce() -> None:1190 assert run_manager is not None # noqa: S1011191 try:1192 if self.rate_limiter:1193 await self.rate_limiter.aacquire(blocking=True)11941195 async for _event in self._aiter_v2_events(1196 input_messages,1197 run_manager=run_manager,1198 stream=stream,1199 stop=stop,1200 **kwargs,1201 ):1202 pass1203 if not stream.done:1204 if stream.has_events:1205 # Native event producers may omit the terminal1206 # `message-finish`. Close the lifecycle here so1207 # `on_llm_end` sees the finalized message. A1208 # truly empty stream remains an error for parity1209 # with `astream()`.1210 stream.dispatch(MessageFinishData(event="message-finish"))1211 else:1212 err = ValueError("No generation chunks were returned")1213 stream.fail(err)1214 await run_manager.on_llm_error(1215 err,1216 response=LLMResult(generations=[]),1217 )1218 return1219 if stream.done and stream.output_message is not None:1220 await run_manager.on_llm_end(1221 LLMResult(1222 generations=[1223 [ChatGeneration(message=stream.output_message)],1224 ],1225 ),1226 )1227 except asyncio.CancelledError as exc:1228 stream.fail(exc)1229 # Close the callback lifecycle so tracing observes a1230 # matching end event for the earlier `on_chat_model_start`.1231 # `on_llm_error` is `@shielded`, so the callback runs to1232 # completion in the background even though the `await`1233 # here re-raises our cancellation.1234 with contextlib.suppress(Exception):1235 await run_manager.on_llm_error(1236 exc,1237 response=LLMResult(generations=[]),1238 )1239 raise1240 except BaseException as exc:1241 stream.fail(exc)1242 await run_manager.on_llm_error(1243 exc,1244 response=LLMResult(generations=[]),1245 )12461247 async def ensure_started() -> None:1248 nonlocal run_manager1249 if stream._producer_task is not None: # noqa: SLF0011250 return12511252 async with start_lock:1253 if stream._producer_task is not None: # noqa: SLF0011254 return # type: ignore[unreachable]12551256 (run_manager,) = await callback_manager.on_chat_model_start(1257 self._serialized,1258 [_format_for_tracing(messages)],1259 invocation_params=params,1260 options=options,1261 name=run_name,1262 run_id=run_id,1263 batch_size=1,1264 )1265 stream.set_message_id("-".join((LC_ID_PREFIX, str(run_manager.run_id))))1266 stream._producer_task = asyncio.get_running_loop().create_task( # noqa: SLF0011267 _produce()1268 )12691270 async def _on_aclose_fail(exc: BaseException) -> None:1271 assert run_manager is not None # noqa: S1011272 # Invoked by `stream.aclose()` only when the producer was1273 # cancelled before `_produce` ran — so `on_llm_error` from1274 # the CancelledError handler never fired. Shielded by the1275 # callback manager; runs to completion even if our caller1276 # is being cancelled.1277 await run_manager.on_llm_error(1278 exc,1279 response=LLMResult(generations=[]),1280 )12811282 stream.set_start(ensure_started)1283 stream._on_aclose_fail = _on_aclose_fail # noqa: SLF0011284 return stream12851286 @overload # type: ignore[override]1287 def stream_events(1288 self,1289 input: LanguageModelInput,1290 config: RunnableConfig | None = None,1291 *,1292 version: Literal["v1", "v2"] = "v2",1293 **kwargs: Any,1294 ) -> Iterator[StreamEvent]: ...12951296 @overload1297 def stream_events(1298 self,1299 input: LanguageModelInput,1300 config: RunnableConfig | None = None,1301 *,1302 version: Literal["v3"],1303 stop: list[str] | None = None,1304 **kwargs: Any,1305 ) -> ChatModelStream: ...13061307 def stream_events(1308 self,1309 input: LanguageModelInput,1310 config: RunnableConfig | None = None,1311 *,1312 version: Literal["v1", "v2", "v3"] = "v2",1313 stop: list[str] | None = None,1314 **kwargs: Any,1315 ) -> Iterator[StreamEvent] | ChatModelStream:1316 """Stream events from this chat model.13171318 For `version="v1"` / `"v2"`, yields `StreamEvent` dicts (see1319 `Runnable.stream_events`). For `version="v3"`, returns a1320 `ChatModelStream` exposing typed projections (`.text`,1321 `.reasoning`, `.tool_calls`, `.output`).13221323 !!! warning "Beta"13241325 `version="v3"` is in beta. The protocol shape, return type,1326 and surface area may change in future releases. Calling it1327 emits a `LangChainBetaWarning` at runtime.13281329 !!! note "v3 always produces v1-shaped content"13301331 `ChatModelStream.output.content` is always a list of v11332 content blocks (text / reasoning / tool_call / image / …),1333 regardless of the model's `output_version` attribute. The1334 setting only affects the legacy `stream()` / `astream()` /1335 `invoke()` paths. If you're mixing1336 `stream_events(version="v3")` with those paths in the same1337 pipeline and need a consistent output shape across them,1338 set `output_version="v1"` on the model.13391340 Args:1341 input: The model input.1342 config: Optional runnable config.1343 version: Streaming-event schema version. `"v3"` selects the1344 content-block-centric streaming protocol.1345 stop: Optional stop sequences. Only used for `version="v3"`;1346 ignored otherwise.1347 **kwargs: Additional keyword arguments. For `version="v3"`,1348 forwarded to the model.13491350 Returns:1351 For `version="v3"`, a `ChatModelStream` with typed1352 projections. Otherwise an `Iterator[StreamEvent]`.1353 """1354 if version == "v3":1355 return self._chat_model_stream_v3(input, config, stop=stop, **kwargs)1356 return super().stream_events(1357 input, config, version=version, stop=stop, **kwargs1358 )13591360 @overload1361 def astream_events(1362 self,1363 input: LanguageModelInput,1364 config: RunnableConfig | None = None,1365 *,1366 version: Literal["v1", "v2"] = "v2",1367 **kwargs: Any,1368 ) -> AsyncIterator[StreamEvent]: ...13691370 @overload1371 def astream_events(1372 self,1373 input: LanguageModelInput,1374 config: RunnableConfig | None = None,1375 *,1376 version: Literal["v3"],1377 stop: list[str] | None = None,1378 **kwargs: Any,1379 ) -> Awaitable[AsyncChatModelStream]: ...13801381 def astream_events(1382 self,1383 input: LanguageModelInput,1384 config: RunnableConfig | None = None,1385 *,1386 version: Literal["v1", "v2", "v3"] = "v2",1387 stop: list[str] | None = None,1388 **kwargs: Any,1389 ) -> AsyncIterator[StreamEvent] | Awaitable[AsyncChatModelStream]:1390 """Async variant of `stream_events`. See `stream_events` for full docs."""1391 if version == "v3":1392 return self._achat_model_stream_v3(input, config, stop=stop, **kwargs)1393 # v1/v2: forward to Runnable.astream_events (async generator).1394 return super().astream_events(1395 input, config, version=version, stop=stop, **kwargs1396 )13971398 # --- Custom methods ---13991400 def _combine_llm_outputs(1401 self, _llm_outputs: list[builtins.dict[str, Any] | None], /1402 ) -> builtins.dict[str, Any]:1403 return {}14041405 def _convert_cached_generations(1406 self, cache_val: list[Generation]1407 ) -> list[ChatGeneration]:1408 """Convert cached Generation objects to ChatGeneration objects.14091410 Handle case where cache contains Generation objects instead of1411 ChatGeneration objects. This can happen due to serialization/deserialization1412 issues or legacy cache data (see #22389).14131414 Args:1415 cache_val: List of cached generation objects.14161417 Returns:1418 List of ChatGeneration objects.14191420 """1421 converted_generations = []1422 for gen in cache_val:1423 if isinstance(gen, Generation) and not isinstance(gen, ChatGeneration):1424 # Convert Generation to ChatGeneration by creating AIMessage1425 # from the text content1426 chat_gen = ChatGeneration(1427 message=AIMessage(content=gen.text),1428 generation_info=gen.generation_info,1429 )1430 converted_generations.append(chat_gen)1431 else:1432 # Already a ChatGeneration or other expected type1433 if hasattr(gen, "message") and isinstance(gen.message, AIMessage):1434 # We zero out cost on cache hits1435 gen.message = gen.message.model_copy(1436 update={1437 "usage_metadata": {1438 **(gen.message.usage_metadata or {}),1439 "total_cost": 0,1440 }1441 }1442 )1443 converted_generations.append(gen)1444 return converted_generations14451446 def _replay_v2_events_for_cache_hit(1447 self,1448 generations: list[ChatGeneration],1449 *,1450 run_manager: CallbackManagerForLLMRun | None,1451 **kwargs: Any,1452 ) -> None:1453 """Replay cached messages as v2 events when a v2 handler is attached.14541455 A warm cache must produce the same `on_stream_event` stream as a1456 cold call so LangGraph-style consumers do not observe behavior1457 that depends on cache state. Gated by1458 `_should_use_protocol_streaming` so a `disable_streaming` config1459 that suppresses v2 on cold calls also suppresses it here.1460 """1461 if run_manager is None or not self._should_use_protocol_streaming(1462 async_api=False, run_manager=run_manager, **kwargs1463 ):1464 return1465 message_id = f"{LC_ID_PREFIX}-{run_manager.run_id}"1466 for gen in generations:1467 msg = getattr(gen, "message", None)1468 if not isinstance(msg, AIMessage):1469 continue1470 for event in message_to_events(msg, message_id=message_id):1471 run_manager.on_stream_event(event)14721473 async def _areplay_v2_events_for_cache_hit(1474 self,1475 generations: list[ChatGeneration],1476 *,1477 run_manager: AsyncCallbackManagerForLLMRun | None,1478 **kwargs: Any,1479 ) -> None:1480 """Async counterpart to `_replay_v2_events_for_cache_hit`."""1481 if run_manager is None or not self._should_use_protocol_streaming(1482 async_api=True, run_manager=run_manager, **kwargs1483 ):1484 return1485 message_id = f"{LC_ID_PREFIX}-{run_manager.run_id}"1486 for gen in generations:1487 msg = getattr(gen, "message", None)1488 if not isinstance(msg, AIMessage):1489 continue1490 async for event in amessage_to_events(msg, message_id=message_id):1491 await run_manager.on_stream_event(event)14921493 def _get_invocation_params(1494 self,1495 stop: list[str] | None = None,1496 **kwargs: Any,1497 ) -> builtins.dict[str, Any]:1498 params = self._dict_for_compat()1499 params["stop"] = stop1500 return {**params, **kwargs}15011502 def _get_ls_params(1503 self,1504 stop: list[str] | None = None,1505 **kwargs: Any,1506 ) -> LangSmithParams:1507 """Get standard params for LangSmith tracing.15081509 Subclasses **should override** this method to populate `ls_provider`1510 and `ls_model_name` from provider-specific attributes (e.g. `self.model`,1511 `self.model_name`, `self.model_id`) and to honor per-call overrides1512 passed via `kwargs["model"]` so that runtime `bind`/`invoke` model1513 changes are reflected in traces.15141515 The implementation here is a best-effort fallback for subclasses that1516 do not override it. It is not part of a stable contract and the1517 derivation rules may change:15181519 - `ls_provider` is derived from the class name by stripping a leading1520 or trailing `"Chat"` and lowercasing the remainder. This produces1521 ugly values for multi-word providers (e.g. `ChatGoogleGenerativeAI`1522 would become `"googlegenerativeai"`).15231524 Override to set a stable, conventional value1525 such as `"google_genai"`.1526 - `ls_model_name` is resolved from `kwargs["model"]`, then1527 `self.model`, then `self.model_name`.15281529 Subclasses whose model attribute has a different name1530 (`model_id`, `deployment_name`, ...) must override.1531 """1532 # get default provider from class name1533 default_provider = self.__class__.__name__1534 if default_provider.startswith("Chat"):1535 default_provider = default_provider[4:].lower()1536 elif default_provider.endswith("Chat"):1537 default_provider = default_provider[:-4]1538 default_provider = default_provider.lower()15391540 ls_params = LangSmithParams(ls_provider=default_provider, ls_model_type="chat")1541 if stop:1542 ls_params["ls_stop"] = stop15431544 # model1545 if "model" in kwargs and isinstance(kwargs["model"], str):1546 ls_params["ls_model_name"] = kwargs["model"]1547 elif hasattr(self, "model") and isinstance(self.model, str):1548 ls_params["ls_model_name"] = self.model1549 elif hasattr(self, "model_name") and isinstance(self.model_name, str):1550 ls_params["ls_model_name"] = self.model_name15511552 # temperature1553 if "temperature" in kwargs and isinstance(kwargs["temperature"], (int, float)):1554 ls_params["ls_temperature"] = kwargs["temperature"]1555 elif hasattr(self, "temperature") and isinstance(1556 self.temperature, (int, float)1557 ):1558 ls_params["ls_temperature"] = self.temperature15591560 # max_tokens1561 if "max_tokens" in kwargs and isinstance(kwargs["max_tokens"], int):1562 ls_params["ls_max_tokens"] = kwargs["max_tokens"]1563 elif hasattr(self, "max_tokens") and isinstance(self.max_tokens, int):1564 ls_params["ls_max_tokens"] = self.max_tokens15651566 return ls_params15671568 def _get_ls_params_with_defaults(1569 self,1570 stop: list[str] | None = None,1571 **kwargs: Any,1572 ) -> LangSmithParams:1573 """Wrap _get_ls_params to always include ls_integration."""1574 ls_params = self._get_ls_params(stop=stop, **kwargs)1575 ls_params["ls_integration"] = "langchain_chat_model"1576 return ls_params15771578 def _get_llm_string(self, stop: list[str] | None = None, **kwargs: Any) -> str:1579 if self.is_lc_serializable():1580 params = {**kwargs, "stop": stop}1581 param_string = str(sorted(params.items()))1582 # This code is not super efficient as it goes back and forth between1583 # json and dict.1584 serialized_repr = self._serialized1585 _cleanup_llm_representation(serialized_repr, 1)1586 llm_string = json.dumps(serialized_repr, sort_keys=True)1587 return llm_string + "---" + param_string1588 params = self._get_invocation_params(stop=stop, **kwargs)1589 params = {**params, **kwargs}1590 return str(sorted(params.items()))15911592 def generate(1593 self,1594 messages: list[list[BaseMessage]],1595 stop: list[str] | None = None,1596 callbacks: Callbacks = None,1597 *,1598 tags: list[str] | None = None,1599 metadata: builtins.dict[str, Any] | None = None,1600 run_name: str | None = None,1601 run_id: uuid.UUID | None = None,1602 **kwargs: Any,1603 ) -> LLMResult:1604 """Pass a sequence of prompts to the model and return model generations.16051606 This method should make use of batched calls for models that expose a batched1607 API.16081609 Use this method when you want to:16101611 1. Take advantage of batched calls,1612 2. Need more output from the model than just the top generated value,1613 3. Are building chains that are agnostic to the underlying language model1614 type (e.g., pure text completion models vs chat models).16151616 Args:1617 messages: List of list of messages.1618 stop: Stop words to use when generating.16191620 Model output is cut off at the first occurrence of any of these1621 substrings.1622 callbacks: `Callbacks` to pass through.16231624 Used for executing additional functionality, such as logging or1625 streaming, throughout generation.1626 tags: The tags to apply.1627 metadata: The metadata to apply.1628 run_name: The name of the run.1629 run_id: The ID of the run.1630 **kwargs: Arbitrary additional keyword arguments.16311632 These are usually passed to the model provider API call.16331634 Returns:1635 An `LLMResult`, which contains a list of candidate `Generations` for each1636 input prompt and additional model provider-specific output.16371638 """1639 ls_structured_output_format = kwargs.pop(1640 "ls_structured_output_format", None1641 ) or kwargs.pop("structured_output_format", None)1642 ls_structured_output_format_dict = _format_ls_structured_output(1643 ls_structured_output_format1644 )16451646 params = self._get_invocation_params(stop=stop, **kwargs)1647 options = {"stop": stop, **ls_structured_output_format_dict}1648 inheritable_metadata = {1649 **(metadata or {}),1650 **self._get_ls_params_with_defaults(stop=stop, **kwargs),1651 }16521653 callback_manager = CallbackManager.configure(1654 callbacks,1655 self.callbacks,1656 self.verbose,1657 tags,1658 self.tags,1659 inheritable_metadata,1660 self.metadata,1661 langsmith_inheritable_metadata=_filter_invocation_params_for_tracing(1662 params1663 ),1664 )1665 messages_to_trace = [1666 _format_for_tracing(message_list) for message_list in messages1667 ]1668 run_managers = callback_manager.on_chat_model_start(1669 self._serialized,1670 messages_to_trace,1671 invocation_params=params,1672 options=options,1673 name=run_name,1674 run_id=run_id,1675 batch_size=len(messages),1676 )1677 results = []1678 input_messages = [1679 _normalize_messages(message_list) for message_list in messages1680 ]1681 for i, m in enumerate(input_messages):1682 try:1683 results.append(1684 self._generate_with_cache(1685 m,1686 stop=stop,1687 run_manager=run_managers[i] if run_managers else None,1688 **kwargs,1689 )1690 )1691 except BaseException as e:1692 if run_managers:1693 generations_with_error_metadata = _generate_response_from_error(e)1694 run_managers[i].on_llm_error(1695 e,1696 response=LLMResult(1697 generations=[generations_with_error_metadata]1698 ),1699 )1700 raise1701 flattened_outputs = [1702 LLMResult(generations=[res.generations], llm_output=res.llm_output)1703 for res in results1704 ]1705 llm_output = self._combine_llm_outputs([res.llm_output for res in results])1706 generations = [res.generations for res in results]1707 output = LLMResult(generations=generations, llm_output=llm_output)1708 if run_managers:1709 run_infos = []1710 for manager, flattened_output in zip(1711 run_managers, flattened_outputs, strict=False1712 ):1713 manager.on_llm_end(flattened_output)1714 run_infos.append(RunInfo(run_id=manager.run_id))1715 output.run = run_infos1716 return output17171718 async def agenerate(1719 self,1720 messages: list[list[BaseMessage]],1721 stop: list[str] | None = None,1722 callbacks: Callbacks = None,1723 *,1724 tags: list[str] | None = None,1725 metadata: builtins.dict[str, Any] | None = None,1726 run_name: str | None = None,1727 run_id: uuid.UUID | None = None,1728 **kwargs: Any,1729 ) -> LLMResult:1730 """Asynchronously pass a sequence of prompts to a model and return generations.17311732 This method should make use of batched calls for models that expose a batched1733 API.17341735 Use this method when you want to:17361737 1. Take advantage of batched calls,1738 2. Need more output from the model than just the top generated value,1739 3. Are building chains that are agnostic to the underlying language model1740 type (e.g., pure text completion models vs chat models).17411742 Args:1743 messages: List of list of messages.1744 stop: Stop words to use when generating.17451746 Model output is cut off at the first occurrence of any of these1747 substrings.1748 callbacks: `Callbacks` to pass through.17491750 Used for executing additional functionality, such as logging or1751 streaming, throughout generation.1752 tags: The tags to apply.1753 metadata: The metadata to apply.1754 run_name: The name of the run.1755 run_id: The ID of the run.1756 **kwargs: Arbitrary additional keyword arguments.17571758 These are usually passed to the model provider API call.17591760 Returns:1761 An `LLMResult`, which contains a list of candidate `Generations` for each1762 input prompt and additional model provider-specific output.17631764 """1765 ls_structured_output_format = kwargs.pop(1766 "ls_structured_output_format", None1767 ) or kwargs.pop("structured_output_format", None)1768 ls_structured_output_format_dict = _format_ls_structured_output(1769 ls_structured_output_format1770 )17711772 params = self._get_invocation_params(stop=stop, **kwargs)1773 options = {"stop": stop, **ls_structured_output_format_dict}1774 inheritable_metadata = {1775 **(metadata or {}),1776 **self._get_ls_params_with_defaults(stop=stop, **kwargs),1777 }17781779 callback_manager = AsyncCallbackManager.configure(1780 callbacks,1781 self.callbacks,1782 self.verbose,1783 tags,1784 self.tags,1785 inheritable_metadata,1786 self.metadata,1787 langsmith_inheritable_metadata=_filter_invocation_params_for_tracing(1788 params1789 ),1790 )17911792 messages_to_trace = [1793 _format_for_tracing(message_list) for message_list in messages1794 ]1795 run_managers = await callback_manager.on_chat_model_start(1796 self._serialized,1797 messages_to_trace,1798 invocation_params=params,1799 options=options,1800 name=run_name,1801 batch_size=len(messages),1802 run_id=run_id,1803 )18041805 input_messages = [1806 _normalize_messages(message_list) for message_list in messages1807 ]1808 results = await asyncio.gather(1809 *[1810 self._agenerate_with_cache(1811 m,1812 stop=stop,1813 run_manager=run_managers[i] if run_managers else None,1814 **kwargs,1815 )1816 for i, m in enumerate(input_messages)1817 ],1818 return_exceptions=True,1819 )1820 exceptions = []1821 for i, res in enumerate(results):1822 if isinstance(res, BaseException):1823 if run_managers:1824 generations_with_error_metadata = _generate_response_from_error(res)1825 await run_managers[i].on_llm_error(1826 res,1827 response=LLMResult(1828 generations=[generations_with_error_metadata]1829 ),1830 )1831 exceptions.append(res)1832 if exceptions:1833 if run_managers:1834 await asyncio.gather(1835 *[1836 run_manager.on_llm_end(1837 LLMResult(1838 generations=[res.generations], # type: ignore[union-attr]1839 llm_output=res.llm_output, # type: ignore[union-attr]1840 )1841 )1842 for run_manager, res in zip(run_managers, results, strict=False)1843 if not isinstance(res, Exception)1844 ]1845 )1846 raise exceptions[0]1847 flattened_outputs = [1848 LLMResult(generations=[res.generations], llm_output=res.llm_output) # type: ignore[union-attr]1849 for res in results1850 ]1851 llm_output = self._combine_llm_outputs([res.llm_output for res in results]) # type: ignore[union-attr]1852 generations = [res.generations for res in results] # type: ignore[union-attr]1853 output = LLMResult(generations=generations, llm_output=llm_output)1854 await asyncio.gather(1855 *[1856 run_manager.on_llm_end(flattened_output)1857 for run_manager, flattened_output in zip(1858 run_managers, flattened_outputs, strict=False1859 )1860 ]1861 )1862 if run_managers:1863 output.run = [1864 RunInfo(run_id=run_manager.run_id) for run_manager in run_managers1865 ]1866 return output18671868 @override1869 def generate_prompt(1870 self,1871 prompts: list[PromptValue],1872 stop: list[str] | None = None,1873 callbacks: Callbacks = None,1874 **kwargs: Any,1875 ) -> LLMResult:1876 prompt_messages = [p.to_messages() for p in prompts]1877 return self.generate(prompt_messages, stop=stop, callbacks=callbacks, **kwargs)18781879 @override1880 async def agenerate_prompt(1881 self,1882 prompts: list[PromptValue],1883 stop: list[str] | None = None,1884 callbacks: Callbacks = None,1885 **kwargs: Any,1886 ) -> LLMResult:1887 prompt_messages = [p.to_messages() for p in prompts]1888 return await self.agenerate(1889 prompt_messages, stop=stop, callbacks=callbacks, **kwargs1890 )18911892 def _generate_with_cache(1893 self,1894 messages: list[BaseMessage],1895 stop: list[str] | None = None,1896 run_manager: CallbackManagerForLLMRun | None = None,1897 **kwargs: Any,1898 ) -> ChatResult:1899 llm_cache = self.cache if isinstance(self.cache, BaseCache) else get_llm_cache()1900 # We should check the cache unless it's explicitly set to False1901 # A None cache means we should use the default global cache1902 # if it's configured.1903 check_cache = self.cache is not False1904 if check_cache:1905 if llm_cache is not None:1906 llm_string = self._get_llm_string(stop=stop, **kwargs)1907 normalized_messages = [1908 (1909 msg.model_copy(update={"id": None})1910 if getattr(msg, "id", None) is not None1911 else msg1912 )1913 for msg in messages1914 ]1915 prompt = dumps(normalized_messages)1916 cache_val = llm_cache.lookup(prompt, llm_string)1917 if isinstance(cache_val, list):1918 converted_generations = self._convert_cached_generations(cache_val)1919 self._replay_v2_events_for_cache_hit(1920 converted_generations,1921 run_manager=run_manager,1922 **kwargs,1923 )1924 return ChatResult(generations=converted_generations)1925 elif self.cache is None:1926 pass1927 else:1928 msg = "Asked to cache, but no cache found at `langchain.cache`."1929 raise ValueError(msg)19301931 # Apply the rate limiter after checking the cache, since1932 # we usually don't want to rate limit cache lookups, but1933 # we do want to rate limit API requests.1934 if self.rate_limiter:1935 self.rate_limiter.acquire(blocking=True)19361937 # v2 streaming: preferred over v1 when any attached handler opts in via1938 # `_V2StreamingCallbackHandler`. Drives the protocol event generator1939 # (native or `_stream` compat bridge) through the shared helper so1940 # `on_stream_event` fires per event, then returns a normal `ChatResult`1941 # so caching / `on_llm_end` stay on the existing generate path.1942 if self._should_use_protocol_streaming(1943 async_api=False,1944 run_manager=run_manager,1945 **kwargs,1946 ):1947 stream_accum = ChatModelStream(1948 message_id=(1949 f"{LC_ID_PREFIX}-{run_manager.run_id}" if run_manager else None1950 )1951 )1952 assert run_manager is not None # noqa: S1011953 for _event in self._iter_v2_events(1954 messages,1955 run_manager=run_manager,1956 stream=stream_accum,1957 stop=stop,1958 **kwargs,1959 ):1960 pass1961 if stream_accum.output_message is None:1962 msg = "v2 stream finished without producing a message"1963 raise RuntimeError(msg)1964 result = ChatResult(1965 generations=[ChatGeneration(message=stream_accum.output_message)]1966 )1967 # If stream is not explicitly set, check if implicitly requested by1968 # astream_events() or astream_log(). Bail out if _stream not implemented1969 elif self._should_stream(1970 async_api=False,1971 run_manager=run_manager,1972 **kwargs,1973 ):1974 chunks: list[ChatGenerationChunk] = []1975 run_id: str | None = (1976 f"{LC_ID_PREFIX}-{run_manager.run_id}" if run_manager else None1977 )1978 yielded = False1979 index = -11980 index_type = ""1981 for chunk in self._stream(messages, stop=stop, **kwargs):1982 chunk.message.response_metadata = _gen_info_and_msg_metadata(chunk)1983 if self.output_version == "v1":1984 # Overwrite .content with .content_blocks1985 chunk.message = _update_message_content_to_blocks(1986 chunk.message, "v1"1987 )1988 for block in cast(1989 "list[types.ContentBlock]", chunk.message.content1990 ):1991 if block["type"] != index_type:1992 index_type = block["type"]1993 index += 11994 if "index" not in block:1995 block["index"] = index1996 if run_manager:1997 if chunk.message.id is None:1998 chunk.message.id = run_id1999 run_manager.on_llm_new_token(chunk.message.content, chunk=chunk)2000 chunks.append(chunk)
Findings
✓ No findings reported for this file.