1"""Base interface for traditional large language models (LLMs) to expose.23These are traditionally older models (newer models generally are chat models).4"""56from __future__ import annotations78import asyncio9import builtins10import functools11import inspect12import json13import logging14from abc import ABC, abstractmethod15from collections.abc import AsyncIterator, Callable, Iterator, Sequence16from pathlib import Path17from typing import (18    TYPE_CHECKING,19    Any,20    cast,21)2223import yaml24from pydantic import ConfigDict25from tenacity import (26    RetryCallState,27    before_sleep_log,28    retry,29    retry_base,30    retry_if_exception_type,31    stop_after_attempt,32    wait_exponential,33)34from typing_extensions import override3536from langchain_core._api import deprecated, suppress_langchain_deprecation_warning37from langchain_core.caches import BaseCache38from langchain_core.callbacks import (39    AsyncCallbackManager,40    AsyncCallbackManagerForLLMRun,41    BaseCallbackManager,42    CallbackManager,43    CallbackManagerForLLMRun,44    Callbacks,45)46from langchain_core.globals import get_llm_cache47from langchain_core.language_models._utils import _filter_invocation_params_for_tracing48from langchain_core.language_models.base import (49    BaseLanguageModel,50    LangSmithParams,51    LanguageModelInput,52)53from langchain_core.load import dumpd54from langchain_core.messages import (55    convert_to_messages,56)57from langchain_core.outputs import Generation, GenerationChunk, LLMResult, RunInfo58from langchain_core.prompt_values import ChatPromptValue, PromptValue, StringPromptValue59from langchain_core.runnables import RunnableConfig, ensure_config, get_config_list60from langchain_core.runnables.config import run_in_executor6162if TYPE_CHECKING:63    import builtins64    import uuid6566logger = logging.getLogger(__name__)6768_background_tasks: set[asyncio.Task[None]] = set()697071@functools.lru_cache72def _log_error_once(msg: str) -> None:73    """Log an error once."""74    logger.error(msg)757677def create_base_retry_decorator(78    error_types: list[type[BaseException]],79    max_retries: int = 1,80    run_manager: AsyncCallbackManagerForLLMRun | CallbackManagerForLLMRun | None = None,81) -> Callable[[Any], Any]:82    """Create a retry decorator for a given LLM and provided a list of error types.8384    Args:85        error_types: List of error types to retry on.86        max_retries: Number of retries.87        run_manager: Callback manager for the run.8889    Returns:90        A retry decorator.9192    Raises:93        ValueError: If the cache is not set and cache is True.94    """95    logging_ = before_sleep_log(logger, logging.WARNING)9697    def _before_sleep(retry_state: RetryCallState) -> None:98        logging_(retry_state)99        if run_manager:100            if isinstance(run_manager, AsyncCallbackManagerForLLMRun):101                coro = run_manager.on_retry(retry_state)102                try:103                    try:104                        loop = asyncio.get_event_loop()105                    except RuntimeError:106                        asyncio.run(coro)107                    else:108                        if loop.is_running():109                            task = loop.create_task(coro)110                            _background_tasks.add(task)111                            task.add_done_callback(_background_tasks.discard)112                        else:113                            asyncio.run(coro)114                except Exception as e:115                    _log_error_once(f"Error in on_retry: {e}")116            else:117                run_manager.on_retry(retry_state)118119    min_seconds = 4120    max_seconds = 10121    # Wait 2^x * 1 second between each retry starting with122    # 4 seconds, then up to 10 seconds, then 10 seconds afterwards123    retry_instance: retry_base = retry_if_exception_type(error_types[0])124    for error in error_types[1:]:125        retry_instance |= retry_if_exception_type(error)126    return retry(127        reraise=True,128        stop=stop_after_attempt(max_retries),129        wait=wait_exponential(multiplier=1, min=min_seconds, max=max_seconds),130        retry=retry_instance,131        before_sleep=_before_sleep,132    )133134135def _resolve_cache(*, cache: BaseCache | bool | None) -> BaseCache | None:136    """Resolve the cache."""137    llm_cache: BaseCache | None138    if isinstance(cache, BaseCache):139        llm_cache = cache140    elif cache is None:141        llm_cache = get_llm_cache()142    elif cache is True:143        llm_cache = get_llm_cache()144        if llm_cache is None:145            msg = (146                "No global cache was configured. Use `set_llm_cache`."147                "to set a global cache if you want to use a global cache."148                "Otherwise either pass a cache object or set cache to False/None"149            )150            raise ValueError(msg)151    elif cache is False:152        llm_cache = None153    else:154        msg = f"Unsupported cache value {cache}"155        raise ValueError(msg)156    return llm_cache157158159def get_prompts(160    params: dict[str, Any],161    prompts: list[str],162    cache: BaseCache | bool | None = None,  # noqa: FBT001163) -> tuple[dict[int, list[Generation]], str, list[int], list[str]]:164    """Get prompts that are already cached.165166    Args:167        params: Dictionary of parameters.168        prompts: List of prompts.169        cache: Cache object.170171    Returns:172        A tuple of existing prompts, llm_string, missing prompt indexes,173            and missing prompts.174175    Raises:176        ValueError: If the cache is not set and cache is True.177    """178    llm_string = str(sorted(params.items()))179    missing_prompts = []180    missing_prompt_idxs = []181    existing_prompts = {}182183    llm_cache = _resolve_cache(cache=cache)184    for i, prompt in enumerate(prompts):185        if llm_cache:186            cache_val = llm_cache.lookup(prompt, llm_string)187            if isinstance(cache_val, list):188                existing_prompts[i] = cache_val189            else:190                missing_prompts.append(prompt)191                missing_prompt_idxs.append(i)192    return existing_prompts, llm_string, missing_prompt_idxs, missing_prompts193194195async def aget_prompts(196    params: dict[str, Any],197    prompts: list[str],198    cache: BaseCache | bool | None = None,  # noqa: FBT001199) -> tuple[dict[int, list[Generation]], str, list[int], list[str]]:200    """Get prompts that are already cached. Async version.201202    Args:203        params: Dictionary of parameters.204        prompts: List of prompts.205        cache: Cache object.206207    Returns:208        A tuple of existing prompts, llm_string, missing prompt indexes,209            and missing prompts.210211    Raises:212        ValueError: If the cache is not set and cache is True.213    """214    llm_string = str(sorted(params.items()))215    missing_prompts = []216    missing_prompt_idxs = []217    existing_prompts = {}218    llm_cache = _resolve_cache(cache=cache)219    for i, prompt in enumerate(prompts):220        if llm_cache:221            cache_val = await llm_cache.alookup(prompt, llm_string)222            if isinstance(cache_val, list):223                existing_prompts[i] = cache_val224            else:225                missing_prompts.append(prompt)226                missing_prompt_idxs.append(i)227    return existing_prompts, llm_string, missing_prompt_idxs, missing_prompts228229230def update_cache(231    cache: BaseCache | bool | None,  # noqa: FBT001232    existing_prompts: dict[int, list[Generation]],233    llm_string: str,234    missing_prompt_idxs: list[int],235    new_results: LLMResult,236    prompts: list[str],237) -> dict[str, Any] | None:238    """Update the cache and get the LLM output.239240    Args:241        cache: Cache object.242        existing_prompts: Dictionary of existing prompts.243        llm_string: LLM string.244        missing_prompt_idxs: List of missing prompt indexes.245        new_results: LLMResult object.246        prompts: List of prompts.247248    Returns:249        LLM output.250251    Raises:252        ValueError: If the cache is not set and cache is True.253    """254    llm_cache = _resolve_cache(cache=cache)255    for i, result in enumerate(new_results.generations):256        existing_prompts[missing_prompt_idxs[i]] = result257        prompt = prompts[missing_prompt_idxs[i]]258        if llm_cache is not None:259            llm_cache.update(prompt, llm_string, result)260    return new_results.llm_output261262263async def aupdate_cache(264    cache: BaseCache | bool | None,  # noqa: FBT001265    existing_prompts: dict[int, list[Generation]],266    llm_string: str,267    missing_prompt_idxs: list[int],268    new_results: LLMResult,269    prompts: list[str],270) -> dict[str, Any] | None:271    """Update the cache and get the LLM output. Async version.272273    Args:274        cache: Cache object.275        existing_prompts: Dictionary of existing prompts.276        llm_string: LLM string.277        missing_prompt_idxs: List of missing prompt indexes.278        new_results: LLMResult object.279        prompts: List of prompts.280281    Returns:282        LLM output.283284    Raises:285        ValueError: If the cache is not set and cache is True.286    """287    llm_cache = _resolve_cache(cache=cache)288    for i, result in enumerate(new_results.generations):289        existing_prompts[missing_prompt_idxs[i]] = result290        prompt = prompts[missing_prompt_idxs[i]]291        if llm_cache:292            await llm_cache.aupdate(prompt, llm_string, result)293    return new_results.llm_output294295296class BaseLLM(BaseLanguageModel[str], ABC):297    """Base LLM abstract interface.298299    It should take in a prompt and return a string.300    """301302    model_config = ConfigDict(303        arbitrary_types_allowed=True,304    )305306    @functools.cached_property307    def _serialized(self) -> builtins.dict[str, Any]:308        # self is always a Serializable object in this case, thus the result is309        # guaranteed to be a dict since dumpd uses the default callback, which uses310        # obj.to_json which always returns TypedDict subclasses311        return cast("builtins.dict[str, Any]", dumpd(self))312313    # --- Runnable methods ---314315    @property316    @override317    def OutputType(self) -> type[str]:318        """Get the output type for this `Runnable`."""319        return str320321    def _convert_input(self, model_input: LanguageModelInput) -> PromptValue:322        if isinstance(model_input, PromptValue):323            return model_input324        if isinstance(model_input, str):325            return StringPromptValue(text=model_input)326        if isinstance(model_input, Sequence):327            return ChatPromptValue(messages=convert_to_messages(model_input))328        msg = (329            f"Invalid input type {type(model_input)}. "330            "Must be a PromptValue, str, or list of BaseMessages."331        )332        raise ValueError(msg)333334    def _get_ls_params(335        self,336        stop: list[str] | None = None,337        **kwargs: Any,338    ) -> LangSmithParams:339        """Get standard params for tracing."""340        # get default provider from class name341        default_provider = self.__class__.__name__342        default_provider = default_provider.removesuffix("LLM")343        default_provider = default_provider.lower()344345        ls_params = LangSmithParams(ls_provider=default_provider, ls_model_type="llm")346        if stop:347            ls_params["ls_stop"] = stop348349        # model350        if "model" in kwargs and isinstance(kwargs["model"], str):351            ls_params["ls_model_name"] = kwargs["model"]352        elif hasattr(self, "model") and isinstance(self.model, str):353            ls_params["ls_model_name"] = self.model354        elif hasattr(self, "model_name") and isinstance(self.model_name, str):355            ls_params["ls_model_name"] = self.model_name356357        # temperature358        if "temperature" in kwargs and isinstance(kwargs["temperature"], (int, float)):359            ls_params["ls_temperature"] = kwargs["temperature"]360        elif hasattr(self, "temperature") and isinstance(361            self.temperature, (int, float)362        ):363            ls_params["ls_temperature"] = self.temperature364365        # max_tokens366        if "max_tokens" in kwargs and isinstance(kwargs["max_tokens"], int):367            ls_params["ls_max_tokens"] = kwargs["max_tokens"]368        elif hasattr(self, "max_tokens") and isinstance(self.max_tokens, int):369            ls_params["ls_max_tokens"] = self.max_tokens370371        return ls_params372373    @override374    def invoke(375        self,376        input: LanguageModelInput,377        config: RunnableConfig | None = None,378        *,379        stop: list[str] | None = None,380        **kwargs: Any,381    ) -> str:382        config = ensure_config(config)383        return (384            self.generate_prompt(385                [self._convert_input(input)],386                stop=stop,387                callbacks=config.get("callbacks"),388                tags=config.get("tags"),389                metadata=config.get("metadata"),390                run_name=config.get("run_name"),391                run_id=config.pop("run_id", None),392                **kwargs,393            )394            .generations[0][0]395            .text396        )397398    @override399    async def ainvoke(400        self,401        input: LanguageModelInput,402        config: RunnableConfig | None = None,403        *,404        stop: list[str] | None = None,405        **kwargs: Any,406    ) -> str:407        config = ensure_config(config)408        llm_result = await self.agenerate_prompt(409            [self._convert_input(input)],410            stop=stop,411            callbacks=config.get("callbacks"),412            tags=config.get("tags"),413            metadata=config.get("metadata"),414            run_name=config.get("run_name"),415            run_id=config.pop("run_id", None),416            **kwargs,417        )418        return llm_result.generations[0][0].text419420    @override421    def batch(422        self,423        inputs: list[LanguageModelInput],424        config: RunnableConfig | list[RunnableConfig] | None = None,425        *,426        return_exceptions: bool = False,427        **kwargs: Any,428    ) -> list[str]:429        if not inputs:430            return []431432        config = get_config_list(config, len(inputs))433        max_concurrency = config[0].get("max_concurrency")434435        if max_concurrency is None:436            try:437                llm_result = self.generate_prompt(438                    [self._convert_input(input_) for input_ in inputs],439                    callbacks=[c.get("callbacks") for c in config],440                    tags=[c.get("tags") for c in config],441                    metadata=[c.get("metadata") for c in config],442                    run_name=[c.get("run_name") for c in config],443                    **kwargs,444                )445                return [g[0].text for g in llm_result.generations]446            except Exception as e:447                if return_exceptions:448                    return cast("list[str]", [e for _ in inputs])449                raise450        else:451            batches = [452                inputs[i : i + max_concurrency]453                for i in range(0, len(inputs), max_concurrency)454            ]455            config = [{**c, "max_concurrency": None} for c in config]456            return [457                output458                for i, batch in enumerate(batches)459                for output in self.batch(460                    batch,461                    config=config[i * max_concurrency : (i + 1) * max_concurrency],462                    return_exceptions=return_exceptions,463                    **kwargs,464                )465            ]466467    @override468    async def abatch(469        self,470        inputs: list[LanguageModelInput],471        config: RunnableConfig | list[RunnableConfig] | None = None,472        *,473        return_exceptions: bool = False,474        **kwargs: Any,475    ) -> list[str]:476        if not inputs:477            return []478        config = get_config_list(config, len(inputs))479        max_concurrency = config[0].get("max_concurrency")480481        if max_concurrency is None:482            try:483                llm_result = await self.agenerate_prompt(484                    [self._convert_input(input_) for input_ in inputs],485                    callbacks=[c.get("callbacks") for c in config],486                    tags=[c.get("tags") for c in config],487                    metadata=[c.get("metadata") for c in config],488                    run_name=[c.get("run_name") for c in config],489                    **kwargs,490                )491                return [g[0].text for g in llm_result.generations]492            except Exception as e:493                if return_exceptions:494                    return cast("list[str]", [e for _ in inputs])495                raise496        else:497            batches = [498                inputs[i : i + max_concurrency]499                for i in range(0, len(inputs), max_concurrency)500            ]501            config = [{**c, "max_concurrency": None} for c in config]502            return [503                output504                for i, batch in enumerate(batches)505                for output in await self.abatch(506                    batch,507                    config=config[i * max_concurrency : (i + 1) * max_concurrency],508                    return_exceptions=return_exceptions,509                    **kwargs,510                )511            ]512513    @override514    def stream(515        self,516        input: LanguageModelInput,517        config: RunnableConfig | None = None,518        *,519        stop: list[str] | None = None,520        **kwargs: Any,521    ) -> Iterator[str]:522        if type(self)._stream == BaseLLM._stream:  # noqa: SLF001523            # model doesn't implement streaming, so use default implementation524            yield self.invoke(input, config=config, stop=stop, **kwargs)525        else:526            prompt = self._convert_input(input).to_string()527            config = ensure_config(config)528            params = self._dict_for_compat()529            params["stop"] = stop530            params = {**params, **kwargs}531            options = {"stop": stop}532            inheritable_metadata = {533                **(config.get("metadata") or {}),534                **self._get_ls_params_with_defaults(stop=stop, **kwargs),535            }536            callback_manager = CallbackManager.configure(537                config.get("callbacks"),538                self.callbacks,539                self.verbose,540                config.get("tags"),541                self.tags,542                inheritable_metadata,543                self.metadata,544                langsmith_inheritable_metadata=_filter_invocation_params_for_tracing(545                    params546                ),547            )548            (run_manager,) = callback_manager.on_llm_start(549                self._serialized,550                [prompt],551                invocation_params=params,552                options=options,553                name=config.get("run_name"),554                run_id=config.pop("run_id", None),555                batch_size=1,556            )557            generation: GenerationChunk | None = None558            try:559                for chunk in self._stream(560                    prompt, stop=stop, run_manager=run_manager, **kwargs561                ):562                    yield chunk.text563                    if generation is None:564                        generation = chunk565                    else:566                        generation += chunk567            except BaseException as e:568                run_manager.on_llm_error(569                    e,570                    response=LLMResult(571                        generations=[[generation]] if generation else []572                    ),573                )574                raise575576            if generation is None:577                err = ValueError("No generation chunks were returned")578                run_manager.on_llm_error(err, response=LLMResult(generations=[]))579                raise err580581            run_manager.on_llm_end(LLMResult(generations=[[generation]]))582583    @override584    async def astream(585        self,586        input: LanguageModelInput,587        config: RunnableConfig | None = None,588        *,589        stop: list[str] | None = None,590        **kwargs: Any,591    ) -> AsyncIterator[str]:592        if (593            type(self)._astream is BaseLLM._astream  # noqa: SLF001594            and type(self)._stream is BaseLLM._stream  # noqa: SLF001595        ):596            yield await self.ainvoke(input, config=config, stop=stop, **kwargs)597            return598599        prompt = self._convert_input(input).to_string()600        config = ensure_config(config)601        params = self._dict_for_compat()602        params["stop"] = stop603        params = {**params, **kwargs}604        options = {"stop": stop}605        inheritable_metadata = {606            **(config.get("metadata") or {}),607            **self._get_ls_params_with_defaults(stop=stop, **kwargs),608        }609        callback_manager = AsyncCallbackManager.configure(610            config.get("callbacks"),611            self.callbacks,612            self.verbose,613            config.get("tags"),614            self.tags,615            inheritable_metadata,616            self.metadata,617            langsmith_inheritable_metadata=_filter_invocation_params_for_tracing(618                params619            ),620        )621        (run_manager,) = await callback_manager.on_llm_start(622            self._serialized,623            [prompt],624            invocation_params=params,625            options=options,626            name=config.get("run_name"),627            run_id=config.pop("run_id", None),628            batch_size=1,629        )630        generation: GenerationChunk | None = None631        try:632            async for chunk in self._astream(633                prompt,634                stop=stop,635                run_manager=run_manager,636                **kwargs,637            ):638                yield chunk.text639                if generation is None:640                    generation = chunk641                else:642                    generation += chunk643        except BaseException as e:644            await run_manager.on_llm_error(645                e,646                response=LLMResult(generations=[[generation]] if generation else []),647            )648            raise649650        if generation is None:651            err = ValueError("No generation chunks were returned")652            await run_manager.on_llm_error(err, response=LLMResult(generations=[]))653            raise err654655        await run_manager.on_llm_end(LLMResult(generations=[[generation]]))656657    # --- Custom methods ---658659    @abstractmethod660    def _generate(661        self,662        prompts: list[str],663        stop: list[str] | None = None,664        run_manager: CallbackManagerForLLMRun | None = None,665        **kwargs: Any,666    ) -> LLMResult:667        """Run the LLM on the given prompts.668669        Args:670            prompts: The prompts to generate from.671            stop: Stop words to use when generating.672673                Model output is cut off at the first occurrence of any of these674                substrings.675676                If stop tokens are not supported consider raising `NotImplementedError`.677            run_manager: Callback manager for the run.678679        Returns:680            The LLM result.681        """682683    async def _agenerate(684        self,685        prompts: list[str],686        stop: list[str] | None = None,687        run_manager: AsyncCallbackManagerForLLMRun | None = None,688        **kwargs: Any,689    ) -> LLMResult:690        """Run the LLM on the given prompts.691692        Args:693            prompts: The prompts to generate from.694            stop: Stop words to use when generating.695696                Model output is cut off at the first occurrence of any of these697                substrings.698699                If stop tokens are not supported consider raising `NotImplementedError`.700            run_manager: Callback manager for the run.701702        Returns:703            The LLM result.704        """705        return await run_in_executor(706            None,707            self._generate,708            prompts,709            stop,710            run_manager.get_sync() if run_manager else None,711            **kwargs,712        )713714    def _stream(715        self,716        prompt: str,717        stop: list[str] | None = None,718        run_manager: CallbackManagerForLLMRun | None = None,719        **kwargs: Any,720    ) -> Iterator[GenerationChunk]:721        """Stream the LLM on the given prompt.722723        This method should be overridden by subclasses that support streaming.724725        If not implemented, the default behavior of calls to stream will be to726        fallback to the non-streaming version of the model and return727        the output as a single chunk.728729        Args:730            prompt: The prompt to generate from.731            stop: Stop words to use when generating.732733                Model output is cut off at the first occurrence of any of these734                substrings.735            run_manager: Callback manager for the run.736            **kwargs: Arbitrary additional keyword arguments.737738                These are usually passed to the model provider API call.739740        Yields:741            Generation chunks.742        """743        raise NotImplementedError744745    async def _astream(746        self,747        prompt: str,748        stop: list[str] | None = None,749        run_manager: AsyncCallbackManagerForLLMRun | None = None,750        **kwargs: Any,751    ) -> AsyncIterator[GenerationChunk]:752        """An async version of the _stream method.753754        The default implementation uses the synchronous _stream method and wraps it in755        an async iterator. Subclasses that need to provide a true async implementation756        should override this method.757758        Args:759            prompt: The prompt to generate from.760            stop: Stop words to use when generating.761762                Model output is cut off at the first occurrence of any of these763                substrings.764            run_manager: Callback manager for the run.765            **kwargs: Arbitrary additional keyword arguments.766767                These are usually passed to the model provider API call.768769        Yields:770            Generation chunks.771        """772        iterator = await run_in_executor(773            None,774            self._stream,775            prompt,776            stop,777            run_manager.get_sync() if run_manager else None,778            **kwargs,779        )780        done = object()781        while True:782            item = await run_in_executor(783                None,784                next,785                iterator,786                done,787            )788            if item is done:789                break790            yield item  # type: ignore[misc]791792    @override793    def generate_prompt(794        self,795        prompts: list[PromptValue],796        stop: list[str] | None = None,797        callbacks: Callbacks | list[Callbacks] | None = None,798        **kwargs: Any,799    ) -> LLMResult:800        prompt_strings = [p.to_string() for p in prompts]801        return self.generate(prompt_strings, stop=stop, callbacks=callbacks, **kwargs)802803    @override804    async def agenerate_prompt(805        self,806        prompts: list[PromptValue],807        stop: list[str] | None = None,808        callbacks: Callbacks | list[Callbacks] | None = None,809        **kwargs: Any,810    ) -> LLMResult:811        prompt_strings = [p.to_string() for p in prompts]812        return await self.agenerate(813            prompt_strings, stop=stop, callbacks=callbacks, **kwargs814        )815816    def _generate_helper(817        self,818        prompts: list[str],819        stop: list[str] | None,820        run_managers: list[CallbackManagerForLLMRun],821        *,822        new_arg_supported: bool,823        **kwargs: Any,824    ) -> LLMResult:825        try:826            output = (827                self._generate(828                    prompts,829                    stop=stop,830                    # TODO: support multiple run managers831                    run_manager=run_managers[0] if run_managers else None,832                    **kwargs,833                )834                if new_arg_supported835                else self._generate(prompts, stop=stop)836            )837        except BaseException as e:838            for run_manager in run_managers:839                run_manager.on_llm_error(e, response=LLMResult(generations=[]))840            raise841        flattened_outputs = output.flatten()842        for manager, flattened_output in zip(843            run_managers, flattened_outputs, strict=False844        ):845            manager.on_llm_end(flattened_output)846        if run_managers:847            output.run = [848                RunInfo(run_id=run_manager.run_id) for run_manager in run_managers849            ]850        return output851852    def generate(853        self,854        prompts: list[str],855        stop: list[str] | None = None,856        callbacks: Callbacks | list[Callbacks] | None = None,857        *,858        tags: list[str] | list[list[str]] | None = None,859        metadata: builtins.dict[str, Any] | list[builtins.dict[str, Any]] | None = None,860        run_name: str | list[str] | None = None,861        run_id: uuid.UUID | list[uuid.UUID | None] | None = None,862        **kwargs: Any,863    ) -> LLMResult:864        """Pass a sequence of prompts to a model and return generations.865866        This method should make use of batched calls for models that expose a batched867        API.868869        Use this method when you want to:870871        1. Take advantage of batched calls,872        2. Need more output from the model than just the top generated value,873        3. Are building chains that are agnostic to the underlying language model874            type (e.g., pure text completion models vs chat models).875876        Args:877            prompts: List of string prompts.878            stop: Stop words to use when generating.879880                Model output is cut off at the first occurrence of any of these881                substrings.882            callbacks: `Callbacks` to pass through.883884                Used for executing additional functionality, such as logging or885                streaming, throughout generation.886            tags: List of tags to associate with each prompt. If provided, the length887                of the list must match the length of the prompts list.888            metadata: List of metadata dictionaries to associate with each prompt. If889                provided, the length of the list must match the length of the prompts890                list.891            run_name: List of run names to associate with each prompt. If provided, the892                length of the list must match the length of the prompts list.893            run_id: List of run IDs to associate with each prompt. If provided, the894                length of the list must match the length of the prompts list.895            **kwargs: Arbitrary additional keyword arguments.896897                These are usually passed to the model provider API call.898899        Raises:900            ValueError: If prompts is not a list.901            ValueError: If the length of `callbacks`, `tags`, `metadata`, or902                `run_name` (if provided) does not match the length of prompts.903904        Returns:905            An `LLMResult`, which contains a list of candidate `Generations` for each906                input prompt and additional model provider-specific output.907        """908        if not isinstance(prompts, list):909            msg = (910                "Argument 'prompts' is expected to be of type list[str], received"911                f" argument of type {type(prompts)}."912            )913            raise ValueError(msg)  # noqa: TRY004914        # Create callback managers915        if isinstance(metadata, list):916            metadata = [917                {918                    **(meta or {}),919                    **self._get_ls_params_with_defaults(stop=stop, **kwargs),920                }921                for meta in metadata922            ]923        elif isinstance(metadata, dict):924            metadata = {925                **(metadata or {}),926                **self._get_ls_params_with_defaults(stop=stop, **kwargs),927            }928        if (929            isinstance(callbacks, list)930            and callbacks931            and (932                isinstance(callbacks[0], (list, BaseCallbackManager))933                or callbacks[0] is None934            )935        ):936            # We've received a list of callbacks args to apply to each input937            if len(callbacks) != len(prompts):938                msg = "callbacks must be the same length as prompts"939                raise ValueError(msg)940            if tags is not None and not (941                isinstance(tags, list) and len(tags) == len(prompts)942            ):943                msg = "tags must be a list of the same length as prompts"944                raise ValueError(msg)945            if metadata is not None and not (946                isinstance(metadata, list) and len(metadata) == len(prompts)947            ):948                msg = "metadata must be a list of the same length as prompts"949                raise ValueError(msg)950            if run_name is not None and not (951                isinstance(run_name, list) and len(run_name) == len(prompts)952            ):953                msg = "run_name must be a list of the same length as prompts"954                raise ValueError(msg)955            tags_list = cast("list[list[str] | None]", tags or ([None] * len(prompts)))956            metadata_list = cast(957                "list[builtins.dict[str, Any] | None]",958                metadata or ([{}] * len(prompts)),959            )960            run_name_list = run_name or cast(961                "list[str | None]", ([None] * len(prompts))962            )963            params = self._dict_for_compat()964            params["stop"] = stop965            callback_managers = [966                CallbackManager.configure(967                    callback,968                    self.callbacks,969                    self.verbose,970                    tag,971                    self.tags,972                    meta,973                    self.metadata,974                    langsmith_inheritable_metadata=_filter_invocation_params_for_tracing(975                        params976                    ),977                )978                for callback, tag, meta in zip(979                    callbacks, tags_list, metadata_list, strict=False980                )981            ]982        else:983            # We've received a single callbacks arg to apply to all inputs984            params = self._dict_for_compat()985            params["stop"] = stop986            callback_managers = [987                CallbackManager.configure(988                    cast("Callbacks", callbacks),989                    self.callbacks,990                    self.verbose,991                    cast("list[str]", tags),992                    self.tags,993                    cast("builtins.dict[str, Any]", metadata),994                    self.metadata,995                    langsmith_inheritable_metadata=_filter_invocation_params_for_tracing(996                        params997                    ),998                )999            ] * len(prompts)1000            run_name_list = [cast("str | None", run_name)] * len(prompts)1001        run_ids_list = self._get_run_ids_list(run_id, prompts)1002        options = {"stop": stop}1003        (1004            existing_prompts,1005            llm_string,1006            missing_prompt_idxs,1007            missing_prompts,1008        ) = get_prompts(params, prompts, self.cache)1009        new_arg_supported = inspect.signature(self._generate).parameters.get(1010            "run_manager"1011        )1012        if (self.cache is None and get_llm_cache() is None) or self.cache is False:1013            run_managers = [1014                callback_manager.on_llm_start(1015                    self._serialized,1016                    [prompt],1017                    invocation_params=params,1018                    options=options,1019                    name=run_name,1020                    batch_size=len(prompts),1021                    run_id=run_id_,1022                )[0]1023                for callback_manager, prompt, run_name, run_id_ in zip(1024                    callback_managers,1025                    prompts,1026                    run_name_list,1027                    run_ids_list,1028                    strict=False,1029                )1030            ]1031            return self._generate_helper(1032                prompts,1033                stop,1034                run_managers,1035                new_arg_supported=bool(new_arg_supported),1036                **kwargs,1037            )1038        if len(missing_prompts) > 0:1039            run_managers = [1040                callback_managers[idx].on_llm_start(1041                    self._serialized,1042                    [prompts[idx]],1043                    invocation_params=params,1044                    options=options,1045                    name=run_name_list[idx],1046                    batch_size=len(missing_prompts),1047                )[0]1048                for idx in missing_prompt_idxs1049            ]1050            new_results = self._generate_helper(1051                missing_prompts,1052                stop,1053                run_managers,1054                new_arg_supported=bool(new_arg_supported),1055                **kwargs,1056            )1057            llm_output = update_cache(1058                self.cache,1059                existing_prompts,1060                llm_string,1061                missing_prompt_idxs,1062                new_results,1063                prompts,1064            )1065            run_info = (1066                [RunInfo(run_id=run_manager.run_id) for run_manager in run_managers]1067                if run_managers1068                else None1069            )1070        else:1071            llm_output = {}1072            run_info = None1073        generations = [existing_prompts[i] for i in range(len(prompts))]1074        return LLMResult(generations=generations, llm_output=llm_output, run=run_info)10751076    @staticmethod1077    def _get_run_ids_list(1078        run_id: uuid.UUID | list[uuid.UUID | None] | None, prompts: list[str]1079    ) -> list[uuid.UUID | None]:1080        if run_id is None:1081            return [None] * len(prompts)1082        if isinstance(run_id, list):1083            if len(run_id) != len(prompts):1084                msg = (1085                    "Number of manually provided run_id's does not match batch length."1086                    f" {len(run_id)} != {len(prompts)}"1087                )1088                raise ValueError(msg)1089            return run_id1090        return [run_id] + [None] * (len(prompts) - 1)10911092    async def _agenerate_helper(1093        self,1094        prompts: list[str],1095        stop: list[str] | None,1096        run_managers: list[AsyncCallbackManagerForLLMRun],1097        *,1098        new_arg_supported: bool,1099        **kwargs: Any,1100    ) -> LLMResult:1101        try:1102            output = (1103                await self._agenerate(1104                    prompts,1105                    stop=stop,1106                    run_manager=run_managers[0] if run_managers else None,1107                    **kwargs,1108                )1109                if new_arg_supported1110                else await self._agenerate(prompts, stop=stop)1111            )1112        except BaseException as e:1113            await asyncio.gather(1114                *[1115                    run_manager.on_llm_error(e, response=LLMResult(generations=[]))1116                    for run_manager in run_managers1117                ]1118            )1119            raise1120        flattened_outputs = output.flatten()1121        await asyncio.gather(1122            *[1123                run_manager.on_llm_end(flattened_output)1124                for run_manager, flattened_output in zip(1125                    run_managers, flattened_outputs, strict=False1126                )1127            ]1128        )1129        if run_managers:1130            output.run = [1131                RunInfo(run_id=run_manager.run_id) for run_manager in run_managers1132            ]1133        return output11341135    async def agenerate(1136        self,1137        prompts: list[str],1138        stop: list[str] | None = None,1139        callbacks: Callbacks | list[Callbacks] | None = None,1140        *,1141        tags: list[str] | list[list[str]] | None = None,1142        metadata: builtins.dict[str, Any] | list[builtins.dict[str, Any]] | None = None,1143        run_name: str | list[str] | None = None,1144        run_id: uuid.UUID | list[uuid.UUID | None] | None = None,1145        **kwargs: Any,1146    ) -> LLMResult:1147        """Asynchronously pass a sequence of prompts to a model and return generations.11481149        This method should make use of batched calls for models that expose a batched1150        API.11511152        Use this method when you want to:11531154        1. Take advantage of batched calls,1155        2. Need more output from the model than just the top generated value,1156        3. Are building chains that are agnostic to the underlying language model1157            type (e.g., pure text completion models vs chat models).11581159        Args:1160            prompts: List of string prompts.1161            stop: Stop words to use when generating.11621163                Model output is cut off at the first occurrence of any of these1164                substrings.1165            callbacks: `Callbacks` to pass through.11661167                Used for executing additional functionality, such as logging or1168                streaming, throughout generation.1169            tags: List of tags to associate with each prompt. If provided, the length1170                of the list must match the length of the prompts list.1171            metadata: List of metadata dictionaries to associate with each prompt. If1172                provided, the length of the list must match the length of the prompts1173                list.1174            run_name: List of run names to associate with each prompt. If provided, the1175                length of the list must match the length of the prompts list.1176            run_id: List of run IDs to associate with each prompt. If provided, the1177                length of the list must match the length of the prompts list.1178            **kwargs: Arbitrary additional keyword arguments.11791180                These are usually passed to the model provider API call.11811182        Raises:1183            ValueError: If the length of `callbacks`, `tags`, `metadata`, or1184                `run_name` (if provided) does not match the length of prompts.11851186        Returns:1187            An `LLMResult`, which contains a list of candidate `Generations` for each1188                input prompt and additional model provider-specific output.1189        """1190        if isinstance(metadata, list):1191            metadata = [1192                {1193                    **(meta or {}),1194                    **self._get_ls_params_with_defaults(stop=stop, **kwargs),1195                }1196                for meta in metadata1197            ]1198        elif isinstance(metadata, dict):1199            metadata = {1200                **(metadata or {}),1201                **self._get_ls_params_with_defaults(stop=stop, **kwargs),1202            }1203        # Create callback managers1204        if isinstance(callbacks, list) and (1205            isinstance(callbacks[0], (list, BaseCallbackManager))1206            or callbacks[0] is None1207        ):1208            # We've received a list of callbacks args to apply to each input1209            if len(callbacks) != len(prompts):1210                msg = "callbacks must be the same length as prompts"1211                raise ValueError(msg)1212            if tags is not None and not (1213                isinstance(tags, list) and len(tags) == len(prompts)1214            ):1215                msg = "tags must be a list of the same length as prompts"1216                raise ValueError(msg)1217            if metadata is not None and not (1218                isinstance(metadata, list) and len(metadata) == len(prompts)1219            ):1220                msg = "metadata must be a list of the same length as prompts"1221                raise ValueError(msg)1222            if run_name is not None and not (1223                isinstance(run_name, list) and len(run_name) == len(prompts)1224            ):1225                msg = "run_name must be a list of the same length as prompts"1226                raise ValueError(msg)1227            tags_list = cast("list[list[str] | None]", tags or ([None] * len(prompts)))1228            metadata_list = cast(1229                "list[builtins.dict[str, Any] | None]",1230                metadata or ([{}] * len(prompts)),1231            )1232            run_name_list = run_name or cast(1233                "list[str | None]", ([None] * len(prompts))1234            )1235            params = self._dict_for_compat()1236            params["stop"] = stop1237            callback_managers = [1238                AsyncCallbackManager.configure(1239                    callback,1240                    self.callbacks,1241                    self.verbose,1242                    tag,1243                    self.tags,1244                    meta,1245                    self.metadata,1246                    langsmith_inheritable_metadata=_filter_invocation_params_for_tracing(1247                        params1248                    ),1249                )1250                for callback, tag, meta in zip(1251                    callbacks, tags_list, metadata_list, strict=False1252                )1253            ]1254        else:1255            # We've received a single callbacks arg to apply to all inputs1256            params = self._dict_for_compat()1257            params["stop"] = stop1258            callback_managers = [1259                AsyncCallbackManager.configure(1260                    callbacks,1261                    self.callbacks,1262                    self.verbose,1263                    cast("list[str]", tags),1264                    self.tags,1265                    cast("builtins.dict[str, Any]", metadata),1266                    self.metadata,1267                    langsmith_inheritable_metadata=_filter_invocation_params_for_tracing(1268                        params1269                    ),1270                )1271            ] * len(prompts)1272            run_name_list = [cast("str | None", run_name)] * len(prompts)1273        run_ids_list = self._get_run_ids_list(run_id, prompts)1274        options = {"stop": stop}1275        (1276            existing_prompts,1277            llm_string,1278            missing_prompt_idxs,1279            missing_prompts,1280        ) = await aget_prompts(params, prompts, self.cache)12811282        # Verify whether the cache is set, and if the cache is set,1283        # verify whether the cache is available.1284        new_arg_supported = inspect.signature(self._agenerate).parameters.get(1285            "run_manager"1286        )1287        if (self.cache is None and get_llm_cache() is None) or self.cache is False:1288            run_managers = await asyncio.gather(1289                *[1290                    callback_manager.on_llm_start(1291                        self._serialized,1292                        [prompt],1293                        invocation_params=params,1294                        options=options,1295                        name=run_name,1296                        batch_size=len(prompts),1297                        run_id=run_id_,1298                    )1299                    for callback_manager, prompt, run_name, run_id_ in zip(1300                        callback_managers,1301                        prompts,1302                        run_name_list,1303                        run_ids_list,1304                        strict=False,1305                    )1306                ]1307            )1308            run_managers = [r[0] for r in run_managers]  # type: ignore[misc]1309            return await self._agenerate_helper(1310                prompts,1311                stop,1312                run_managers,  # type: ignore[arg-type]1313                new_arg_supported=bool(new_arg_supported),1314                **kwargs,1315            )1316        if len(missing_prompts) > 0:1317            run_managers = await asyncio.gather(1318                *[1319                    callback_managers[idx].on_llm_start(1320                        self._serialized,1321                        [prompts[idx]],1322                        invocation_params=params,1323                        options=options,1324                        name=run_name_list[idx],1325                        batch_size=len(missing_prompts),1326                    )1327                    for idx in missing_prompt_idxs1328                ]1329            )1330            run_managers = [r[0] for r in run_managers]  # type: ignore[misc]1331            new_results = await self._agenerate_helper(1332                missing_prompts,1333                stop,1334                run_managers,  # type: ignore[arg-type]1335                new_arg_supported=bool(new_arg_supported),1336                **kwargs,1337            )1338            llm_output = await aupdate_cache(1339                self.cache,1340                existing_prompts,1341                llm_string,1342                missing_prompt_idxs,1343                new_results,1344                prompts,1345            )1346            run_info = (1347                [RunInfo(run_id=run_manager.run_id) for run_manager in run_managers]  # type: ignore[attr-defined]1348                if run_managers1349                else None1350            )1351        else:1352            llm_output = {}1353            run_info = None1354        generations = [existing_prompts[i] for i in range(len(prompts))]1355        return LLMResult(generations=generations, llm_output=llm_output, run=run_info)13561357    async def _call_async(1358        self,1359        prompt: str,1360        stop: list[str] | None = None,1361        callbacks: Callbacks = None,1362        *,1363        tags: list[str] | None = None,1364        metadata: builtins.dict[str, Any] | None = None,1365        **kwargs: Any,1366    ) -> str:1367        """Check Cache and run the LLM on the given prompt and input."""1368        result = await self.agenerate(1369            [prompt],1370            stop=stop,1371            callbacks=callbacks,1372            tags=tags,1373            metadata=metadata,1374            **kwargs,1375        )1376        return result.generations[0][0].text13771378    def __str__(self) -> str:1379        """Return a string representation of the object for printing."""1380        cls_name = f"\033[1m{self.__class__.__name__}\033[0m"1381        return f"{cls_name}\nParams: {self._identifying_params}"13821383    @property1384    @abstractmethod1385    def _llm_type(self) -> str:1386        """Return type of llm."""13871388    @deprecated("1.4.2", alternative="asdict", removal="2.0.0")1389    @override1390    def dict(self, **_kwargs: Any) -> builtins.dict[str, Any]:1391        """DEPRECATED - use `asdict()` instead.13921393        Return a dictionary representation of the LLM.1394        """1395        return self.asdict()13961397    def asdict(self) -> builtins.dict[str, Any]:1398        """Return a dictionary representation of the LLM."""1399        starter_dict = dict(self._identifying_params)1400        starter_dict["_type"] = self._llm_type1401        return starter_dict14021403    def _dict_for_compat(self) -> builtins.dict[str, Any]:1404        """Return the LLM dictionary while preserving deprecated overrides."""1405        with suppress_langchain_deprecation_warning():1406            return self.dict()14071408    def save(self, file_path: Path | str) -> None:1409        """Save the LLM.14101411        Args:1412            file_path: Path to file to save the LLM to.14131414        Raises:1415            ValueError: If the file path is not a string or Path object.14161417        Example:1418            ```python1419            llm.save(file_path="path/llm.yaml")1420            ```1421        """1422        # Convert file to Path object.1423        save_path = Path(file_path)14241425        directory_path = save_path.parent1426        directory_path.mkdir(parents=True, exist_ok=True)14271428        # Fetch dictionary to save1429        prompt_dict = self._dict_for_compat()14301431        if save_path.suffix == ".json":1432            with save_path.open("w", encoding="utf-8") as f:1433                json.dump(prompt_dict, f, indent=4)1434        elif save_path.suffix.endswith((".yaml", ".yml")):1435            with save_path.open("w", encoding="utf-8") as f:1436                yaml.dump(prompt_dict, f, default_flow_style=False)1437        else:1438            msg = f"{save_path} must be json or yaml"1439            raise ValueError(msg)144014411442class LLM(BaseLLM):1443    """Simple interface for implementing a custom LLM.14441445    You should subclass this class and implement the following:14461447    - `_call` method: Run the LLM on the given prompt and input (used by `invoke`).1448    - `_identifying_params` property: Return a dictionary of the identifying parameters1449        This is critical for caching and tracing purposes. Identifying parameters1450        is a dict that identifies the LLM.1451        It should mostly include a `model_name`.14521453    Optional: Override the following methods to provide more optimizations:14541455    - `_acall`: Provide a native async version of the `_call` method.1456        If not provided, will delegate to the synchronous version using1457        `run_in_executor`. (Used by `ainvoke`).1458    - `_stream`: Stream the LLM on the given prompt and input.1459        `stream` will use `_stream` if provided, otherwise it1460        use `_call` and output will arrive in one chunk.1461    - `_astream`: Override to provide a native async version of the `_stream` method.1462        `astream` will use `_astream` if provided, otherwise it will implement1463        a fallback behavior that will use `_stream` if `_stream` is implemented,1464        and use `_acall` if `_stream` is not implemented.1465    """14661467    @abstractmethod1468    def _call(1469        self,1470        prompt: str,1471        stop: list[str] | None = None,1472        run_manager: CallbackManagerForLLMRun | None = None,1473        **kwargs: Any,1474    ) -> str:1475        """Run the LLM on the given input.14761477        Override this method to implement the LLM logic.14781479        Args:1480            prompt: The prompt to generate from.1481            stop: Stop words to use when generating.14821483                Model output is cut off at the first occurrence of any of these1484                substrings.14851486                If stop tokens are not supported consider raising `NotImplementedError`.1487            run_manager: Callback manager for the run.1488            **kwargs: Arbitrary additional keyword arguments.14891490                These are usually passed to the model provider API call.14911492        Returns:1493            The model output as a string. SHOULD NOT include the prompt.1494        """14951496    async def _acall(1497        self,1498        prompt: str,1499        stop: list[str] | None = None,1500        run_manager: AsyncCallbackManagerForLLMRun | None = None,1501        **kwargs: Any,1502    ) -> str:1503        """Async version of the _call method.15041505        The default implementation delegates to the synchronous _call method using1506        `run_in_executor`. Subclasses that need to provide a true async implementation1507        should override this method to reduce the overhead of using `run_in_executor`.15081509        Args:1510            prompt: The prompt to generate from.1511            stop: Stop words to use when generating.15121513                Model output is cut off at the first occurrence of any of these1514                substrings.15151516                If stop tokens are not supported consider raising `NotImplementedError`.1517            run_manager: Callback manager for the run.1518            **kwargs: Arbitrary additional keyword arguments.15191520                These are usually passed to the model provider API call.15211522        Returns:1523            The model output as a string. SHOULD NOT include the prompt.1524        """1525        return await run_in_executor(1526            None,1527            self._call,1528            prompt,1529            stop,1530            run_manager.get_sync() if run_manager else None,1531            **kwargs,1532        )15331534    def _generate(1535        self,1536        prompts: list[str],1537        stop: list[str] | None = None,1538        run_manager: CallbackManagerForLLMRun | None = None,1539        **kwargs: Any,1540    ) -> LLMResult:1541        # TODO: add caching here.1542        generations = []1543        new_arg_supported = inspect.signature(self._call).parameters.get("run_manager")1544        for prompt in prompts:1545            text = (1546                self._call(prompt, stop=stop, run_manager=run_manager, **kwargs)1547                if new_arg_supported1548                else self._call(prompt, stop=stop, **kwargs)1549            )1550            generations.append([Generation(text=text)])1551        return LLMResult(generations=generations)15521553    async def _agenerate(1554        self,1555        prompts: list[str],1556        stop: list[str] | None = None,1557        run_manager: AsyncCallbackManagerForLLMRun | None = None,1558        **kwargs: Any,1559    ) -> LLMResult:1560        generations = []1561        new_arg_supported = inspect.signature(self._acall).parameters.get("run_manager")1562        for prompt in prompts:1563            text = (1564                await self._acall(prompt, stop=stop, run_manager=run_manager, **kwargs)1565                if new_arg_supported1566                else await self._acall(prompt, stop=stop, **kwargs)1567            )1568            generations.append([Generation(text=text)])1569        return LLMResult(generations=generations)