1"""Base interface for traditional large language models (LLMs) to expose.23These are traditionally older models (newer models generally are chat models).4"""56from __future__ import annotations78import asyncio9import builtins10import functools11import json12import logging13from abc import ABC, abstractmethod14from collections.abc import AsyncIterator, Callable, Iterator, Sequence15from pathlib import Path16from typing import (17    TYPE_CHECKING,18    Any,19    cast,20)2122import yaml23from pydantic import ConfigDict24from tenacity import (25    RetryCallState,26    before_sleep_log,27    retry,28    retry_base,29    retry_if_exception_type,30    stop_after_attempt,31    wait_exponential,32)33from typing_extensions import override3435from langchain_core._api import deprecated, suppress_langchain_deprecation_warning36from langchain_core.caches import BaseCache37from langchain_core.callbacks import (38    AsyncCallbackManager,39    AsyncCallbackManagerForLLMRun,40    BaseCallbackManager,41    CallbackManager,42    CallbackManagerForLLMRun,43    Callbacks,44)45from langchain_core.globals import get_llm_cache46from langchain_core.language_models._utils import _filter_invocation_params_for_tracing47from langchain_core.language_models.base import (48    BaseLanguageModel,49    LangSmithParams,50    LanguageModelInput,51)52from langchain_core.load import dumpd53from langchain_core.messages import (54    convert_to_messages,55)56from langchain_core.outputs import Generation, GenerationChunk, LLMResult, RunInfo57from langchain_core.prompt_values import ChatPromptValue, PromptValue, StringPromptValue58from langchain_core.runnables import RunnableConfig, ensure_config, get_config_list59from langchain_core.runnables.config import run_in_executor60from langchain_core.utils._signature import _signature_parameters6162if TYPE_CHECKING:63    import builtins64    import uuid6566logger = logging.getLogger(__name__)6768_background_tasks: set[asyncio.Task[None]] = set()697071@functools.lru_cache72def _log_error_once(msg: str) -> None:73    """Log an error once."""74    logger.error(msg)757677def create_base_retry_decorator(78    error_types: list[type[BaseException]],79    max_retries: int = 1,80    run_manager: AsyncCallbackManagerForLLMRun | CallbackManagerForLLMRun | None = None,81) -> Callable[[Any], Any]:82    """Create a retry decorator for a given LLM and provided a list of error types.8384    Args:85        error_types: List of error types to retry on.86        max_retries: Number of retries.87        run_manager: Callback manager for the run.8889    Returns:90        A retry decorator.9192    Raises:93        ValueError: If the cache is not set and cache is True.94    """95    logging_ = before_sleep_log(logger, logging.WARNING)9697    def _before_sleep(retry_state: RetryCallState) -> None:98        logging_(retry_state)99        if run_manager:100            if isinstance(run_manager, AsyncCallbackManagerForLLMRun):101                coro = run_manager.on_retry(retry_state)102                try:103                    try:104                        loop = asyncio.get_event_loop()105                    except RuntimeError:106                        asyncio.run(coro)107                    else:108                        if loop.is_running():109                            task = loop.create_task(coro)110                            _background_tasks.add(task)111                            task.add_done_callback(_background_tasks.discard)112                        else:113                            asyncio.run(coro)114                except Exception as e:115                    _log_error_once(f"Error in on_retry: {e}")116            else:117                run_manager.on_retry(retry_state)118119    min_seconds = 4120    max_seconds = 10121    # Wait 2^x * 1 second between each retry starting with122    # 4 seconds, then up to 10 seconds, then 10 seconds afterwards123    retry_instance: retry_base = retry_if_exception_type(error_types[0])124    for error in error_types[1:]:125        retry_instance |= retry_if_exception_type(error)126    return retry(127        reraise=True,128        stop=stop_after_attempt(max_retries),129        wait=wait_exponential(multiplier=1, min=min_seconds, max=max_seconds),130        retry=retry_instance,131        before_sleep=_before_sleep,132    )133134135def _resolve_cache(*, cache: BaseCache | bool | None) -> BaseCache | None:136    """Resolve the cache."""137    llm_cache: BaseCache | None138    if isinstance(cache, BaseCache):139        llm_cache = cache140    elif cache is None:141        llm_cache = get_llm_cache()142    elif cache is True:143        llm_cache = get_llm_cache()144        if llm_cache is None:145            msg = (146                "No global cache was configured. Use `set_llm_cache`."147                "to set a global cache if you want to use a global cache."148                "Otherwise either pass a cache object or set cache to False/None"149            )150            raise ValueError(msg)151    elif cache is False:152        llm_cache = None153    else:154        msg = f"Unsupported cache value {cache}"  # type: ignore[unreachable]155        raise ValueError(msg)156    return llm_cache157158159def get_prompts(160    params: dict[str, Any],161    prompts: list[str],162    cache: BaseCache | bool | None = None,  # noqa: FBT001163) -> tuple[dict[int, list[Generation]], str, list[int], list[str]]:164    """Get prompts that are already cached.165166    Args:167        params: Dictionary of parameters.168        prompts: List of prompts.169        cache: Cache object.170171    Returns:172        A tuple of existing prompts, llm_string, missing prompt indexes,173            and missing prompts.174175    Raises:176        ValueError: If the cache is not set and cache is True.177    """178    llm_string = str(sorted(params.items()))179    missing_prompts = []180    missing_prompt_idxs = []181    existing_prompts = {}182183    llm_cache = _resolve_cache(cache=cache)184    for i, prompt in enumerate(prompts):185        if llm_cache is not None:186            cache_val = llm_cache.lookup(prompt, llm_string)187            if isinstance(cache_val, list):188                existing_prompts[i] = cache_val189                continue190        missing_prompts.append(prompt)191        missing_prompt_idxs.append(i)192    return existing_prompts, llm_string, missing_prompt_idxs, missing_prompts193194195async def aget_prompts(196    params: dict[str, Any],197    prompts: list[str],198    cache: BaseCache | bool | None = None,  # noqa: FBT001199) -> tuple[dict[int, list[Generation]], str, list[int], list[str]]:200    """Get prompts that are already cached. Async version.201202    Args:203        params: Dictionary of parameters.204        prompts: List of prompts.205        cache: Cache object.206207    Returns:208        A tuple of existing prompts, llm_string, missing prompt indexes,209            and missing prompts.210211    Raises:212        ValueError: If the cache is not set and cache is True.213    """214    llm_string = str(sorted(params.items()))215    missing_prompts = []216    missing_prompt_idxs = []217    existing_prompts = {}218    llm_cache = _resolve_cache(cache=cache)219    for i, prompt in enumerate(prompts):220        if llm_cache is not None:221            cache_val = await llm_cache.alookup(prompt, llm_string)222            if isinstance(cache_val, list):223                existing_prompts[i] = cache_val224                continue225        missing_prompts.append(prompt)226        missing_prompt_idxs.append(i)227    return existing_prompts, llm_string, missing_prompt_idxs, missing_prompts228229230def update_cache(231    cache: BaseCache | bool | None,  # noqa: FBT001232    existing_prompts: dict[int, list[Generation]],233    llm_string: str,234    missing_prompt_idxs: list[int],235    new_results: LLMResult,236    prompts: list[str],237) -> dict[str, Any] | None:238    """Update the cache and get the LLM output.239240    Args:241        cache: Cache object.242        existing_prompts: Dictionary of existing prompts.243        llm_string: LLM string.244        missing_prompt_idxs: List of missing prompt indexes.245        new_results: LLMResult object.246        prompts: List of prompts.247248    Returns:249        LLM output.250251    Raises:252        ValueError: If the cache is not set and cache is True.253    """254    llm_cache = _resolve_cache(cache=cache)255    for i, result in enumerate(new_results.generations):256        existing_prompts[missing_prompt_idxs[i]] = result257        prompt = prompts[missing_prompt_idxs[i]]258        if llm_cache is not None:259            llm_cache.update(prompt, llm_string, result)260    return new_results.llm_output261262263async def aupdate_cache(264    cache: BaseCache | bool | None,  # noqa: FBT001265    existing_prompts: dict[int, list[Generation]],266    llm_string: str,267    missing_prompt_idxs: list[int],268    new_results: LLMResult,269    prompts: list[str],270) -> dict[str, Any] | None:271    """Update the cache and get the LLM output. Async version.272273    Args:274        cache: Cache object.275        existing_prompts: Dictionary of existing prompts.276        llm_string: LLM string.277        missing_prompt_idxs: List of missing prompt indexes.278        new_results: LLMResult object.279        prompts: List of prompts.280281    Returns:282        LLM output.283284    Raises:285        ValueError: If the cache is not set and cache is True.286    """287    llm_cache = _resolve_cache(cache=cache)288    for i, result in enumerate(new_results.generations):289        existing_prompts[missing_prompt_idxs[i]] = result290        prompt = prompts[missing_prompt_idxs[i]]291        if llm_cache is not None:292            await llm_cache.aupdate(prompt, llm_string, result)293    return new_results.llm_output294295296class BaseLLM(BaseLanguageModel[str], ABC):297    """Base LLM abstract interface.298299    It should take in a prompt and return a string.300    """301302    model_config = ConfigDict(303        arbitrary_types_allowed=True,304    )305306    @functools.cached_property307    def _serialized(self) -> builtins.dict[str, Any]:308        # self is always a Serializable object in this case, thus the result is309        # guaranteed to be a dict since dumpd uses the default callback, which uses310        # obj.to_json which always returns TypedDict subclasses311        return cast("builtins.dict[str, Any]", dumpd(self))312313    # --- Runnable methods ---314315    @property316    @override317    def OutputType(self) -> type[str]:318        """Get the output type for this `Runnable`."""319        return str320321    def _convert_input(self, model_input: LanguageModelInput) -> PromptValue:322        if isinstance(model_input, PromptValue):323            return model_input324        if isinstance(model_input, str):325            return StringPromptValue(text=model_input)326        if isinstance(model_input, Sequence):327            return ChatPromptValue(messages=convert_to_messages(model_input))328        msg = (  # type: ignore[unreachable]329            f"Invalid input type {type(model_input)}. "330            "Must be a PromptValue, str, or list of BaseMessages."331        )332        raise ValueError(msg)333334    def _get_ls_params(335        self,336        stop: list[str] | None = None,337        **kwargs: Any,338    ) -> LangSmithParams:339        """Get standard params for tracing."""340        # get default provider from class name341        default_provider = self.__class__.__name__342        default_provider = default_provider.removesuffix("LLM")343        default_provider = default_provider.lower()344345        ls_params = LangSmithParams(ls_provider=default_provider, ls_model_type="llm")346        if stop:347            ls_params["ls_stop"] = stop348349        # model350        if "model" in kwargs and isinstance(kwargs["model"], str):351            ls_params["ls_model_name"] = kwargs["model"]352        elif hasattr(self, "model") and isinstance(self.model, str):353            ls_params["ls_model_name"] = self.model354        elif hasattr(self, "model_name") and isinstance(self.model_name, str):355            ls_params["ls_model_name"] = self.model_name356357        # temperature358        if "temperature" in kwargs and isinstance(kwargs["temperature"], (int, float)):359            ls_params["ls_temperature"] = kwargs["temperature"]360        elif hasattr(self, "temperature") and isinstance(361            self.temperature, (int, float)362        ):363            ls_params["ls_temperature"] = self.temperature364365        # max_tokens366        if "max_tokens" in kwargs and isinstance(kwargs["max_tokens"], int):367            ls_params["ls_max_tokens"] = kwargs["max_tokens"]368        elif hasattr(self, "max_tokens") and isinstance(self.max_tokens, int):369            ls_params["ls_max_tokens"] = self.max_tokens370371        return ls_params372373    @override374    def invoke(375        self,376        input: LanguageModelInput,377        config: RunnableConfig | None = None,378        *,379        stop: list[str] | None = None,380        **kwargs: Any,381    ) -> str:382        config = ensure_config(config)383        return (384            self.generate_prompt(385                [self._convert_input(input)],386                stop=stop,387                callbacks=config.get("callbacks"),388                tags=config.get("tags"),389                metadata=config.get("metadata"),390                run_name=config.get("run_name"),391                run_id=config.pop("run_id", None),392                **kwargs,393            )394            .generations[0][0]395            .text396        )397398    @override399    async def ainvoke(400        self,401        input: LanguageModelInput,402        config: RunnableConfig | None = None,403        *,404        stop: list[str] | None = None,405        **kwargs: Any,406    ) -> str:407        config = ensure_config(config)408        llm_result = await self.agenerate_prompt(409            [self._convert_input(input)],410            stop=stop,411            callbacks=config.get("callbacks"),412            tags=config.get("tags"),413            metadata=config.get("metadata"),414            run_name=config.get("run_name"),415            run_id=config.pop("run_id", None),416            **kwargs,417        )418        return llm_result.generations[0][0].text419420    @override421    def batch(422        self,423        inputs: list[LanguageModelInput],424        config: RunnableConfig | list[RunnableConfig] | None = None,425        *,426        return_exceptions: bool = False,427        **kwargs: Any,428    ) -> list[str]:429        if not inputs:430            return []431432        config = get_config_list(config, len(inputs))433        max_concurrency = config[0].get("max_concurrency")434435        if max_concurrency is None:436            try:437                llm_result = self.generate_prompt(438                    [self._convert_input(input_) for input_ in inputs],439                    callbacks=[c.get("callbacks") for c in config],440                    tags=[c.get("tags") for c in config],441                    metadata=[c.get("metadata") for c in config],442                    run_name=[c.get("run_name") for c in config],443                    **kwargs,444                )445                return [g[0].text for g in llm_result.generations]446            except Exception as e:447                if return_exceptions:448                    return cast("list[str]", [e for _ in inputs])449                raise450        else:451            batches = [452                inputs[i : i + max_concurrency]453                for i in range(0, len(inputs), max_concurrency)454            ]455            config = [{**c, "max_concurrency": None} for c in config]456            return [457                output458                for i, batch in enumerate(batches)459                for output in self.batch(460                    batch,461                    config=config[i * max_concurrency : (i + 1) * max_concurrency],462                    return_exceptions=return_exceptions,463                    **kwargs,464                )465            ]466467    @override468    async def abatch(469        self,470        inputs: list[LanguageModelInput],471        config: RunnableConfig | list[RunnableConfig] | None = None,472        *,473        return_exceptions: bool = False,474        **kwargs: Any,475    ) -> list[str]:476        if not inputs:477            return []478        config = get_config_list(config, len(inputs))479        max_concurrency = config[0].get("max_concurrency")480481        if max_concurrency is None:482            try:483                llm_result = await self.agenerate_prompt(484                    [self._convert_input(input_) for input_ in inputs],485                    callbacks=[c.get("callbacks") for c in config],486                    tags=[c.get("tags") for c in config],487                    metadata=[c.get("metadata") for c in config],488                    run_name=[c.get("run_name") for c in config],489                    **kwargs,490                )491                return [g[0].text for g in llm_result.generations]492            except Exception as e:493                if return_exceptions:494                    return cast("list[str]", [e for _ in inputs])495                raise496        else:497            batches = [498                inputs[i : i + max_concurrency]499                for i in range(0, len(inputs), max_concurrency)500            ]501            config = [{**c, "max_concurrency": None} for c in config]502            return [503                output504                for i, batch in enumerate(batches)505                for output in await self.abatch(506                    batch,507                    config=config[i * max_concurrency : (i + 1) * max_concurrency],508                    return_exceptions=return_exceptions,509                    **kwargs,510                )511            ]512513    @override514    def stream(515        self,516        input: LanguageModelInput,517        config: RunnableConfig | None = None,518        *,519        stop: list[str] | None = None,520        **kwargs: Any,521    ) -> Iterator[str]:522        if type(self)._stream == BaseLLM._stream:  # noqa: SLF001523            # model doesn't implement streaming, so use default implementation524            yield self.invoke(input, config=config, stop=stop, **kwargs)525        else:526            prompt = self._convert_input(input).to_string()527            config = ensure_config(config)528            params = self._dict_for_compat()529            params["stop"] = stop530            params = {**params, **kwargs}531            options = {"stop": stop}532            inheritable_metadata = {533                **(config.get("metadata") or {}),534                **self._get_ls_params_with_defaults(stop=stop, **kwargs),535            }536            callback_manager = CallbackManager.configure(537                config.get("callbacks"),538                self.callbacks,539                self.verbose,540                config.get("tags"),541                self.tags,542                inheritable_metadata,543                self.metadata,544                langsmith_inheritable_metadata=_filter_invocation_params_for_tracing(545                    params546                ),547            )548            (run_manager,) = callback_manager.on_llm_start(549                self._serialized,550                [prompt],551                invocation_params=params,552                options=options,553                name=config.get("run_name"),554                run_id=config.pop("run_id", None),555                batch_size=1,556            )557            generation: GenerationChunk | None = None558            try:559                for chunk in self._stream(560                    prompt, stop=stop, run_manager=run_manager, **kwargs561                ):562                    yield chunk.text563                    if generation is None:564                        generation = chunk565                    else:566                        generation += chunk567            except BaseException as e:568                run_manager.on_llm_error(569                    e,570                    response=LLMResult(571                        generations=[[generation]] if generation else []572                    ),573                )574                raise575576            if generation is None:577                err = ValueError("No generation chunks were returned")578                run_manager.on_llm_error(err, response=LLMResult(generations=[]))579                raise err580581            run_manager.on_llm_end(LLMResult(generations=[[generation]]))582583    @override584    async def astream(585        self,586        input: LanguageModelInput,587        config: RunnableConfig | None = None,588        *,589        stop: list[str] | None = None,590        **kwargs: Any,591    ) -> AsyncIterator[str]:592        if (593            type(self)._astream is BaseLLM._astream  # noqa: SLF001594            and type(self)._stream is BaseLLM._stream  # noqa: SLF001595        ):596            yield await self.ainvoke(input, config=config, stop=stop, **kwargs)597            return598599        prompt = self._convert_input(input).to_string()600        config = ensure_config(config)601        params = self._dict_for_compat()602        params["stop"] = stop603        params = {**params, **kwargs}604        options = {"stop": stop}605        inheritable_metadata = {606            **(config.get("metadata") or {}),607            **self._get_ls_params_with_defaults(stop=stop, **kwargs),608        }609        callback_manager = AsyncCallbackManager.configure(610            config.get("callbacks"),611            self.callbacks,612            self.verbose,613            config.get("tags"),614            self.tags,615            inheritable_metadata,616            self.metadata,617            langsmith_inheritable_metadata=_filter_invocation_params_for_tracing(618                params619            ),620        )621        (run_manager,) = await callback_manager.on_llm_start(622            self._serialized,623            [prompt],624            invocation_params=params,625            options=options,626            name=config.get("run_name"),627            run_id=config.pop("run_id", None),628            batch_size=1,629        )630        generation: GenerationChunk | None = None631        try:632            async for chunk in self._astream(633                prompt,634                stop=stop,635                run_manager=run_manager,636                **kwargs,637            ):638                yield chunk.text639                if generation is None:640                    generation = chunk641                else:642                    generation += chunk643        except BaseException as e:644            await run_manager.on_llm_error(645                e,646                response=LLMResult(generations=[[generation]] if generation else []),647            )648            raise649650        if generation is None:651            err = ValueError("No generation chunks were returned")652            await run_manager.on_llm_error(err, response=LLMResult(generations=[]))653            raise err654655        await run_manager.on_llm_end(LLMResult(generations=[[generation]]))656657    # --- Custom methods ---658659    @abstractmethod660    def _generate(661        self,662        prompts: list[str],663        stop: list[str] | None = None,664        run_manager: CallbackManagerForLLMRun | None = None,665        **kwargs: Any,666    ) -> LLMResult:667        """Run the LLM on the given prompts.668669        Args:670            prompts: The prompts to generate from.671            stop: Stop words to use when generating.672673                Model output is cut off at the first occurrence of any of these674                substrings.675676                If stop tokens are not supported consider raising `NotImplementedError`.677            run_manager: Callback manager for the run.678679        Returns:680            The LLM result.681        """682683    async def _agenerate(684        self,685        prompts: list[str],686        stop: list[str] | None = None,687        run_manager: AsyncCallbackManagerForLLMRun | None = None,688        **kwargs: Any,689    ) -> LLMResult:690        """Run the LLM on the given prompts.691692        Args:693            prompts: The prompts to generate from.694            stop: Stop words to use when generating.695696                Model output is cut off at the first occurrence of any of these697                substrings.698699                If stop tokens are not supported consider raising `NotImplementedError`.700            run_manager: Callback manager for the run.701702        Returns:703            The LLM result.704        """705        return await run_in_executor(706            None,707            self._generate,708            prompts,709            stop,710            run_manager.get_sync() if run_manager else None,711            **kwargs,712        )713714    def _stream(715        self,716        prompt: str,717        stop: list[str] | None = None,718        run_manager: CallbackManagerForLLMRun | None = None,719        **kwargs: Any,720    ) -> Iterator[GenerationChunk]:721        """Stream the LLM on the given prompt.722723        This method should be overridden by subclasses that support streaming.724725        If not implemented, the default behavior of calls to stream will be to726        fallback to the non-streaming version of the model and return727        the output as a single chunk.728729        Args:730            prompt: The prompt to generate from.731            stop: Stop words to use when generating.732733                Model output is cut off at the first occurrence of any of these734                substrings.735            run_manager: Callback manager for the run.736            **kwargs: Arbitrary additional keyword arguments.737738                These are usually passed to the model provider API call.739740        Yields:741            Generation chunks.742        """743        raise NotImplementedError744745    async def _astream(746        self,747        prompt: str,748        stop: list[str] | None = None,749        run_manager: AsyncCallbackManagerForLLMRun | None = None,750        **kwargs: Any,751    ) -> AsyncIterator[GenerationChunk]:752        """An async version of the _stream method.753754        The default implementation uses the synchronous _stream method and wraps it in755        an async iterator. Subclasses that need to provide a true async implementation756        should override this method.757758        Args:759            prompt: The prompt to generate from.760            stop: Stop words to use when generating.761762                Model output is cut off at the first occurrence of any of these763                substrings.764            run_manager: Callback manager for the run.765            **kwargs: Arbitrary additional keyword arguments.766767                These are usually passed to the model provider API call.768769        Yields:770            Generation chunks.771        """772        iterator = await run_in_executor(773            None,774            self._stream,775            prompt,776            stop,777            run_manager.get_sync() if run_manager else None,778            **kwargs,779        )780        done = object()781        while True:782            item = await run_in_executor(783                None,784                next,785                iterator,786                done,787            )788            if item is done:789                break790            yield item  # type: ignore[misc]791792    @override793    def generate_prompt(794        self,795        prompts: list[PromptValue],796        stop: list[str] | None = None,797        callbacks: Callbacks | list[Callbacks] | None = None,798        **kwargs: Any,799    ) -> LLMResult:800        prompt_strings = [p.to_string() for p in prompts]801        return self.generate(prompt_strings, stop=stop, callbacks=callbacks, **kwargs)802803    @override804    async def agenerate_prompt(805        self,806        prompts: list[PromptValue],807        stop: list[str] | None = None,808        callbacks: Callbacks | list[Callbacks] | None = None,809        **kwargs: Any,810    ) -> LLMResult:811        prompt_strings = [p.to_string() for p in prompts]812        return await self.agenerate(813            prompt_strings, stop=stop, callbacks=callbacks, **kwargs814        )815816    def _generate_helper(817        self,818        prompts: list[str],819        stop: list[str] | None,820        run_managers: list[CallbackManagerForLLMRun],821        *,822        new_arg_supported: bool,823        **kwargs: Any,824    ) -> LLMResult:825        try:826            output = (827                self._generate(828                    prompts,829                    stop=stop,830                    # TODO: support multiple run managers831                    run_manager=run_managers[0] if run_managers else None,832                    **kwargs,833                )834                if new_arg_supported835                else self._generate(prompts, stop=stop)836            )837        except BaseException as e:838            for run_manager in run_managers:839                run_manager.on_llm_error(e, response=LLMResult(generations=[]))840            raise841        flattened_outputs = output.flatten()842        for manager, flattened_output in zip(843            run_managers, flattened_outputs, strict=False844        ):845            manager.on_llm_end(flattened_output)846        if run_managers:847            output.run = [848                RunInfo(run_id=run_manager.run_id) for run_manager in run_managers849            ]850        return output851852    def generate(853        self,854        prompts: list[str],855        stop: list[str] | None = None,856        callbacks: Callbacks | list[Callbacks] | None = None,857        *,858        tags: list[str] | list[list[str]] | None = None,859        metadata: builtins.dict[str, Any] | list[builtins.dict[str, Any]] | None = None,860        run_name: str | list[str] | None = None,861        run_id: uuid.UUID | list[uuid.UUID | None] | None = None,862        **kwargs: Any,863    ) -> LLMResult:864        """Pass a sequence of prompts to a model and return generations.865866        This method should make use of batched calls for models that expose a batched867        API.868869        Use this method when you want to:870871        1. Take advantage of batched calls,872        2. Need more output from the model than just the top generated value,873        3. Are building chains that are agnostic to the underlying language model874            type (e.g., pure text completion models vs chat models).875876        Args:877            prompts: List of string prompts.878            stop: Stop words to use when generating.879880                Model output is cut off at the first occurrence of any of these881                substrings.882            callbacks: `Callbacks` to pass through.883884                Used for executing additional functionality, such as logging or885                streaming, throughout generation.886            tags: List of tags to associate with each prompt. If provided, the length887                of the list must match the length of the prompts list.888            metadata: List of metadata dictionaries to associate with each prompt. If889                provided, the length of the list must match the length of the prompts890                list.891            run_name: List of run names to associate with each prompt. If provided, the892                length of the list must match the length of the prompts list.893            run_id: List of run IDs to associate with each prompt. If provided, the894                length of the list must match the length of the prompts list.895            **kwargs: Arbitrary additional keyword arguments.896897                These are usually passed to the model provider API call.898899        Raises:900            ValueError: If prompts is not a list.901            ValueError: If the length of `callbacks`, `tags`, `metadata`, or902                `run_name` (if provided) does not match the length of prompts.903904        Returns:905            An `LLMResult`, which contains a list of candidate `Generations` for each906                input prompt and additional model provider-specific output.907        """908        if not isinstance(prompts, list):909            msg = (  # type: ignore[unreachable]910                "Argument 'prompts' is expected to be of type list[str], received"911                f" argument of type {type(prompts)}."912            )913            raise ValueError(msg)  # noqa: TRY004914        # Create callback managers915        if isinstance(metadata, list):916            metadata = [917                {918                    **(meta or {}),919                    **self._get_ls_params_with_defaults(stop=stop, **kwargs),920                }921                for meta in metadata922            ]923        elif isinstance(metadata, dict):924            metadata = {925                **(metadata or {}),926                **self._get_ls_params_with_defaults(stop=stop, **kwargs),927            }928        if (929            isinstance(callbacks, list)930            and callbacks931            and (932                isinstance(callbacks[0], (list, BaseCallbackManager))933                or callbacks[0] is None934            )935        ):936            # We've received a list of callbacks args to apply to each input937            if len(callbacks) != len(prompts):938                msg = "callbacks must be the same length as prompts"939                raise ValueError(msg)940            if tags is not None and not (941                isinstance(tags, list) and len(tags) == len(prompts)942            ):943                msg = "tags must be a list of the same length as prompts"944                raise ValueError(msg)945            if metadata is not None and not (946                isinstance(metadata, list) and len(metadata) == len(prompts)947            ):948                msg = "metadata must be a list of the same length as prompts"949                raise ValueError(msg)950            if run_name is not None and not (951                isinstance(run_name, list) and len(run_name) == len(prompts)952            ):953                msg = "run_name must be a list of the same length as prompts"954                raise ValueError(msg)955            tags_list = cast("list[list[str] | None]", tags or ([None] * len(prompts)))956            metadata_list = cast(957                "list[builtins.dict[str, Any] | None]",958                metadata or ([{}] * len(prompts)),959            )960            run_name_list = run_name or cast(961                "list[str | None]", ([None] * len(prompts))962            )963            params = self._dict_for_compat()964            params["stop"] = stop965            callback_managers = [966                CallbackManager.configure(967                    callback,968                    self.callbacks,969                    self.verbose,970                    tag,971                    self.tags,972                    meta,973                    self.metadata,974                    langsmith_inheritable_metadata=_filter_invocation_params_for_tracing(975                        params976                    ),977                )978                for callback, tag, meta in zip(979                    callbacks, tags_list, metadata_list, strict=False980                )981            ]982        else:983            # We've received a single callbacks arg to apply to all inputs984            params = self._dict_for_compat()985            params["stop"] = stop986            callback_managers = [987                CallbackManager.configure(988                    cast("Callbacks", callbacks),989                    self.callbacks,990                    self.verbose,991                    cast("list[str]", tags),992                    self.tags,993                    cast("builtins.dict[str, Any]", metadata),994                    self.metadata,995                    langsmith_inheritable_metadata=_filter_invocation_params_for_tracing(996                        params997                    ),998                )999            ] * len(prompts)1000            run_name_list = [cast("str | None", run_name)] * len(prompts)1001        run_ids_list = self._get_run_ids_list(run_id, prompts)1002        options = {"stop": stop}1003        (1004            existing_prompts,1005            llm_string,1006            missing_prompt_idxs,1007            missing_prompts,1008        ) = get_prompts(params, prompts, self.cache)1009        new_arg_supported = _signature_parameters(self._generate).get("run_manager")1010        if (self.cache is None and get_llm_cache() is None) or self.cache is False:1011            run_managers = [1012                callback_manager.on_llm_start(1013                    self._serialized,1014                    [prompt],1015                    invocation_params=params,1016                    options=options,1017                    name=run_name,1018                    batch_size=len(prompts),1019                    run_id=run_id_,1020                )[0]1021                for callback_manager, prompt, run_name, run_id_ in zip(1022                    callback_managers,1023                    prompts,1024                    run_name_list,1025                    run_ids_list,1026                    strict=False,1027                )1028            ]1029            return self._generate_helper(1030                prompts,1031                stop,1032                run_managers,1033                new_arg_supported=bool(new_arg_supported),1034                **kwargs,1035            )1036        if len(missing_prompts) > 0:1037            run_managers = [1038                callback_managers[idx].on_llm_start(1039                    self._serialized,1040                    [prompts[idx]],1041                    invocation_params=params,1042                    options=options,1043                    name=run_name_list[idx],1044                    batch_size=len(missing_prompts),1045                )[0]1046                for idx in missing_prompt_idxs1047            ]1048            new_results = self._generate_helper(1049                missing_prompts,1050                stop,1051                run_managers,1052                new_arg_supported=bool(new_arg_supported),1053                **kwargs,1054            )1055            llm_output = update_cache(1056                self.cache,1057                existing_prompts,1058                llm_string,1059                missing_prompt_idxs,1060                new_results,1061                prompts,1062            )1063            run_info = (1064                [RunInfo(run_id=run_manager.run_id) for run_manager in run_managers]1065                if run_managers1066                else None1067            )1068        else:1069            llm_output = {}1070            run_info = None1071        generations = [existing_prompts[i] for i in range(len(prompts))]1072        return LLMResult(generations=generations, llm_output=llm_output, run=run_info)10731074    @staticmethod1075    def _get_run_ids_list(1076        run_id: uuid.UUID | list[uuid.UUID | None] | None, prompts: list[str]1077    ) -> list[uuid.UUID | None]:1078        if run_id is None:1079            return [None] * len(prompts)1080        if isinstance(run_id, list):1081            if len(run_id) != len(prompts):1082                msg = (1083                    "Number of manually provided run_id's does not match batch length."1084                    f" {len(run_id)} != {len(prompts)}"1085                )1086                raise ValueError(msg)1087            return run_id1088        return [run_id] + [None] * (len(prompts) - 1)10891090    async def _agenerate_helper(1091        self,1092        prompts: list[str],1093        stop: list[str] | None,1094        run_managers: list[AsyncCallbackManagerForLLMRun],1095        *,1096        new_arg_supported: bool,1097        **kwargs: Any,1098    ) -> LLMResult:1099        try:1100            output = (1101                await self._agenerate(1102                    prompts,1103                    stop=stop,1104                    run_manager=run_managers[0] if run_managers else None,1105                    **kwargs,1106                )1107                if new_arg_supported1108                else await self._agenerate(prompts, stop=stop)1109            )1110        except BaseException as e:1111            await asyncio.gather(1112                *[1113                    run_manager.on_llm_error(e, response=LLMResult(generations=[]))1114                    for run_manager in run_managers1115                ]1116            )1117            raise1118        flattened_outputs = output.flatten()1119        await asyncio.gather(1120            *[1121                run_manager.on_llm_end(flattened_output)1122                for run_manager, flattened_output in zip(1123                    run_managers, flattened_outputs, strict=False1124                )1125            ]1126        )1127        if run_managers:1128            output.run = [1129                RunInfo(run_id=run_manager.run_id) for run_manager in run_managers1130            ]1131        return output11321133    async def agenerate(1134        self,1135        prompts: list[str],1136        stop: list[str] | None = None,1137        callbacks: Callbacks | list[Callbacks] | None = None,1138        *,1139        tags: list[str] | list[list[str]] | None = None,1140        metadata: builtins.dict[str, Any] | list[builtins.dict[str, Any]] | None = None,1141        run_name: str | list[str] | None = None,1142        run_id: uuid.UUID | list[uuid.UUID | None] | None = None,1143        **kwargs: Any,1144    ) -> LLMResult:1145        """Asynchronously pass a sequence of prompts to a model and return generations.11461147        This method should make use of batched calls for models that expose a batched1148        API.11491150        Use this method when you want to:11511152        1. Take advantage of batched calls,1153        2. Need more output from the model than just the top generated value,1154        3. Are building chains that are agnostic to the underlying language model1155            type (e.g., pure text completion models vs chat models).11561157        Args:1158            prompts: List of string prompts.1159            stop: Stop words to use when generating.11601161                Model output is cut off at the first occurrence of any of these1162                substrings.1163            callbacks: `Callbacks` to pass through.11641165                Used for executing additional functionality, such as logging or1166                streaming, throughout generation.1167            tags: List of tags to associate with each prompt. If provided, the length1168                of the list must match the length of the prompts list.1169            metadata: List of metadata dictionaries to associate with each prompt. If1170                provided, the length of the list must match the length of the prompts1171                list.1172            run_name: List of run names to associate with each prompt. If provided, the1173                length of the list must match the length of the prompts list.1174            run_id: List of run IDs to associate with each prompt. If provided, the1175                length of the list must match the length of the prompts list.1176            **kwargs: Arbitrary additional keyword arguments.11771178                These are usually passed to the model provider API call.11791180        Raises:1181            ValueError: If the length of `callbacks`, `tags`, `metadata`, or1182                `run_name` (if provided) does not match the length of prompts.11831184        Returns:1185            An `LLMResult`, which contains a list of candidate `Generations` for each1186                input prompt and additional model provider-specific output.1187        """1188        if isinstance(metadata, list):1189            metadata = [1190                {1191                    **(meta or {}),1192                    **self._get_ls_params_with_defaults(stop=stop, **kwargs),1193                }1194                for meta in metadata1195            ]1196        elif isinstance(metadata, dict):1197            metadata = {1198                **(metadata or {}),1199                **self._get_ls_params_with_defaults(stop=stop, **kwargs),1200            }1201        # Create callback managers1202        if isinstance(callbacks, list) and (1203            isinstance(callbacks[0], (list, BaseCallbackManager))1204            or callbacks[0] is None1205        ):1206            # We've received a list of callbacks args to apply to each input1207            if len(callbacks) != len(prompts):1208                msg = "callbacks must be the same length as prompts"1209                raise ValueError(msg)1210            if tags is not None and not (1211                isinstance(tags, list) and len(tags) == len(prompts)1212            ):1213                msg = "tags must be a list of the same length as prompts"1214                raise ValueError(msg)1215            if metadata is not None and not (1216                isinstance(metadata, list) and len(metadata) == len(prompts)1217            ):1218                msg = "metadata must be a list of the same length as prompts"1219                raise ValueError(msg)1220            if run_name is not None and not (1221                isinstance(run_name, list) and len(run_name) == len(prompts)1222            ):1223                msg = "run_name must be a list of the same length as prompts"1224                raise ValueError(msg)1225            tags_list = cast("list[list[str] | None]", tags or ([None] * len(prompts)))1226            metadata_list = cast(1227                "list[builtins.dict[str, Any] | None]",1228                metadata or ([{}] * len(prompts)),1229            )1230            run_name_list = run_name or cast(1231                "list[str | None]", ([None] * len(prompts))1232            )1233            params = self._dict_for_compat()1234            params["stop"] = stop1235            callback_managers = [1236                AsyncCallbackManager.configure(1237                    callback,1238                    self.callbacks,1239                    self.verbose,1240                    tag,1241                    self.tags,1242                    meta,1243                    self.metadata,1244                    langsmith_inheritable_metadata=_filter_invocation_params_for_tracing(1245                        params1246                    ),1247                )1248                for callback, tag, meta in zip(1249                    callbacks, tags_list, metadata_list, strict=False1250                )1251            ]1252        else:1253            # We've received a single callbacks arg to apply to all inputs1254            params = self._dict_for_compat()1255            params["stop"] = stop1256            callback_managers = [1257                AsyncCallbackManager.configure(1258                    callbacks,1259                    self.callbacks,1260                    self.verbose,1261                    cast("list[str]", tags),1262                    self.tags,1263                    cast("builtins.dict[str, Any]", metadata),1264                    self.metadata,1265                    langsmith_inheritable_metadata=_filter_invocation_params_for_tracing(1266                        params1267                    ),1268                )1269            ] * len(prompts)1270            run_name_list = [cast("str | None", run_name)] * len(prompts)1271        run_ids_list = self._get_run_ids_list(run_id, prompts)1272        options = {"stop": stop}1273        (1274            existing_prompts,1275            llm_string,1276            missing_prompt_idxs,1277            missing_prompts,1278        ) = await aget_prompts(params, prompts, self.cache)12791280        # Verify whether the cache is set, and if the cache is set,1281        # verify whether the cache is available.1282        new_arg_supported = _signature_parameters(self._agenerate).get("run_manager")1283        if (self.cache is None and get_llm_cache() is None) or self.cache is False:1284            run_managers = await asyncio.gather(1285                *[1286                    callback_manager.on_llm_start(1287                        self._serialized,1288                        [prompt],1289                        invocation_params=params,1290                        options=options,1291                        name=run_name,1292                        batch_size=len(prompts),1293                        run_id=run_id_,1294                    )1295                    for callback_manager, prompt, run_name, run_id_ in zip(1296                        callback_managers,1297                        prompts,1298                        run_name_list,1299                        run_ids_list,1300                        strict=False,1301                    )1302                ]1303            )1304            run_managers = [r[0] for r in run_managers]  # type: ignore[misc]1305            return await self._agenerate_helper(1306                prompts,1307                stop,1308                run_managers,  # type: ignore[arg-type]1309                new_arg_supported=bool(new_arg_supported),1310                **kwargs,1311            )1312        if len(missing_prompts) > 0:1313            run_managers = await asyncio.gather(1314                *[1315                    callback_managers[idx].on_llm_start(1316                        self._serialized,1317                        [prompts[idx]],1318                        invocation_params=params,1319                        options=options,1320                        name=run_name_list[idx],1321                        batch_size=len(missing_prompts),1322                    )1323                    for idx in missing_prompt_idxs1324                ]1325            )1326            run_managers = [r[0] for r in run_managers]  # type: ignore[misc]1327            new_results = await self._agenerate_helper(1328                missing_prompts,1329                stop,1330                run_managers,  # type: ignore[arg-type]1331                new_arg_supported=bool(new_arg_supported),1332                **kwargs,1333            )1334            llm_output = await aupdate_cache(1335                self.cache,1336                existing_prompts,1337                llm_string,1338                missing_prompt_idxs,1339                new_results,1340                prompts,1341            )1342            run_info = (1343                [RunInfo(run_id=run_manager.run_id) for run_manager in run_managers]  # type: ignore[attr-defined]1344                if run_managers1345                else None1346            )1347        else:1348            llm_output = {}1349            run_info = None1350        generations = [existing_prompts[i] for i in range(len(prompts))]1351        return LLMResult(generations=generations, llm_output=llm_output, run=run_info)13521353    async def _call_async(1354        self,1355        prompt: str,1356        stop: list[str] | None = None,1357        callbacks: Callbacks = None,1358        *,1359        tags: list[str] | None = None,1360        metadata: builtins.dict[str, Any] | None = None,1361        **kwargs: Any,1362    ) -> str:1363        """Check Cache and run the LLM on the given prompt and input."""1364        result = await self.agenerate(1365            [prompt],1366            stop=stop,1367            callbacks=callbacks,1368            tags=tags,1369            metadata=metadata,1370            **kwargs,1371        )1372        return result.generations[0][0].text13731374    def __str__(self) -> str:1375        """Return a string representation of the object for printing."""1376        cls_name = f"\033[1m{self.__class__.__name__}\033[0m"1377        return f"{cls_name}\nParams: {self._identifying_params}"13781379    @property1380    @abstractmethod1381    def _llm_type(self) -> str:1382        """Return type of llm."""13831384    @deprecated("1.4.2", alternative="asdict", removal="2.0.0")1385    @override1386    def dict(self, **_kwargs: Any) -> builtins.dict[str, Any]:1387        """DEPRECATED - use `asdict()` instead.13881389        Return a dictionary representation of the LLM.1390        """1391        return self.asdict()13921393    def asdict(self) -> builtins.dict[str, Any]:1394        """Return a dictionary representation of the LLM."""1395        starter_dict = dict(self._identifying_params)1396        starter_dict["_type"] = self._llm_type1397        return starter_dict13981399    def _dict_for_compat(self) -> builtins.dict[str, Any]:1400        """Return the LLM dictionary while preserving deprecated overrides."""1401        with suppress_langchain_deprecation_warning():1402            return self.dict()14031404    def save(self, file_path: Path | str) -> None:1405        """Save the LLM.14061407        Args:1408            file_path: Path to file to save the LLM to.14091410        Raises:1411            ValueError: If the file path is not a string or Path object.14121413        Example:1414            ```python1415            llm.save(file_path="path/llm.yaml")1416            ```1417        """1418        # Convert file to Path object.1419        save_path = Path(file_path)14201421        directory_path = save_path.parent1422        directory_path.mkdir(parents=True, exist_ok=True)14231424        # Fetch dictionary to save1425        prompt_dict = self._dict_for_compat()14261427        if save_path.suffix == ".json":1428            with save_path.open("w", encoding="utf-8") as f:1429                json.dump(prompt_dict, f, indent=4)1430        elif save_path.suffix.endswith((".yaml", ".yml")):1431            with save_path.open("w", encoding="utf-8") as f:1432                yaml.dump(prompt_dict, f, default_flow_style=False)1433        else:1434            msg = f"{save_path} must be json or yaml"1435            raise ValueError(msg)143614371438class LLM(BaseLLM):1439    """Simple interface for implementing a custom LLM.14401441    You should subclass this class and implement the following:14421443    - `_call` method: Run the LLM on the given prompt and input (used by `invoke`).1444    - `_identifying_params` property: Return a dictionary of the identifying parameters1445        This is critical for caching and tracing purposes. Identifying parameters1446        is a dict that identifies the LLM.1447        It should mostly include a `model_name`.14481449    Optional: Override the following methods to provide more optimizations:14501451    - `_acall`: Provide a native async version of the `_call` method.1452        If not provided, will delegate to the synchronous version using1453        `run_in_executor`. (Used by `ainvoke`).1454    - `_stream`: Stream the LLM on the given prompt and input.1455        `stream` will use `_stream` if provided, otherwise it1456        use `_call` and output will arrive in one chunk.1457    - `_astream`: Override to provide a native async version of the `_stream` method.1458        `astream` will use `_astream` if provided, otherwise it will implement1459        a fallback behavior that will use `_stream` if `_stream` is implemented,1460        and use `_acall` if `_stream` is not implemented.1461    """14621463    @abstractmethod1464    def _call(1465        self,1466        prompt: str,1467        stop: list[str] | None = None,1468        run_manager: CallbackManagerForLLMRun | None = None,1469        **kwargs: Any,1470    ) -> str:1471        """Run the LLM on the given input.14721473        Override this method to implement the LLM logic.14741475        Args:1476            prompt: The prompt to generate from.1477            stop: Stop words to use when generating.14781479                Model output is cut off at the first occurrence of any of these1480                substrings.14811482                If stop tokens are not supported consider raising `NotImplementedError`.1483            run_manager: Callback manager for the run.1484            **kwargs: Arbitrary additional keyword arguments.14851486                These are usually passed to the model provider API call.14871488        Returns:1489            The model output as a string. SHOULD NOT include the prompt.1490        """14911492    async def _acall(1493        self,1494        prompt: str,1495        stop: list[str] | None = None,1496        run_manager: AsyncCallbackManagerForLLMRun | None = None,1497        **kwargs: Any,1498    ) -> str:1499        """Async version of the _call method.15001501        The default implementation delegates to the synchronous _call method using1502        `run_in_executor`. Subclasses that need to provide a true async implementation1503        should override this method to reduce the overhead of using `run_in_executor`.15041505        Args:1506            prompt: The prompt to generate from.1507            stop: Stop words to use when generating.15081509                Model output is cut off at the first occurrence of any of these1510                substrings.15111512                If stop tokens are not supported consider raising `NotImplementedError`.1513            run_manager: Callback manager for the run.1514            **kwargs: Arbitrary additional keyword arguments.15151516                These are usually passed to the model provider API call.15171518        Returns:1519            The model output as a string. SHOULD NOT include the prompt.1520        """1521        return await run_in_executor(1522            None,1523            self._call,1524            prompt,1525            stop,1526            run_manager.get_sync() if run_manager else None,1527            **kwargs,1528        )15291530    def _generate(1531        self,1532        prompts: list[str],1533        stop: list[str] | None = None,1534        run_manager: CallbackManagerForLLMRun | None = None,1535        **kwargs: Any,1536    ) -> LLMResult:1537        # TODO: add caching here.1538        generations = []1539        new_arg_supported = _signature_parameters(self._call).get("run_manager")1540        for prompt in prompts:1541            text = (1542                self._call(prompt, stop=stop, run_manager=run_manager, **kwargs)1543                if new_arg_supported1544                else self._call(prompt, stop=stop, **kwargs)1545            )1546            generations.append([Generation(text=text)])1547        return LLMResult(generations=generations)15481549    async def _agenerate(1550        self,1551        prompts: list[str],1552        stop: list[str] | None = None,1553        run_manager: AsyncCallbackManagerForLLMRun | None = None,1554        **kwargs: Any,1555    ) -> LLMResult:1556        generations = []1557        new_arg_supported = _signature_parameters(self._acall).get("run_manager")1558        for prompt in prompts:1559            text = (1560                await self._acall(prompt, stop=stop, run_manager=run_manager, **kwargs)1561                if new_arg_supported1562                else await self._acall(prompt, stop=stop, **kwargs)1563            )1564            generations.append([Generation(text=text)])1565        return LLMResult(generations=generations)