libs/partners/huggingface/langchain_huggingface/chat_models/huggingface.py PYTHON 1,254 lines View on github.com → Search inside
1"""Hugging Face Chat Wrapper."""23from __future__ import annotations45import contextlib6import json7from collections.abc import AsyncIterator, Callable, Iterator, Mapping, Sequence8from dataclasses import dataclass9from operator import itemgetter10from typing import TYPE_CHECKING, Any, Literal, cast1112if TYPE_CHECKING:13    from langchain_huggingface.llms.huggingface_endpoint import HuggingFaceEndpoint14    from langchain_huggingface.llms.huggingface_pipeline import HuggingFacePipeline1516from typing import Self1718from langchain_core.callbacks.manager import (19    AsyncCallbackManagerForLLMRun,20    CallbackManagerForLLMRun,21)22from langchain_core.language_models import (23    LanguageModelInput,24    ModelProfile,25    ModelProfileRegistry,26)27from langchain_core.language_models.chat_models import (28    BaseChatModel,29    agenerate_from_stream,30    generate_from_stream,31)32from langchain_core.messages import (33    AIMessage,34    AIMessageChunk,35    BaseMessage,36    BaseMessageChunk,37    ChatMessage,38    ChatMessageChunk,39    FunctionMessage,40    FunctionMessageChunk,41    HumanMessage,42    HumanMessageChunk,43    InvalidToolCall,44    SystemMessage,45    SystemMessageChunk,46    ToolCall,47    ToolMessage,48    ToolMessageChunk,49)50from langchain_core.messages.tool import ToolCallChunk51from langchain_core.messages.tool import tool_call_chunk as create_tool_call_chunk52from langchain_core.output_parsers import JsonOutputParser53from langchain_core.output_parsers.openai_tools import (54    JsonOutputKeyToolsParser,55    make_invalid_tool_call,56    parse_tool_call,57)58from langchain_core.outputs import (59    ChatGeneration,60    ChatGenerationChunk,61    ChatResult,62    LLMResult,63)64from langchain_core.runnables import Runnable, RunnableMap, RunnablePassthrough65from langchain_core.tools import BaseTool66from langchain_core.utils.function_calling import (67    convert_to_json_schema,68    convert_to_openai_tool,69)70from langchain_core.utils.pydantic import is_basemodel_subclass71from pydantic import BaseModel, Field, model_validator7273from langchain_huggingface._version import __version__74from langchain_huggingface.data._profiles import _PROFILES75from langchain_huggingface.llms.huggingface_endpoint import HuggingFaceEndpoint76from langchain_huggingface.llms.huggingface_pipeline import HuggingFacePipeline7778_MODEL_PROFILES = cast("ModelProfileRegistry", _PROFILES)798081def _get_default_model_profile(model_name: str) -> ModelProfile:82    default = _MODEL_PROFILES.get(model_name) or {}83    return default.copy()848586@dataclass87class TGI_RESPONSE:88    """Response from the TextGenInference API."""8990    choices: list[Any]91    usage: dict929394@dataclass95class TGI_MESSAGE:96    """Message to send to the TextGenInference API."""9798    role: str99    content: str100    tool_calls: list[dict]101102103def _lc_tool_call_to_hf_tool_call(tool_call: ToolCall) -> dict:104    return {105        "type": "function",106        "id": tool_call["id"],107        "function": {108            "name": tool_call["name"],109            "arguments": json.dumps(tool_call["args"], ensure_ascii=False),110        },111    }112113114def _lc_invalid_tool_call_to_hf_tool_call(115    invalid_tool_call: InvalidToolCall,116) -> dict:117    return {118        "type": "function",119        "id": invalid_tool_call["id"],120        "function": {121            "name": invalid_tool_call["name"],122            "arguments": invalid_tool_call["args"],123        },124    }125126127def _convert_message_to_dict(message: BaseMessage) -> dict:128    """Convert a LangChain message to a dictionary.129130    Args:131        message: The LangChain message.132133    Returns:134        The dictionary.135136    """137    message_dict: dict[str, Any]138    if isinstance(message, ChatMessage):139        message_dict = {"role": message.role, "content": message.content}140    elif isinstance(message, HumanMessage):141        message_dict = {"role": "user", "content": message.content}142    elif isinstance(message, AIMessage):143        message_dict = {"role": "assistant", "content": message.content}144        if "function_call" in message.additional_kwargs:145            message_dict["function_call"] = message.additional_kwargs["function_call"]146            # If function call only, content is None not empty string147            if message_dict["content"] == "":148                message_dict["content"] = None149        if message.tool_calls or message.invalid_tool_calls:150            message_dict["tool_calls"] = [151                _lc_tool_call_to_hf_tool_call(tc) for tc in message.tool_calls152            ] + [153                _lc_invalid_tool_call_to_hf_tool_call(tc)154                for tc in message.invalid_tool_calls155            ]156        elif "tool_calls" in message.additional_kwargs:157            message_dict["tool_calls"] = message.additional_kwargs["tool_calls"]158        # If tool calls only, content is None not empty string159        if "tool_calls" in message_dict and message_dict["content"] == "":160            message_dict["content"] = None161        else:162            pass163    elif isinstance(message, SystemMessage):164        message_dict = {"role": "system", "content": message.content}165    elif isinstance(message, FunctionMessage):166        message_dict = {167            "role": "function",168            "content": message.content,169            "name": message.name,170        }171    elif isinstance(message, ToolMessage):172        message_dict = {173            "role": "tool",174            "content": message.content,175            "tool_call_id": message.tool_call_id,176        }177    else:178        msg = f"Got unknown type {message}"179        raise TypeError(msg)180    if "name" in message.additional_kwargs:181        message_dict["name"] = message.additional_kwargs["name"]182    return message_dict183184185def _convert_dict_to_message(_dict: Mapping[str, Any]) -> BaseMessage:186    """Convert a dictionary to a LangChain message.187188    Args:189        _dict: The dictionary.190191    Returns:192        The LangChain message.193194    """195    role = _dict.get("role")196    if role == "user":197        return HumanMessage(content=_dict.get("content", ""))198    if role == "assistant":199        content = _dict.get("content", "") or ""200        additional_kwargs: dict = {}201        if function_call := _dict.get("function_call"):202            additional_kwargs["function_call"] = dict(function_call)203        tool_calls = []204        invalid_tool_calls = []205        if raw_tool_calls := _dict.get("tool_calls"):206            additional_kwargs["tool_calls"] = raw_tool_calls207            for raw_tool_call in raw_tool_calls:208                try:209                    tool_calls.append(parse_tool_call(raw_tool_call, return_id=True))210                except Exception as e:211                    invalid_tool_calls.append(212                        dict(make_invalid_tool_call(raw_tool_call, str(e)))213                    )214        return AIMessage(215            content=content,216            additional_kwargs=additional_kwargs,217            tool_calls=tool_calls,218            invalid_tool_calls=invalid_tool_calls,219        )220    if role == "system":221        return SystemMessage(content=_dict.get("content", ""))222    if role == "function":223        return FunctionMessage(224            content=_dict.get("content", ""), name=_dict.get("name", "")225        )226    if role == "tool":227        additional_kwargs = {}228        if "name" in _dict:229            additional_kwargs["name"] = _dict["name"]230        return ToolMessage(231            content=_dict.get("content", ""),232            tool_call_id=_dict.get("tool_call_id", ""),233            additional_kwargs=additional_kwargs,234        )235    return ChatMessage(content=_dict.get("content", ""), role=role or "")236237238def _is_huggingface_hub(llm: Any) -> bool:239    try:240        from langchain_community.llms.huggingface_hub import (241            HuggingFaceHub,  # type: ignore[import-not-found]242        )243244        return isinstance(llm, HuggingFaceHub)245    except ImportError:246        # if no langchain community, it is not a HuggingFaceHub247        return False248249250def _convert_chunk_to_message_chunk(251    chunk: Mapping[str, Any], default_class: type[BaseMessageChunk]252) -> BaseMessageChunk:253    choice = chunk["choices"][0]254    _dict = choice["delta"]255    role = cast(str, _dict.get("role"))256    content = cast(str, _dict.get("content") or "")257    additional_kwargs: dict = {}258    tool_call_chunks: list[ToolCallChunk] = []259    if _dict.get("function_call"):260        function_call = dict(_dict["function_call"])261        if "name" in function_call and function_call["name"] is None:262            function_call["name"] = ""263        additional_kwargs["function_call"] = function_call264    if raw_tool_calls := _dict.get("tool_calls"):265        additional_kwargs["tool_calls"] = raw_tool_calls266        for rtc in raw_tool_calls:267            with contextlib.suppress(KeyError):268                tool_call_chunks.append(269                    create_tool_call_chunk(270                        name=rtc["function"].get("name"),271                        args=rtc["function"].get("arguments"),272                        id=rtc.get("id"),273                        index=rtc.get("index"),274                    )275                )276    if role == "user" or default_class == HumanMessageChunk:277        return HumanMessageChunk(content=content)278    if role == "assistant" or default_class == AIMessageChunk:279        if usage := chunk.get("usage"):280            input_tokens = usage.get("prompt_tokens", 0)281            output_tokens = usage.get("completion_tokens", 0)282            usage_metadata = {283                "input_tokens": input_tokens,284                "output_tokens": output_tokens,285                "total_tokens": usage.get("total_tokens", input_tokens + output_tokens),286            }287        else:288            usage_metadata = None289        return AIMessageChunk(290            content=content,291            additional_kwargs=additional_kwargs,292            tool_call_chunks=tool_call_chunks,293            usage_metadata=usage_metadata,  # type: ignore[arg-type]294        )295    if role == "system" or default_class == SystemMessageChunk:296        return SystemMessageChunk(content=content)297    if role == "function" or default_class == FunctionMessageChunk:298        return FunctionMessageChunk(content=content, name=_dict["name"])299    if role == "tool" or default_class == ToolMessageChunk:300        return ToolMessageChunk(content=content, tool_call_id=_dict["tool_call_id"])301    if role or default_class == ChatMessageChunk:302        return ChatMessageChunk(content=content, role=role)303    return default_class(content=content)  # type: ignore[call-arg]304305306def _is_huggingface_textgen_inference(llm: Any) -> bool:307    try:308        from langchain_community.llms.huggingface_text_gen_inference import (309            HuggingFaceTextGenInference,  # type: ignore[import-not-found]310        )311312        return isinstance(llm, HuggingFaceTextGenInference)313    except ImportError:314        # if no langchain community, it is not a HuggingFaceTextGenInference315        return False316317318def _is_huggingface_endpoint(llm: Any) -> bool:319    return isinstance(llm, HuggingFaceEndpoint)320321322def _is_huggingface_pipeline(llm: Any) -> bool:323    return isinstance(llm, HuggingFacePipeline)324325326class ChatHuggingFace(BaseChatModel):327    r"""Hugging Face LLM's as ChatModels.328329    Works with `HuggingFaceTextGenInference`, `HuggingFaceEndpoint`,330    `HuggingFaceHub`, and `HuggingFacePipeline` LLMs.331332    Upon instantiating this class, the model_id is resolved from the url333    provided to the LLM, and the appropriate tokenizer is loaded from334    the HuggingFace Hub.335336    Setup:337        Install `langchain-huggingface` and ensure your Hugging Face token338        is saved.339340        ```bash341        pip install langchain-huggingface342        ```343344        ```python345        from huggingface_hub import login346347        login()  # You will be prompted for your HF key, which will then be saved locally348        ```349350    Key init args — completion params:351        llm:352            LLM to be used.353354    Key init args — client params:355        custom_get_token_ids:356            Optional encoder to use for counting tokens.357        metadata:358            Metadata to add to the run trace.359        tags:360            Tags to add to the run trace.361        verbose:362            Whether to print out response text.363364    See full list of supported init args and their descriptions in the params365    section.366367    Instantiate:368        ```python369        from langchain_huggingface import HuggingFaceEndpoint,370        ChatHuggingFace371372        model = HuggingFaceEndpoint(373            repo_id="microsoft/Phi-3-mini-4k-instruct",374            task="text-generation",375            max_new_tokens=512,376            do_sample=False,377            repetition_penalty=1.03,378        )379380        chat = ChatHuggingFace(llm=model, verbose=True)381        ```382383    Invoke:384        ```python385        messages = [386            ("system", "You are a helpful translator. Translate the user387            sentence to French."),388            ("human", "I love programming."),389        ]390391        chat(...).invoke(messages)392        ```393394        ```python395        AIMessage(content='Je ai une passion pour le programme.\n\nIn396        French, we use "ai" for masculine subjects and "a" for feminine397        subjects. Since "programming" is gender-neutral in English, we398        will go with the masculine "programme".\n\nConfirmation: "J\'aime399        le programme." is more commonly used. The sentence above is400        technically accurate, but less commonly used in spoken French as401        "ai" is used less frequently in everyday speech.',402        response_metadata={'token_usage': ChatCompletionOutputUsage403        (completion_tokens=100, prompt_tokens=55, total_tokens=155),404        'model': '', 'finish_reason': 'length'},405        id='run-874c24b7-0272-4c99-b259-5d6d7facbc56-0')406        ```407408    Stream:409        ```python410        for chunk in chat.stream(messages):411            print(chunk)412        ```413414        ```python415        content='Je ai une passion pour le programme.\n\nIn French, we use416        "ai" for masculine subjects and "a" for feminine subjects.417        Since "programming" is gender-neutral in English,418        we will go with the masculine "programme".\n\nConfirmation:419        "J\'aime le programme." is more commonly used. The sentence420        above is technically accurate, but less commonly used in spoken421        French as "ai" is used less frequently in everyday speech.'422        response_metadata={'token_usage': ChatCompletionOutputUsage423        (completion_tokens=100, prompt_tokens=55, total_tokens=155),424        'model': '', 'finish_reason': 'length'}425        id='run-7d7b1967-9612-4f9a-911a-b2b5ca85046a-0'426        ```427428    Async:429        ```python430        await chat.ainvoke(messages)431        ```432433        ```python434        AIMessage(content='Je déaime le programming.\n\nLittérale : Je435        (j\'aime) déaime (le) programming.\n\nNote: "Programming" in436        French is "programmation". But here, I used "programming" instead437        of "programmation" because the user said "I love programming"438        instead of "I love programming (in French)", which would be439        "J\'aime la programmation". By translating the sentence440        literally, I preserved the original meaning of the user\'s441        sentence.', id='run-fd850318-e299-4735-b4c6-3496dc930b1d-0')442        ```443444    Tool calling:445        ```python446        from pydantic import BaseModel, Field447448        class GetWeather(BaseModel):449            '''Get the current weather in a given location'''450451            location: str = Field(..., description="The city and state,452            e.g. San Francisco, CA")453454        class GetPopulation(BaseModel):455            '''Get the current population in a given location'''456457            location: str = Field(..., description="The city and state,458            e.g. San Francisco, CA")459460        chat_with_tools = chat.bind_tools([GetWeather, GetPopulation])461        ai_msg = chat_with_tools.invoke("Which city is hotter today and462        which is bigger: LA or NY?")463        ai_msg.tool_calls464        ```465466        ```python467        [468            {469                "name": "GetPopulation",470                "args": {"location": "Los Angeles, CA"},471                "id": "0",472            }473        ]474        ```475476    Response metadata477        ```python478        ai_msg = chat.invoke(messages)479        ai_msg.response_metadata480        ```481482        ```python483        {484            "token_usage": ChatCompletionOutputUsage(485                completion_tokens=100, prompt_tokens=8, total_tokens=108486            ),487            "model": "",488            "finish_reason": "length",489        }490        ```491    """  # noqa: E501492493    llm: Any494    """LLM, must be of type HuggingFaceTextGenInference, HuggingFaceEndpoint,495        HuggingFaceHub, or HuggingFacePipeline."""496    tokenizer: Any = None497    """Tokenizer for the model. Only used for HuggingFacePipeline."""498    model_id: str | None = None499    """Model ID for the model. Only used for HuggingFaceEndpoint."""500    temperature: float | None = None501    """What sampling temperature to use."""502    stop: str | list[str] | None = Field(default=None, alias="stop_sequences")503    """Default stop sequences."""504    presence_penalty: float | None = None505    """Penalizes repeated tokens."""506    frequency_penalty: float | None = None507    """Penalizes repeated tokens according to frequency."""508    seed: int | None = None509    """Seed for generation"""510    logprobs: bool | None = None511    """Whether to return logprobs."""512    top_logprobs: int | None = None513    """Number of most likely tokens to return at each token position, each with514     an associated log probability. `logprobs` must be set to true515     if this parameter is used."""516    logit_bias: dict[int, int] | None = None517    """Modify the likelihood of specified tokens appearing in the completion."""518    streaming: bool = False519    """Whether to stream the results or not."""520    stream_usage: bool | None = None521    """Whether to include usage metadata in streaming output. If True, an additional522    message chunk will be generated during the stream including usage metadata."""523    n: int | None = None524    """Number of chat completions to generate for each prompt."""525    top_p: float | None = None526    """Total probability mass of tokens to consider at each step."""527    max_tokens: int | None = None528    """Maximum number of tokens to generate."""529    model_kwargs: dict[str, Any] = Field(default_factory=dict)530    """Holds any model parameters valid for `create` call not explicitly specified."""531532    def __init__(self, **kwargs: Any):533        super().__init__(**kwargs)534535        # Inherit properties from the LLM if they weren't explicitly set536        self._inherit_llm_properties()537538        self._resolve_model_id()539540    def _inherit_llm_properties(self) -> None:541        """Inherit properties from the wrapped LLM instance if not explicitly set."""542        if not hasattr(self, "llm") or self.llm is None:543            return544545        # Map of ChatHuggingFace properties to LLM properties546        property_mappings = {547            "temperature": "temperature",548            "max_tokens": "max_new_tokens",  # Different naming convention549            "top_p": "top_p",550            "seed": "seed",551            "streaming": "streaming",552            "stop": "stop_sequences",553        }554555        # Inherit properties from LLM and not explicitly set here556        for chat_prop, llm_prop in property_mappings.items():557            if hasattr(self.llm, llm_prop):558                llm_value = getattr(self.llm, llm_prop)559                chat_value = getattr(self, chat_prop, None)560                if not chat_value and llm_value:561                    setattr(self, chat_prop, llm_value)562563        # Handle special cases for HuggingFaceEndpoint564        if _is_huggingface_endpoint(self.llm):565            # Inherit additional HuggingFaceEndpoint specific properties566            endpoint_mappings = {567                "frequency_penalty": "repetition_penalty",568            }569570            for chat_prop, llm_prop in endpoint_mappings.items():571                if hasattr(self.llm, llm_prop):572                    llm_value = getattr(self.llm, llm_prop)573                    chat_value = getattr(self, chat_prop, None)574                    if chat_value is None and llm_value is not None:575                        setattr(self, chat_prop, llm_value)576577        # Inherit model_kwargs if not explicitly set578        if (579            not self.model_kwargs580            and hasattr(self.llm, "model_kwargs")581            and isinstance(self.llm.model_kwargs, dict)582        ):583            self.model_kwargs = self.llm.model_kwargs.copy()584585    @model_validator(mode="after")586    def _set_huggingface_version(self) -> Self:587        """Set package version in metadata."""588        self._add_version("langchain-huggingface", __version__)589        return self590591    @model_validator(mode="after")592    def validate_llm(self) -> Self:593        if (594            not _is_huggingface_hub(self.llm)595            and not _is_huggingface_textgen_inference(self.llm)596            and not _is_huggingface_endpoint(self.llm)597            and not _is_huggingface_pipeline(self.llm)598        ):599            msg = (600                "Expected llm to be one of HuggingFaceTextGenInference, "601                "HuggingFaceEndpoint, HuggingFaceHub, HuggingFacePipeline "602                f"received {type(self.llm)}"603            )604            raise TypeError(msg)605        return self606607    def _resolve_model_profile(self) -> ModelProfile | None:608        if self.model_id:609            return _get_default_model_profile(self.model_id) or None610        return None611612    @classmethod613    def from_model_id(614        cls,615        model_id: str,616        task: str | None = None,617        backend: Literal["pipeline", "endpoint", "text-gen"] = "pipeline",618        **kwargs: Any,619    ) -> ChatHuggingFace:620        """Construct a ChatHuggingFace model from a model_id.621622        Args:623            model_id: The model ID of the Hugging Face model.624            task: The task to perform (e.g., "text-generation").625            backend: The backend to use. One of "pipeline", "endpoint", "text-gen".626            **kwargs: Additional arguments to pass to the backend or ChatHuggingFace.627        """628        llm: (629            Any  # HuggingFacePipeline, HuggingFaceEndpoint, HuggingFaceTextGenInference630        )631        if backend == "pipeline":632            from langchain_huggingface.llms.huggingface_pipeline import (633                HuggingFacePipeline,634            )635636            task = task if task is not None else "text-generation"637638            # Separate pipeline-specific kwargs from ChatHuggingFace kwargs639            # Parameters that should go to HuggingFacePipeline.from_model_id640            pipeline_specific_kwargs = {}641642            # Extract pipeline-specific parameters643            pipeline_keys = [644                "backend",645                "device",646                "device_map",647                "model_kwargs",648                "pipeline_kwargs",649                "batch_size",650            ]651            for key in pipeline_keys:652                if key in kwargs:653                    pipeline_specific_kwargs[key] = kwargs.pop(key)654655            # Remaining kwargs (temperature, max_tokens, etc.) should go to656            # pipeline_kwargs for generation parameters, which ChatHuggingFace657            # will inherit from the LLM658            if "pipeline_kwargs" not in pipeline_specific_kwargs:659                pipeline_specific_kwargs["pipeline_kwargs"] = {}660661            # Add generation parameters to pipeline_kwargs662            # Map max_tokens to max_new_tokens for HuggingFace pipeline663            generation_params = {}664            for k, v in list(kwargs.items()):665                if k == "max_tokens":666                    generation_params["max_new_tokens"] = v667                    kwargs.pop(k)668                elif k in (669                    "temperature",670                    "max_new_tokens",671                    "top_p",672                    "top_k",673                    "repetition_penalty",674                    "do_sample",675                ):676                    generation_params[k] = v677                    kwargs.pop(k)678679            pipeline_specific_kwargs["pipeline_kwargs"].update(generation_params)680681            # Create the HuggingFacePipeline682            llm = HuggingFacePipeline.from_model_id(683                model_id=model_id, task=task, **pipeline_specific_kwargs684            )685        elif backend == "endpoint":686            from langchain_huggingface.llms.huggingface_endpoint import (687                HuggingFaceEndpoint,688            )689690            llm = HuggingFaceEndpoint(repo_id=model_id, task=task, **kwargs)691        elif backend == "text-gen":692            from langchain_community.llms.huggingface_text_gen_inference import (  # type: ignore[import-not-found]693                HuggingFaceTextGenInference,694            )695696            llm = HuggingFaceTextGenInference(inference_server_url=model_id, **kwargs)697        else:698            msg = f"Unknown backend: {backend}"699            raise ValueError(msg)700701        return cls(llm=llm, **kwargs)702703    def _create_chat_result(self, response: dict) -> ChatResult:704        generations = []705        token_usage = response.get("usage", {})706        for res in response["choices"]:707            message = _convert_dict_to_message(res["message"])708            if token_usage and isinstance(message, AIMessage):709                message.usage_metadata = {710                    "input_tokens": token_usage.get("prompt_tokens", 0),711                    "output_tokens": token_usage.get("completion_tokens", 0),712                    "total_tokens": token_usage.get("total_tokens", 0),713                }714            generation_info = {"finish_reason": res.get("finish_reason")}715            if "logprobs" in res:716                generation_info["logprobs"] = res["logprobs"]717            gen = ChatGeneration(718                message=message,719                generation_info=generation_info,720            )721            generations.append(gen)722        llm_output = {723            "token_usage": token_usage,724            "model_name": self.model_id,725            "system_fingerprint": response.get("system_fingerprint", ""),726        }727        return ChatResult(generations=generations, llm_output=llm_output)728729    def _generate(730        self,731        messages: list[BaseMessage],732        stop: list[str] | None = None,733        run_manager: CallbackManagerForLLMRun | None = None,734        stream: bool | None = None,  # noqa: FBT001735        **kwargs: Any,736    ) -> ChatResult:737        should_stream = stream if stream is not None else self.streaming738739        if _is_huggingface_textgen_inference(self.llm):740            message_dicts, params = self._create_message_dicts(messages, stop)741            answer = self.llm.client.chat(messages=message_dicts, **kwargs)742            return self._create_chat_result(answer)743        if _is_huggingface_endpoint(self.llm):744            if should_stream:745                stream_iter = self._stream(746                    messages, stop=stop, run_manager=run_manager, **kwargs747                )748                return generate_from_stream(stream_iter)749            message_dicts, params = self._create_message_dicts(messages, stop)750            params = {751                "stop": stop,752                **params,753                **({"stream": stream} if stream is not None else {}),754                **kwargs,755            }756            answer = self.llm.client.chat_completion(messages=message_dicts, **params)757            return self._create_chat_result(answer)758        llm_input = self._to_chat_prompt(messages)759760        if should_stream:761            stream_iter = self.llm._stream(762                llm_input, stop=stop, run_manager=run_manager, **kwargs763            )764            return generate_from_stream(stream_iter)765        llm_result = self.llm._generate(766            prompts=[llm_input], stop=stop, run_manager=run_manager, **kwargs767        )768        return self._to_chat_result(llm_result)769770    async def _agenerate(771        self,772        messages: list[BaseMessage],773        stop: list[str] | None = None,774        run_manager: AsyncCallbackManagerForLLMRun | None = None,775        stream: bool | None = None,  # noqa: FBT001776        **kwargs: Any,777    ) -> ChatResult:778        if _is_huggingface_textgen_inference(self.llm):779            message_dicts, params = self._create_message_dicts(messages, stop)780            answer = await self.llm.async_client.chat(messages=message_dicts, **kwargs)781            return self._create_chat_result(answer)782        if _is_huggingface_endpoint(self.llm):783            should_stream = stream if stream is not None else self.streaming784            if should_stream:785                stream_iter = self._astream(786                    messages, stop=stop, run_manager=run_manager, **kwargs787                )788                return await agenerate_from_stream(stream_iter)789            message_dicts, params = self._create_message_dicts(messages, stop)790            params = {791                **params,792                **({"stream": stream} if stream is not None else {}),793                **kwargs,794            }795796            answer = await self.llm.async_client.chat_completion(797                messages=message_dicts, **params798            )799            return self._create_chat_result(answer)800        if _is_huggingface_pipeline(self.llm):801            msg = "async generation is not supported with HuggingFacePipeline"802            raise NotImplementedError(msg)803        llm_input = self._to_chat_prompt(messages)804        llm_result = await self.llm._agenerate(805            prompts=[llm_input], stop=stop, run_manager=run_manager, **kwargs806        )807        return self._to_chat_result(llm_result)808809    def _should_stream_usage(810        self, *, stream_usage: bool | None = None, **kwargs: Any811    ) -> bool | None:812        """Determine whether to include usage metadata in streaming output.813814        For backwards compatibility, we check for `stream_options` passed815        explicitly to kwargs or in the model_kwargs and override self.stream_usage.816        """817        stream_usage_sources = [  # order of precedence818            stream_usage,819            kwargs.get("stream_options", {}).get("include_usage"),820            self.model_kwargs.get("stream_options", {}).get("include_usage"),821            self.stream_usage,822        ]823        for source in stream_usage_sources:824            if isinstance(source, bool):825                return source826        return self.stream_usage827828    def _stream(829        self,830        messages: list[BaseMessage],831        stop: list[str] | None = None,832        run_manager: CallbackManagerForLLMRun | None = None,833        *,834        stream_usage: bool | None = None,835        **kwargs: Any,836    ) -> Iterator[ChatGenerationChunk]:837        if _is_huggingface_endpoint(self.llm):838            stream_usage = self._should_stream_usage(839                stream_usage=stream_usage, **kwargs840            )841            if stream_usage:842                kwargs["stream_options"] = {"include_usage": stream_usage}843            message_dicts, params = self._create_message_dicts(messages, stop)844            params = {**params, **kwargs, "stream": True}845846            default_chunk_class: type[BaseMessageChunk] = AIMessageChunk847            for chunk in self.llm.client.chat_completion(848                messages=message_dicts, **params849            ):850                if len(chunk["choices"]) == 0:851                    if usage := chunk.get("usage"):852                        usage_msg = AIMessageChunk(853                            content="",854                            additional_kwargs={},855                            response_metadata={},856                            usage_metadata={857                                "input_tokens": usage.get("prompt_tokens", 0),858                                "output_tokens": usage.get("completion_tokens", 0),859                                "total_tokens": usage.get("total_tokens", 0),860                            },861                        )862                        yield ChatGenerationChunk(message=usage_msg)863                    continue864865                choice = chunk["choices"][0]866                message_chunk = _convert_chunk_to_message_chunk(867                    chunk, default_chunk_class868                )869                generation_info = {}870                if finish_reason := choice.get("finish_reason"):871                    generation_info["finish_reason"] = finish_reason872                    generation_info["model_name"] = self.model_id873                logprobs = choice.get("logprobs")874                if logprobs:875                    generation_info["logprobs"] = logprobs876                default_chunk_class = message_chunk.__class__877                generation_chunk = ChatGenerationChunk(878                    message=message_chunk, generation_info=generation_info or None879                )880                if run_manager:881                    run_manager.on_llm_new_token(882                        generation_chunk.text, chunk=generation_chunk, logprobs=logprobs883                    )884                yield generation_chunk885        else:886            llm_input = self._to_chat_prompt(messages)887            stream_iter = self.llm._stream(888                llm_input, stop=stop, run_manager=run_manager, **kwargs889            )890            for chunk in stream_iter:  # chunk is a GenerationChunk891                chat_chunk = ChatGenerationChunk(892                    message=AIMessageChunk(content=chunk.text),893                    generation_info=chunk.generation_info,894                )895                yield chat_chunk896897    async def _astream(898        self,899        messages: list[BaseMessage],900        stop: list[str] | None = None,901        run_manager: AsyncCallbackManagerForLLMRun | None = None,902        *,903        stream_usage: bool | None = None,904        **kwargs: Any,905    ) -> AsyncIterator[ChatGenerationChunk]:906        stream_usage = self._should_stream_usage(stream_usage=stream_usage, **kwargs)907        if stream_usage:908            kwargs["stream_options"] = {"include_usage": stream_usage}909        message_dicts, params = self._create_message_dicts(messages, stop)910        params = {**params, **kwargs, "stream": True}911912        default_chunk_class: type[BaseMessageChunk] = AIMessageChunk913914        async for chunk in await self.llm.async_client.chat_completion(915            messages=message_dicts, **params916        ):917            if len(chunk["choices"]) == 0:918                if usage := chunk.get("usage"):919                    usage_msg = AIMessageChunk(920                        content="",921                        additional_kwargs={},922                        response_metadata={},923                        usage_metadata={924                            "input_tokens": usage.get("prompt_tokens", 0),925                            "output_tokens": usage.get("completion_tokens", 0),926                            "total_tokens": usage.get("total_tokens", 0),927                        },928                    )929                    yield ChatGenerationChunk(message=usage_msg)930                continue931932            choice = chunk["choices"][0]933            message_chunk = _convert_chunk_to_message_chunk(chunk, default_chunk_class)934            generation_info = {}935            if finish_reason := choice.get("finish_reason"):936                generation_info["finish_reason"] = finish_reason937                generation_info["model_name"] = self.model_id938            logprobs = choice.get("logprobs")939            if logprobs:940                generation_info["logprobs"] = logprobs941            default_chunk_class = message_chunk.__class__942            generation_chunk = ChatGenerationChunk(943                message=message_chunk, generation_info=generation_info or None944            )945            if run_manager:946                await run_manager.on_llm_new_token(947                    token=generation_chunk.text,948                    chunk=generation_chunk,949                    logprobs=logprobs,950                )951            yield generation_chunk952953    def _to_chat_prompt(954        self,955        messages: list[BaseMessage],956    ) -> str:957        """Convert a list of messages into a prompt format expected by wrapped LLM."""958        if not messages:959            msg = "At least one HumanMessage must be provided!"960            raise ValueError(msg)961962        if not isinstance(messages[-1], HumanMessage):963            msg = "Last message must be a HumanMessage!"964            raise ValueError(msg)965966        messages_dicts = [self._to_chatml_format(m) for m in messages]967968        return self.tokenizer.apply_chat_template(969            messages_dicts, tokenize=False, add_generation_prompt=True970        )971972    def _to_chatml_format(self, message: BaseMessage) -> dict:973        """Convert LangChain message to ChatML format."""974        if isinstance(message, SystemMessage):975            role = "system"976        elif isinstance(message, AIMessage):977            role = "assistant"978        elif isinstance(message, HumanMessage):979            role = "user"980        else:981            msg = f"Unknown message type: {type(message)}"982            raise ValueError(msg)983984        return {"role": role, "content": message.content}985986    @staticmethod987    def _to_chat_result(llm_result: LLMResult) -> ChatResult:988        chat_generations = []989990        for g in llm_result.generations[0]:991            chat_generation = ChatGeneration(992                message=AIMessage(content=g.text), generation_info=g.generation_info993            )994            chat_generations.append(chat_generation)995996        return ChatResult(997            generations=chat_generations, llm_output=llm_result.llm_output998        )9991000    def _resolve_model_id(self) -> None:1001        """Resolve the model_id from the LLM's inference_server_url."""1002        from huggingface_hub import list_inference_endpoints  # type: ignore[import]10031004        if _is_huggingface_hub(self.llm) or (1005            hasattr(self.llm, "repo_id") and self.llm.repo_id1006        ):1007            self.model_id = self.llm.repo_id1008            return1009        if _is_huggingface_textgen_inference(self.llm):1010            endpoint_url: str | None = self.llm.inference_server_url1011        if _is_huggingface_pipeline(self.llm):1012            from transformers import AutoTokenizer  # type: ignore[import]10131014            self.model_id = self.model_id or self.llm.model_id1015            self.tokenizer = (1016                AutoTokenizer.from_pretrained(self.model_id)1017                if self.tokenizer is None1018                else self.tokenizer1019            )1020            return1021        if _is_huggingface_endpoint(self.llm):1022            self.model_id = self.llm.repo_id or self.llm.model1023            return1024        endpoint_url = self.llm.endpoint_url1025        available_endpoints = list_inference_endpoints("*")1026        for endpoint in available_endpoints:1027            if endpoint.url == endpoint_url:1028                self.model_id = endpoint.repository10291030        if not self.model_id:1031            msg = (1032                "Failed to resolve model_id:"1033                f"Could not find model id for inference server: {endpoint_url}"1034                "Make sure that your Hugging Face token has access to the endpoint."1035            )1036            raise ValueError(msg)10371038    def bind_tools(1039        self,1040        tools: Sequence[dict[str, Any] | type | Callable | BaseTool],1041        *,1042        tool_choice: dict | str | bool | None = None,1043        **kwargs: Any,1044    ) -> Runnable[LanguageModelInput, AIMessage]:1045        """Bind tool-like objects to this chat model.10461047        Assumes model is compatible with OpenAI tool-calling API.10481049        Args:1050            tools: A list of tool definitions to bind to this chat model.10511052                Supports any tool definition handled by [`convert_to_openai_tool`][langchain_core.utils.function_calling.convert_to_openai_tool].1053            tool_choice: Which tool to require the model to call.1054                Must be the name of the single provided function or1055                `'auto'` to automatically determine which function to call1056                (if any), or a dict of the form:1057                {"type": "function", "function": {"name": <<tool_name>>}}.1058            **kwargs: Any additional parameters to pass to the1059                `langchain.runnable.Runnable` constructor.1060        """  # noqa: E5011061        formatted_tools = [convert_to_openai_tool(tool) for tool in tools]1062        if tool_choice is not None and tool_choice:1063            if len(formatted_tools) != 1:1064                msg = (1065                    "When specifying `tool_choice`, you must provide exactly one "1066                    f"tool. Received {len(formatted_tools)} tools."1067                )1068                raise ValueError(msg)1069            if isinstance(tool_choice, str):1070                if tool_choice not in ("auto", "none", "required"):1071                    tool_choice = {1072                        "type": "function",1073                        "function": {"name": tool_choice},1074                    }1075            elif isinstance(tool_choice, bool):1076                tool_choice = formatted_tools[0]1077            elif isinstance(tool_choice, dict):1078                if (1079                    formatted_tools[0]["function"]["name"]1080                    != tool_choice["function"]["name"]1081                ):1082                    msg = (1083                        f"Tool choice {tool_choice} was specified, but the only "1084                        f"provided tool was {formatted_tools[0]['function']['name']}."1085                    )1086                    raise ValueError(msg)1087            else:1088                msg = (1089                    f"Unrecognized tool_choice type. Expected str, bool or dict. "1090                    f"Received: {tool_choice}"1091                )1092                raise ValueError(msg)1093            kwargs["tool_choice"] = tool_choice1094        return super().bind(tools=formatted_tools, **kwargs)10951096    def with_structured_output(1097        self,1098        schema: dict | type[BaseModel] | None = None,1099        *,1100        method: Literal[1101            "function_calling", "json_mode", "json_schema"1102        ] = "function_calling",1103        include_raw: bool = False,1104        **kwargs: Any,1105    ) -> Runnable[LanguageModelInput, dict | BaseModel]:1106        """Model wrapper that returns outputs formatted to match the given schema.11071108        Args:1109            schema: The output schema. Can be passed in as:11101111                - An OpenAI function/tool schema,1112                - A JSON Schema,1113                - A `TypedDict` class11141115                Pydantic class is currently supported.11161117            method: The method for steering model generation, one of:11181119                - `'function_calling'`: uses tool-calling features.1120                - `'json_schema'`: uses dedicated structured output features.1121                - `'json_mode'`: uses JSON mode.11221123            include_raw:1124                If `False` then only the parsed structured output is returned.11251126                If an error occurs during model output parsing it will be raised.11271128                If `True` then both the raw model response (a `BaseMessage`) and the1129                parsed model response will be returned.11301131                If an error occurs during output parsing it will be caught and returned1132                as well.11331134                The final output is always a `dict` with keys `'raw'`, `'parsed'`, and1135                `'parsing_error'`.11361137            kwargs:1138                Additional parameters to pass to the underlying LLM's1139                `langchain_core.language_models.chat.BaseChatModel.bind`1140                method, such as `response_format` or `ls_structured_output_format`.11411142        Returns:1143            A `Runnable` that takes same inputs as a1144                `langchain_core.language_models.chat.BaseChatModel`. If `include_raw` is1145                `False` and `schema` is a Pydantic class, `Runnable` outputs an instance1146                of `schema` (i.e., a Pydantic object). Otherwise, if `include_raw` is1147                `False` then `Runnable` outputs a `dict`.11481149                If `include_raw` is `True`, then `Runnable` outputs a `dict` with keys:11501151                - `'raw'`: `BaseMessage`1152                - `'parsed'`: `None` if there was a parsing error, otherwise the type1153                    depends on the `schema` as described above.1154                - `'parsing_error'`: `BaseException | None`1155        """1156        _ = kwargs.pop("strict", None)1157        if kwargs:1158            msg = f"Received unsupported arguments {kwargs}"1159            raise ValueError(msg)1160        is_pydantic_schema = isinstance(schema, type) and is_basemodel_subclass(schema)1161        if method == "function_calling":1162            if schema is None:1163                msg = (1164                    "schema must be specified when method is 'function_calling'. "1165                    "Received None."1166                )1167                raise ValueError(msg)1168            formatted_tool = convert_to_openai_tool(schema)1169            tool_name = formatted_tool["function"]["name"]1170            llm = self.bind_tools(1171                [schema],1172                tool_choice=tool_name,1173                ls_structured_output_format={1174                    "kwargs": {"method": "function_calling"},1175                    "schema": formatted_tool,1176                },1177            )1178            if is_pydantic_schema:1179                msg = "Pydantic schema is not supported for function calling"1180                raise NotImplementedError(msg)1181            output_parser: JsonOutputKeyToolsParser | JsonOutputParser = (1182                JsonOutputKeyToolsParser(key_name=tool_name, first_tool_only=True)1183            )1184        elif method == "json_schema":1185            if schema is None:1186                msg = (1187                    "schema must be specified when method is 'json_schema'. "1188                    "Received None."1189                )1190                raise ValueError(msg)1191            formatted_schema = convert_to_json_schema(schema)1192            llm = self.bind(1193                response_format={"type": "json_object", "schema": formatted_schema},1194                ls_structured_output_format={1195                    "kwargs": {"method": "json_schema"},1196                    "schema": schema,1197                },1198            )1199            output_parser = JsonOutputParser()  # type: ignore[arg-type]1200        elif method == "json_mode":1201            llm = self.bind(1202                response_format={"type": "json_object"},1203                ls_structured_output_format={1204                    "kwargs": {"method": "json_mode"},1205                    "schema": schema,1206                },1207            )1208            output_parser = JsonOutputParser()  # type: ignore[arg-type]1209        else:1210            msg = (1211                f"Unrecognized method argument. Expected one of 'function_calling' or "1212                f"'json_mode'. Received: '{method}'"1213            )1214            raise ValueError(msg)12151216        if include_raw:1217            parser_assign = RunnablePassthrough.assign(1218                parsed=itemgetter("raw") | output_parser, parsing_error=lambda _: None1219            )1220            parser_none = RunnablePassthrough.assign(parsed=lambda _: None)1221            parser_with_fallback = parser_assign.with_fallbacks(1222                [parser_none], exception_key="parsing_error"1223            )1224            return RunnableMap(raw=llm) | parser_with_fallback1225        return llm | output_parser12261227    def _create_message_dicts(1228        self, messages: list[BaseMessage], stop: list[str] | None1229    ) -> tuple[list[dict[str, Any]], dict[str, Any]]:1230        params = self._default_params1231        if stop is not None:1232            params["stop"] = stop1233        message_dicts = [_convert_message_to_dict(m) for m in messages]1234        return message_dicts, params12351236    @property1237    def _default_params(self) -> dict[str, Any]:1238        """Get default parameters for calling Hugging Face Inference Providers API."""1239        params = {1240            "model": self.model_id,1241            "stream": self.streaming,1242            "n": self.n,1243            "temperature": self.temperature,1244            "stop": self.stop,1245            **(self.model_kwargs if self.model_kwargs else {}),1246        }1247        if self.max_tokens is not None:1248            params["max_tokens"] = self.max_tokens1249        return params12501251    @property1252    def _llm_type(self) -> str:1253        return "huggingface-chat-wrapper"

Code quality findings 1

Ensure functions have docstrings for documentation
info maintainability missing-docstring
def validate_llm(self) -> Self:

Get this view in your editor

Same data, no extra tab — call code_get_file + code_get_findings over MCP from Claude/Cursor/Copilot.

Generated by searchcode.com · powered by code_get_file + code_get_findings