Ensure functions have docstrings for documentation
def validate_llm(self) -> Self:
1"""Hugging Face Chat Wrapper."""23from __future__ import annotations45import contextlib6import json7from collections.abc import AsyncIterator, Callable, Iterator, Mapping, Sequence8from dataclasses import dataclass9from operator import itemgetter10from typing import TYPE_CHECKING, Any, Literal, cast1112if TYPE_CHECKING:13 from langchain_huggingface.llms.huggingface_endpoint import HuggingFaceEndpoint14 from langchain_huggingface.llms.huggingface_pipeline import HuggingFacePipeline1516from typing import Self1718from langchain_core.callbacks.manager import (19 AsyncCallbackManagerForLLMRun,20 CallbackManagerForLLMRun,21)22from langchain_core.language_models import (23 LanguageModelInput,24 ModelProfile,25 ModelProfileRegistry,26)27from langchain_core.language_models.chat_models import (28 BaseChatModel,29 agenerate_from_stream,30 generate_from_stream,31)32from langchain_core.messages import (33 AIMessage,34 AIMessageChunk,35 BaseMessage,36 BaseMessageChunk,37 ChatMessage,38 ChatMessageChunk,39 FunctionMessage,40 FunctionMessageChunk,41 HumanMessage,42 HumanMessageChunk,43 InvalidToolCall,44 SystemMessage,45 SystemMessageChunk,46 ToolCall,47 ToolMessage,48 ToolMessageChunk,49)50from langchain_core.messages.tool import ToolCallChunk51from langchain_core.messages.tool import tool_call_chunk as create_tool_call_chunk52from langchain_core.output_parsers import JsonOutputParser53from langchain_core.output_parsers.openai_tools import (54 JsonOutputKeyToolsParser,55 make_invalid_tool_call,56 parse_tool_call,57)58from langchain_core.outputs import (59 ChatGeneration,60 ChatGenerationChunk,61 ChatResult,62 LLMResult,63)64from langchain_core.runnables import Runnable, RunnableMap, RunnablePassthrough65from langchain_core.tools import BaseTool66from langchain_core.utils.function_calling import (67 convert_to_json_schema,68 convert_to_openai_tool,69)70from langchain_core.utils.pydantic import is_basemodel_subclass71from pydantic import BaseModel, Field, model_validator7273from langchain_huggingface._version import __version__74from langchain_huggingface.data._profiles import _PROFILES75from langchain_huggingface.llms.huggingface_endpoint import HuggingFaceEndpoint76from langchain_huggingface.llms.huggingface_pipeline import HuggingFacePipeline7778_MODEL_PROFILES = cast("ModelProfileRegistry", _PROFILES)798081def _get_default_model_profile(model_name: str) -> ModelProfile:82 default = _MODEL_PROFILES.get(model_name) or {}83 return default.copy()848586@dataclass87class TGI_RESPONSE:88 """Response from the TextGenInference API."""8990 choices: list[Any]91 usage: dict929394@dataclass95class TGI_MESSAGE:96 """Message to send to the TextGenInference API."""9798 role: str99 content: str100 tool_calls: list[dict]101102103def _lc_tool_call_to_hf_tool_call(tool_call: ToolCall) -> dict:104 return {105 "type": "function",106 "id": tool_call["id"],107 "function": {108 "name": tool_call["name"],109 "arguments": json.dumps(tool_call["args"], ensure_ascii=False),110 },111 }112113114def _lc_invalid_tool_call_to_hf_tool_call(115 invalid_tool_call: InvalidToolCall,116) -> dict:117 return {118 "type": "function",119 "id": invalid_tool_call["id"],120 "function": {121 "name": invalid_tool_call["name"],122 "arguments": invalid_tool_call["args"],123 },124 }125126127def _convert_message_to_dict(message: BaseMessage) -> dict:128 """Convert a LangChain message to a dictionary.129130 Args:131 message: The LangChain message.132133 Returns:134 The dictionary.135136 """137 message_dict: dict[str, Any]138 if isinstance(message, ChatMessage):139 message_dict = {"role": message.role, "content": message.content}140 elif isinstance(message, HumanMessage):141 message_dict = {"role": "user", "content": message.content}142 elif isinstance(message, AIMessage):143 message_dict = {"role": "assistant", "content": message.content}144 if "function_call" in message.additional_kwargs:145 message_dict["function_call"] = message.additional_kwargs["function_call"]146 # If function call only, content is None not empty string147 if message_dict["content"] == "":148 message_dict["content"] = None149 if message.tool_calls or message.invalid_tool_calls:150 message_dict["tool_calls"] = [151 _lc_tool_call_to_hf_tool_call(tc) for tc in message.tool_calls152 ] + [153 _lc_invalid_tool_call_to_hf_tool_call(tc)154 for tc in message.invalid_tool_calls155 ]156 elif "tool_calls" in message.additional_kwargs:157 message_dict["tool_calls"] = message.additional_kwargs["tool_calls"]158 # If tool calls only, content is None not empty string159 if "tool_calls" in message_dict and message_dict["content"] == "":160 message_dict["content"] = None161 else:162 pass163 elif isinstance(message, SystemMessage):164 message_dict = {"role": "system", "content": message.content}165 elif isinstance(message, FunctionMessage):166 message_dict = {167 "role": "function",168 "content": message.content,169 "name": message.name,170 }171 elif isinstance(message, ToolMessage):172 message_dict = {173 "role": "tool",174 "content": message.content,175 "tool_call_id": message.tool_call_id,176 }177 else:178 msg = f"Got unknown type {message}"179 raise TypeError(msg)180 if "name" in message.additional_kwargs:181 message_dict["name"] = message.additional_kwargs["name"]182 return message_dict183184185def _convert_dict_to_message(_dict: Mapping[str, Any]) -> BaseMessage:186 """Convert a dictionary to a LangChain message.187188 Args:189 _dict: The dictionary.190191 Returns:192 The LangChain message.193194 """195 role = _dict.get("role")196 if role == "user":197 return HumanMessage(content=_dict.get("content", ""))198 if role == "assistant":199 content = _dict.get("content", "") or ""200 additional_kwargs: dict = {}201 if function_call := _dict.get("function_call"):202 additional_kwargs["function_call"] = dict(function_call)203 tool_calls = []204 invalid_tool_calls = []205 if raw_tool_calls := _dict.get("tool_calls"):206 additional_kwargs["tool_calls"] = raw_tool_calls207 for raw_tool_call in raw_tool_calls:208 try:209 tool_calls.append(parse_tool_call(raw_tool_call, return_id=True))210 except Exception as e:211 invalid_tool_calls.append(212 dict(make_invalid_tool_call(raw_tool_call, str(e)))213 )214 return AIMessage(215 content=content,216 additional_kwargs=additional_kwargs,217 tool_calls=tool_calls,218 invalid_tool_calls=invalid_tool_calls,219 )220 if role == "system":221 return SystemMessage(content=_dict.get("content", ""))222 if role == "function":223 return FunctionMessage(224 content=_dict.get("content", ""), name=_dict.get("name", "")225 )226 if role == "tool":227 additional_kwargs = {}228 if "name" in _dict:229 additional_kwargs["name"] = _dict["name"]230 return ToolMessage(231 content=_dict.get("content", ""),232 tool_call_id=_dict.get("tool_call_id", ""),233 additional_kwargs=additional_kwargs,234 )235 return ChatMessage(content=_dict.get("content", ""), role=role or "")236237238def _is_huggingface_hub(llm: Any) -> bool:239 try:240 from langchain_community.llms.huggingface_hub import (241 HuggingFaceHub, # type: ignore[import-not-found]242 )243244 return isinstance(llm, HuggingFaceHub)245 except ImportError:246 # if no langchain community, it is not a HuggingFaceHub247 return False248249250def _convert_chunk_to_message_chunk(251 chunk: Mapping[str, Any], default_class: type[BaseMessageChunk]252) -> BaseMessageChunk:253 choice = chunk["choices"][0]254 _dict = choice["delta"]255 role = cast(str, _dict.get("role"))256 content = cast(str, _dict.get("content") or "")257 additional_kwargs: dict = {}258 tool_call_chunks: list[ToolCallChunk] = []259 if _dict.get("function_call"):260 function_call = dict(_dict["function_call"])261 if "name" in function_call and function_call["name"] is None:262 function_call["name"] = ""263 additional_kwargs["function_call"] = function_call264 if raw_tool_calls := _dict.get("tool_calls"):265 additional_kwargs["tool_calls"] = raw_tool_calls266 for rtc in raw_tool_calls:267 with contextlib.suppress(KeyError):268 tool_call_chunks.append(269 create_tool_call_chunk(270 name=rtc["function"].get("name"),271 args=rtc["function"].get("arguments"),272 id=rtc.get("id"),273 index=rtc.get("index"),274 )275 )276 if role == "user" or default_class == HumanMessageChunk:277 return HumanMessageChunk(content=content)278 if role == "assistant" or default_class == AIMessageChunk:279 if usage := chunk.get("usage"):280 input_tokens = usage.get("prompt_tokens", 0)281 output_tokens = usage.get("completion_tokens", 0)282 usage_metadata = {283 "input_tokens": input_tokens,284 "output_tokens": output_tokens,285 "total_tokens": usage.get("total_tokens", input_tokens + output_tokens),286 }287 else:288 usage_metadata = None289 return AIMessageChunk(290 content=content,291 additional_kwargs=additional_kwargs,292 tool_call_chunks=tool_call_chunks,293 usage_metadata=usage_metadata, # type: ignore[arg-type]294 )295 if role == "system" or default_class == SystemMessageChunk:296 return SystemMessageChunk(content=content)297 if role == "function" or default_class == FunctionMessageChunk:298 return FunctionMessageChunk(content=content, name=_dict["name"])299 if role == "tool" or default_class == ToolMessageChunk:300 return ToolMessageChunk(content=content, tool_call_id=_dict["tool_call_id"])301 if role or default_class == ChatMessageChunk:302 return ChatMessageChunk(content=content, role=role)303 return default_class(content=content) # type: ignore[call-arg]304305306def _is_huggingface_textgen_inference(llm: Any) -> bool:307 try:308 from langchain_community.llms.huggingface_text_gen_inference import (309 HuggingFaceTextGenInference, # type: ignore[import-not-found]310 )311312 return isinstance(llm, HuggingFaceTextGenInference)313 except ImportError:314 # if no langchain community, it is not a HuggingFaceTextGenInference315 return False316317318def _is_huggingface_endpoint(llm: Any) -> bool:319 return isinstance(llm, HuggingFaceEndpoint)320321322def _is_huggingface_pipeline(llm: Any) -> bool:323 return isinstance(llm, HuggingFacePipeline)324325326class ChatHuggingFace(BaseChatModel):327 r"""Hugging Face LLM's as ChatModels.328329 Works with `HuggingFaceTextGenInference`, `HuggingFaceEndpoint`,330 `HuggingFaceHub`, and `HuggingFacePipeline` LLMs.331332 Upon instantiating this class, the model_id is resolved from the url333 provided to the LLM, and the appropriate tokenizer is loaded from334 the HuggingFace Hub.335336 Setup:337 Install `langchain-huggingface` and ensure your Hugging Face token338 is saved.339340 ```bash341 pip install langchain-huggingface342 ```343344 ```python345 from huggingface_hub import login346347 login() # You will be prompted for your HF key, which will then be saved locally348 ```349350 Key init args — completion params:351 llm:352 LLM to be used.353354 Key init args — client params:355 custom_get_token_ids:356 Optional encoder to use for counting tokens.357 metadata:358 Metadata to add to the run trace.359 tags:360 Tags to add to the run trace.361 verbose:362 Whether to print out response text.363364 See full list of supported init args and their descriptions in the params365 section.366367 Instantiate:368 ```python369 from langchain_huggingface import HuggingFaceEndpoint,370 ChatHuggingFace371372 model = HuggingFaceEndpoint(373 repo_id="microsoft/Phi-3-mini-4k-instruct",374 task="text-generation",375 max_new_tokens=512,376 do_sample=False,377 repetition_penalty=1.03,378 )379380 chat = ChatHuggingFace(llm=model, verbose=True)381 ```382383 Invoke:384 ```python385 messages = [386 ("system", "You are a helpful translator. Translate the user387 sentence to French."),388 ("human", "I love programming."),389 ]390391 chat(...).invoke(messages)392 ```393394 ```python395 AIMessage(content='Je ai une passion pour le programme.\n\nIn396 French, we use "ai" for masculine subjects and "a" for feminine397 subjects. Since "programming" is gender-neutral in English, we398 will go with the masculine "programme".\n\nConfirmation: "J\'aime399 le programme." is more commonly used. The sentence above is400 technically accurate, but less commonly used in spoken French as401 "ai" is used less frequently in everyday speech.',402 response_metadata={'token_usage': ChatCompletionOutputUsage403 (completion_tokens=100, prompt_tokens=55, total_tokens=155),404 'model': '', 'finish_reason': 'length'},405 id='run-874c24b7-0272-4c99-b259-5d6d7facbc56-0')406 ```407408 Stream:409 ```python410 for chunk in chat.stream(messages):411 print(chunk)412 ```413414 ```python415 content='Je ai une passion pour le programme.\n\nIn French, we use416 "ai" for masculine subjects and "a" for feminine subjects.417 Since "programming" is gender-neutral in English,418 we will go with the masculine "programme".\n\nConfirmation:419 "J\'aime le programme." is more commonly used. The sentence420 above is technically accurate, but less commonly used in spoken421 French as "ai" is used less frequently in everyday speech.'422 response_metadata={'token_usage': ChatCompletionOutputUsage423 (completion_tokens=100, prompt_tokens=55, total_tokens=155),424 'model': '', 'finish_reason': 'length'}425 id='run-7d7b1967-9612-4f9a-911a-b2b5ca85046a-0'426 ```427428 Async:429 ```python430 await chat.ainvoke(messages)431 ```432433 ```python434 AIMessage(content='Je déaime le programming.\n\nLittérale : Je435 (j\'aime) déaime (le) programming.\n\nNote: "Programming" in436 French is "programmation". But here, I used "programming" instead437 of "programmation" because the user said "I love programming"438 instead of "I love programming (in French)", which would be439 "J\'aime la programmation". By translating the sentence440 literally, I preserved the original meaning of the user\'s441 sentence.', id='run-fd850318-e299-4735-b4c6-3496dc930b1d-0')442 ```443444 Tool calling:445 ```python446 from pydantic import BaseModel, Field447448 class GetWeather(BaseModel):449 '''Get the current weather in a given location'''450451 location: str = Field(..., description="The city and state,452 e.g. San Francisco, CA")453454 class GetPopulation(BaseModel):455 '''Get the current population in a given location'''456457 location: str = Field(..., description="The city and state,458 e.g. San Francisco, CA")459460 chat_with_tools = chat.bind_tools([GetWeather, GetPopulation])461 ai_msg = chat_with_tools.invoke("Which city is hotter today and462 which is bigger: LA or NY?")463 ai_msg.tool_calls464 ```465466 ```python467 [468 {469 "name": "GetPopulation",470 "args": {"location": "Los Angeles, CA"},471 "id": "0",472 }473 ]474 ```475476 Response metadata477 ```python478 ai_msg = chat.invoke(messages)479 ai_msg.response_metadata480 ```481482 ```python483 {484 "token_usage": ChatCompletionOutputUsage(485 completion_tokens=100, prompt_tokens=8, total_tokens=108486 ),487 "model": "",488 "finish_reason": "length",489 }490 ```491 """ # noqa: E501492493 llm: Any494 """LLM, must be of type HuggingFaceTextGenInference, HuggingFaceEndpoint,495 HuggingFaceHub, or HuggingFacePipeline."""496 tokenizer: Any = None497 """Tokenizer for the model. Only used for HuggingFacePipeline."""498 model_id: str | None = None499 """Model ID for the model. Only used for HuggingFaceEndpoint."""500 temperature: float | None = None501 """What sampling temperature to use."""502 stop: str | list[str] | None = Field(default=None, alias="stop_sequences")503 """Default stop sequences."""504 presence_penalty: float | None = None505 """Penalizes repeated tokens."""506 frequency_penalty: float | None = None507 """Penalizes repeated tokens according to frequency."""508 seed: int | None = None509 """Seed for generation"""510 logprobs: bool | None = None511 """Whether to return logprobs."""512 top_logprobs: int | None = None513 """Number of most likely tokens to return at each token position, each with514 an associated log probability. `logprobs` must be set to true515 if this parameter is used."""516 logit_bias: dict[int, int] | None = None517 """Modify the likelihood of specified tokens appearing in the completion."""518 streaming: bool = False519 """Whether to stream the results or not."""520 stream_usage: bool | None = None521 """Whether to include usage metadata in streaming output. If True, an additional522 message chunk will be generated during the stream including usage metadata."""523 n: int | None = None524 """Number of chat completions to generate for each prompt."""525 top_p: float | None = None526 """Total probability mass of tokens to consider at each step."""527 max_tokens: int | None = None528 """Maximum number of tokens to generate."""529 model_kwargs: dict[str, Any] = Field(default_factory=dict)530 """Holds any model parameters valid for `create` call not explicitly specified."""531532 def __init__(self, **kwargs: Any):533 super().__init__(**kwargs)534535 # Inherit properties from the LLM if they weren't explicitly set536 self._inherit_llm_properties()537538 self._resolve_model_id()539540 def _inherit_llm_properties(self) -> None:541 """Inherit properties from the wrapped LLM instance if not explicitly set."""542 if not hasattr(self, "llm") or self.llm is None:543 return544545 # Map of ChatHuggingFace properties to LLM properties546 property_mappings = {547 "temperature": "temperature",548 "max_tokens": "max_new_tokens", # Different naming convention549 "top_p": "top_p",550 "seed": "seed",551 "streaming": "streaming",552 "stop": "stop_sequences",553 }554555 # Inherit properties from LLM and not explicitly set here556 for chat_prop, llm_prop in property_mappings.items():557 if hasattr(self.llm, llm_prop):558 llm_value = getattr(self.llm, llm_prop)559 chat_value = getattr(self, chat_prop, None)560 if not chat_value and llm_value:561 setattr(self, chat_prop, llm_value)562563 # Handle special cases for HuggingFaceEndpoint564 if _is_huggingface_endpoint(self.llm):565 # Inherit additional HuggingFaceEndpoint specific properties566 endpoint_mappings = {567 "frequency_penalty": "repetition_penalty",568 }569570 for chat_prop, llm_prop in endpoint_mappings.items():571 if hasattr(self.llm, llm_prop):572 llm_value = getattr(self.llm, llm_prop)573 chat_value = getattr(self, chat_prop, None)574 if chat_value is None and llm_value is not None:575 setattr(self, chat_prop, llm_value)576577 # Inherit model_kwargs if not explicitly set578 if (579 not self.model_kwargs580 and hasattr(self.llm, "model_kwargs")581 and isinstance(self.llm.model_kwargs, dict)582 ):583 self.model_kwargs = self.llm.model_kwargs.copy()584585 @model_validator(mode="after")586 def _set_huggingface_version(self) -> Self:587 """Set package version in metadata."""588 self._add_version("langchain-huggingface", __version__)589 return self590591 @model_validator(mode="after")592 def validate_llm(self) -> Self:593 if (594 not _is_huggingface_hub(self.llm)595 and not _is_huggingface_textgen_inference(self.llm)596 and not _is_huggingface_endpoint(self.llm)597 and not _is_huggingface_pipeline(self.llm)598 ):599 msg = (600 "Expected llm to be one of HuggingFaceTextGenInference, "601 "HuggingFaceEndpoint, HuggingFaceHub, HuggingFacePipeline "602 f"received {type(self.llm)}"603 )604 raise TypeError(msg)605 return self606607 def _resolve_model_profile(self) -> ModelProfile | None:608 if self.model_id:609 return _get_default_model_profile(self.model_id) or None610 return None611612 @classmethod613 def from_model_id(614 cls,615 model_id: str,616 task: str | None = None,617 backend: Literal["pipeline", "endpoint", "text-gen"] = "pipeline",618 **kwargs: Any,619 ) -> ChatHuggingFace:620 """Construct a ChatHuggingFace model from a model_id.621622 Args:623 model_id: The model ID of the Hugging Face model.624 task: The task to perform (e.g., "text-generation").625 backend: The backend to use. One of "pipeline", "endpoint", "text-gen".626 **kwargs: Additional arguments to pass to the backend or ChatHuggingFace.627 """628 llm: (629 Any # HuggingFacePipeline, HuggingFaceEndpoint, HuggingFaceTextGenInference630 )631 if backend == "pipeline":632 from langchain_huggingface.llms.huggingface_pipeline import (633 HuggingFacePipeline,634 )635636 task = task if task is not None else "text-generation"637638 # Separate pipeline-specific kwargs from ChatHuggingFace kwargs639 # Parameters that should go to HuggingFacePipeline.from_model_id640 pipeline_specific_kwargs = {}641642 # Extract pipeline-specific parameters643 pipeline_keys = [644 "backend",645 "device",646 "device_map",647 "model_kwargs",648 "pipeline_kwargs",649 "batch_size",650 ]651 for key in pipeline_keys:652 if key in kwargs:653 pipeline_specific_kwargs[key] = kwargs.pop(key)654655 # Remaining kwargs (temperature, max_tokens, etc.) should go to656 # pipeline_kwargs for generation parameters, which ChatHuggingFace657 # will inherit from the LLM658 if "pipeline_kwargs" not in pipeline_specific_kwargs:659 pipeline_specific_kwargs["pipeline_kwargs"] = {}660661 # Add generation parameters to pipeline_kwargs662 # Map max_tokens to max_new_tokens for HuggingFace pipeline663 generation_params = {}664 for k, v in list(kwargs.items()):665 if k == "max_tokens":666 generation_params["max_new_tokens"] = v667 kwargs.pop(k)668 elif k in (669 "temperature",670 "max_new_tokens",671 "top_p",672 "top_k",673 "repetition_penalty",674 "do_sample",675 ):676 generation_params[k] = v677 kwargs.pop(k)678679 pipeline_specific_kwargs["pipeline_kwargs"].update(generation_params)680681 # Create the HuggingFacePipeline682 llm = HuggingFacePipeline.from_model_id(683 model_id=model_id, task=task, **pipeline_specific_kwargs684 )685 elif backend == "endpoint":686 from langchain_huggingface.llms.huggingface_endpoint import (687 HuggingFaceEndpoint,688 )689690 llm = HuggingFaceEndpoint(repo_id=model_id, task=task, **kwargs)691 elif backend == "text-gen":692 from langchain_community.llms.huggingface_text_gen_inference import ( # type: ignore[import-not-found]693 HuggingFaceTextGenInference,694 )695696 llm = HuggingFaceTextGenInference(inference_server_url=model_id, **kwargs)697 else:698 msg = f"Unknown backend: {backend}"699 raise ValueError(msg)700701 return cls(llm=llm, **kwargs)702703 def _create_chat_result(self, response: dict) -> ChatResult:704 generations = []705 token_usage = response.get("usage", {})706 for res in response["choices"]:707 message = _convert_dict_to_message(res["message"])708 if token_usage and isinstance(message, AIMessage):709 message.usage_metadata = {710 "input_tokens": token_usage.get("prompt_tokens", 0),711 "output_tokens": token_usage.get("completion_tokens", 0),712 "total_tokens": token_usage.get("total_tokens", 0),713 }714 generation_info = {"finish_reason": res.get("finish_reason")}715 if "logprobs" in res:716 generation_info["logprobs"] = res["logprobs"]717 gen = ChatGeneration(718 message=message,719 generation_info=generation_info,720 )721 generations.append(gen)722 llm_output = {723 "token_usage": token_usage,724 "model_name": self.model_id,725 "system_fingerprint": response.get("system_fingerprint", ""),726 }727 return ChatResult(generations=generations, llm_output=llm_output)728729 def _generate(730 self,731 messages: list[BaseMessage],732 stop: list[str] | None = None,733 run_manager: CallbackManagerForLLMRun | None = None,734 stream: bool | None = None, # noqa: FBT001735 **kwargs: Any,736 ) -> ChatResult:737 should_stream = stream if stream is not None else self.streaming738739 if _is_huggingface_textgen_inference(self.llm):740 message_dicts, params = self._create_message_dicts(messages, stop)741 answer = self.llm.client.chat(messages=message_dicts, **kwargs)742 return self._create_chat_result(answer)743 if _is_huggingface_endpoint(self.llm):744 if should_stream:745 stream_iter = self._stream(746 messages, stop=stop, run_manager=run_manager, **kwargs747 )748 return generate_from_stream(stream_iter)749 message_dicts, params = self._create_message_dicts(messages, stop)750 params = {751 "stop": stop,752 **params,753 **({"stream": stream} if stream is not None else {}),754 **kwargs,755 }756 answer = self.llm.client.chat_completion(messages=message_dicts, **params)757 return self._create_chat_result(answer)758 llm_input = self._to_chat_prompt(messages)759760 if should_stream:761 stream_iter = self.llm._stream(762 llm_input, stop=stop, run_manager=run_manager, **kwargs763 )764 return generate_from_stream(stream_iter)765 llm_result = self.llm._generate(766 prompts=[llm_input], stop=stop, run_manager=run_manager, **kwargs767 )768 return self._to_chat_result(llm_result)769770 async def _agenerate(771 self,772 messages: list[BaseMessage],773 stop: list[str] | None = None,774 run_manager: AsyncCallbackManagerForLLMRun | None = None,775 stream: bool | None = None, # noqa: FBT001776 **kwargs: Any,777 ) -> ChatResult:778 if _is_huggingface_textgen_inference(self.llm):779 message_dicts, params = self._create_message_dicts(messages, stop)780 answer = await self.llm.async_client.chat(messages=message_dicts, **kwargs)781 return self._create_chat_result(answer)782 if _is_huggingface_endpoint(self.llm):783 should_stream = stream if stream is not None else self.streaming784 if should_stream:785 stream_iter = self._astream(786 messages, stop=stop, run_manager=run_manager, **kwargs787 )788 return await agenerate_from_stream(stream_iter)789 message_dicts, params = self._create_message_dicts(messages, stop)790 params = {791 **params,792 **({"stream": stream} if stream is not None else {}),793 **kwargs,794 }795796 answer = await self.llm.async_client.chat_completion(797 messages=message_dicts, **params798 )799 return self._create_chat_result(answer)800 if _is_huggingface_pipeline(self.llm):801 msg = "async generation is not supported with HuggingFacePipeline"802 raise NotImplementedError(msg)803 llm_input = self._to_chat_prompt(messages)804 llm_result = await self.llm._agenerate(805 prompts=[llm_input], stop=stop, run_manager=run_manager, **kwargs806 )807 return self._to_chat_result(llm_result)808809 def _should_stream_usage(810 self, *, stream_usage: bool | None = None, **kwargs: Any811 ) -> bool | None:812 """Determine whether to include usage metadata in streaming output.813814 For backwards compatibility, we check for `stream_options` passed815 explicitly to kwargs or in the model_kwargs and override self.stream_usage.816 """817 stream_usage_sources = [ # order of precedence818 stream_usage,819 kwargs.get("stream_options", {}).get("include_usage"),820 self.model_kwargs.get("stream_options", {}).get("include_usage"),821 self.stream_usage,822 ]823 for source in stream_usage_sources:824 if isinstance(source, bool):825 return source826 return self.stream_usage827828 def _stream(829 self,830 messages: list[BaseMessage],831 stop: list[str] | None = None,832 run_manager: CallbackManagerForLLMRun | None = None,833 *,834 stream_usage: bool | None = None,835 **kwargs: Any,836 ) -> Iterator[ChatGenerationChunk]:837 if _is_huggingface_endpoint(self.llm):838 stream_usage = self._should_stream_usage(839 stream_usage=stream_usage, **kwargs840 )841 if stream_usage:842 kwargs["stream_options"] = {"include_usage": stream_usage}843 message_dicts, params = self._create_message_dicts(messages, stop)844 params = {**params, **kwargs, "stream": True}845846 default_chunk_class: type[BaseMessageChunk] = AIMessageChunk847 for chunk in self.llm.client.chat_completion(848 messages=message_dicts, **params849 ):850 if len(chunk["choices"]) == 0:851 if usage := chunk.get("usage"):852 usage_msg = AIMessageChunk(853 content="",854 additional_kwargs={},855 response_metadata={},856 usage_metadata={857 "input_tokens": usage.get("prompt_tokens", 0),858 "output_tokens": usage.get("completion_tokens", 0),859 "total_tokens": usage.get("total_tokens", 0),860 },861 )862 yield ChatGenerationChunk(message=usage_msg)863 continue864865 choice = chunk["choices"][0]866 message_chunk = _convert_chunk_to_message_chunk(867 chunk, default_chunk_class868 )869 generation_info = {}870 if finish_reason := choice.get("finish_reason"):871 generation_info["finish_reason"] = finish_reason872 generation_info["model_name"] = self.model_id873 logprobs = choice.get("logprobs")874 if logprobs:875 generation_info["logprobs"] = logprobs876 default_chunk_class = message_chunk.__class__877 generation_chunk = ChatGenerationChunk(878 message=message_chunk, generation_info=generation_info or None879 )880 if run_manager:881 run_manager.on_llm_new_token(882 generation_chunk.text, chunk=generation_chunk, logprobs=logprobs883 )884 yield generation_chunk885 else:886 llm_input = self._to_chat_prompt(messages)887 stream_iter = self.llm._stream(888 llm_input, stop=stop, run_manager=run_manager, **kwargs889 )890 for chunk in stream_iter: # chunk is a GenerationChunk891 chat_chunk = ChatGenerationChunk(892 message=AIMessageChunk(content=chunk.text),893 generation_info=chunk.generation_info,894 )895 yield chat_chunk896897 async def _astream(898 self,899 messages: list[BaseMessage],900 stop: list[str] | None = None,901 run_manager: AsyncCallbackManagerForLLMRun | None = None,902 *,903 stream_usage: bool | None = None,904 **kwargs: Any,905 ) -> AsyncIterator[ChatGenerationChunk]:906 stream_usage = self._should_stream_usage(stream_usage=stream_usage, **kwargs)907 if stream_usage:908 kwargs["stream_options"] = {"include_usage": stream_usage}909 message_dicts, params = self._create_message_dicts(messages, stop)910 params = {**params, **kwargs, "stream": True}911912 default_chunk_class: type[BaseMessageChunk] = AIMessageChunk913914 async for chunk in await self.llm.async_client.chat_completion(915 messages=message_dicts, **params916 ):917 if len(chunk["choices"]) == 0:918 if usage := chunk.get("usage"):919 usage_msg = AIMessageChunk(920 content="",921 additional_kwargs={},922 response_metadata={},923 usage_metadata={924 "input_tokens": usage.get("prompt_tokens", 0),925 "output_tokens": usage.get("completion_tokens", 0),926 "total_tokens": usage.get("total_tokens", 0),927 },928 )929 yield ChatGenerationChunk(message=usage_msg)930 continue931932 choice = chunk["choices"][0]933 message_chunk = _convert_chunk_to_message_chunk(chunk, default_chunk_class)934 generation_info = {}935 if finish_reason := choice.get("finish_reason"):936 generation_info["finish_reason"] = finish_reason937 generation_info["model_name"] = self.model_id938 logprobs = choice.get("logprobs")939 if logprobs:940 generation_info["logprobs"] = logprobs941 default_chunk_class = message_chunk.__class__942 generation_chunk = ChatGenerationChunk(943 message=message_chunk, generation_info=generation_info or None944 )945 if run_manager:946 await run_manager.on_llm_new_token(947 token=generation_chunk.text,948 chunk=generation_chunk,949 logprobs=logprobs,950 )951 yield generation_chunk952953 def _to_chat_prompt(954 self,955 messages: list[BaseMessage],956 ) -> str:957 """Convert a list of messages into a prompt format expected by wrapped LLM."""958 if not messages:959 msg = "At least one HumanMessage must be provided!"960 raise ValueError(msg)961962 if not isinstance(messages[-1], HumanMessage):963 msg = "Last message must be a HumanMessage!"964 raise ValueError(msg)965966 messages_dicts = [self._to_chatml_format(m) for m in messages]967968 return self.tokenizer.apply_chat_template(969 messages_dicts, tokenize=False, add_generation_prompt=True970 )971972 def _to_chatml_format(self, message: BaseMessage) -> dict:973 """Convert LangChain message to ChatML format."""974 if isinstance(message, SystemMessage):975 role = "system"976 elif isinstance(message, AIMessage):977 role = "assistant"978 elif isinstance(message, HumanMessage):979 role = "user"980 else:981 msg = f"Unknown message type: {type(message)}"982 raise ValueError(msg)983984 return {"role": role, "content": message.content}985986 @staticmethod987 def _to_chat_result(llm_result: LLMResult) -> ChatResult:988 chat_generations = []989990 for g in llm_result.generations[0]:991 chat_generation = ChatGeneration(992 message=AIMessage(content=g.text), generation_info=g.generation_info993 )994 chat_generations.append(chat_generation)995996 return ChatResult(997 generations=chat_generations, llm_output=llm_result.llm_output998 )9991000 def _resolve_model_id(self) -> None:1001 """Resolve the model_id from the LLM's inference_server_url."""1002 from huggingface_hub import list_inference_endpoints # type: ignore[import]10031004 if _is_huggingface_hub(self.llm) or (1005 hasattr(self.llm, "repo_id") and self.llm.repo_id1006 ):1007 self.model_id = self.llm.repo_id1008 return1009 if _is_huggingface_textgen_inference(self.llm):1010 endpoint_url: str | None = self.llm.inference_server_url1011 if _is_huggingface_pipeline(self.llm):1012 from transformers import AutoTokenizer # type: ignore[import]10131014 self.model_id = self.model_id or self.llm.model_id1015 self.tokenizer = (1016 AutoTokenizer.from_pretrained(self.model_id)1017 if self.tokenizer is None1018 else self.tokenizer1019 )1020 return1021 if _is_huggingface_endpoint(self.llm):1022 self.model_id = self.llm.repo_id or self.llm.model1023 return1024 endpoint_url = self.llm.endpoint_url1025 available_endpoints = list_inference_endpoints("*")1026 for endpoint in available_endpoints:1027 if endpoint.url == endpoint_url:1028 self.model_id = endpoint.repository10291030 if not self.model_id:1031 msg = (1032 "Failed to resolve model_id:"1033 f"Could not find model id for inference server: {endpoint_url}"1034 "Make sure that your Hugging Face token has access to the endpoint."1035 )1036 raise ValueError(msg)10371038 def bind_tools(1039 self,1040 tools: Sequence[dict[str, Any] | type | Callable | BaseTool],1041 *,1042 tool_choice: dict | str | bool | None = None,1043 **kwargs: Any,1044 ) -> Runnable[LanguageModelInput, AIMessage]:1045 """Bind tool-like objects to this chat model.10461047 Assumes model is compatible with OpenAI tool-calling API.10481049 Args:1050 tools: A list of tool definitions to bind to this chat model.10511052 Supports any tool definition handled by [`convert_to_openai_tool`][langchain_core.utils.function_calling.convert_to_openai_tool].1053 tool_choice: Which tool to require the model to call.1054 Must be the name of the single provided function or1055 `'auto'` to automatically determine which function to call1056 (if any), or a dict of the form:1057 {"type": "function", "function": {"name": <<tool_name>>}}.1058 **kwargs: Any additional parameters to pass to the1059 `langchain.runnable.Runnable` constructor.1060 """ # noqa: E5011061 formatted_tools = [convert_to_openai_tool(tool) for tool in tools]1062 if tool_choice is not None and tool_choice:1063 if len(formatted_tools) != 1:1064 msg = (1065 "When specifying `tool_choice`, you must provide exactly one "1066 f"tool. Received {len(formatted_tools)} tools."1067 )1068 raise ValueError(msg)1069 if isinstance(tool_choice, str):1070 if tool_choice not in ("auto", "none", "required"):1071 tool_choice = {1072 "type": "function",1073 "function": {"name": tool_choice},1074 }1075 elif isinstance(tool_choice, bool):1076 tool_choice = formatted_tools[0]1077 elif isinstance(tool_choice, dict):1078 if (1079 formatted_tools[0]["function"]["name"]1080 != tool_choice["function"]["name"]1081 ):1082 msg = (1083 f"Tool choice {tool_choice} was specified, but the only "1084 f"provided tool was {formatted_tools[0]['function']['name']}."1085 )1086 raise ValueError(msg)1087 else:1088 msg = (1089 f"Unrecognized tool_choice type. Expected str, bool or dict. "1090 f"Received: {tool_choice}"1091 )1092 raise ValueError(msg)1093 kwargs["tool_choice"] = tool_choice1094 return super().bind(tools=formatted_tools, **kwargs)10951096 def with_structured_output(1097 self,1098 schema: dict | type[BaseModel] | None = None,1099 *,1100 method: Literal[1101 "function_calling", "json_mode", "json_schema"1102 ] = "function_calling",1103 include_raw: bool = False,1104 **kwargs: Any,1105 ) -> Runnable[LanguageModelInput, dict | BaseModel]:1106 """Model wrapper that returns outputs formatted to match the given schema.11071108 Args:1109 schema: The output schema. Can be passed in as:11101111 - An OpenAI function/tool schema,1112 - A JSON Schema,1113 - A `TypedDict` class11141115 Pydantic class is currently supported.11161117 method: The method for steering model generation, one of:11181119 - `'function_calling'`: uses tool-calling features.1120 - `'json_schema'`: uses dedicated structured output features.1121 - `'json_mode'`: uses JSON mode.11221123 include_raw:1124 If `False` then only the parsed structured output is returned.11251126 If an error occurs during model output parsing it will be raised.11271128 If `True` then both the raw model response (a `BaseMessage`) and the1129 parsed model response will be returned.11301131 If an error occurs during output parsing it will be caught and returned1132 as well.11331134 The final output is always a `dict` with keys `'raw'`, `'parsed'`, and1135 `'parsing_error'`.11361137 kwargs:1138 Additional parameters to pass to the underlying LLM's1139 `langchain_core.language_models.chat.BaseChatModel.bind`1140 method, such as `response_format` or `ls_structured_output_format`.11411142 Returns:1143 A `Runnable` that takes same inputs as a1144 `langchain_core.language_models.chat.BaseChatModel`. If `include_raw` is1145 `False` and `schema` is a Pydantic class, `Runnable` outputs an instance1146 of `schema` (i.e., a Pydantic object). Otherwise, if `include_raw` is1147 `False` then `Runnable` outputs a `dict`.11481149 If `include_raw` is `True`, then `Runnable` outputs a `dict` with keys:11501151 - `'raw'`: `BaseMessage`1152 - `'parsed'`: `None` if there was a parsing error, otherwise the type1153 depends on the `schema` as described above.1154 - `'parsing_error'`: `BaseException | None`1155 """1156 _ = kwargs.pop("strict", None)1157 if kwargs:1158 msg = f"Received unsupported arguments {kwargs}"1159 raise ValueError(msg)1160 is_pydantic_schema = isinstance(schema, type) and is_basemodel_subclass(schema)1161 if method == "function_calling":1162 if schema is None:1163 msg = (1164 "schema must be specified when method is 'function_calling'. "1165 "Received None."1166 )1167 raise ValueError(msg)1168 formatted_tool = convert_to_openai_tool(schema)1169 tool_name = formatted_tool["function"]["name"]1170 llm = self.bind_tools(1171 [schema],1172 tool_choice=tool_name,1173 ls_structured_output_format={1174 "kwargs": {"method": "function_calling"},1175 "schema": formatted_tool,1176 },1177 )1178 if is_pydantic_schema:1179 msg = "Pydantic schema is not supported for function calling"1180 raise NotImplementedError(msg)1181 output_parser: JsonOutputKeyToolsParser | JsonOutputParser = (1182 JsonOutputKeyToolsParser(key_name=tool_name, first_tool_only=True)1183 )1184 elif method == "json_schema":1185 if schema is None:1186 msg = (1187 "schema must be specified when method is 'json_schema'. "1188 "Received None."1189 )1190 raise ValueError(msg)1191 formatted_schema = convert_to_json_schema(schema)1192 llm = self.bind(1193 response_format={"type": "json_object", "schema": formatted_schema},1194 ls_structured_output_format={1195 "kwargs": {"method": "json_schema"},1196 "schema": schema,1197 },1198 )1199 output_parser = JsonOutputParser() # type: ignore[arg-type]1200 elif method == "json_mode":1201 llm = self.bind(1202 response_format={"type": "json_object"},1203 ls_structured_output_format={1204 "kwargs": {"method": "json_mode"},1205 "schema": schema,1206 },1207 )1208 output_parser = JsonOutputParser() # type: ignore[arg-type]1209 else:1210 msg = (1211 f"Unrecognized method argument. Expected one of 'function_calling' or "1212 f"'json_mode'. Received: '{method}'"1213 )1214 raise ValueError(msg)12151216 if include_raw:1217 parser_assign = RunnablePassthrough.assign(1218 parsed=itemgetter("raw") | output_parser, parsing_error=lambda _: None1219 )1220 parser_none = RunnablePassthrough.assign(parsed=lambda _: None)1221 parser_with_fallback = parser_assign.with_fallbacks(1222 [parser_none], exception_key="parsing_error"1223 )1224 return RunnableMap(raw=llm) | parser_with_fallback1225 return llm | output_parser12261227 def _create_message_dicts(1228 self, messages: list[BaseMessage], stop: list[str] | None1229 ) -> tuple[list[dict[str, Any]], dict[str, Any]]:1230 params = self._default_params1231 if stop is not None:1232 params["stop"] = stop1233 message_dicts = [_convert_message_to_dict(m) for m in messages]1234 return message_dicts, params12351236 @property1237 def _default_params(self) -> dict[str, Any]:1238 """Get default parameters for calling Hugging Face Inference Providers API."""1239 params = {1240 "model": self.model_id,1241 "stream": self.streaming,1242 "n": self.n,1243 "temperature": self.temperature,1244 "stop": self.stop,1245 **(self.model_kwargs if self.model_kwargs else {}),1246 }1247 if self.max_tokens is not None:1248 params["max_tokens"] = self.max_tokens1249 return params12501251 @property1252 def _llm_type(self) -> str:1253 return "huggingface-chat-wrapper"
Same data, no extra tab — call code_get_file + code_get_findings over MCP from Claude/Cursor/Copilot.
Generated by searchcode.com · powered by code_get_file + code_get_findings