1"""Utilities for running language models or Chains over datasets."""23from __future__ import annotations45import concurrent.futures6import dataclasses7import functools8import inspect9import logging10import re11import unicodedata12import uuid13from collections.abc import Callable14from datetime import datetime, timezone15from typing import (16    TYPE_CHECKING,17    Any,18    cast,19)20from urllib.parse import urlsplit, urlunsplit2122from langchain_core._api import warn_deprecated23from langchain_core.callbacks import Callbacks24from langchain_core.language_models import BaseLanguageModel25from langchain_core.messages import BaseMessage, messages_from_dict26from langchain_core.outputs import ChatResult, LLMResult27from langchain_core.runnables import Runnable, RunnableConfig, RunnableLambda28from langchain_core.runnables import config as runnable_config29from langchain_core.runnables import utils as runnable_utils30from langchain_core.tracers.evaluation import (31    EvaluatorCallbackHandler,32    wait_for_all_evaluators,33)34from langchain_core.tracers.langchain import LangChainTracer35from langsmith.client import Client36from langsmith.env import get_git_info, get_langchain_env_var_metadata37from langsmith.evaluation import (38    EvaluationResult,39    RunEvaluator,40)41from langsmith.evaluation import (42    run_evaluator as run_evaluator_dec,43)44from langsmith.run_helpers import as_runnable, is_traceable_function45from langsmith.schemas import Dataset, DataType, Example, Run, TracerSession46from langsmith.utils import LangSmithError47from requests import HTTPError48from typing_extensions import TypedDict4950from langchain_classic.chains.base import Chain51from langchain_classic.evaluation.loading import load_evaluator52from langchain_classic.evaluation.schema import (53    EvaluatorType,54    PairwiseStringEvaluator,55    StringEvaluator,56)57from langchain_classic.smith import evaluation as smith_eval58from langchain_classic.smith.evaluation import config as smith_eval_config59from langchain_classic.smith.evaluation import name_generation, progress6061if TYPE_CHECKING:62    import pandas as pd6364logger = logging.getLogger(__name__)6566MODEL_OR_CHAIN_FACTORY = (67    Callable[[], Chain | Runnable]68    | BaseLanguageModel69    | Callable[[dict], Any]70    | Runnable71    | Chain72)73MCF = Callable[[], Chain | Runnable] | BaseLanguageModel7475_GIT_REMOTE_URL_SCHEMES = {"git", "http", "https", "ssh"}76_SCP_STYLE_GIT_REMOTE_RE = re.compile(77    r"^(?:[^@/:\s]+@)?(?P<host>[^@/:\s]+):(?P<path>[^\s:][^\s]*)$"78)798081def _sanitize_git_remote_url(remote_url: object) -> str | None:82    if not isinstance(remote_url, str) or not remote_url:83        return None84    if any(85        character.isspace() or unicodedata.category(character) == "Cc"86        for character in remote_url87    ):88        return None8990    if "://" not in remote_url:91        match = _SCP_STYLE_GIT_REMOTE_RE.fullmatch(remote_url)92        if match is None:93            return None94        host = match.group("host")95        path = match.group("path")96        if host.lower() in _GIT_REMOTE_URL_SCHEMES or (97            len(host) == 198            and host.isascii()99            and host.isalpha()100            and path.startswith(("/", "\\"))101        ):102            return None103        return f"{host}:{path}"104105    try:106        parsed = urlsplit(remote_url)107        host = parsed.hostname108        if (109            parsed.scheme not in _GIT_REMOTE_URL_SCHEMES110            or host is None111            or "%" in host112            or not parsed.path113            or parsed.query114            or parsed.fragment115        ):116            return None117        port = parsed.port118    except ValueError:119        return None120121    if ":" in host:122        host = f"[{host}]"123    netloc = f"{host}:{port}" if port is not None else host124    return urlunsplit((parsed.scheme, netloc, parsed.path, "", ""))125126127def _format_git_tags(git_info: dict[str, Any]) -> list[str]:128    tags = []129    for key, value in git_info.items():130        if key == "remote_url":131            sanitized_remote_url = _sanitize_git_remote_url(value)132            if sanitized_remote_url is None:133                continue134            tags.append(f"git:{key}={sanitized_remote_url}")135        else:136            tags.append(f"git:{key}={value}")137    return tags138139140class InputFormatError(Exception):141    """Raised when the input format is invalid."""142143144## Shared Utilities145146147class TestResult(dict):148    """A dictionary of the results of a single test run."""149150    def get_aggregate_feedback(151        self,152    ) -> pd.DataFrame:153        """Return quantiles for the feedback scores.154155        This method calculates and prints the quantiles for the feedback scores156        across all feedback keys.157158        Returns:159            A DataFrame containing the quantiles for each feedback key.160        """161        df = self.to_dataframe()162        # Drop all things starting with inputs., outputs., and reference163        to_drop = [164            col165            for col in df.columns166            if col.startswith(("inputs.", "outputs.", "reference"))167            or col in {"input", "output"}168        ]169        return df.describe(include="all").drop(to_drop, axis=1)170171    def to_dataframe(self) -> pd.DataFrame:172        """Convert the results to a dataframe."""173        try:174            import pandas as pd175        except ImportError as e:176            msg = (177                "Pandas is required to convert the results to a dataframe."178                " to install pandas, run `pip install pandas`."179            )180            raise ImportError(msg) from e181182        indices = []183        records = []184        for example_id, result in self["results"].items():185            feedback = result["feedback"]186            output_ = result.get("output")187            if isinstance(output_, dict):188                output = {f"outputs.{k}": v for k, v in output_.items()}189            elif output_ is None:190                output = {}191            else:192                output = {"output": output_}193194            r = {195                **{f"inputs.{k}": v for k, v in result["input"].items()},196                **output,197            }198            if "reference" in result:199                if isinstance(result["reference"], dict):200                    r.update(201                        {f"reference.{k}": v for k, v in result["reference"].items()},202                    )203                else:204                    r["reference"] = result["reference"]205            r.update(206                {207                    **{f"feedback.{f.key}": f.score for f in feedback},208                    "error": result.get("Error"),209                    "execution_time": result["execution_time"],210                    "run_id": result.get("run_id"),211                },212            )213            records.append(r)214            indices.append(example_id)215216        return pd.DataFrame(records, index=indices)217218219class EvalError(dict):220    """Your architecture raised an error."""221222    def __init__(self, Error: BaseException, **kwargs: Any) -> None:  # noqa: N803223        """Initialize the `EvalError` with an error and additional attributes.224225        Args:226            Error: The error that occurred.227            **kwargs: Additional attributes to include in the error.228        """229        super().__init__(Error=Error, **kwargs)230231    def __getattr__(self, name: str) -> Any:232        """Get an attribute from the `EvalError`.233234        Args:235            name: The name of the attribute to get.236237        Returns:238            The value of the attribute.239240        Raises:241            AttributeError: If the attribute does not exist.242        """243        try:244            return self[name]245        except KeyError as e:246            msg = f"'EvalError' object has no attribute '{name}'"247            raise AttributeError(msg) from e248249250def _wrap_in_chain_factory(251    llm_or_chain_factory: MODEL_OR_CHAIN_FACTORY,252    dataset_name: str = "<my_dataset>",253) -> MCF:254    """Wrap in a chain factory.255256    Forgive the user if they pass in a chain without memory instead of a chain257    factory. It's a common mistake. Raise a more helpful error message as well.258    """259    if isinstance(llm_or_chain_factory, Chain):260        chain = llm_or_chain_factory261        chain_class = chain.__class__.__name__262        if llm_or_chain_factory.memory is not None:263            memory_class = chain.memory.__class__.__name__264            msg = (265                "Cannot directly evaluate a chain with stateful memory."266                " To evaluate this chain, pass in a chain constructor"267                " that initializes fresh memory each time it is called."268                "  This will safeguard against information"269                " leakage between dataset examples."270                "\nFor example:\n\n"271                "def chain_constructor():\n"272                f"    new_memory = {memory_class}(...)\n"273                f"    return {chain_class}"274                "(memory=new_memory, ...)\n\n"275                f'run_on_dataset("{dataset_name}", chain_constructor, ...)'276            )277            raise ValueError(msg)278        return lambda: chain279    if isinstance(llm_or_chain_factory, BaseLanguageModel):280        return llm_or_chain_factory281    if isinstance(llm_or_chain_factory, Runnable):282        # Memory may exist here, but it's not elegant to check all those cases.283        lcf = llm_or_chain_factory284        return lambda: lcf285    if callable(llm_or_chain_factory):286        if is_traceable_function(llm_or_chain_factory):287            runnable_ = as_runnable(cast("Callable", llm_or_chain_factory))288            return lambda: runnable_289        try:290            _model = llm_or_chain_factory()  # type: ignore[call-arg]291        except TypeError:292            # It's an arbitrary function, wrap it in a RunnableLambda293            user_func = cast("Callable", llm_or_chain_factory)294            sig = inspect.signature(user_func)295            logger.info("Wrapping function %s as RunnableLambda.", sig)296            wrapped = RunnableLambda(user_func)297            return lambda: wrapped298        constructor = cast("Callable", llm_or_chain_factory)299        if isinstance(_model, BaseLanguageModel):300            # It's not uncommon to do an LLM constructor instead of raw LLM,301            # so we'll unpack it for the user.302            return _model303        if is_traceable_function(cast("Callable", _model)):304            runnable_ = as_runnable(cast("Callable", _model))305            return lambda: runnable_306        if not isinstance(_model, Runnable):307            # This is unlikely to happen - a constructor for a model function308            return lambda: RunnableLambda(constructor)309        # Typical correct case310        return constructor311    return llm_or_chain_factory  # type: ignore[unreachable]312313314def _get_prompt(inputs: dict[str, Any]) -> str:315    """Get prompt from inputs.316317    Args:318        inputs: The input dictionary.319320    Returns:321        A string prompt.322323    Raises:324        InputFormatError: If the input format is invalid.325    """326    if not inputs:327        msg = "Inputs should not be empty."328        raise InputFormatError(msg)329330    prompts = []331    if "prompt" in inputs:332        if not isinstance(inputs["prompt"], str):333            msg = f"Expected string for 'prompt', got {type(inputs['prompt']).__name__}"334            raise InputFormatError(msg)335        prompts = [inputs["prompt"]]336    elif "prompts" in inputs:337        if not isinstance(inputs["prompts"], list) or not all(338            isinstance(i, str) for i in inputs["prompts"]339        ):340            msg = (341                "Expected list of strings for 'prompts',"342                f" got {type(inputs['prompts']).__name__}"343            )344            raise InputFormatError(msg)345        prompts = inputs["prompts"]346    elif len(inputs) == 1:347        prompt_ = next(iter(inputs.values()))348        if isinstance(prompt_, str):349            prompts = [prompt_]350        elif isinstance(prompt_, list) and all(isinstance(i, str) for i in prompt_):351            prompts = prompt_352        else:353            msg = f"LLM Run expects string prompt input. Got {inputs}"354            raise InputFormatError(msg)355    else:356        msg = f"LLM Run expects 'prompt' or 'prompts' in inputs. Got {inputs}"357        raise InputFormatError(msg)358    if len(prompts) == 1:359        return prompts[0]360    msg = f"LLM Run expects single prompt input. Got {len(prompts)} prompts."361    raise InputFormatError(msg)362363364class ChatModelInput(TypedDict):365    """Input for a chat model."""366367    messages: list[BaseMessage]368369370def _get_messages(inputs: dict[str, Any]) -> dict:371    """Get Chat Messages from inputs.372373    Args:374        inputs: The input dictionary.375376    Returns:377        A list of chat messages.378379    Raises:380        InputFormatError: If the input format is invalid.381    """382    if not inputs:383        msg = "Inputs should not be empty."384        raise InputFormatError(msg)385    input_copy = inputs.copy()386    if "messages" in inputs:387        input_copy["input"] = input_copy.pop("messages")388    elif len(inputs) == 1:389        input_copy["input"] = next(iter(inputs.values()))390    if "input" in input_copy:391        raw_messages = input_copy["input"]392        if isinstance(raw_messages, list) and all(393            isinstance(i, dict) for i in raw_messages394        ):395            raw_messages = [raw_messages]396        if len(raw_messages) == 1:397            input_copy["input"] = messages_from_dict(raw_messages[0])398        else:399            msg = (400                "Batch messages not supported. Please provide a"401                " single list of messages."402            )403            raise InputFormatError(msg)404        return input_copy405    msg = (406        f"Chat Run expects single List[dict] or List[List[dict]] 'messages'"407        f" input. Got {inputs}"408    )409    raise InputFormatError(msg)410411412## Shared data validation utilities413def _validate_example_inputs_for_language_model(414    first_example: Example,415    input_mapper: Callable[[dict], Any] | None,416) -> None:417    if input_mapper:418        prompt_input = input_mapper(first_example.inputs or {})419        if not isinstance(prompt_input, str) and not (420            isinstance(prompt_input, list)421            and all(isinstance(msg, BaseMessage) for msg in prompt_input)422        ):423            msg = (424                "When using an input_mapper to prepare dataset example inputs"425                " for an LLM or chat model, the output must a single string or"426                " a list of chat messages."427                f"\nGot: {prompt_input} of type {type(prompt_input)}."428            )429            raise InputFormatError(msg)430    else:431        try:432            _get_prompt(first_example.inputs or {})433        except InputFormatError:434            try:435                _get_messages(first_example.inputs or {})436            except InputFormatError as err2:437                msg = (438                    "Example inputs do not match language model input format. "439                    "Expected a dictionary with messages or a single prompt."440                    f" Got: {first_example.inputs}"441                    " Please update your dataset OR provide an input_mapper"442                    " to convert the example.inputs to a compatible format"443                    " for the llm or chat model you wish to evaluate."444                )445                raise InputFormatError(msg) from err2446447448def _validate_example_inputs_for_chain(449    first_example: Example,450    chain: Chain,451    input_mapper: Callable[[dict], Any] | None,452) -> None:453    """Validate that the example inputs match the chain input keys."""454    if input_mapper:455        first_inputs = input_mapper(first_example.inputs or {})456        missing_keys = set(chain.input_keys).difference(first_inputs)457        if not isinstance(first_inputs, dict):458            msg = (459                "When using an input_mapper to prepare dataset example"460                " inputs for a chain, the mapped value must be a dictionary."461                f"\nGot: {first_inputs} of type {type(first_inputs)}."462            )463            raise InputFormatError(msg)464        if missing_keys:465            msg = (466                "Missing keys after loading example using input_mapper."467                f"\nExpected: {chain.input_keys}. Got: {first_inputs.keys()}"468            )469            raise InputFormatError(msg)470    else:471        first_inputs = first_example.inputs or {}472        missing_keys = set(chain.input_keys).difference(first_inputs)473        if len(first_inputs) == 1 and len(chain.input_keys) == 1:474            # We can pass this through the run method.475            # Refrain from calling to validate.476            pass477        elif missing_keys:478            msg = (479                "Example inputs missing expected chain input keys."480                " Please provide an input_mapper to convert the example.inputs"481                " to a compatible format for the chain you wish to evaluate."482                f"Expected: {chain.input_keys}. "483                f"Got: {first_inputs.keys()}"484            )485            raise InputFormatError(msg)486487488def _validate_example_inputs(489    example: Example,490    llm_or_chain_factory: MCF,491    input_mapper: Callable[[dict], Any] | None,492) -> None:493    """Validate that the example inputs are valid for the model."""494    if isinstance(llm_or_chain_factory, BaseLanguageModel):495        _validate_example_inputs_for_language_model(example, input_mapper)496    else:497        chain = llm_or_chain_factory()498        if isinstance(chain, Chain):499            # Otherwise it's a runnable500            _validate_example_inputs_for_chain(example, chain, input_mapper)501        elif isinstance(chain, Runnable):502            logger.debug("Skipping input validation for %s", chain)503504505## Shared Evaluator Setup Utilities506507508def _setup_evaluation(509    llm_or_chain_factory: MCF,510    examples: list[Example],511    evaluation: smith_eval.RunEvalConfig | None,512    data_type: DataType,513) -> list[RunEvaluator] | None:514    """Configure the evaluators to run on the results of the chain."""515    if evaluation:516        if isinstance(llm_or_chain_factory, BaseLanguageModel):517            run_inputs, run_outputs = None, None518            run_type = "llm"519        else:520            run_type = "chain"521            chain = llm_or_chain_factory()522            run_inputs = chain.input_keys if isinstance(chain, Chain) else None523            run_outputs = chain.output_keys if isinstance(chain, Chain) else None524        run_evaluators = _load_run_evaluators(525            evaluation,526            run_type,527            data_type,528            list(examples[0].outputs) if examples[0].outputs else None,529            run_inputs,530            run_outputs,531        )532    else:533        # TODO: Create a default helpfulness evaluator534        run_evaluators = None535    return run_evaluators536537538def _determine_input_key(539    config: smith_eval.RunEvalConfig,540    run_inputs: list[str] | None,541) -> str | None:542    input_key = None543    if config.input_key:544        input_key = config.input_key545        if run_inputs and input_key not in run_inputs:546            logger.warning(547                "Input key %s not in chain's specified input keys %s. "548                "Evaluation behavior may be undefined.",549                input_key,550                run_inputs,551            )552    elif run_inputs and len(run_inputs) == 1:553        input_key = run_inputs[0]554    elif run_inputs is not None and len(run_inputs) > 1:555        logger.warning(556            "Chain expects multiple input keys: %s,"557            " Evaluator is likely to fail. Evaluation behavior may be undefined."558            " Specify an input_key in the RunEvalConfig to avoid this warning.",559            run_inputs,560        )561562    return input_key563564565def _determine_prediction_key(566    config: smith_eval.RunEvalConfig,567    run_outputs: list[str] | None,568) -> str | None:569    prediction_key = None570    if config.prediction_key:571        prediction_key = config.prediction_key572        if run_outputs and prediction_key not in run_outputs:573            logger.warning(574                "Prediction key %s not in chain's specified output keys %s. "575                "Evaluation behavior may be undefined.",576                prediction_key,577                run_outputs,578            )579    elif run_outputs and len(run_outputs) == 1:580        prediction_key = run_outputs[0]581    elif run_outputs is not None and len(run_outputs) > 1:582        logger.warning(583            "Chain expects multiple output keys: %s,"584            " Evaluation behavior may be undefined. Specify a prediction_key"585            " in the RunEvalConfig to avoid this warning.",586            run_outputs,587        )588    return prediction_key589590591def _determine_reference_key(592    config: smith_eval.RunEvalConfig,593    example_outputs: list[str] | None,594) -> str | None:595    if config.reference_key:596        reference_key = config.reference_key597        if example_outputs and reference_key not in example_outputs:598            msg = (599                f"Reference key {reference_key} not in Dataset"600                f" example outputs: {example_outputs}"601            )602            raise ValueError(msg)603    elif example_outputs and len(example_outputs) == 1:604        reference_key = next(iter(example_outputs))605    else:606        reference_key = None607    return reference_key608609610def _construct_run_evaluator(611    eval_config: smith_eval_config.SINGLE_EVAL_CONFIG_TYPE612    | smith_eval_config.CUSTOM_EVALUATOR_TYPE,613    eval_llm: BaseLanguageModel | None,614    run_type: str,615    data_type: DataType,616    example_outputs: list[str] | None,617    reference_key: str | None,618    input_key: str | None,619    prediction_key: str | None,620) -> RunEvaluator:621    if isinstance(eval_config, RunEvaluator):622        return eval_config623    if isinstance(eval_config, (EvaluatorType, str)):624        if not isinstance(eval_config, EvaluatorType):625            eval_config = EvaluatorType(eval_config)626        evaluator_ = load_evaluator(eval_config, llm=eval_llm)627        eval_type_tag = eval_config.value628    elif isinstance(eval_config, smith_eval_config.EvalConfig):629        kwargs = {"llm": eval_llm, **eval_config.get_kwargs()}630        evaluator_ = load_evaluator(eval_config.evaluator_type, **kwargs)631        eval_type_tag = eval_config.evaluator_type.value632        # Override keys if specified in the config633        if isinstance(eval_config, smith_eval_config.SingleKeyEvalConfig):634            input_key = eval_config.input_key or input_key635            prediction_key = eval_config.prediction_key or prediction_key636            reference_key = eval_config.reference_key or reference_key637    elif callable(eval_config):638        # Assume we can decorate639        return run_evaluator_dec(eval_config)640    else:641        msg = f"Unknown evaluator type: {type(eval_config)}"642        raise ValueError(msg)  # noqa: TRY004643644    if isinstance(evaluator_, StringEvaluator):645        if evaluator_.requires_reference and reference_key is None:646            msg = (647                f"Must specify reference_key in smith_eval.RunEvalConfig to use"648                f" evaluator of type {eval_type_tag} with"649                f" dataset with multiple output keys: {example_outputs}."650            )651            raise ValueError(msg)652        run_evaluator = smith_eval.StringRunEvaluatorChain.from_run_and_data_type(653            evaluator_,654            run_type,655            data_type,656            input_key=input_key,657            prediction_key=prediction_key,658            reference_key=reference_key,659            tags=[eval_type_tag],660        )661    elif isinstance(evaluator_, PairwiseStringEvaluator):662        msg = (663            f"Run evaluator for {eval_type_tag} is not implemented."664            " PairwiseStringEvaluators compare the outputs of two different models"665            " rather than the output of a single model."666            " Did you mean to use a StringEvaluator instead?"667            "\nSee: https://python.langchain.com/docs/guides/evaluation/string/"668        )669        raise NotImplementedError(msg)670671    else:672        msg = f"Run evaluator for {eval_type_tag} is not implemented"673        raise NotImplementedError(msg)674    return run_evaluator675676677def _get_keys(678    config: smith_eval.RunEvalConfig,679    run_inputs: list[str] | None,680    run_outputs: list[str] | None,681    example_outputs: list[str] | None,682) -> tuple[str | None, str | None, str | None]:683    input_key = _determine_input_key(config, run_inputs)684    prediction_key = _determine_prediction_key(config, run_outputs)685    reference_key = _determine_reference_key(config, example_outputs)686    return input_key, prediction_key, reference_key687688689def _load_run_evaluators(690    config: smith_eval.RunEvalConfig,691    run_type: str,692    data_type: DataType,693    example_outputs: list[str] | None,694    run_inputs: list[str] | None,695    run_outputs: list[str] | None,696) -> list[RunEvaluator]:697    """Load run evaluators from a configuration.698699    Args:700        config: Configuration for the run evaluators.701        run_type: The type of run.702        data_type: The type of dataset used in the run.703        example_outputs: The example outputs.704        run_inputs: The input keys for the run.705        run_outputs: The output keys for the run.706707    Returns:708        A list of run evaluators.709    """710    run_evaluators = []711    input_key, prediction_key, reference_key = None, None, None712    if config.evaluators or (713        config.custom_evaluators714        and any(isinstance(e, StringEvaluator) for e in config.custom_evaluators)715    ):716        input_key, prediction_key, reference_key = _get_keys(717            config,718            run_inputs,719            run_outputs,720            example_outputs,721        )722    for eval_config in config.evaluators:723        run_evaluator = _construct_run_evaluator(724            eval_config,725            config.eval_llm,726            run_type,727            data_type,728            example_outputs,729            reference_key,730            input_key,731            prediction_key,732        )733        run_evaluators.append(run_evaluator)734    custom_evaluators = config.custom_evaluators or []735    for custom_evaluator in custom_evaluators:736        if isinstance(custom_evaluator, RunEvaluator):737            run_evaluators.append(custom_evaluator)738        elif isinstance(custom_evaluator, StringEvaluator):739            run_evaluators.append(740                smith_eval.StringRunEvaluatorChain.from_run_and_data_type(741                    custom_evaluator,742                    run_type,743                    data_type,744                    input_key=input_key,745                    prediction_key=prediction_key,746                    reference_key=reference_key,747                ),748            )749        elif callable(custom_evaluator):750            run_evaluators.append(run_evaluator_dec(custom_evaluator))751        else:752            msg = (  # type: ignore[unreachable]753                f"Unsupported custom evaluator: {custom_evaluator}."754                f" Expected RunEvaluator or StringEvaluator."755            )756            raise ValueError(msg)  # noqa: TRY004757758    return run_evaluators759760761### Async Helpers762763764async def _arun_llm(765    llm: BaseLanguageModel,766    inputs: dict[str, Any],767    *,768    tags: list[str] | None = None,769    callbacks: Callbacks = None,770    input_mapper: Callable[[dict], Any] | None = None,771    metadata: dict[str, Any] | None = None,772) -> str | BaseMessage:773    """Asynchronously run the language model.774775    Args:776        llm: The language model to run.777        inputs: The input dictionary.778        tags: Optional tags to add to the run.779        callbacks: Optional callbacks to use during the run.780        input_mapper: Optional function to map inputs to the expected format.781        metadata: Optional metadata to add to the run.782783    Returns:784        The LLMResult or ChatResult.785786    Raises:787        ValueError: If the LLM type is unsupported.788        InputFormatError: If the input format is invalid.789    """790    if input_mapper is not None:791        prompt_or_messages = input_mapper(inputs)792        if isinstance(prompt_or_messages, str) or (793            isinstance(prompt_or_messages, list)794            and all(isinstance(msg, BaseMessage) for msg in prompt_or_messages)795        ):796            return await llm.ainvoke(797                prompt_or_messages,798                config=RunnableConfig(799                    callbacks=callbacks,800                    tags=tags or [],801                    metadata=metadata or {},802                ),803            )804        msg = (805            "Input mapper returned invalid format"806            f" {prompt_or_messages}"807            "\nExpected a single string or list of chat messages."808        )809        raise InputFormatError(msg)810811    try:812        prompt = _get_prompt(inputs)813        llm_output: str | BaseMessage = await llm.ainvoke(814            prompt,815            config=RunnableConfig(816                callbacks=callbacks,817                tags=tags or [],818                metadata=metadata or {},819            ),820        )821    except InputFormatError:822        llm_inputs = _get_messages(inputs)823        llm_output = await llm.ainvoke(824            **llm_inputs,825            config=RunnableConfig(826                callbacks=callbacks,827                tags=tags or [],828                metadata=metadata or {},829            ),830        )831    return llm_output832833834async def _arun_chain(835    chain: Chain | Runnable,836    inputs: dict[str, Any],837    callbacks: Callbacks,838    *,839    tags: list[str] | None = None,840    input_mapper: Callable[[dict], Any] | None = None,841    metadata: dict[str, Any] | None = None,842) -> dict | str:843    """Run a chain asynchronously on inputs."""844    inputs_ = inputs if input_mapper is None else input_mapper(inputs)845    if (846        isinstance(chain, Chain)847        and isinstance(inputs_, dict)848        and len(inputs_) == 1849        and chain.input_keys850    ):851        val = next(iter(inputs_.values()))852        output = await chain.ainvoke(853            val,854            config=RunnableConfig(855                callbacks=callbacks,856                tags=tags or [],857                metadata=metadata or {},858            ),859        )860    else:861        runnable_config = RunnableConfig(862            tags=tags or [],863            callbacks=callbacks,864            metadata=metadata or {},865        )866        output = await chain.ainvoke(inputs_, config=runnable_config)867    return output868869870async def _arun_llm_or_chain(871    example: Example,872    config: RunnableConfig,873    *,874    llm_or_chain_factory: MCF,875    input_mapper: Callable[[dict], Any] | None = None,876) -> dict | str | LLMResult | ChatResult:877    """Asynchronously run the Chain or language model.878879    Args:880        example: The example to run.881        config: The configuration for the run.882        llm_or_chain_factory: The Chain or language model constructor to run.883        input_mapper: Optional function to map the input to the expected format.884885    Returns:886        A list of outputs.887    """888    chain_or_llm = (889        "LLM" if isinstance(llm_or_chain_factory, BaseLanguageModel) else "Chain"890    )891    result = None892    try:893        if isinstance(llm_or_chain_factory, BaseLanguageModel):894            output: Any = await _arun_llm(895                llm_or_chain_factory,896                example.inputs or {},897                tags=config["tags"],898                callbacks=config["callbacks"],899                input_mapper=input_mapper,900                metadata=config.get("metadata"),901            )902        else:903            chain = llm_or_chain_factory()904            output = await _arun_chain(905                chain,906                example.inputs or {},907                tags=config["tags"],908                callbacks=config["callbacks"],909                input_mapper=input_mapper,910                metadata=config.get("metadata"),911            )912        result = output913    except Exception as e:  # noqa: BLE001914        logger.warning(915            "%s failed for example %s with inputs %s\n%s",916            chain_or_llm,917            example.id,918            example.inputs,919            e,920        )921        result = EvalError(Error=e)922    return result923924925## Sync Utilities926927928def _run_llm(929    llm: BaseLanguageModel,930    inputs: dict[str, Any],931    callbacks: Callbacks,932    *,933    tags: list[str] | None = None,934    input_mapper: Callable[[dict], Any] | None = None,935    metadata: dict[str, Any] | None = None,936) -> str | BaseMessage:937    """Run the language model on the example.938939    Args:940        llm: The language model to run.941        inputs: The input dictionary.942        callbacks: The callbacks to use during the run.943        tags: Optional tags to add to the run.944        input_mapper: function to map to the inputs dictionary from an Example945        metadata: Optional metadata to add to the run.946947    Returns:948        The LLMResult or ChatResult.949950    Raises:951        ValueError: If the LLM type is unsupported.952        InputFormatError: If the input format is invalid.953    """954    # Most of this is legacy code; we could probably remove a lot of it.955    if input_mapper is not None:956        prompt_or_messages = input_mapper(inputs)957        if isinstance(prompt_or_messages, str) or (958            isinstance(prompt_or_messages, list)959            and all(isinstance(msg, BaseMessage) for msg in prompt_or_messages)960        ):961            llm_output: str | BaseMessage = llm.invoke(962                prompt_or_messages,963                config=RunnableConfig(964                    callbacks=callbacks,965                    tags=tags or [],966                    metadata=metadata or {},967                ),968            )969        else:970            msg = (971                "Input mapper returned invalid format: "972                f" {prompt_or_messages}"973                "\nExpected a single string or list of chat messages."974            )975            raise InputFormatError(msg)976    else:977        try:978            llm_prompts = _get_prompt(inputs)979            llm_output = llm.invoke(980                llm_prompts,981                config=RunnableConfig(982                    callbacks=callbacks,983                    tags=tags or [],984                    metadata=metadata or {},985                ),986            )987        except InputFormatError:988            llm_inputs = _get_messages(inputs)989            llm_output = llm.invoke(990                **llm_inputs,991                config=RunnableConfig(callbacks=callbacks, metadata=metadata or {}),992            )993    return llm_output994995996def _run_chain(997    chain: Chain | Runnable,998    inputs: dict[str, Any],999    callbacks: Callbacks,1000    *,1001    tags: list[str] | None = None,1002    input_mapper: Callable[[dict], Any] | None = None,1003    metadata: dict[str, Any] | None = None,1004) -> dict | str:1005    """Run a chain on inputs."""1006    inputs_ = inputs if input_mapper is None else input_mapper(inputs)1007    if (1008        isinstance(chain, Chain)1009        and isinstance(inputs_, dict)1010        and len(inputs_) == 11011        and chain.input_keys1012    ):1013        val = next(iter(inputs_.values()))1014        output = chain.invoke(1015            val,1016            config=RunnableConfig(1017                callbacks=callbacks,1018                tags=tags or [],1019                metadata=metadata or {},1020            ),1021        )1022    else:1023        runnable_config = RunnableConfig(1024            tags=tags or [],1025            callbacks=callbacks,1026            metadata=metadata or {},1027        )1028        output = chain.invoke(inputs_, config=runnable_config)1029    return output103010311032def _run_llm_or_chain(1033    example: Example,1034    config: RunnableConfig,1035    *,1036    llm_or_chain_factory: MCF,1037    input_mapper: Callable[[dict], Any] | None = None,1038) -> dict | str | LLMResult | ChatResult:1039    """Run the Chain or language model synchronously.10401041    Args:1042        example: The example to run.1043        config: The configuration for the run.1044        llm_or_chain_factory: The Chain or language model constructor to run.1045        input_mapper: Optional function to map the input to the expected format.10461047    Returns:1048        The outputs of the model or chain.1049    """1050    chain_or_llm = (1051        "LLM" if isinstance(llm_or_chain_factory, BaseLanguageModel) else "Chain"1052    )1053    result = None1054    try:1055        if isinstance(llm_or_chain_factory, BaseLanguageModel):1056            output: Any = _run_llm(1057                llm_or_chain_factory,1058                example.inputs or {},1059                config["callbacks"],1060                tags=config["tags"],1061                input_mapper=input_mapper,1062                metadata=config.get("metadata"),1063            )1064        else:1065            chain = llm_or_chain_factory()1066            output = _run_chain(1067                chain,1068                example.inputs or {},1069                config["callbacks"],1070                tags=config["tags"],1071                input_mapper=input_mapper,1072                metadata=config.get("metadata"),1073            )1074        result = output1075    except Exception as e:  # noqa: BLE0011076        error_type = type(e).__name__1077        logger.warning(1078            "%s failed for example %s with inputs %s\nError Type: %s, Message: %s",1079            chain_or_llm,1080            example.id,1081            example.inputs,1082            error_type,1083            e,1084        )1085        result = EvalError(Error=e)1086    return result108710881089def _prepare_eval_run(1090    client: Client,1091    dataset_name: str,1092    llm_or_chain_factory: MODEL_OR_CHAIN_FACTORY,1093    project_name: str,1094    project_metadata: dict[str, Any] | None = None,1095    tags: list[str] | None = None,1096    dataset_version: str | datetime | None = None,1097) -> tuple[MCF, TracerSession, Dataset, list[Example]]:1098    wrapped_model = _wrap_in_chain_factory(llm_or_chain_factory, dataset_name)1099    dataset = client.read_dataset(dataset_name=dataset_name)11001101    examples = list(client.list_examples(dataset_id=dataset.id, as_of=dataset_version))1102    if not examples:1103        msg = f"Dataset {dataset_name} has no example rows."1104        raise ValueError(msg)1105    modified_at = [ex.modified_at for ex in examples if ex.modified_at]1106    # Should always be defined in practice when fetched,1107    # but the typing permits None1108    max_modified_at = max(modified_at) if modified_at else None1109    inferred_version = max_modified_at.isoformat() if max_modified_at else None11101111    try:1112        project_metadata = project_metadata or {}1113        git_info = get_git_info()1114        if git_info:1115            project_metadata = {1116                **project_metadata,1117                "git": git_info,1118            }11191120        project_metadata["dataset_version"] = inferred_version1121        project = client.create_project(1122            project_name,1123            reference_dataset_id=dataset.id,1124            project_extra={"tags": tags} if tags else {},1125            metadata=project_metadata,1126        )1127    except (HTTPError, ValueError, LangSmithError) as e:1128        if "already exists " not in str(e):1129            raise1130        uid = uuid.uuid4()1131        example_msg = f"""1132run_on_dataset(1133    ...1134    project_name="{project_name} - {uid}", # Update since {project_name} already exists1135)1136"""1137        msg = (1138            f"Test project {project_name} already exists. Please use a different name:"1139            f"\n\n{example_msg}"1140        )1141        raise ValueError(msg) from e1142    comparison_url = dataset.url + f"/compare?selectedSessions={project.id}"1143    print(  # noqa: T2011144        f"View the evaluation results for project '{project_name}'"1145        f" at:\n{comparison_url}\n\n"1146        f"View all tests for Dataset {dataset_name} at:\n{dataset.url}",1147        flush=True,1148    )1149    return wrapped_model, project, dataset, examples115011511152class _RowResult(TypedDict, total=False):1153    """A dictionary of the results for a single example row."""11541155    feedback: list[EvaluationResult] | None1156    execution_time: float | None1157    run_id: str | None115811591160@dataclasses.dataclass1161class _DatasetRunContainer:1162    """A container to help manage the state of a eval run."""11631164    client: Client1165    project: TracerSession1166    wrapped_model: MCF1167    examples: list[Example]1168    configs: list[RunnableConfig]1169    batch_evaluators: list[smith_eval_config.BATCH_EVALUATOR_LIKE] | None = None11701171    def _merge_test_outputs(1172        self,1173        batch_results: list,1174        all_eval_results: dict[str, _RowResult],1175    ) -> dict:1176        results: dict = {}1177        for example, output in zip(self.examples, batch_results, strict=False):1178            row_result = all_eval_results.get(str(example.id), {})1179            results[str(example.id)] = {1180                "input": example.inputs,1181                "feedback": row_result.get("feedback", []),1182                "execution_time": row_result.get("execution_time"),1183                "run_id": row_result.get("run_id"),1184            }1185            if isinstance(output, EvalError):1186                results[str(example.id)]["Error"] = output.Error1187            else:1188                results[str(example.id)]["output"] = output1189            if example.outputs:1190                results[str(example.id)]["reference"] = example.outputs1191        return results11921193    def _run_batch_evaluators(self, runs: dict[str, Run]) -> list[dict]:1194        evaluators = self.batch_evaluators1195        if not evaluators:1196            return []1197        runs_list = [runs[str(example.id)] for example in self.examples]1198        aggregate_feedback = []1199        with concurrent.futures.ThreadPoolExecutor() as executor:1200            for evaluator in evaluators:1201                try:1202                    result = evaluator(runs_list, self.examples)1203                    if isinstance(result, EvaluationResult):1204                        result = result.model_dump()1205                    aggregate_feedback.append(cast("dict", result))1206                    executor.submit(1207                        self.client.create_feedback,1208                        **result,1209                        run_id=None,1210                        project_id=self.project.id,1211                    )1212                except Exception:1213                    logger.exception(1214                        "Error running batch evaluator %s", repr(evaluator)1215                    )1216        return aggregate_feedback12171218    def _collect_metrics(self) -> tuple[dict[str, _RowResult], dict[str, Run]]:1219        all_eval_results: dict = {}1220        all_runs: dict = {}1221        for c in self.configs:1222            for callback in cast("list", c["callbacks"]):1223                if isinstance(callback, EvaluatorCallbackHandler):1224                    eval_results = callback.logged_eval_results1225                    for (_, example_id), v in eval_results.items():1226                        all_eval_results.setdefault(str(example_id), {}).update(1227                            {"feedback": v},1228                        )1229                elif isinstance(callback, LangChainTracer):1230                    run = callback.latest_run1231                    execution_time = (1232                        (run.end_time - run.start_time).total_seconds()1233                        if run and run.end_time1234                        else None1235                    )1236                    run_id = str(run.id) if run else None1237                    all_eval_results.setdefault(str(callback.example_id), {}).update(1238                        {1239                            "execution_time": execution_time,1240                            "run_id": run_id,1241                            "run": run,1242                        },1243                    )1244                    all_runs[str(callback.example_id)] = run1245        return cast("dict[str, _RowResult]", all_eval_results), all_runs12461247    def _collect_test_results(1248        self,1249        batch_results: list[dict | str | LLMResult | ChatResult],1250    ) -> TestResult:1251        logger.info("Waiting for evaluators to complete.")1252        wait_for_all_evaluators()1253        all_eval_results, all_runs = self._collect_metrics()1254        aggregate_feedback = None1255        if self.batch_evaluators:1256            logger.info("Running session evaluators.")1257            aggregate_feedback = self._run_batch_evaluators(all_runs)1258        results = self._merge_test_outputs(batch_results, all_eval_results)1259        return TestResult(1260            project_name=self.project.name,1261            results=results,1262            aggregate_metrics=aggregate_feedback,1263        )12641265    def finish(1266        self,1267        batch_results: list,1268        verbose: bool = False,  # noqa: FBT001,FBT0021269    ) -> TestResult:1270        results = self._collect_test_results(batch_results)1271        if verbose:1272            try:1273                agg_feedback = results.get_aggregate_feedback()1274                _display_aggregate_results(agg_feedback)1275            except Exception as e:  # noqa: BLE0011276                logger.debug("Failed to print aggregate feedback: %s", e, exc_info=True)1277        try:1278            # Closing the project permits name changing and metric optimizations1279            self.client.update_project(1280                self.project.id,1281                end_time=datetime.now(timezone.utc),1282            )1283        except Exception as e:  # noqa: BLE0011284            logger.debug("Failed to close project: %s", e, exc_info=True)1285        return results12861287    @classmethod1288    def prepare(1289        cls,1290        client: Client,1291        dataset_name: str,1292        llm_or_chain_factory: MODEL_OR_CHAIN_FACTORY,1293        project_name: str | None,1294        evaluation: smith_eval.RunEvalConfig | None = None,1295        tags: list[str] | None = None,1296        input_mapper: Callable[[dict], Any] | None = None,1297        concurrency_level: int = 5,1298        project_metadata: dict[str, Any] | None = None,1299        revision_id: str | None = None,1300        dataset_version: datetime | str | None = None,1301    ) -> _DatasetRunContainer:1302        project_name = project_name or name_generation.random_name()1303        if revision_id:1304            if not project_metadata:1305                project_metadata = {}1306            project_metadata.update({"revision_id": revision_id})1307        wrapped_model, project, dataset, examples = _prepare_eval_run(1308            client,1309            dataset_name,1310            llm_or_chain_factory,1311            project_name,1312            project_metadata=project_metadata,1313            tags=tags,1314            dataset_version=dataset_version,1315        )1316        tags = tags or []1317        tags.extend(_format_git_tags(project.metadata.get("git") or {}))1318        run_metadata = {"dataset_version": project.metadata["dataset_version"]}1319        if revision_id:1320            run_metadata["revision_id"] = revision_id1321        wrapped_model = _wrap_in_chain_factory(llm_or_chain_factory)1322        run_evaluators = _setup_evaluation(1323            wrapped_model,1324            examples,1325            evaluation,1326            dataset.data_type or DataType.kv,1327        )1328        _validate_example_inputs(examples[0], wrapped_model, input_mapper)1329        progress_bar = progress.ProgressBarCallback(len(examples))1330        configs = [1331            RunnableConfig(1332                callbacks=[1333                    LangChainTracer(1334                        project_name=project.name,1335                        client=client,1336                        example_id=example.id,1337                    ),1338                    EvaluatorCallbackHandler(1339                        evaluators=run_evaluators or [],1340                        client=client,1341                        example_id=example.id,1342                        max_concurrency=0,1343                    ),1344                    progress_bar,1345                ],1346                tags=tags,1347                max_concurrency=concurrency_level,1348                metadata=run_metadata,1349            )1350            for example in examples1351        ]1352        return cls(1353            client=client,1354            project=project,1355            wrapped_model=wrapped_model,1356            examples=examples,1357            configs=configs,1358            batch_evaluators=evaluation.batch_evaluators if evaluation else None,1359        )136013611362def _is_jupyter_environment() -> bool:1363    try:1364        from IPython.core.getipython import get_ipython13651366        res = get_ipython()  # type: ignore[no-untyped-call]1367        return res is not None and "zmqshell" in str(type(res))1368    except ImportError:1369        return False137013711372def _display_aggregate_results(aggregate_results: pd.DataFrame) -> None:1373    if _is_jupyter_environment():1374        from IPython.display import HTML, display13751376        display(HTML("<h3>Experiment Results:</h3>"))  # type: ignore[no-untyped-call]1377        display(aggregate_results)  # type: ignore[no-untyped-call]1378    else:1379        formatted_string = aggregate_results.to_string(1380            float_format=lambda x: f"{x:.2f}",1381            justify="right",1382        )1383        print("\n Experiment Results:")  # noqa: T2011384        print(formatted_string)  # noqa: T201138513861387_INPUT_MAPPER_DEP_WARNING = (1388    "The input_mapper argument is deprecated and "1389    "will be removed in a future release. Please add a "1390    " RunnableLambda to your chain to map inputs to the expected format"1391    " instead. Example:\n"1392    "def construct_chain():\n"1393    "    my_chain = ...\n"1394    "    input_mapper = {'other_key': 'MyOtherInput', 'my_input_key': x}\n"1395    "    return input_mapper | my_chain\n"1396    "run_on_dataset(..., llm_or_chain_factory=construct_chain)\n"1397    "(See https://api.python.langchain.com/en/latest/schema/"1398    "langchain.schema.runnable.base.RunnableLambda.html)"1399)14001401## Public API140214031404async def arun_on_dataset(1405    client: Client | None,1406    dataset_name: str,1407    llm_or_chain_factory: MODEL_OR_CHAIN_FACTORY,1408    *,1409    evaluation: smith_eval.RunEvalConfig | None = None,1410    dataset_version: datetime | str | None = None,1411    concurrency_level: int = 5,1412    project_name: str | None = None,1413    project_metadata: dict[str, Any] | None = None,1414    verbose: bool = False,1415    revision_id: str | None = None,1416    **kwargs: Any,1417) -> dict[str, Any]:1418    """Run on dataset.14191420    Run the Chain or language model on a dataset and store traces1421    to the specified project name.14221423    For the (usually faster) async version of this function,1424    see `arun_on_dataset`.14251426    Args:1427        dataset_name: Name of the dataset to run the chain on.1428        llm_or_chain_factory: Language model or Chain constructor to run1429            over the dataset. The Chain constructor is used to permit1430            independent calls on each example without carrying over state.1431        evaluation: Configuration for evaluators to run on the1432            results of the chain.1433        dataset_version: Optional version of the dataset.1434        concurrency_level: The number of async tasks to run concurrently.1435        project_name: Name of the project to store the traces in.1436            Defaults to `{dataset_name}-{chain class name}-{datetime}`.1437        project_metadata: Optional metadata to add to the project.1438            Useful for storing information the test variant.1439            (prompt version, model version, etc.)1440        client: LangSmith client to use to access the dataset and to1441            log feedback and run traces.1442        verbose: Whether to print progress.1443        revision_id: Optional revision identifier to assign this test run to1444            track the performance of different versions of your system.1445        **kwargs: Should not be used, but is provided for backwards compatibility.14461447    Returns:1448        `dict` containing the run's project name and the resulting model outputs.14491450    Examples:1451    ```python1452    from langsmith import Client1453    from langchain_openai import ChatOpenAI1454    from langchain_classic.chains import LLMChain1455    from langchain_classic.smith import smith_eval.RunEvalConfig, run_on_dataset14561457    # Chains may have memory. Passing in a constructor function lets the1458    # evaluation framework avoid cross-contamination between runs.1459    def construct_chain():1460        model = ChatOpenAI(temperature=0)1461        chain = LLMChain.from_string(1462            model,1463            "What's the answer to {your_input_key}"1464        )1465        return chain14661467    # Load off-the-shelf evaluators via config or the EvaluatorType (string or enum)1468    evaluation_config = smith_eval.RunEvalConfig(1469        evaluators=[1470            "qa",  # "Correctness" against a reference answer1471            "embedding_distance",1472            smith_eval.RunEvalConfig.Criteria("helpfulness"),1473            smith_eval.RunEvalConfig.Criteria({1474                "fifth-grader-score": "Do you have to be smarter than a fifth "1475                "grader to answer this question?"1476            }),1477        ]1478    )14791480    client = Client()1481    await arun_on_dataset(1482        client,1483        dataset_name="<my_dataset_name>",1484        llm_or_chain_factory=construct_chain,1485        evaluation=evaluation_config,1486    )1487    ```1488    You can also create custom evaluators by subclassing the `StringEvaluator or1489    LangSmith's `RunEvaluator` classes.14901491    ```python1492    from typing import Optional1493    from langchain_classic.evaluation import StringEvaluator149414951496    class MyStringEvaluator(StringEvaluator):1497        @property1498        def requires_input(self) -> bool:1499            return False15001501        @property1502        def requires_reference(self) -> bool:1503            return True15041505        @property1506        def evaluation_name(self) -> str:1507            return "exact_match"15081509        def _evaluate_strings(1510            self, prediction, reference=None, input=None, **kwargs1511        ) -> dict:1512            return {"score": prediction == reference}151315141515    evaluation_config = smith_eval.RunEvalConfig(1516        custom_evaluators=[MyStringEvaluator()],1517    )15181519    await arun_on_dataset(1520        client,1521        dataset_name="<my_dataset_name>",1522        llm_or_chain_factory=construct_chain,1523        evaluation=evaluation_config,1524    )1525    ```1526    """1527    input_mapper = kwargs.pop("input_mapper", None)1528    if input_mapper:1529        warn_deprecated("0.0.305", message=_INPUT_MAPPER_DEP_WARNING, pending=True)1530    if revision_id is None:1531        revision_id = get_langchain_env_var_metadata().get("revision_id")1532    tags = kwargs.pop("tags", None)1533    if tags:1534        warn_deprecated(1535            "0.1.9",1536            message="The tags argument is deprecated and will be"1537            " removed in a future release. Please specify project_metadata instead.",1538            pending=True,1539        )15401541    if kwargs:1542        warn_deprecated(1543            "0.0.305",1544            message="The following arguments are deprecated and "1545            "will be removed in a future release: "1546            f"{kwargs.keys()}.",1547            removal="0.0.305",1548        )1549    client = client or Client()1550    container = _DatasetRunContainer.prepare(1551        client,1552        dataset_name,1553        llm_or_chain_factory,1554        project_name,1555        evaluation,1556        tags,1557        input_mapper,1558        concurrency_level,1559        project_metadata=project_metadata,1560        revision_id=revision_id,1561        dataset_version=dataset_version,1562    )1563    batch_results = await runnable_utils.gather_with_concurrency(1564        container.configs[0].get("max_concurrency"),1565        *map(1566            functools.partial(1567                _arun_llm_or_chain,1568                llm_or_chain_factory=container.wrapped_model,1569                input_mapper=input_mapper,1570            ),1571            container.examples,1572            container.configs,1573        ),1574    )1575    return container.finish(batch_results, verbose=verbose)157615771578def run_on_dataset(1579    client: Client | None,1580    dataset_name: str,1581    llm_or_chain_factory: MODEL_OR_CHAIN_FACTORY,1582    *,1583    evaluation: smith_eval.RunEvalConfig | None = None,1584    dataset_version: datetime | str | None = None,1585    concurrency_level: int = 5,1586    project_name: str | None = None,1587    project_metadata: dict[str, Any] | None = None,1588    verbose: bool = False,1589    revision_id: str | None = None,1590    **kwargs: Any,1591) -> dict[str, Any]:1592    """Run on dataset.15931594    Run the Chain or language model on a dataset and store traces1595    to the specified project name.15961597    For the (usually faster) async version of this function,1598    see `arun_on_dataset`.15991600    Args:1601        dataset_name: Name of the dataset to run the chain on.1602        llm_or_chain_factory: Language model or Chain constructor to run1603            over the dataset. The Chain constructor is used to permit1604            independent calls on each example without carrying over state.1605        evaluation: Configuration for evaluators to run on the1606            results of the chain.1607        dataset_version: Optional version of the dataset.1608        concurrency_level: The number of async tasks to run concurrently.1609        project_name: Name of the project to store the traces in.1610            Defaults to `{dataset_name}-{chain class name}-{datetime}`.1611        project_metadata: Optional metadata to add to the project.1612            Useful for storing information the test variant.1613            (prompt version, model version, etc.)1614        client: LangSmith client to use to access the dataset and to1615            log feedback and run traces.1616        verbose: Whether to print progress.1617        revision_id: Optional revision identifier to assign this test run to1618            track the performance of different versions of your system.1619        **kwargs: Should not be used, but is provided for backwards compatibility.16201621    Returns:1622        `dict` containing the run's project name and the resulting model outputs.16231624    Examples:1625    ```python1626    from langsmith import Client1627    from langchain_openai import ChatOpenAI1628    from langchain_classic.chains import LLMChain1629    from langchain_classic.smith import smith_eval.RunEvalConfig, run_on_dataset16301631    # Chains may have memory. Passing in a constructor function lets the1632    # evaluation framework avoid cross-contamination between runs.1633    def construct_chain():1634        model = ChatOpenAI(temperature=0)1635        chain = LLMChain.from_string(1636            model,1637            "What's the answer to {your_input_key}"1638        )1639        return chain16401641    # Load off-the-shelf evaluators via config or the EvaluatorType (string or enum)1642    evaluation_config = smith_eval.RunEvalConfig(1643        evaluators=[1644            "qa",  # "Correctness" against a reference answer1645            "embedding_distance",1646            smith_eval.RunEvalConfig.Criteria("helpfulness"),1647            smith_eval.RunEvalConfig.Criteria({1648                "fifth-grader-score": "Do you have to be smarter than a fifth "1649                "grader to answer this question?"1650            }),1651        ]1652    )16531654    client = Client()1655    run_on_dataset(1656        client,1657        dataset_name="<my_dataset_name>",1658        llm_or_chain_factory=construct_chain,1659        evaluation=evaluation_config,1660    )1661    ```16621663    You can also create custom evaluators by subclassing the `StringEvaluator` or1664    LangSmith's `RunEvaluator` classes.16651666    ```python1667    from typing import Optional1668    from langchain_classic.evaluation import StringEvaluator166916701671    class MyStringEvaluator(StringEvaluator):1672        @property1673        def requires_input(self) -> bool:1674            return False16751676        @property1677        def requires_reference(self) -> bool:1678            return True16791680        @property1681        def evaluation_name(self) -> str:1682            return "exact_match"16831684        def _evaluate_strings(1685            self, prediction, reference=None, input=None, **kwargs1686        ) -> dict:1687            return {"score": prediction == reference}168816891690    evaluation_config = smith_eval.RunEvalConfig(1691        custom_evaluators=[MyStringEvaluator()],1692    )16931694    run_on_dataset(1695        client,1696        dataset_name="<my_dataset_name>",1697        llm_or_chain_factory=construct_chain,1698        evaluation=evaluation_config,1699    )1700    ```1701    """1702    input_mapper = kwargs.pop("input_mapper", None)1703    if input_mapper:1704        warn_deprecated("0.0.305", message=_INPUT_MAPPER_DEP_WARNING, pending=True)1705    tags = kwargs.pop("tags", None)1706    if tags:1707        warn_deprecated(1708            "0.1.9",1709            message="The tags argument is deprecated and will be"1710            " removed in a future release. Please specify project_metadata instead.",1711            pending=True,1712        )1713    if revision_id is None:1714        revision_id = get_langchain_env_var_metadata().get("revision_id")17151716    if kwargs:1717        warn_deprecated(1718            "0.0.305",1719            message="The following arguments are deprecated and "1720            "will be removed in a future release: "1721            f"{kwargs.keys()}.",1722            removal="0.0.305",1723        )1724    client = client or Client()1725    container = _DatasetRunContainer.prepare(1726        client,1727        dataset_name,1728        llm_or_chain_factory,1729        project_name,1730        evaluation,1731        tags,1732        input_mapper,1733        concurrency_level,1734        project_metadata=project_metadata,1735        revision_id=revision_id,1736        dataset_version=dataset_version,1737    )1738    if concurrency_level == 0:1739        batch_results = [1740            _run_llm_or_chain(1741                example,1742                config,1743                llm_or_chain_factory=container.wrapped_model,1744                input_mapper=input_mapper,1745            )1746            for example, config in zip(1747                container.examples, container.configs, strict=False1748            )1749        ]1750    else:1751        with runnable_config.get_executor_for_config(container.configs[0]) as executor:1752            batch_results = list(1753                executor.map(1754                    functools.partial(1755                        _run_llm_or_chain,1756                        llm_or_chain_factory=container.wrapped_model,1757                        input_mapper=input_mapper,1758                    ),1759                    container.examples,1760                    container.configs,1761                ),1762            )17631764    return container.finish(batch_results, verbose=verbose)