a
    •˜ëj  ã                (   @   s   U d Z ddlmZ ddlmZmZmZ ddlmZ ddl	m
Z
 ddlmZ ddlmZ ddlmZ dd	lmZmZ dd
lmZmZ ddlmZ ddlmZmZ ddlmZ ddlmZ ddl m!Z!m"Z"m#Z# ddl$m%Z% ddl&m'Z'm(Z(m)Z) ddl*m+Z+m,Z, ddl-m.Z.m/Z/ e0e1e2 dœdd„Z3e'j4e#e'j5e"e'j6e!e'j7ee'j8e,e'j9ee'j:e+e'j;ee'j<ee'j=ee'j>e/e'j?e.e'j@ee'jAee'jBee'jCee'jDee'jEee'jFe%e'jGeiZHe2e'eeIe( eIe
 eIe) f f eJd< ddœe'ee eee
e)f dœdd„ZKdddœee' ee ee2 ee1ee
e)f  dœdd „ZLdS )!z Loading datasets and evaluators.é    )ÚSequence)ÚAnyÚOptionalÚUnion)ÚBaseLanguageModel)ÚChain)ÚTrajectoryEvalChain)ÚPairwiseStringEvalChain)ÚLabeledPairwiseStringEvalChain)ÚCriteriaEvalChainÚLabeledCriteriaEvalChain)ÚEmbeddingDistanceEvalChainÚ"PairwiseEmbeddingDistanceEvalChain)ÚExactMatchStringEvaluator)ÚJsonEqualityEvaluatorÚJsonValidityEvaluator)ÚJsonEditDistanceEvaluator)ÚJsonSchemaEvaluator)ÚContextQAEvalChainÚCotQAEvalChainÚQAEvalChain)ÚRegexMatchStringEvaluator)ÚEvaluatorTypeÚLLMEvalChainÚStringEvaluator)ÚLabeledScoreStringEvalChainÚScoreStringEvalChain)ÚPairwiseStringDistanceEvalChainÚStringDistanceEvalChain)ÚuriÚreturnc              
   C   s^   zddl m} W n2 tyB } zd}t|ƒ|‚W Y d}~n
d}~0 0 |d| › �ƒ}t|d ƒS )aà  Load a dataset from the `LangChainDatasets on HuggingFace <https://huggingface.co/LangChainDatasets>`_.

    Args:
        uri: The uri of the dataset to load.

    Returns:
        A list of dictionaries, each representing a row in the dataset.

    **Prerequisites**

    .. code-block:: shell

        pip install datasets

    Examples:
    --------
    .. code-block:: python

        from langchain.evaluation import load_dataset

        ds = load_dataset("llm-math")

    r   )Úload_datasetzXload_dataset requires the `datasets` package. Please install with `pip install datasets`NzLangChainDatasets/Ztrain)Zdatasetsr!   ÚImportErrorÚlist)r   r!   ÚeÚmsgZdataset© r&   ú[/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/langchain/evaluation/loading.pyr!   (   s    ÿ r!   Ú_EVALUATOR_MAPN)Úllm)Ú	evaluatorr)   Úkwargsr    c                K   s  | t vr(d| › dtt  ¡ ƒ› �}t|ƒ‚t |  }t|tƒ�r
z~zddlm} W nV ty¤   zddl	m} W n2 tyž } zd}t|ƒ|‚W Y d}~n
d}~0 0 Y n0 |p¶|dddd	�}W n: t
yô } z"d
|› d�}t|ƒ|‚W Y d}~n
d}~0 0 |jf d|i|¤ŽS |f i |¤ŽS )a>  Load the requested evaluation chain specified by a string.

    Parameters
    ----------
    evaluator : EvaluatorType
        The type of evaluator to load.
    llm : BaseLanguageModel, optional
        The language model to use for evaluation, by default None
    **kwargs : Any
        Additional keyword arguments to pass to the evaluator.

    Returns:
    -------
    Chain
        The loaded evaluation chain.

    Examples:
    --------
    >>> from langchain.evaluation import load_evaluator, EvaluatorType
    >>> evaluator = load_evaluator(EvaluatorType.QA)
    zUnknown evaluator type: z
Valid types are: r   )Ú
ChatOpenAIzåCould not import langchain_openai or fallback onto langchain_community. Please install langchain_openai or specify a language model explicitly. It's recommended to install langchain_openai AND specify a language model explicitly.Nzgpt-4é*   )ÚmodelÚseedZtemperaturezEvaluation with the zŸ requires a language model to function. Failed to create the default 'gpt-4' model. Please manually provide an evaluation LLM or check your openai credentials.r)   )r(   r#   ÚkeysÚ
ValueErrorÚ
issubclassr   Zlangchain_openair,   r"   Z&langchain_community.chat_models.openaiÚ	ExceptionZfrom_llm)r*   r)   r+   r%   Zevaluator_clsr,   r$   r&   r&   r'   Úload_evaluatorh   s2    
ÿÿÿ&
ÿ r4   )r)   Úconfig)Ú
evaluatorsr)   r5   r+   r    c                K   sH   g }| D ]:}|r|  |i ¡ni }| t|fd|ii |¥|¥¤Ž¡ q|S )ag  Load evaluators specified by a list of evaluator types.

    Parameters
    ----------
    evaluators : Sequence[EvaluatorType]
        The list of evaluator types to load.
    llm : BaseLanguageModel, optional
        The language model to use for evaluation, if none is provided, a default
        ChatOpenAI gpt-4 model will be used.
    config : dict, optional
        A dictionary mapping evaluator types to additional keyword arguments,
        by default None
    **kwargs : Any
        Additional keyword arguments to pass to all evaluators.

    Returns:
    -------
    List[Chain]
        The loaded evaluators.

    Examples:
    --------
    >>> from langchain.evaluation import load_evaluators, EvaluatorType
    >>> evaluators = [EvaluatorType.QA, EvaluatorType.CRITERIA]
    >>> loaded_evaluators = load_evaluators(evaluators, criteria="helpfulness")
    r)   )ÚgetÚappendr4   )r6   r)   r5   r+   Zloadedr*   Ú_kwargsr&   r&   r'   Úload_evaluators«   s
    !$r:   )MÚ__doc__Úcollections.abcr   Útypingr   r   r   Zlangchain_core.language_modelsr   Zlangchain.chains.baser   Z1langchain.evaluation.agents.trajectory_eval_chainr   Zlangchain.evaluation.comparisonr	   Z*langchain.evaluation.comparison.eval_chainr
   Z(langchain.evaluation.criteria.eval_chainr   r   Z,langchain.evaluation.embedding_distance.baser   r   Z%langchain.evaluation.exact_match.baser   Z!langchain.evaluation.parsing.baser   r   Z*langchain.evaluation.parsing.json_distancer   Z(langchain.evaluation.parsing.json_schemar   Zlangchain.evaluation.qar   r   r   Z%langchain.evaluation.regex_match.baser   Zlangchain.evaluation.schemar   r   r   Z'langchain.evaluation.scoring.eval_chainr   r   Z)langchain.evaluation.string_distance.baser   r   Ústrr#   Údictr!   ZQAZCOT_QAZ
CONTEXT_QAZPAIRWISE_STRINGZSCORE_STRINGZLABELED_PAIRWISE_STRINGZLABELED_SCORE_STRINGZAGENT_TRAJECTORYZCRITERIAZLABELED_CRITERIAZSTRING_DISTANCEZPAIRWISE_STRING_DISTANCEZEMBEDDING_DISTANCEZPAIRWISE_EMBEDDING_DISTANCEZJSON_VALIDITYZJSON_EQUALITYZJSON_EDIT_DISTANCEZJSON_SCHEMA_VALIDATIONZREGEX_MATCHZEXACT_MATCHr(   ÚtypeÚ__annotations__r4   r:   r&   r&   r&   r'   Ú<module>   sz   )ìýÿÿý
ûFüú