a
    •˜ëj-  ã                   @   s`  d Z ddlmZ ddlmZmZmZmZ ddlm	Z	 ddl
mZ ddlmZ ddlmZ ddlmZmZ dd	lmZmZ dd
lmZmZmZ ddlmZ ddlmZ ddlmZ  ddl!m"Z"m#Z# ddl$m%Z& eeee geeee'f f Z(eee eee  geeee'f f Z)G dd„ deƒZ*G dd„ de*ƒZ+ee(ee#f Z,ee"e-e*f Z.G dd„ deƒZ/dS )z!Configuration for run evaluators.é    )ÚSequence)ÚAnyÚCallableÚOptionalÚUnion)Ú
Embeddings)ÚBaseLanguageModel)ÚBasePromptTemplate)ÚRunEvaluator)ÚEvaluationResultÚEvaluationResults)ÚExampleÚRun)Ú	BaseModelÚ
ConfigDictÚField)Úoverride)ÚCRITERIA_TYPE)ÚEmbeddingDistance)ÚEvaluatorTypeÚStringEvaluator)ÚStringDistancec                   @   s0   e Zd ZU dZeed< eeef dœdd„Z	dS )Ú
EvalConfiga  Configuration for a given run evaluator.

    Parameters
    ----------
    evaluator_type : EvaluatorType
        The type of evaluator to use.

    Methods:
    -------
    get_kwargs()
        Get the keyword arguments for the evaluator configuration.

    Úevaluator_type©Úreturnc                 C   s0   i }| D ]"\}}|dks|du r"q|||< q|S )z¸Get the keyword arguments for the load_evaluator call.

        Returns:
        -------
        Dict[str, Any]
            The keyword arguments for the load_evaluator call.

        r   N© )ÚselfÚkwargsÚfieldÚvalr   r   ú`/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/langchain/smith/evaluation/config.pyÚ
get_kwargs3   s    	
zEvalConfig.get_kwargsN)
Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   Ú__annotations__ÚdictÚstrr   r"   r   r   r   r!   r   "   s   
r   c                       sd   e Zd ZU dZdZee ed< dZee ed< dZ	ee ed< e
eeef dœ‡ fdd„ƒZ‡  ZS )	ÚSingleKeyEvalConfigzBConfiguration for a run evaluator that only requires a single key.NÚreference_keyÚprediction_keyÚ	input_keyr   c                    s$   t ƒ  ¡ }dD ]}| |d ¡ q|S )N)r+   r,   r-   )Úsuperr"   Úpop)r   r   Úkey©Ú	__class__r   r!   r"   R   s    
zSingleKeyEvalConfig.get_kwargs)r#   r$   r%   r&   r+   r   r)   r'   r,   r-   r   r(   r   r"   Ú__classcell__r   r   r1   r!   r*   D   s   
r*   c                   @   sr  e Zd ZU dZeed�Zeeee	f  e
d< dZeee	  e
d< dZeee  e
d< dZee e
d< dZee e
d< dZee e
d	< dZee e
d
< edd�ZG dd„ deƒZG dd„ deƒZG dd„ deƒZG dd„ deƒZG dd„ deƒZG dd„ deƒZG dd„ deƒZG dd„ deƒZG dd„ de ƒZ!G dd „ d eƒZ"G d!d"„ d"eƒZ#G d#d$„ d$eƒZ$G d%d&„ d&e$ƒZ%dS )'ÚRunEvalConfigz#Configuration for a run evaluation.)Údefault_factoryÚ
evaluatorsNÚcustom_evaluatorsÚbatch_evaluatorsr+   r,   r-   Úeval_llmT©Zarbitrary_types_allowedc                   @   s@   e Zd ZU dZdZee ed< dZee	 ed< e
jZe
ed< dS )zRunEvalConfig.Criteriaa"  Configuration for a reference-free criteria evaluator.

        Parameters
        ----------
        criteria : Optional[CRITERIA_TYPE]
            The criteria to evaluate.
        llm : Optional[BaseLanguageModel]
            The language model to use for the evaluation chain.

        NÚcriteriaÚllmr   )r#   r$   r%   r&   r;   r   r   r'   r<   r   r   ZCRITERIAr   r   r   r   r!   ÚCriteria‰   s   
r=   c                   @   s@   e Zd ZU dZdZee ed< dZee	 ed< e
jZe
ed< dS )zRunEvalConfig.LabeledCriteriaa,  Configuration for a labeled (with references) criteria evaluator.

        Parameters
        ----------
        criteria : Optional[CRITERIA_TYPE]
            The criteria to evaluate.
        llm : Optional[BaseLanguageModel]
            The language model to use for the evaluation chain.
        Nr;   r<   r   )r#   r$   r%   r&   r;   r   r   r'   r<   r   r   ZLABELED_CRITERIAr   r   r   r   r!   ÚLabeledCriteria™   s   

r>   c                   @   sJ   e Zd ZU dZejZeed< dZe	e
 ed< dZe	e ed< edd�ZdS )zRunEvalConfig.EmbeddingDistanceaI  Configuration for an embedding distance evaluator.

        Parameters
        ----------
        embeddings : Optional[Embeddings]
            The embeddings to use for computing the distance.

        distance_metric : Optional[EmbeddingDistanceEnum]
            The distance metric to use for computing the distance.

        r   NÚ
embeddingsÚdistance_metricTr:   )r#   r$   r%   r&   r   ZEMBEDDING_DISTANCEr   r'   r?   r   r   r@   ÚEmbeddingDistanceEnumr   Úmodel_configr   r   r   r!   r   ¨   s   
ÿr   c                   @   s<   e Zd ZU dZejZeed< dZe	e
 ed< dZeed< dS )zRunEvalConfig.StringDistancez¾Configuration for a string distance evaluator.

        Parameters
        ----------
        distance : Optional[StringDistanceEnum]
            The string distance metric to use.

        r   NÚdistanceTÚnormalize_score)r#   r$   r%   r&   r   ZSTRING_DISTANCEr   r'   rC   r   ÚStringDistanceEnumrD   Úboolr   r   r   r!   r   ½   s
   
	r   c                   @   s@   e Zd ZU dZejZeed< dZe	e
 ed< dZe	e ed< dS )zRunEvalConfig.QAa-  Configuration for a QA evaluator.

        Parameters
        ----------
        prompt : Optional[BasePromptTemplate]
            The prompt template to use for generating the question.
        llm : Optional[BaseLanguageModel]
            The language model to use for the evaluation chain.
        r   Nr<   Úprompt)r#   r$   r%   r&   r   ÚQAr   r'   r<   r   r   rG   r	   r   r   r   r!   rH   Ó   s   

rH   c                   @   s@   e Zd ZU dZejZeed< dZe	e
 ed< dZe	e ed< dS )zRunEvalConfig.ContextQAá<  Configuration for a context-based QA evaluator.

        Parameters
        ----------
        prompt : Optional[BasePromptTemplate]
            The prompt template to use for generating the question.
        llm : Optional[BaseLanguageModel]
            The language model to use for the evaluation chain.

        r   Nr<   rG   ©r#   r$   r%   r&   r   Z
CONTEXT_QAr   r'   r<   r   r   rG   r	   r   r   r   r!   Ú	ContextQAâ   s   
rK   c                   @   s@   e Zd ZU dZejZeed< dZe	e
 ed< dZe	e ed< dS )zRunEvalConfig.CoTQArI   r   Nr<   rG   rJ   r   r   r   r!   ÚCoTQAò   s   
rL   c                   @   s    e Zd ZU dZejZeed< dS )zRunEvalConfig.JsonValidityz\Configuration for a json validity evaluator.

        Parameters
        ----------
        r   N)r#   r$   r%   r&   r   ZJSON_VALIDITYr   r'   r   r   r   r!   ÚJsonValidity  s   
rM   c                   @   s    e Zd ZU dZejZeed< dS )z#RunEvalConfig.JsonEqualityEvaluatorz\Configuration for a json equality evaluator.

        Parameters
        ----------
        r   N)r#   r$   r%   r&   r   ZJSON_EQUALITYr   r'   r   r   r   r!   ÚJsonEqualityEvaluator  s   
rN   c                   @   sD   e Zd ZU dZejZeed< dZe	ed< dZ
e	ed< dZe	ed< dS )zRunEvalConfig.ExactMatchax  Configuration for an exact match string evaluator.

        Parameters
        ----------
        ignore_case : bool
            Whether to ignore case when comparing strings.
        ignore_punctuation : bool
            Whether to ignore punctuation when comparing strings.
        ignore_numbers : bool
            Whether to ignore numbers when comparing strings.
        r   FÚignore_caseÚignore_punctuationÚignore_numbersN)r#   r$   r%   r&   r   ZEXACT_MATCHr   r'   rO   rF   rP   rQ   r   r   r   r!   Ú
ExactMatch  s
   
rR   c                   @   s,   e Zd ZU dZejZeed< dZe	ed< dS )zRunEvalConfig.RegexMatchz¹Configuration for a regex match string evaluator.

        Parameters
        ----------
        flags : int
            The flags to pass to the regex. Example: re.IGNORECASE.
        r   r   ÚflagsN)
r#   r$   r%   r&   r   ZREGEX_MATCHr   r'   rS   Úintr   r   r   r!   Ú
RegexMatch&  s   
rU   c                   @   s`   e Zd ZU dZejZeed< dZe	e
 ed< dZe	e ed< dZe	e ed< dZe	e ed< dS )zRunEvalConfig.ScoreStringaã  Configuration for a score string evaluator.

        This is like the criteria evaluator but it is configured by
        default to return a score on the scale from 1-10.

        It is recommended to normalize these scores
        by setting `normalize_by` to 10.

        Parameters
        ----------
        criteria : Optional[CRITERIA_TYPE]
            The criteria to evaluate.
        llm : Optional[BaseLanguageModel]
            The language model to use for the evaluation chain.
        normalize_by: Optional[int] = None
            If you want to normalize the score, the denominator to use.
            If not provided, the score will be between 1 and 10 (by default).
        prompt : Optional[BasePromptTemplate]

        r   Nr;   r<   Únormalize_byrG   )r#   r$   r%   r&   r   ZSCORE_STRINGr   r'   r;   r   r   r<   r   rV   ÚfloatrG   r	   r   r   r   r!   ÚScoreString2  s   
rX   c                   @   s    e Zd ZU dZejZeed< dS )z RunEvalConfig.LabeledScoreStringz3Configuration for a labeled score string evaluator.r   N)r#   r$   r%   r&   r   ZLABELED_SCORE_STRINGr   r'   r   r   r   r!   ÚLabeledScoreStringN  s   
rY   )&r#   r$   r%   r&   r   Úlistr6   r   ÚSINGLE_EVAL_CONFIG_TYPEÚCUSTOM_EVALUATOR_TYPEr'   r7   r   r8   ÚBATCH_EVALUATOR_LIKEr+   r)   r,   r-   r9   r   r   rB   r*   r=   r>   r   r   rH   rK   rL   rM   r   rN   rR   rU   rX   rY   r   r   r   r!   r4   _   s>   
ûÿÿÿÿ		r4   N)0r&   Úcollections.abcr   Útypingr   r   r   r   Zlangchain_core.embeddingsr   Zlangchain_core.language_modelsr   Zlangchain_core.promptsr	   Z	langsmithr
   Zlangsmith.evaluation.evaluatorr   r   Zlangsmith.schemasr   r   Zpydanticr   r   r   Ztyping_extensionsr   Z(langchain.evaluation.criteria.eval_chainr   Z,langchain.evaluation.embedding_distance.baser   rA   Zlangchain.evaluation.schemar   r   Z)langchain.evaluation.string_distance.baser   rE   r(   ZRUN_EVALUATOR_LIKEr]   r   r*   r\   r)   r[   r4   r   r   r   r!   Ú<module>   s:   
ÿÿÿÿ"