a
    •˜ëjPH  ã                   @  s  d Z ddlmZ ddlZddlmZmZ ddlmZ ddl	m
Z
 ddlmZmZmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ e e¡ZG dd„ dee
ƒZG dd„ deƒZG dd„ dƒZG dd„ deeƒZ G dd„ deeƒZ!G dd„ deeƒZ"dS )z3Interfaces to be implemented by general evaluators.é    )ÚannotationsN)ÚABCÚabstractmethod)ÚSequence)ÚEnum)ÚAnyÚOptionalÚUnion)Úwarn)ÚAgentAction)ÚBaseLanguageModel)Úrun_in_executor)ÚChainc                   @  s`   e Zd ZdZdZdZdZdZdZdZ	dZ
d	Zd
ZdZdZdZdZdZdZdZdZdZdZdZdS )ÚEvaluatorTypezThe types of the evaluators.ZqaZcot_qaZ
context_qaZpairwise_stringZscore_stringZlabeled_pairwise_stringZlabeled_score_stringZ
trajectoryZcriteriaZlabeled_criteriaZstring_distanceZexact_matchZregex_matchZpairwise_string_distanceZembedding_distanceZpairwise_embedding_distanceZjson_validityZjson_equalityZjson_edit_distanceZjson_schema_validationN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__ZQAZCOT_QAZ
CONTEXT_QAZPAIRWISE_STRINGZSCORE_STRINGZLABELED_PAIRWISE_STRINGZLABELED_SCORE_STRINGZAGENT_TRAJECTORYZCRITERIAZLABELED_CRITERIAZSTRING_DISTANCEZEXACT_MATCHZREGEX_MATCHZPAIRWISE_STRING_DISTANCEZEMBEDDING_DISTANCEZPAIRWISE_EMBEDDING_DISTANCEZJSON_VALIDITYZJSON_EQUALITYZJSON_EDIT_DISTANCEZJSON_SCHEMA_VALIDATION© r   r   úZ/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/langchain/evaluation/schema.pyr      s,   r   c                   @  s*   e Zd ZdZeeddd dœdd„ƒƒZdS )ÚLLMEvalChainz,A base class for evaluators that use an LLM.r   r   )ÚllmÚkwargsÚreturnc                 K  s   dS )z#Create a new evaluator from an LLM.Nr   )Úclsr   r   r   r   r   Úfrom_llmN   s    zLLMEvalChain.from_llmN)r   r   r   r   Úclassmethodr   r   r   r   r   r   r   K   s   r   c                   @  sl   e Zd ZdZeddœdd„ƒZeddœdd„ƒZeddœd	d
„ƒZeddœdd„ƒZdddddœdd„Z	dS )Ú_EvalArgsMixinz(Mixin for checking evaluation arguments.Úbool©r   c                 C  s   dS ©z2Whether this evaluator requires a reference label.Fr   ©Úselfr   r   r   Úrequires_referenceW   s    z!_EvalArgsMixin.requires_referencec                 C  s   dS )ú0Whether this evaluator requires an input string.Fr   r!   r   r   r   Úrequires_input\   s    z_EvalArgsMixin.requires_inputÚstrc                 C  s   d| j j› d�S )z&Warning to show when input is ignored.zIgnoring input in ú, as it is not expected.©Ú	__class__r   r!   r   r   r   Ú_skip_input_warninga   s    z"_EvalArgsMixin._skip_input_warningc                 C  s   d| j j› d�S )z*Warning to show when reference is ignored.zIgnoring reference in r'   r(   r!   r   r   r   Ú_skip_reference_warningf   s    ÿz&_EvalArgsMixin._skip_reference_warningNúOptional[str]ÚNone)Ú	referenceÚinput_r   c                 C  s„   | j r$|du r$| jj› d�}t|ƒ‚|dur@| j s@t| jdd� | jrd|du rd| jj› d�}t|ƒ‚|dur€| js€t| jdd� dS )aˆ  Check if the evaluation arguments are valid.

        Args:
            reference (Optional[str], optional): The reference label.
            input_ (Optional[str], optional): The input string.

        Raises:
            ValueError: If the evaluator requires an input string but none is provided,
                or if the evaluator requires a reference label but none is provided.
        Nz requires an input string.é   )Ú
stacklevelz requires a reference string.)r%   r)   r   Ú
ValueErrorr
   r*   r#   r+   )r"   r.   r/   Úmsgr   r   r   Ú_check_evaluation_argsm   s    z%_EvalArgsMixin._check_evaluation_args)NN)
r   r   r   r   Úpropertyr#   r%   r*   r+   r4   r   r   r   r   r   T   s     ýr   c                   @  s°   e Zd ZdZeddœdd„ƒZeddœdd„ƒZed	d	d
œddddddœdd„ƒZd	d	d
œddddddœdd„Z	d	d	d
œddddddœdd„Z
d	d	d
œddddddœdd„Zd	S )ÚStringEvaluatorz‰String evaluator interface.

    Grade, tag, or otherwise evaluate predictions relative to their inputs
    and/or reference labels.
    r&   r   c                 C  s   | j jS )zThe name of the evaluation.r(   r!   r   r   r   Úevaluation_name�   s    zStringEvaluator.evaluation_namer   c                 C  s   dS r    r   r!   r   r   r   r#   ”   s    z"StringEvaluator.requires_referenceN©r.   ÚinputzUnion[str, Any]zOptional[Union[str, Any]]r   Údict)Ú
predictionr.   r9   r   r   c                K  s   dS )a;  Evaluate Chain or LLM output, based on optional input and label.

        Args:
            prediction (str): The LLM or chain prediction to evaluate.
            reference (Optional[str], optional): The reference label to evaluate against.
            input (Optional[str], optional): The input to consider during evaluation.
            kwargs: Additional keyword arguments, including callbacks, tags, etc.

        Returns:
            dict: The evaluation results containing the score or value.
                It is recommended that the dictionary contain the following keys:
                     - score: the score of the evaluation, if applicable.
                     - value: the string value of the evaluation, if applicable.
                     - reasoning: the reasoning for the evaluation, if applicable.
        Nr   ©r"   r;   r.   r9   r   r   r   r   Ú_evaluate_strings™   s    	z!StringEvaluator._evaluate_stringsc                Ë  s"   t d| jf|||dœ|¤ŽI dH S )aJ  Asynchronously evaluate Chain or LLM output, based on optional input and label.

        Args:
            prediction (str): The LLM or chain prediction to evaluate.
            reference (Optional[str], optional): The reference label to evaluate against.
            input (Optional[str], optional): The input to consider during evaluation.
            kwargs: Additional keyword arguments, including callbacks, tags, etc.

        Returns:
            dict: The evaluation results containing the score or value.
                It is recommended that the dictionary contain the following keys:
                     - score: the score of the evaluation, if applicable.
                     - value: the string value of the evaluation, if applicable.
                     - reasoning: the reasoning for the evaluation, if applicable.
        N©r;   r.   r9   )r   r=   r<   r   r   r   Ú_aevaluate_strings²   s    þûúz"StringEvaluator._aevaluate_stringsr,   c                K  s&   | j ||d� | jf |||dœ|¤ŽS )aû  Evaluate Chain or LLM output, based on optional input and label.

        Args:
            prediction (str): The LLM or chain prediction to evaluate.
            reference (Optional[str], optional): The reference label to evaluate against.
            input (Optional[str], optional): The input to consider during evaluation.
            kwargs: Additional keyword arguments, including callbacks, tags, etc.

        Returns:
            dict: The evaluation results containing the score or value.
        ©r.   r/   r>   )r4   r=   r<   r   r   r   Úevaluate_stringsÒ   s    ýüz StringEvaluator.evaluate_stringsc                Ë  s,   | j ||d� | jf |||dœ|¤ŽI dH S )a
  Asynchronously evaluate Chain or LLM output, based on optional input and label.

        Args:
            prediction (str): The LLM or chain prediction to evaluate.
            reference (Optional[str], optional): The reference label to evaluate against.
            input (Optional[str], optional): The input to consider during evaluation.
            kwargs: Additional keyword arguments, including callbacks, tags, etc.

        Returns:
            dict: The evaluation results containing the score or value.
        r@   r>   N)r4   r?   r<   r   r   r   Úaevaluate_stringsí   s    ýüz!StringEvaluator.aevaluate_strings)r   r   r   r   r5   r7   r#   r   r=   r?   rA   rB   r   r   r   r   r6   ˆ   s$   ûû$ûûr6   c                	   @  s”   e Zd ZdZedddœdddddddœd	d
„ƒZdddœdddddddœdd„Zdddœdddddddœdd„Zdddœdddddddœdd„ZdS )ÚPairwiseStringEvaluatorzDCompare the output of two models (or two outputs of the same model).Nr8   r&   r,   r   r:   )r;   Úprediction_br.   r9   r   r   c                K  s   dS )á2  Evaluate the output string pairs.

        Args:
            prediction (str): The output string from the first model.
            prediction_b (str): The output string from the second model.
            reference (Optional[str], optional): The expected output / reference string.
            input (Optional[str], optional): The input string.
            kwargs: Additional keyword arguments, such as callbacks and optional reference strings.

        Returns:
            dict: A dictionary containing the preference, scores, and/or other information.
        Nr   ©r"   r;   rD   r.   r9   r   r   r   r   Ú_evaluate_string_pairs  s    
z.PairwiseStringEvaluator._evaluate_string_pairsc                Ë  s$   t d| jf||||dœ|¤ŽI dH S )áA  Asynchronously evaluate the output string pairs.

        Args:
            prediction (str): The output string from the first model.
            prediction_b (str): The output string from the second model.
            reference (Optional[str], optional): The expected output / reference string.
            input (Optional[str], optional): The input string.
            kwargs: Additional keyword arguments, such as callbacks and optional reference strings.

        Returns:
            dict: A dictionary containing the preference, scores, and/or other information.
        N©r;   rD   r.   r9   )r   rG   rF   r   r   r   Ú_aevaluate_string_pairs#  s    þúùz/PairwiseStringEvaluator._aevaluate_string_pairsc                K  s(   | j ||d� | jf ||||dœ|¤ŽS )rE   r@   rI   )r4   rG   rF   r   r   r   Úevaluate_string_pairsB  s    üûz-PairwiseStringEvaluator.evaluate_string_pairsc                Ë  s.   | j ||d� | jf ||||dœ|¤ŽI dH S )rH   r@   rI   N)r4   rJ   rF   r   r   r   Úaevaluate_string_pairs`  s    üûz.PairwiseStringEvaluator.aevaluate_string_pairs)	r   r   r   r   r   rG   rJ   rK   rL   r   r   r   r   rC   	  s   úú$ú#úrC   c                	   @  sž   e Zd ZdZeddœdd„ƒZeddœdd	dd
dddœdd„ƒZddœdd	dd
dddœdd„Zddœdd	dd
dddœdd„Z	ddœdd	dd
dddœdd„Z
dS )ÚAgentTrajectoryEvaluatorz,Interface for evaluating agent trajectories.r   r   c                 C  s   dS )r$   Tr   r!   r   r   r   r%   ‚  s    z'AgentTrajectoryEvaluator.requires_inputN)r.   r&   z!Sequence[tuple[AgentAction, str]]r,   r   r:   )r;   Úagent_trajectoryr9   r.   r   r   c                K  s   dS )áˆ  Evaluate a trajectory.

        Args:
            prediction: The final predicted response.
            agent_trajectory:
                The intermediate steps forming the agent trajectory.
            input: The input to the agent.
            reference: The reference answer.
            **kwargs: Additional keyword arguments.

        Returns:
            The evaluation result.
        Nr   ©r"   r;   rN   r9   r.   r   r   r   r   Ú_evaluate_agent_trajectory‡  s    
z3AgentTrajectoryEvaluator._evaluate_agent_trajectoryc                Ë  s$   t d| jf||||dœ|¤ŽI dH S )á—  Asynchronously evaluate a trajectory.

        Args:
            prediction: The final predicted response.
            agent_trajectory:
                The intermediate steps forming the agent trajectory.
            input: The input to the agent.
            reference: The reference answer.
            **kwargs: Additional keyword arguments.

        Returns:
            The evaluation result.
        N)r;   rN   r.   r9   )r   rQ   rP   r   r   r   Ú_aevaluate_agent_trajectoryŸ  s    þúùz4AgentTrajectoryEvaluator._aevaluate_agent_trajectoryc                K  s(   | j ||d� | jf ||||dœ|¤ŽS )rO   r@   ©r;   r9   rN   r.   )r4   rQ   rP   r   r   r   Úevaluate_agent_trajectory¿  s    üûz2AgentTrajectoryEvaluator.evaluate_agent_trajectoryc                Ë  s.   | j ||d� | jf ||||dœ|¤ŽI dH S )rR   r@   rT   N)r4   rS   rP   r   r   r   Úaevaluate_agent_trajectoryÞ  s    üûz3AgentTrajectoryEvaluator.aevaluate_agent_trajectory)r   r   r   r   r5   r%   r   rQ   rS   rU   rV   r   r   r   r   rM     s   úú&ú%úrM   )#r   Ú
__future__r   ÚloggingÚabcr   r   Úcollections.abcr   Úenumr   Útypingr   r   r	   Úwarningsr
   Zlangchain_core.agentsr   Zlangchain_core.language_modelsr   Zlangchain_core.runnables.configr   Zlangchain.chains.baser   Ú	getLoggerr   Úloggerr&   r   r   r   r6   rC   rM   r   r   r   r   Ú<module>   s&   
6	4 v