a
    ”˜ëj²’  ã                   @  sd  d Z ddlmZ ddlZddlZddlZddlmZ ddlm	Z	m
Z
 ddlmZmZmZmZmZmZ ddlmZ ddlmZ dd	lmZ zdd
lmZmZmZmZ W n* eyÎ   dd
lmZmZmZmZ Y n0 ddlZddl m!Z! ddl"m#Z#m$Z$m%Z%m&Z& e 'e(¡Z)G dd„ deƒZ*G dd„ dedd�Z+G dd„ deƒZ,G dd„ dedd�Z-G dd„ dƒZ.ee,e-e/f Z0G dd„ deƒZ1ee1e/f Z2G dd„ de.ƒZ3ddœdd „Z4d!Z5d"d#œd$d%„Z6G d&d'„ d'ƒZ7d(d'd)œd*d+„Z8d,d-d)œd.d/„Z9d,d0d)œd1d2„Z:d3d4d5œd6d7„Z;eee
ej& e
ej% gee,e-f f ee<ej& e<ej% gee,e-f f f Z=d,d8d)œd9d:„Z>dS );z?This module contains the evaluator classes for evaluating runs.é    )ÚannotationsN)Úabstractmethod)Ú	AwaitableÚSequence)ÚAnyÚCallableÚLiteralÚOptionalÚUnionÚcast)Ú	TypedDict©Úrun_helpers)Úschemas)Ú	BaseModelÚFieldÚValidationErrorÚ	validator)Úwraps)Ú
SCORE_TYPEÚ
VALUE_TYPEÚExampleÚRunc                   @  s"   e Zd ZU dZded< ded< dS )ÚCategoryz$A category for categorical feedback.úOptional[Union[float, int]]ÚvalueÚstrÚlabelN©Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú__annotations__© r$   r$   ú]/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/langsmith/evaluation/evaluator.pyr   /   s   
r   c                   @  s2   e Zd ZU dZded< ded< ded< ded< d	S )
ÚFeedbackConfigziConfiguration to define a type of feedback.

    Applied on on the first creation of a feedback_key.
    z0Literal['continuous', 'categorical', 'freeform']Útyper   ÚminÚmaxz%Optional[list[Union[Category, dict]]]Ú
categoriesNr   r$   r$   r$   r%   r&   8   s
   
r&   F)Útotalc                   @  s®   e Zd ZU dZded< dZded< dZded< dZd	ed
< dZded< e	e
d�Zded< dZded< dZded< dZded< dZded< G dd„ dƒZeddd�dd„ ƒZdS )ÚEvaluationResultzEvaluation result.r   ÚkeyNr   Úscorer   r   zOptional[str]ÚcommentzOptional[dict]Ú
correction)Údefault_factoryÚdictÚevaluator_infoz%Optional[Union[FeedbackConfig, dict]]Úfeedback_configúOptional[Union[uuid.UUID, str]]Úsource_run_idÚtarget_run_idÚextrac                   @  s   e Zd ZdZdZdS )zEvaluationResult.ConfigzPydantic model configuration.FN)r   r    r!   r"   Zallow_extrar$   r$   r$   r%   ÚConfigb   s   r9   T)Úprec                 C  s6   d|vs|d du r2t |ttfƒr2t d|› �¡ |S )z$Check that the value is not numeric.r.   NzJNumeric values should be provided in the 'score' field, not 'value'. Got: )Ú
isinstanceÚintÚfloatÚloggerÚwarning)ÚclsÚvÚvaluesr$   r$   r%   Úcheck_value_non_numericg   s    þÿz(EvaluationResult.check_value_non_numeric)r   r    r!   r"   r#   r.   r   r/   r0   r   r2   r3   r4   r6   r7   r8   r9   r   rC   r$   r$   r$   r%   r,   G   s   

r,   c                   @  s   e Zd ZU dZded< dS )ÚEvaluationResultszqBatch evaluation results.

    This makes it easy for your evaluator to return multiple
    metrics at once.
    zlist[EvaluationResult]ÚresultsNr   r$   r$   r$   r%   rD   v   s   
rD   c                   @  s@   e Zd ZdZeddddddœdd	„ƒZddddddœd
d„ZdS )ÚRunEvaluatorzEvaluator interface class.Nr   úOptional[Example]úOptional[uuid.UUID]ú*Union[EvaluationResult, EvaluationResults]©ÚrunÚexampleÚevaluator_run_idÚreturnc                 C  s   dS )zEvaluate an example.Nr$   )ÚselfrK   rL   rM   r$   r$   r%   Úevaluate_run„   s    zRunEvaluator.evaluate_runc                 ƒ  s2   t  ¡ ‰ ‡ ‡‡‡‡fdd„}t ¡  d|¡I dH S )z#Evaluate an example asynchronously.c                     sB   t jf i ˆ ¤Ž� ˆ ˆˆˆ¡W  d   ƒ S 1 s40    Y  d S ©N)ÚrhZtracing_contextrP   r$   ©Zcurrent_contextrM   rL   rK   rO   r$   r%   Ú_run_with_context–   s    z5RunEvaluator.aevaluate_run.<locals>._run_with_contextN)rR   Zget_tracing_contextÚasyncioZget_running_loopZrun_in_executor)rO   rK   rL   rM   rT   r$   rS   r%   Úaevaluate_run�   s    zRunEvaluator.aevaluate_run)NN)NN)r   r    r!   r"   r   rP   rV   r$   r$   r$   r%   rF   �   s     ü  ürF   c                   @  s:   e Zd ZU dZded< ded< dZded< dZd	ed
< dS )ÚComparisonEvaluationResultz°Feedback scores for the results of comparative evaluations.

    These are generated by functions that compare two or more runs,
    returning a ranking or other feedback.
    r   r-   z'dict[Union[uuid.UUID, str], SCORE_TYPE]ÚscoresNr5   r6   z6Optional[Union[str, dict[Union[uuid.UUID, str], str]]]r/   )r   r    r!   r"   r#   r6   r/   r$   r$   r$   r%   rW       s   
rW   c                      sÂ   e Zd ZdZd+dddœdd„Zd,d	d
dddœdd„Zdd
ddœdd„Zdd
ddœdd„Zeddœdd„ƒZ	d-dddddœd d!„Z
d.dddd"œ‡ fd#d$„Zd/dddd%œd&d'„Zd(dœd)d*„Z‡  ZS )0ÚDynamicRunEvaluatora¶  A dynamic evaluator that wraps a function and transforms it into a `RunEvaluator`.

    This class is designed to be used with the `@run_evaluator` decorator, allowing
    functions that take a `Run` and an optional `Example` as arguments, and return
    an `EvaluationResult` or `EvaluationResults`, to be used as instances of `RunEvaluator`.

    Attributes:
        func (Callable): The function that is wrapped by this evaluator.
    NúXCallable[[Run, Optional[Example]], Union[_RUNNABLE_OUTPUT, Awaitable[_RUNNABLE_OUTPUT]]]zIOptional[Callable[[Run, Optional[Example]], Awaitable[_RUNNABLE_OUTPUT]]]©ÚfuncÚafuncc                   sà   t |ƒ\}‰ |rt |ƒ\}‰ dddœ‡ fdd„}t|ƒ| ƒ ddlm} |durn|j||d�| _t|d	d
ƒ| _t 	|¡r¨|durˆt
dƒ‚|j||d�| _t|d	d
ƒ| _n4|jttttt gtf |ƒ|d�| _t|d	d
ƒ| _dS )zôInitialize the DynamicRunEvaluator with a given function.

        Args:
            func (Callable): A function that takes a `Run` and an optional `Example` as
            arguments, and returns a dict or `ComparisonEvaluationResult`.
        r2   ©ÚinputsrN   c                   s,   ˆ d u r| S ˆ |   d¡|   d¡ƒ\}}}|S )NrK   rL   ©Úget©r_   Ú_Útraced_inputs©Zprepare_inputsr$   r%   Úprocess_inputsØ   s    ÿ
z4DynamicRunEvaluator.__init__.<locals>.process_inputsr   r   N©rf   r   rY   ú�Func was provided as a coroutine function, but afunc was also provided. If providing both, func should be a regular function to avoid ambiguity.)Ú_normalize_evaluator_funcr   Ú	langsmithr   Úensure_traceabler]   ÚgetattrÚ_nameÚinspectÚiscoroutinefunctionÚ	TypeErrorr   r   r   r	   r   Ú_RUNNABLE_OUTPUTr\   ©rO   r\   r]   rf   r   r$   re   r%   Ú__init__À   s2    ÿ
ÿÿþzDynamicRunEvaluator.__init__FzUnion[EvaluationResult, dict]ú	uuid.UUIDÚboolr,   )Úresultr6   Úallow_no_keyrN   c              
     s¸   t ˆ tƒrˆ js|ˆ _ˆ S zdˆ s.tdˆ › �ƒ‚dˆ vrD|rD| jˆ d< t‡ fdd„dD ƒƒrhtdˆ › �ƒ‚tf i d|iˆ ¥¤ŽW S  ty² } ztdˆ › �ƒ|‚W Y d }~n
d }~0 0 d S )	NziExpected an EvaluationResult object, or dict with a metric 'key' and optional 'score'; got empty result: r-   c                 3  s   | ]}|ˆ vV  qd S rQ   r$   )Ú.0Úk©rv   r$   r%   Ú	<genexpr>  ó    z@DynamicRunEvaluator._coerce_evaluation_result.<locals>.<genexpr>)r.   r   r/   zrExpected an EvaluationResult object, or dict with a metric 'key' and optional 'score' or categorical 'value'; got r6   z[Expected an EvaluationResult object, or dict with a metric 'key' and optional 'score'; got )r;   r,   r6   Ú
ValueErrorrm   Úallr   )rO   rv   r6   rw   Úer$   rz   r%   Ú_coerce_evaluation_resultú   s8    
ÿÿ
ÿÿÿÿýz-DynamicRunEvaluator._coerce_evaluation_resultzUnion[dict, EvaluationResults]rI   )rE   r6   rN   c                   sP   d|v r:|  ¡ }‡ ‡fdd„|d D ƒ|d< tf i |¤ŽS ˆ jtt|ƒˆdd�S )NrE   c                   s   g | ]}ˆ j |ˆd �‘qS ))r6   )r€   )rx   Úr©rO   r6   r$   r%   Ú
<listcomp>  s   ÿzBDynamicRunEvaluator._coerce_evaluation_results.<locals>.<listcomp>T)r6   rw   )ÚcopyrD   r€   r   r2   )rO   rE   r6   Úcpr$   r‚   r%   Ú_coerce_evaluation_results  s    þ
ÿz.DynamicRunEvaluator._coerce_evaluation_resultszMUnion[EvaluationResult, EvaluationResults, dict, str, int, bool, float, list])rv   r6   rN   c                 C  s.   t |tƒr|js||_|S t|ƒ}|  ||¡S rQ   )r;   r,   r6   Ú_format_evaluator_resultr†   )rO   rv   r6   r$   r$   r%   Ú_format_result)  s    
z"DynamicRunEvaluator._format_result©rN   c                 C  s
   t | dƒS ©zžCheck if the evaluator function is asynchronous.

        Returns:
            bool: True if the evaluator function is asynchronous, False otherwise.
        r]   ©Úhasattr©rO   r$   r$   r%   Úis_async7  s    zDynamicRunEvaluator.is_asyncr   rG   rH   rJ   c                 C  sŒ   t | dƒs6t ¡ }| ¡ r$tdƒ‚n| |  ||¡¡S |du rFt ¡ }d|j	i}t
|ddƒrjt|jƒ|d< | j||||dœd�}|  ||¡S )	a�  Evaluate a run using the wrapped function.

        This method directly invokes the wrapped function with the provided arguments.

        Args:
            run (Run): The run to be evaluated.
            example (Optional[Example]): An optional example to be used in the evaluation.

        Returns:
            Union[EvaluationResult, EvaluationResults]: The result of the evaluation.
        r\   útCannot call `evaluate_run` on an async run evaluator from within an running event loop. Use `aevaluate_run` instead.Nr7   Ú
session_idÚ
experiment©Úrun_idÚmetadata©Zlangsmith_extra)rŒ   rU   Úget_event_loopÚ
is_runningÚRuntimeErrorÚrun_until_completerV   ÚuuidÚuuid4Úidrl   r   r�   r\   rˆ   )rO   rK   rL   rM   Úrunning_loopr”   rv   r$   r$   r%   rP   @  s$    
ÿ
ýz DynamicRunEvaluator.evaluate_run)rK   rL   rM   c                 ƒ  sz   t | dƒstƒ  ||¡I dH S |du r.t ¡ }d|ji}t|ddƒrRt|jƒ|d< | j	||||dœd�I dH }|  
||¡S )aÄ  Evaluate a run asynchronously using the wrapped async function.

        This method directly invokes the wrapped async function with the
            provided arguments.

        Args:
            run (Run): The run to be evaluated.
            example (Optional[Example]): An optional example to be used
                in the evaluation.

        Returns:
            Union[EvaluationResult, EvaluationResults]: The result of the evaluation.
        r]   Nr7   r�   r‘   r’   r•   )rŒ   ÚsuperrV   rš   r›   rœ   rl   r   r�   r]   rˆ   )rO   rK   rL   rM   r”   rv   ©Ú	__class__r$   r%   rV   f  s    

ýz!DynamicRunEvaluator.aevaluate_run©rK   rL   rN   c                 C  s   |   ||¡S )aÉ  Make the evaluator callable, allowing it to be used like a function.

        This method enables the evaluator instance to be called directly, forwarding the
        call to `evaluate_run`.

        Args:
            run (Run): The run to be evaluated.
            example (Optional[Example]): An optional example to be used in the evaluation.

        Returns:
            Union[EvaluationResult, EvaluationResults]: The result of the evaluation.
        )rP   )rO   rK   rL   r$   r$   r%   Ú__call__‡  s    zDynamicRunEvaluator.__call__r   c                 C  s   d| j › d�S )ú)Represent the DynamicRunEvaluator object.z<DynamicRunEvaluator ú>©rm   r�   r$   r$   r%   Ú__repr__˜  s    zDynamicRunEvaluator.__repr__)N)F)NN)NN)N)r   r    r!   r"   rs   r€   r†   rˆ   ÚpropertyrŽ   rP   rV   r¢   r¦   Ú__classcell__r$   r$   rŸ   r%   rY   µ   s$    ô> ü  ü)  ü" ÿrY   rZ   ©r\   c                 C  s   t | ƒS )zmCreate a run evaluator from a function.

    Decorator that transforms a function into a `RunEvaluator`.
    )rY   r©   r$   r$   r%   Úrun_evaluator�  s    	rª   i'  r   )Úobjc                 C  s,   t | ƒ}t|ƒtkr(|d td … d }|S )Né   z...))ÚreprÚlenÚ_MAXSIZE)r«   Úsr$   r$   r%   Ú_maxsize_repr¬  s    r±   c                   @  s¦   e Zd ZdZd"dddœdd„Zedd	œd
d„ƒZd#ddddœdd„Zd$ddddœdd„Zd%ddddœdd„Z	dd	œdd„Z
edddœdd„ƒZdddddœd d!„ZdS )&ÚDynamicComparisonRunEvaluatorz4Compare predictions (as traces) from 2 or more runs.NúfCallable[[Sequence[Run], Optional[Example]], Union[_COMPARISON_OUTPUT, Awaitable[_COMPARISON_OUTPUT]]]zUOptional[Callable[[Sequence[Run], Optional[Example]], Awaitable[_COMPARISON_OUTPUT]]]r[   c                   sä   t |ƒ\}‰ |rt |ƒ\}‰ dddœ‡ fdd„}t|ƒ| ƒ ddlm} |durn|j||d�| _t|d	d
ƒ| _t 	|¡r¨|durˆt
dƒ‚|j||d�| _t|d	d
ƒ| _n8|jtttt tt gtf |ƒ|d�| _t|d	d
ƒ| _dS )zúInitialize the DynamicRunEvaluator with a given function.

        Args:
            func (Callable): A function that takes a `Run` and an optional `Example` as
            arguments, and returns an `EvaluationResult` or `EvaluationResults`.
        r2   r^   c                   s,   ˆ d u r| S ˆ |   d¡|   d¡ƒ\}}}|S )NÚrunsrL   r`   rb   re   r$   r%   rf   Î  s    ÿ
z>DynamicComparisonRunEvaluator.__init__.<locals>.process_inputsr   r   Nrg   r   rY   rh   )Ú$_normalize_comparison_evaluator_funcr   rj   r   rk   r]   rl   rm   rn   ro   rp   r   r   r   r   r	   r   Ú_COMPARISON_OUTPUTr\   rr   r$   re   r%   rs   ¶  s@    ÿ
ÿÿÿÿûø
z&DynamicComparisonRunEvaluator.__init__ru   r‰   c                 C  s
   t | dƒS rŠ   r‹   r�   r$   r$   r%   rŽ   ö  s    z&DynamicComparisonRunEvaluator.is_asyncúSequence[Run]rG   rW   ©r´   rL   rN   c                 C  sl   t | dƒs6t ¡ }| ¡ r$tdƒ‚n| |  ||¡¡S t ¡ }|  	|¡}| j
||||dœd�}|  |||¡S )z¯Compare runs to score preferences.

        Args:
            runs: A list of runs to compare.
            example: An optional example to be used in the evaluation.

        r\   r�   ©r“   Útagsr•   )rŒ   rU   r–   r—   r˜   r™   Úacompare_runsrš   r›   Ú	_get_tagsr\   Ú_format_results)rO   r´   rL   r�   r6   rº   rv   r$   r$   r%   Úcompare_runsÿ  s"    

ÿ
ÿ
ýz*DynamicComparisonRunEvaluator.compare_runsc                 Ã  sR   t | dƒs|  ||¡S t ¡ }|  |¡}| j||||dœd�I dH }|  |||¡S )a¶  Evaluate a run asynchronously using the wrapped async function.

        This method directly invokes the wrapped async function with the
            provided arguments.

        Args:
            runs (Run): The runs to be evaluated.
            example (Optional[Example]): An optional example to be used
                in the evaluation.

        Returns:
            ComparisonEvaluationResult: The result of the evaluation.
        r]   r¹   r•   N)rŒ   r¾   rš   r›   r¼   r]   r½   )rO   r´   rL   r6   rº   rv   r$   r$   r%   r»     s    

ýz+DynamicComparisonRunEvaluator.acompare_runsc                 C  s   |   ||¡S )a¹  Make the evaluator callable, allowing it to be used like a function.

        This method enables the evaluator instance to be called directly, forwarding the
        call to `evaluate_run`.

        Args:
            run (Run): The run to be evaluated.
            example (Optional[Example]): An optional example to be used in the evaluation.

        Returns:
            ComparisonEvaluationResult: The result of the evaluation.
        )r¾   )rO   r´   rL   r$   r$   r%   r¢   :  s    z&DynamicComparisonRunEvaluator.__call__r   c                 C  s   d| j › d�S )r£   z<DynamicComparisonRunEvaluator r¤   r¥   r�   r$   r$   r%   r¦   K  s    z&DynamicComparisonRunEvaluator.__repr__z	list[str])r´   rN   c                 C  sF   g }| D ]8}|  dt|jƒ ¡ t|ddƒr|  dt|jƒ ¡ q|S )zExtract tags from runs.zrun:r�   Nzexperiment:)Úappendr   rœ   rl   r�   )r´   rº   rK   r$   r$   r%   r¼   O  s    z'DynamicComparisonRunEvaluator._get_tagsz-Union[dict, list, ComparisonEvaluationResult]rt   )rv   r6   r´   rN   c              
   C  sÆ   t |tƒr|js||_|S t |tƒrDdd„ t||ƒD ƒ| j|dœ}n0t |tƒrbd|vrt| j|d< nd|›�}t|ƒ‚ztf i d|i|¥¤ŽW S  tyÀ } ztd|› �ƒ|‚W Y d }~n
d }~0 0 d S )Nc                 S  s   i | ]\}}|j |“qS r$   )rœ   )rx   rK   r.   r$   r$   r%   Ú
<dictcomp>f  r|   zADynamicComparisonRunEvaluator._format_results.<locals>.<dictcomp>)rX   r-   r6   r-   zXExpected 'dict', 'list' or 'ComparisonEvaluationResult' result object. Received: result=r6   zˆExpected a dictionary with a 'key' and dictionary of scores mappingrun IDs to numeric scores, or ComparisonEvaluationResult object, got )	r;   rW   r6   ÚlistÚziprm   r2   r}   r   )rO   rv   r6   r´   Úmsgr   r$   r$   r%   r½   Z  s:    

ý
ÿÿ
ÿþÿüz-DynamicComparisonRunEvaluator._format_results)N)N)N)N)r   r    r!   r"   rs   r§   rŽ   r¾   r»   r¢   r¦   Ústaticmethodr¼   r½   r$   r$   r$   r%   r²   ³  s    ô@	 ÿ  ÿ ÿ
r²   r³   )r\   rN   c                 C  s   t | ƒS )z.Create a comaprison evaluator from a function.)r²   r©   r$   r$   r%   Úcomparison_evaluator  s    rÅ   r   z¢tuple[Union[Callable[[Run, Optional[Example]], _RUNNABLE_OUTPUT], Callable[[Run, Optional[Example]], Awaitable[_RUNNABLE_OUTPUT]]], Optional[Callable[..., dict]]]c                   sd  d‰t  ˆ¡‰dd„ ˆj ¡ D ƒ}dd„ ˆj ¡ D ƒ‰|rlt‡‡fdd„|D ƒƒs‚t‡fdd„|D ƒƒdkr‚d	ˆ› d
�}t|ƒ‚nÞt‡‡fdd„|D ƒƒr¦|ddgkr®ˆd fS t  ˆ¡�rddddœ‡fdd„‰ ddddœ‡ ‡fdd„}tˆdƒrüt	ˆdƒn|j
|_
|ˆ fS ddddœ‡fdd„‰ ddddœ‡ ‡fdd„}tˆdƒ�rPt	ˆdƒn|j
|_
|ˆ fS d S )N)rK   rL   r_   ÚoutputsÚreference_outputsÚattachmentsc                 S  s    g | ]\}}|j |jkr|‘qS r$   ©ÚkindÚVAR_KEYWORD©rx   ÚpnameÚpr$   r$   r%   rƒ   ›  r|   z-_normalize_evaluator_func.<locals>.<listcomp>c                 S  s"   g | ]\}}|j tjjur|‘qS r$   ©Údefaultrn   Ú	ParameterÚemptyrÌ   r$   r$   r%   rƒ   œ  s   þc                 3  s   | ]}|ˆv p|ˆ v V  qd S rQ   r$   ©rx   rÍ   ©Úargs_with_defaultsÚsupported_argsr$   r%   r{   ¢  s   z,_normalize_evaluator_func.<locals>.<genexpr>c                   s   g | ]}|ˆ vr|‘qS r$   r$   ©rx   Úa©rÕ   r$   r%   rƒ   ¥  r|   é   úUInvalid evaluator function. Must have at least one argument. Supported arguments are ú€. Please see https://docs.smith.langchain.com/evaluation/how_to_guides/evaluation/evaluate_llm_application#use-custom-evaluatorsc                 3  s   | ]}|ˆv p|ˆ v V  qd S rQ   r$   rÓ   rÔ   r$   r%   r{   °  s   rK   rL   r   rG   útuple[list, dict, dict]r¡   c                   s¾   | ||r|j ni | jpi |r&|jp(i ni |r6|jp8i ni dœ}i }g }i }ˆ j ¡ D ]^\}}||v rT|j|j|jfv r†| || ¡ n|| ||< |dv r¦t	|| ƒn|| ||< qT|||fS ©N)rK   rL   r_   rÆ   rÈ   rÇ   )rK   rL   ©
r_   rÆ   rÈ   Ú
parametersÚitemsrÊ   ÚPOSITIONAL_OR_KEYWORDÚPOSITIONAL_ONLYr¿   r±   ©rK   rL   Úarg_mapÚkwargsÚargsrd   Ú
param_nameÚparam©Úsigr$   r%   Ú_prepare_inputsº  s.    úþÿýz2_normalize_evaluator_func.<locals>._prepare_inputsrq   c                 “  s$   ˆ | |ƒ\}}}ˆ|i |¤ŽI d H S rQ   r$   ©rK   rL   rç   ræ   rc   ©rì   r\   r$   r%   ÚawrapperÙ  s    z+_normalize_evaluator_func.<locals>.awrapperr   c                   s¾   | ||r|j ni | jpi |r&|jp(i ni |r6|jp8i ni dœ}i }g }i }ˆ j ¡ D ]^\}}||v rT|j|j|jfv r†| || ¡ n|| ||< |dv r¦t	|| ƒn|| ||< qT|||fS rÞ   rß   rä   rê   r$   r%   rì   è  s.    úþÿýc                   s   ˆ | |ƒ\}}}ˆ|i |¤ŽS rQ   r$   rí   rî   r$   r%   Úwrapper  s    z*_normalize_evaluator_func.<locals>.wrapper©rn   Ú	signaturerà   rá   r~   r®   r}   ro   rŒ   rl   r   ©r\   Úall_argsrÃ   rï   rð   r$   ©rì   rÕ   r\   rë   rÖ   r%   ri   ‰  sT    	
þÿÿüÿÿ
ÿþþÿýÿýri   zºtuple[Union[Callable[[Sequence[Run], Optional[Example]], _COMPARISON_OUTPUT], Callable[[Sequence[Run], Optional[Example]], Awaitable[_COMPARISON_OUTPUT]]], Optional[Callable[..., dict]]]c                   sd  d‰t  ˆ¡‰dd„ ˆj ¡ D ƒ}dd„ ˆj ¡ D ƒ‰|rlt‡‡fdd„|D ƒƒs‚t‡fdd„|D ƒƒdkr‚d	ˆ› d
�}t|ƒ‚nÞt‡‡fdd„|D ƒƒr¦|ddgkr®ˆd fS t  ˆ¡�rddddœ‡fdd„‰ ddddœ‡ ‡fdd„}tˆdƒrüt	ˆdƒn|j
|_
|ˆ fS ddddœ‡fdd„‰ ddddœ‡ ‡fdd„}tˆdƒ�rPt	ˆdƒn|j
|_
|ˆ fS d S )N©r´   rL   r_   rÆ   rÇ   c                 S  s    g | ]\}}|j |jkr|‘qS r$   rÉ   rÌ   r$   r$   r%   rƒ     r|   z8_normalize_comparison_evaluator_func.<locals>.<listcomp>c                 S  s"   g | ]\}}|j tjjur|‘qS r$   rÏ   rÌ   r$   r$   r%   rƒ     s   þc                 3  s   | ]}|ˆv p|ˆ v V  qd S rQ   r$   rÓ   rÔ   r$   r%   r{   %  s   z7_normalize_comparison_evaluator_func.<locals>.<genexpr>c                   s   g | ]}|ˆ vr|‘qS r$   r$   r×   rÙ   r$   r%   rƒ   (  r|   rÚ   rÛ   rÜ   c                 3  s   | ]}|ˆv p|ˆ v V  qd S rQ   r$   rÓ   rÔ   r$   r%   r{   3  s   r´   rL   r·   rG   rÝ   r¸   c                   s²   | ||r|j ni dd„ | D ƒ|r*|jp,i ni dœ}i }g }i }ˆ j ¡ D ]^\}}||v rH|j|j|jfv rz| || ¡ n|| ||< |dv ršt|| ƒn|| ||< qH|||fS )Nc                 S  s   g | ]}|j pi ‘qS r$   ©rÆ   ©rx   rK   r$   r$   r%   rƒ   D  r|   úQ_normalize_comparison_evaluator_func.<locals>._prepare_inputs.<locals>.<listcomp>rö   ©r´   rL   ©	r_   rÆ   rà   rá   rÊ   râ   rã   r¿   r±   ©r´   rL   rå   ræ   rç   rd   rè   ré   rê   r$   r%   rì   =  s,    ûþÿýz=_normalize_comparison_evaluator_func.<locals>._prepare_inputsr¶   c                 “  s$   ˆ | |ƒ\}}}ˆ|i |¤ŽI d H S rQ   r$   ©r´   rL   rç   ræ   rc   rî   r$   r%   rï   [  s    z6_normalize_comparison_evaluator_func.<locals>.awrapperr   c                   s²   | ||r|j ni dd„ | D ƒ|r*|jp,i ni dœ}i }g }i }ˆ j ¡ D ]^\}}||v rH|j|j|jfv rz| || ¡ n|| ||< |dv ršt|| ƒn|| ||< qH|||fS )Nc                 S  s   g | ]}|j pi ‘qS r$   r÷   rø   r$   r$   r%   rƒ   q  r|   rù   rö   rú   rû   rü   rê   r$   r%   rì   j  s,    ûþÿýc                   s   ˆ | |ƒ\}}}ˆ|i |¤ŽS rQ   r$   rý   rî   r$   r%   rð   ˆ  s    z5_normalize_comparison_evaluator_func.<locals>.wrapperrñ   ró   r$   rõ   r%   rµ     sT    	
þÿÿüÿÿ
ÿþþÿýÿýrµ   z;Union[EvaluationResults, dict, str, int, bool, float, list]zUnion[EvaluationResults, dict])rv   rN   c                 C  s–   t | tttfƒrd| i} nx| s.td| › �ƒ‚ndt | tƒrdtdd„ | D ƒƒsZtd| › d�ƒ‚d| i} n.t | tƒrxd| i} nt | tƒr„ntd	| › �ƒ‚| S )
Nr.   zdExpected a non-empty dict, str, bool, int, float, list, EvaluationResult, or EvaluationResults. Got c                 s  s   | ]}t |tƒV  qd S rQ   )r;   r2   )rx   Úxr$   r$   r%   r{   ¡  r|   z+_format_evaluator_result.<locals>.<genexpr>z8Expected a list of dicts or EvaluationResults. Received Ú.rE   r   zZExpected a dict, str, bool, int, float, list, EvaluationResult, or EvaluationResults. Got )	r;   ru   r=   r<   r}   rÁ   r~   r   r2   rz   r$   r$   r%   r‡   –  s0    
ÿÿ

ÿ



ÿÿr‡   ÚSUMMARY_EVALUATOR_Tc                   sø   d‰t  ˆ¡‰dd„ ˆj ¡ D ƒ}dd„ ˆj ¡ D ƒ‰ |rlt‡ ‡fdd„|D ƒƒs–t‡ fdd„|D ƒƒdkr–d	ˆ› d
�}|rŒ|d|› d
�7 }t|ƒ‚n^t‡fdd„|D ƒƒr¸|ddgkr¼ˆS ddddœ‡‡fdd„}tˆdƒrètˆdƒn|j	|_	|S d S )N©r´   Úexamplesr_   rÆ   rÇ   c                 S  s   g | ]\}}|‘qS r$   r$   rÌ   r$   r$   r%   rƒ   Á  r|   z0_normalize_summary_evaluator.<locals>.<listcomp>c                 S  s"   g | ]\}}|j tjjur|‘qS r$   rÏ   rÌ   r$   r$   r%   rƒ   Â  s   þc                 3  s   | ]}|ˆv p|ˆ v V  qd S rQ   r$   rÓ   rÔ   r$   r%   r{   È  s   z/_normalize_summary_evaluator.<locals>.<genexpr>c                   s   g | ]}|ˆ vr|‘qS r$   r$   r×   rÙ   r$   r%   rƒ   Ë  r|   rÚ   rÛ   rÿ   z Received arguments c                 3  s   | ]}|ˆ v V  qd S rQ   r$   rÓ   )rÖ   r$   r%   r{   Ö  r|   r´   r  zSequence[schemas.Run]zSequence[schemas.Example]rI   )r´   r  rN   c                   s¤   | |dd„ |D ƒdd„ | D ƒdd„ |D ƒdœ}i }g }ˆj  ¡ D ]>\}}||v r@|j|j|jfv rr| || ¡ q@|| ||< q@ˆ |i |¤Ž}t|tƒrœ|S t|ƒS )Nc                 S  s   g | ]
}|j ‘qS r$   )r_   ©rx   rL   r$   r$   r%   rƒ   ã  r|   zA_normalize_summary_evaluator.<locals>.wrapper.<locals>.<listcomp>c                 S  s   g | ]}|j pi ‘qS r$   r÷   rø   r$   r$   r%   rƒ   ä  r|   c                 S  s   g | ]}|j pi ‘qS r$   r÷   r  r$   r$   r%   rƒ   å  r|   r  )	rà   rá   rÊ   râ   rã   r¿   r;   r,   r‡   )r´   r  rå   ræ   rç   rè   ré   rv   )r\   rë   r$   r%   rð   Ý  s(    ûþ
z-_normalize_summary_evaluator.<locals>.wrapperr   )
rn   rò   rà   rá   r~   r®   r}   rŒ   rl   r   )r\   rô   rÃ   rð   r$   )rÕ   r\   rë   rÖ   r%   Ú_normalize_summary_evaluator¾  s:    
þÿÿüÿÿ
þÿr  )?r"   Ú
__future__r   rU   rn   rš   Úabcr   Úcollections.abcr   r   Útypingr   r   r   r	   r
   r   Ztyping_extensionsr   rj   r   rR   r   Zpydantic.v1r   r   r   r   ÚImportErrorZpydanticÚloggingÚ	functoolsr   Zlangsmith.schemasr   r   r   r   Ú	getLoggerr   r>   r   r&   r,   rD   rF   r2   rq   rW   r¶   rY   rª   r¯   r±   r²   rÅ   ri   rµ   r‡   rÁ   r   r  r$   r$   r$   r%   Ú<module>   sj    	
	/ i M
  
ÿÿ
ÿÿüÿ