a
    ”˜ëjê   ã                   @  s  U d Z ddlmZ ddlZddlZddlZddlmZmZm	Z	 ddl
mZmZmZmZmZ ddlmZ ddlZddlmZmZ ddlmZ dd	lmZ dd
lmZ ddlmZ erÊddlm Z  ddl!m"Z" e #e$¡Z%e &¡ Z'de(d< ddœdd„Z)G dd„ deƒZ*dS )z2A tracer that runs evaluators over completed runs.é    )ÚannotationsN)ÚFutureÚThreadPoolExecutorÚwait)ÚTYPE_CHECKINGÚAnyÚOptionalÚUnionÚcast)ÚUUID)ÚEvaluationResultÚEvaluationResults)Ú	langchain)Ú
BaseTracer)Útracing_v2_enabled)Ú_get_executor)ÚSequence)ÚRunz)weakref.WeakSet[EvaluatorCallbackHandler]Ú_TRACERSÚNone©Úreturnc                  C  s"   t tƒD ]} | dur|  ¡  qdS )zWait for all tracers to finish.N)Úlistr   Úwait_for_futures)Ztracer© r   ú`/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/langchain_core/tracers/evaluation.pyÚwait_for_all_evaluators   s    r   c                
      sþ   e Zd ZU dZdZded< dZded< ded	< d
Zded< dZded< e	 
¡ Zded< dZded< dZded< ded< ded< d5dddddddddœ‡ fd d!„Zd"d#dd$œd%d&„Zd'd(d)œd*d+„Zd6d'd"dd(d,œd-d.„Zd"dd/œd0d1„Zdd2œd3d4„Z‡  ZS )7ÚEvaluatorCallbackHandlerz³Tracer that runs a run evaluator whenever a run is persisted.

    Attributes:
        client : Client
            The LangSmith client instance used for evaluating the runs.
    Zevaluator_callback_handlerÚstrÚnameNzOptional[UUID]Ú
example_idzlangsmith.ClientÚclientr   z Sequence[langsmith.RunEvaluator]Ú
evaluatorszOptional[ThreadPoolExecutor]Úexecutorzweakref.WeakSet[Future]ÚfuturesTÚboolÚskip_unfinishedzOptional[str]Úproject_namez-dict[tuple[str, str], list[EvaluationResult]]Úlogged_eval_resultszthreading.LockÚlockzOptional[langsmith.Client]zOptional[Union[UUID, str]]zOptional[int]r   r   )r"   r!   r    r&   r'   Úmax_concurrencyÚkwargsr   c                   s´   t ƒ jf i |¤Ž t|tƒr$t|ƒn|ˆ _|p4t ¡ ˆ _|ˆ _	|du rPt
ƒ ˆ _n0|dkrzt|d�ˆ _t ˆ ‡ fdd„¡ ndˆ _t ¡ ˆ _|ˆ _|ˆ _i ˆ _t ¡ ˆ _t ˆ ¡ dS )a(  Create an EvaluatorCallbackHandler.

        Args:
            evaluators : Sequence[RunEvaluator]
                The run evaluators to apply to all top level runs.
            client : LangSmith Client, optional
                The LangSmith client instance to use for evaluating the runs.
                If not specified, a new instance will be created.
            example_id : Union[UUID, str], optional
                The example ID to be associated with the runs.
            skip_unfinished: bool, optional
                Whether to skip unfinished runs.
            project_name : str, optional
                The LangSmith project name to be organize eval chain runs under.
            max_concurrency : int, optional
                The maximum number of concurrent evaluators to run.
        Nr   )Úmax_workersc                     s   t dˆ jƒjdd�S )Nr   T)r   )r
   r#   Úshutdownr   ©Úselfr   r   Ú<lambda>f   ó    z3EvaluatorCallbackHandler.__init__.<locals>.<lambda>)ÚsuperÚ__init__Ú
isinstancer   r   r    Úlangchain_tracerZ
get_clientr!   r"   r   r#   r   ÚweakrefÚfinalizeÚWeakSetr$   r&   r'   r(   Ú	threadingÚLockr)   r   Úadd)r/   r"   r!   r    r&   r'   r*   r+   ©Ú	__class__r.   r   r3   ?   s(    ÿ

þ

z!EvaluatorCallbackHandler.__init__r   zlangsmith.RunEvaluator)ÚrunÚ	evaluatorr   c                 C  s.  zš| j du r | j ||¡}|g}t| j dg| jd��T}|jrJ| j |j¡nd}|j||d�}| j|||jrr|jjndd�}W d  ƒ n1 sŽ0    Y  W n( t	yÂ   t
 d|j|jj¡ ‚ Y n0 t|jƒ}| j�D |D ].}	tt|	d|jƒƒ}
| j |
|fg ¡ |	¡ qÚW d  ƒ n1 �s 0    Y  dS )z¦Evaluate the run in the project.

        Args:
            run: The run to be evaluated.
            evaluator: The evaluator to use for evaluating the run.
        NÚeval)r'   Útagsr!   )Zexample)Úsource_run_idzError evaluating run %s with %sÚtarget_run_id)r'   r!   Zevaluate_runr   Úreference_example_idZread_exampleÚ_log_evaluation_feedbackZ
latest_runÚidÚ	ExceptionÚloggerÚ	exceptionr=   Ú__name__r   r)   Úgetattrr(   Ú
setdefaultÚappend)r/   r>   r?   Zeval_resultZeval_resultsÚcbZreference_exampleZevaluation_resultr    ÚresZrun_idr   r   r   Ú_evaluate_in_projectq   sF    
ÿÿýýý(ý
ÿz-EvaluatorCallbackHandler._evaluate_in_projectz*Union[EvaluationResult, EvaluationResults]zlist[EvaluationResult])Úresultsr   c                 C  sJ   t |tƒr|g}n4t |tƒr.d|v r.|d }ndt|ƒ› d�}t|ƒ‚|S )NrQ   zInvalid evaluation result type z1. Expected EvaluationResult or EvaluationResults.)r4   r   ÚdictÚtypeÚ	TypeError)r/   rQ   Zresults_Úmsgr   r   r   Ú_select_eval_results�   s    

ÿz-EvaluatorCallbackHandler._select_eval_results)Úevaluator_responser>   rB   r   c                 C  s~   |   |¡}|D ]j}i }|jr*i |j¥|¥}t|dd ƒ}|d u rD|j}| jj||j|j|j|j	|j
||jpj|tjjjd�	 q|S )NrC   )ÚscoreÚvalueÚcommentÚ
correctionZsource_inforB   Zfeedback_source_type)rV   Zevaluator_inforK   rF   r!   Zcreate_feedbackÚkeyrX   rY   rZ   r[   rB   Ú	langsmithZschemasZFeedbackSourceTypeZMODEL)r/   rW   r>   rB   rQ   rO   Zsource_info_Zrun_id_r   r   r   rE   ­   s(    
÷z1EvaluatorCallbackHandler._log_evaluation_feedback)r>   r   c                 C  sp   | j r|jst d|j¡ dS | ¡ }| j|_| jD ]6}| j	du rP|  
||¡ q4| j | j	 | j
||¡¡ q4dS )z_Run the evaluator on the run.

        Args:
            run: The run to be evaluated.
        zSkipping unfinished run %sN)r&   ÚoutputsrH   ÚdebugrF   Úcopyr    rD   r"   r#   rP   r$   r;   Zsubmit)r/   r>   Zrun_r?   r   r   r   Ú_persist_runÈ   s    

ÿz%EvaluatorCallbackHandler._persist_runr   c                 C  s   t | jƒ dS )z!Wait for all futures to complete.N)r   r$   r.   r   r   r   r   Û   s    z)EvaluatorCallbackHandler.wait_for_futures)NNTr"   N)N)rJ   Ú
__module__Ú__qualname__Ú__doc__r   Ú__annotations__r    r"   r#   r6   r8   r$   r&   r'   r3   rP   rV   rE   ra   r   Ú__classcell__r   r   r<   r   r   %   s.   
     ù"2, ür   )+rd   Ú
__future__r   Úloggingr9   r6   Úconcurrent.futuresr   r   r   Útypingr   r   r   r	   r
   Úuuidr   r]   Zlangsmith.evaluation.evaluatorr   r   Zlangchain_core.tracersr   r5   Zlangchain_core.tracers.baser   Zlangchain_core.tracers.contextr   Z langchain_core.tracers.langchainr   Úcollections.abcr   Zlangchain_core.tracers.schemasr   Ú	getLoggerrJ   rH   r8   r   re   r   r   r   r   r   r   Ú<module>   s(   
