a
    •˜ëj‰ì  ã                   @  s"  d Z ddlmZ ddlZddlZddlZddlZddlZddl	Z	ddl
m
Z
mZ ddlmZmZmZmZmZmZ ddlmZ ddlmZ ddlmZ dd	lmZmZ dd
lmZmZ ddlm Z m!Z!m"Z" ddlm#Z$ ddlm%Z& ddl'm(Z(m)Z) ddl*m+Z+ ddl,m-Z- ddl.m/Z/m0Z0 ddl1m2Z2m3Z3 ddl1m4Z5 ddl6m7Z7m8Z8 ddl9m:Z:m;Z;m<Z<m=Z=m>Z> ddl?m@Z@ ddlAmBZB ddlCmDZD ddlEmFZF ddlGmHZH ddlImJZJmKZKmLZL ddlMmNZO ddlPm#ZQ ddlPmRZRmSZS e�ræddlTZUe VeW¡ZXeeg eeFe f f eeeYgef e eFf ZZeeg eeFe f f ef Z[G dd„ de\ƒZ]G d d!„ d!eYƒZ^G d"d#„ d#eYƒZ_d�d%d&d'd(œd)d*„Z`d+d&d,œd-d.„ZaG d/d0„ d0eDƒZbd+d1d,œd2d3„Zcd4d5d6d7œd8d9„Zdd4d:d5d6d;œd<d=„Zed4d'd5d6d>œd?d@„Zfd'dAdBdCdDdEœdFdG„ZgdHdIdJdKœdLdM„ZhdHdIdJdNœdOdP„ZidHdIdJdQœdRdS„ZjdTdUd&dCdIdJdJdJdVdWœ	dXdY„ZkdHdIdIdIdZd[œd\d]„ZldHd&dCdIdIdId^d_œd`da„Zmdddddbœdcd+dIddd5dedfdgœdhdi„Znddddjœdkd+dddId5dedldmœdndo„Zoddpœd4dqd'd5drdsœdtdu„Zpddddjœdcd+dddId5dedfdvœdwdx„Zqddddjœdkd+dddId5dedldmœdydz„Zrddpœd4dqd'd5drdsœd{d|„Zsdžd}d&d%d&dedId~dd€œd�d‚„ZtG dƒd„„ d„eDd…d†�ZuejvG d‡dˆ„ dˆƒƒZwd‰dŠœd‹dŒ„Zxd�d6dŽœd�d�„Zyd‘Zzddd’ddd…dd“œd”d&d%dBd•d–dJded‰dJd—d+d˜œd™dš„Z{ddd’ddd…dd“œd”d&d%dBd•d–dJded‰dJd—d+d˜œd›dœ„Z|dS )Ÿz>Utilities for running language models or Chains over datasets.é    )ÚannotationsN)ÚdatetimeÚtimezone)ÚTYPE_CHECKINGÚAnyÚCallableÚOptionalÚUnionÚcast)Úwarn_deprecated)Ú	Callbacks)ÚBaseLanguageModel)ÚBaseMessageÚmessages_from_dict)Ú
ChatResultÚ	LLMResult)ÚRunnableÚRunnableConfigÚRunnableLambda©Úconfig)Úutils)ÚEvaluatorCallbackHandlerÚwait_for_all_evaluators)ÚLangChainTracer)ÚClient)Úget_git_infoÚget_langchain_env_var_metadata)ÚEvaluationResultÚRunEvaluator)Úrun_evaluator)Úas_runnableÚis_traceable_function)ÚDatasetÚDataTypeÚExampleÚRunÚTracerSession)ÚLangSmithError)Ú	HTTPError)Ú	TypedDict)ÚChain)Úload_evaluator)ÚEvaluatorTypeÚPairwiseStringEvaluatorÚStringEvaluator)Ú
evaluation)Úname_generationÚprogressc                   @  s   e Zd ZdZdS )ÚInputFormatErrorz(Raised when the input format is invalid.N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__© r8   r8   úf/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/langchain/smith/evaluation/runner_utils.pyr3   K   s   r3   c                   @  s,   e Zd ZdZddœdd„Zddœdd„ZdS )	Ú
TestResultz1A dictionary of the results of a single test run.úpd.DataFrame©Úreturnc                 C  s.   |   ¡ }dd„ |jD ƒ}|jdd�j|dd�S )zÿReturn quantiles for the feedback scores.

        This method calculates and prints the quantiles for the feedback scores
        across all feedback keys.

        Returns:
            A DataFrame containing the quantiles for each feedback key.
        c                 S  s"   g | ]}|  d ¡s|dv r|‘qS ))úinputs.úoutputs.Ú	reference>   ÚoutputÚinput)Ú
startswith)Ú.0Úcolr8   r8   r9   Ú
<listcomp>b   s   
ýz5TestResult.get_aggregate_feedback.<locals>.<listcomp>Úall)Úincludeé   )Zaxis)Úto_dataframeÚcolumnsZdescribeZdrop)ÚselfZdfZto_dropr8   r8   r9   Úget_aggregate_feedbackU   s
    þz!TestResult.get_aggregate_feedbackc              
   C  sZ  zddl }W n2 ty> } zd}t|ƒ|‚W Y d}~n
d}~0 0 g }g }| d  ¡ D ]ö\}}|d }| d¡}	t|	tƒrŒdd„ |	 ¡ D ƒ}
n|	du rši }
nd|	i}
i d	d„ |d
  ¡ D ƒ¥|
¥}d|v �rt|d tƒrö| dd„ |d  ¡ D ƒ¡ n|d |d< | i dd„ |D ƒ¥| d¡|d | d¡dœ¥¡ | |¡ | |¡ qT|j||d�S )z#Convert the results to a dataframe.r   NzfPandas is required to convert the results to a dataframe. to install pandas, run `pip install pandas`.ÚresultsÚfeedbackrA   c                 S  s   i | ]\}}d |› �|“qS )r?   r8   ©rD   ÚkÚvr8   r8   r9   Ú
<dictcomp>{   ó    z+TestResult.to_dataframe.<locals>.<dictcomp>c                 S  s   i | ]\}}d |› �|“qS )r>   r8   rP   r8   r8   r9   rS   ‚   rT   rB   r@   c                 S  s   i | ]\}}d |› �|“qS )z
reference.r8   rP   r8   r8   r9   rS   ˆ   rT   c                 S  s   i | ]}d |j › �|j“qS )z	feedback.)ÚkeyZscore)rD   Úfr8   r8   r9   rS   Ž   rT   ÚErrorÚexecution_timeÚrun_id)ÚerrorrX   rY   )Úindex)	ÚpandasÚImportErrorÚitemsÚgetÚ
isinstanceÚdictÚupdateÚappendZ	DataFrame)rL   ÚpdÚeÚmsgÚindicesÚrecordsÚ
example_idÚresultrO   Zoutput_rA   Úrr8   r8   r9   rJ   j   sN    ÿ 

ÿþ
ÿÿüÿ
zTestResult.to_dataframeN)r4   r5   r6   r7   rM   rJ   r8   r8   r8   r9   r:   R   s   r:   c                      s:   e Zd ZdZddddœ‡ fdd„Zddd	œd
d„Z‡  ZS )Ú	EvalErrorz"Your architecture raised an error.ÚBaseExceptionr   ÚNone)rW   Úkwargsr=   c                   s   t ƒ jf d|i|¤Ž dS )zÊInitialize the EvalError with an error and additional attributes.

        Args:
            Error: The error that occurred.
            **kwargs: Additional attributes to include in the error.
        rW   N)ÚsuperÚ__init__)rL   rW   ro   ©Ú	__class__r8   r9   rq   �   s    zEvalError.__init__Ústr)Únamer=   c              
   C  sJ   z
| | W S  t yD } z"d|› d�}t|ƒ|‚W Y d}~n
d}~0 0 dS )zøGet an attribute from the EvalError.

        Args:
            name: The name of the attribute to get.

        Returns:
            The value of the attribute.

        Raises:
            AttributeError: If the attribute does not exist.
        z%'EvalError' object has no attribute 'ú'N)ÚKeyErrorÚAttributeError)rL   ru   re   rf   r8   r8   r9   Ú__getattr__¦   s
    
zEvalError.__getattr__)r4   r5   r6   r7   rq   ry   Ú__classcell__r8   r8   rr   r9   rl   š   s   	rl   ú<my_dataset>ÚMODEL_OR_CHAIN_FACTORYrt   ÚMCF)Úllm_or_chain_factoryÚdataset_namer=   c                   s`  t | tƒrV| ‰ ˆ jj}| jdurJˆ jjj}d|› d|› d|› d�}t|ƒ‚‡ fdd„S t | tƒrd| S t | tƒr~| ‰‡fdd„S t| ƒ�r\t	| ƒrªt
td	| ƒƒ‰‡fd
d„S z
| ƒ }W nF tyú   td	| ƒ}t |¡}t d|¡ t|ƒ‰‡fdd„ Y S 0 td	| ƒ‰t |tƒ�r|S t	td	|ƒƒ�r@t
td	|ƒƒ‰‡fdd„S t |tƒ�sX‡fdd„S ˆS | S )z½Wrap in a chain factory.

    Forgive the user if they pass in a chain without memory instead of a chain
    factory. It's a common mistake. Raise a more helpful error message as well.
    Na$  Cannot directly evaluate a chain with stateful memory. To evaluate this chain, pass in a chain constructor that initializes fresh memory each time it is called.  This will safegaurd against information leakage between dataset examples.
For example:

def chain_constructor():
    new_memory = z(...)
    return z*(memory=new_memory, ...)

run_on_dataset("z", chain_constructor, ...)c                     s   ˆ S ©Nr8   r8   )Úchainr8   r9   Ú<lambda>Õ   rT   z(_wrap_in_chain_factory.<locals>.<lambda>c                     s   ˆ S r€   r8   r8   )Úlcfr8   r9   r‚   Û   rT   r   c                     s   ˆ S r€   r8   r8   ©Ú	runnable_r8   r9   r‚   ß   rT   z'Wrapping function %s as RunnableLambda.c                     s   ˆ S r€   r8   r8   )Úwrappedr8   r9   r‚   è   rT   c                     s   ˆ S r€   r8   r8   r„   r8   r9   r‚   ð   rT   c                     s   t ˆ ƒS r€   )r   r8   )Úconstructorr8   r9   r‚   ó   rT   )r`   r+   rs   r4   ZmemoryÚ
ValueErrorr   r   Úcallabler"   r!   r
   Ú	TypeErrorÚinspectÚ	signatureÚloggerÚinfor   )r~   r   Zchain_classZmemory_classrf   Ú_modelZ	user_funcÚsigr8   )r�   r‡   rƒ   r…   r†   r9   Ú_wrap_in_chain_factory¹   sT    	


ùø
öÿ






r‘   zdict[str, Any])Úinputsr=   c                 C  sN  | sd}t |ƒ‚g }d| v rRt| d tƒsFdt| d ƒj› �}t |ƒ‚| d g}nÊd| v r¤t| d tƒr~tdd„ | d D ƒƒsšdt| d ƒj› �}t |ƒ‚| d }nxt| ƒdk�r
tt	|  
¡ ƒƒ}t|tƒrÔ|g}n4t|tƒrötd	d„ |D ƒƒrö|}nd
| › �}t |ƒ‚nd| › �}t |ƒ‚t|ƒdk�r2|d S dt|ƒ› d�}t |ƒ‚dS )z»Get prompt from inputs.

    Args:
        inputs: The input dictionary.

    Returns:
        A string prompt.

    Raises:
        InputFormatError: If the input format is invalid.
    úInputs should not be empty.Úpromptz"Expected string for 'prompt', got Úpromptsc                 s  s   | ]}t |tƒV  qd S r€   ©r`   rt   ©rD   Úir8   r8   r9   Ú	<genexpr>  s   z_get_prompt.<locals>.<genexpr>z,Expected list of strings for 'prompts', got rI   c                 s  s   | ]}t |tƒV  qd S r€   r–   r—   r8   r8   r9   r™     rT   z)LLM Run expects string prompt input. Got z5LLM Run expects 'prompt' or 'prompts' in inputs. Got r   z)LLM Run expects single prompt input. Got z	 prompts.N)r3   r`   rt   Útyper4   ÚlistrG   ÚlenÚnextÚiterÚvalues)r’   rf   r•   Zprompt_r8   r8   r9   Ú_get_promptù   sB    ÿÿÿ




r    c                   @  s   e Zd ZU dZded< dS )ÚChatModelInputzPInput for a chat model.

    Args:
        messages: List of chat messages.
    zlist[BaseMessage]ÚmessagesN©r4   r5   r6   r7   Ú__annotations__r8   r8   r8   r9   r¡   +  s   
r¡   ra   c                 C  sÆ   | sd}t |ƒ‚|  ¡ }d| v r0| d¡|d< n t| ƒdkrPtt|  ¡ ƒƒ|d< d|v r°|d }t|tƒr‚t	dd„ |D ƒƒr‚|g}t|ƒdkr t
|d ƒ|d< nd}t |ƒ‚|S d	| › �}t |ƒ‚d
S )zÊGet Chat Messages from inputs.

    Args:
        inputs: The input dictionary.

    Returns:
        A list of chat messages.

    Raises:
        InputFormatError: If the input format is invalid.
    r“   r¢   rB   rI   c                 s  s   | ]}t |tƒV  qd S r€   )r`   ra   r—   r8   r8   r9   r™   K  s   z _get_messages.<locals>.<genexpr>r   zGBatch messages not supported. Please provide a single list of messages.zMChat Run expects single List[dict] or List[List[dict]] 'messages' input. Got N)r3   ÚcopyÚpoprœ   r�   rž   rŸ   r`   r›   rG   r   )r’   rf   Z
input_copyZraw_messagesr8   r8   r9   Ú_get_messages5  s2    ÿÿÿÿr§   r%   úOptional[Callable[[dict], Any]]rn   )Úfirst_exampleÚinput_mapperr=   c                 C  sÒ   |rX|| j pi ƒ}t|tƒsÎt|tƒr8tdd„ |D ƒƒsÎd|› dt|ƒ› d�}t|ƒ‚nvzt| j pdi ƒ W nb tyÌ   zt| j p„i ƒ W n< tyÆ } z$d| j › d�}t|ƒ|‚W Y d }~n
d }~0 0 Y n0 d S )Nc                 s  s   | ]}t |tƒV  qd S r€   ©r`   r   ©rD   rf   r8   r8   r9   r™   h  rT   z>_validate_example_inputs_for_language_model.<locals>.<genexpr>z˜When using an input_mapper to prepare dataset example inputs for an LLM or chat model, the output must a single string or a list of chat messages.
Got: ú	 of type Ú.zvExample inputs do not match language model input format. Expected a dictionary with messages or a single prompt. Got: z› Please update your dataset OR provide an input_mapper to convert the example.inputs to a compatible format for the llm or chat model you wish to evaluate.)	r’   r`   rt   r›   rG   rš   r3   r    r§   )r©   rª   Zprompt_inputrf   Zerr2r8   r8   r9   Ú+_validate_example_inputs_for_language_model`  s2    
ÿþýýÿ
þÿr¯   r+   )r©   r�   rª   r=   c                 C  sÆ   |rn|| j pi ƒ}t|jƒ |¡}t|tƒsJd|› dt|ƒ› d�}t|ƒ‚|rÂd|j› d| ¡ › �}t|ƒ‚nT| j }t|jƒ |¡}t	|ƒdkr t	|jƒdkr n"|rÂd|j› d| ¡ › �}t|ƒ‚dS )	z<Validate that the example inputs match the chain input keys.zvWhen using an input_mapper to prepare dataset example inputs for a chain, the mapped value must be a dictionary.
Got: r­   r®   zAMissing keys after loading example using input_mapper.
Expected: z. Got: rI   z³Example inputs missing expected chain input keys. Please provide an input_mapper to convert the example.inputs to a compatible format for the chain you wish to evaluate.Expected: N)
r’   ÚsetÚ
input_keysÚ
differencer`   ra   rš   r3   Úkeysrœ   )r©   r�   rª   Zfirst_inputsZmissing_keysrf   r8   r8   r9   Ú"_validate_example_inputs_for_chainƒ  s>    
þþÿÿÿÿ
ýüÿr´   )Úexampler~   rª   r=   c                 C  sN   t |tƒrt| |ƒ n4|ƒ }t |tƒr4t| ||ƒ nt |tƒrJt d|¡ dS )z9Validate that the example inputs are valid for the model.z Skipping input validation for %sN)r`   r   r¯   r+   r´   r   r�   Údebug)rµ   r~   rª   r�   r8   r8   r9   Ú_validate_example_inputs«  s    


r·   úlist[Example]ú"Optional[smith_eval.RunEvalConfig]r$   zOptional[list[RunEvaluator]])r~   Úexamplesr0   Ú	data_typer=   c           	      C  s‚   |rzt | tƒrd\}}d}n2d}| ƒ }t |tƒr6|jnd}t |tƒrJ|jnd}t||||d jrnt|d jƒnd||ƒ}nd}|S )z<Configure the evaluators to run on the results of the chain.)NNÚllmr�   Nr   )r`   r   r+   r±   Zoutput_keysÚ_load_run_evaluatorsÚoutputsr›   )	r~   rº   r0   r»   Ú
run_inputsÚrun_outputsÚrun_typer�   Úrun_evaluatorsr8   r8   r9   Ú_setup_evaluation¿  s$    
ú
rÃ   zsmith_eval.RunEvalConfigúOptional[list[str]]úOptional[str])r   r¿   r=   c                 C  sj   d }| j r,| j }|rf||vrft d||¡ n:|rFt|ƒdkrF|d }n |d urft|ƒdkrft d|¡ |S )NzZInput key %s not in chain's specified input keys %s. Evaluation behavior may be undefined.rI   r   zªChain expects multiple input keys: %s, Evaluator is likely to fail. Evaluation behavior may be undefined. Specify an input_key in the RunEvalConfig to avoid this warning.)Ú	input_keyr�   Úwarningrœ   )r   r¿   rÆ   r8   r8   r9   Ú_determine_input_keyÝ  s"    ü
ürÈ   )r   rÀ   r=   c                 C  sj   d }| j r,| j }|rf||vrft d||¡ n:|rFt|ƒdkrF|d }n |d urft|ƒdkrft d|¡ |S )Nz`Prediction key %s not in chain's specified output keys %s. Evaluation behavior may be undefined.rI   r   z’Chain expects multiple output keys: %s, Evaluation behavior may be undefined. Specify a prediction_key in the RunEvalConfig to avoid this warning.)Úprediction_keyr�   rÇ   rœ   )r   rÀ   rÉ   r8   r8   r9   Ú_determine_prediction_keyø  s"    ü
ürÊ   )r   Úexample_outputsr=   c                 C  sX   | j r2| j }|rT||vrTd|› d|› �}t|ƒ‚n"|rPt|ƒdkrPtt|ƒƒ}nd }|S )NzReference key z! not in Dataset example outputs: rI   )Úreference_keyrˆ   rœ   r�   rž   )r   rË   rÌ   rf   r8   r8   r9   Ú_determine_reference_key  s    ÿÿ
rÍ   zYUnion[smith_eval_config.SINGLE_EVAL_CONFIG_TYPE, smith_eval_config.CUSTOM_EVALUATOR_TYPE]zOptional[BaseLanguageModel]r   )	Úeval_configÚeval_llmrÁ   r»   rË   rÌ   rÆ   rÉ   r=   c              	   C  sZ  t | tƒr| S t | ttfƒrBt | tƒs.t| ƒ} t| |d�}| j}	nˆt | tjƒr¤d|i|  ¡ ¥}
t| j	fi |
¤Ž}| j	j}	t | tj
ƒrÊ| jpŒ|}| jp–|}| jp |}n&t| ƒr´t| ƒS dt| ƒ› �}t|ƒ‚t |tƒ�r |j�r|d u �rd|	› d|› d�}t|ƒ‚tjj|||||||	gd�}n6t |tƒ�rBd|	› d	�}t|ƒ‚nd|	› d
�}t|ƒ‚|S )N)r¼   r¼   zUnknown evaluator type: zPMust specify reference_key in smith_eval.RunEvalConfig to use evaluator of type z) with dataset with multiple output keys: r®   )rÆ   rÉ   rÌ   ÚtagszRun evaluator for zó is not implemented. PairwiseStringEvaluators compare the outputs of two different models rather than the output of a single model. Did you mean to use a StringEvaluator instead?
See: https://python.langchain.com/docs/guides/evaluation/string/z is not implemented)r`   r   r-   rt   r,   ÚvalueÚsmith_eval_configZ
EvalConfigZ
get_kwargsZevaluator_typeZSingleKeyEvalConfigrÆ   rÉ   rÌ   r‰   Úrun_evaluator_decrš   rˆ   r/   Zrequires_referenceÚ
smith_evalÚStringRunEvaluatorChainÚfrom_run_and_data_typer.   ÚNotImplementedError)rÎ   rÏ   rÁ   r»   rË   rÌ   rÆ   rÉ   Z
evaluator_Zeval_type_tagro   rf   r    r8   r8   r9   Ú_construct_run_evaluator%  sX    



ÿþÿù	
ÿ
rØ   z2tuple[Optional[str], Optional[str], Optional[str]])r   r¿   rÀ   rË   r=   c                 C  s(   t | |ƒ}t| |ƒ}t| |ƒ}|||fS r€   )rÈ   rÊ   rÍ   )r   r¿   rÀ   rË   rÆ   rÉ   rÌ   r8   r8   r9   Ú	_get_keysj  s    


rÙ   zlist[RunEvaluator])r   rÁ   r»   rË   r¿   rÀ   r=   c                 C  sô   g }d\}}}	| j s.| jrBtdd„ | jD ƒƒrBt| |||ƒ\}}}	| j D ]&}
t|
| j||||	||ƒ}| |¡ qH| jpxg }|D ]p}t|tƒr˜| |¡ q~t|t	ƒrÂ| t
jj||||||	d�¡ q~t|ƒrÚ| t|ƒ¡ q~d|› d�}t|ƒ‚q~|S )aŠ  Load run evaluators from a configuration.

    Args:
        config: Configuration for the run evaluators.
        run_type: The type of run.
        data_type: The type of dataset used in the run.
        example_outputs: The example outputs.
        run_inputs: The input keys for the run.
        run_outputs: The output keys for the run.

    Returns:
        A list of run evaluators.
    )NNNc                 s  s   | ]}t |tƒV  qd S r€   )r`   r/   )rD   re   r8   r8   r9   r™   �  rT   z'_load_run_evaluators.<locals>.<genexpr>)rÆ   rÉ   rÌ   zUnsupported custom evaluator: z+. Expected RunEvaluator or StringEvaluator.)Ú
evaluatorsÚcustom_evaluatorsÚanyrÙ   rØ   rÏ   rc   r`   r   r/   rÔ   rÕ   rÖ   r‰   rÓ   rˆ   )r   rÁ   r»   rË   r¿   rÀ   rÂ   rÆ   rÉ   rÌ   rÎ   r    rÛ   Zcustom_evaluatorrf   r8   r8   r9   r½   v  s\    
ÿþü

ø



úÿ

ÿ
r½   ©rÐ   Ú	callbacksrª   Úmetadatar   r   úOptional[dict[str, Any]]zUnion[str, BaseMessage])r¼   r’   rÐ   rÞ   rª   rß   r=   c                Ã  sò   |durp||ƒ}t |tƒs6t |tƒr\tdd„ |D ƒƒr\| j|t||pFg |pLi d�d�I dH S d|› d�}t|ƒ‚z2t|ƒ}| j|t||pŠg |p�i d�d�I dH }	W nJ tyì   t|ƒ}
| jf i |
¤dt||pÐg |pÖi d�i¤ŽI dH }	Y n0 |	S )	a#  Asynchronously run the language model.

    Args:
        llm: The language model to run.
        inputs: The input dictionary.
        tags: Optional tags to add to the run.
        callbacks: Optional callbacks to use during the run.
        input_mapper: Optional function to map inputs to the expected format.
        metadata: Optional metadata to add to the run.

    Returns:
        The LLMResult or ChatResult.

    Raises:
        ValueError: If the LLM type is unsupported.
        InputFormatError: If the input format is invalid.
    Nc                 s  s   | ]}t |tƒV  qd S r€   r«   r¬   r8   r8   r9   r™   ß  rT   z_arun_llm.<locals>.<genexpr>©rÞ   rÐ   rß   r   z%Input mapper returned invalid format ú3
Expected a single string or list of chat messages.r   )	r`   rt   r›   rG   Úainvoker   r3   r    r§   )r¼   r’   rÐ   rÞ   rª   rß   Úprompt_or_messagesrf   r”   Ú
llm_outputÚ
llm_inputsr8   r8   r9   Ú	_arun_llmÁ  sT    
ÿþýþ	ÿÿýþÿýþrç   ©rÐ   rª   rß   zUnion[Chain, Runnable]zUnion[dict, str])r�   r’   rÞ   rÐ   rª   rß   r=   c          
      Ã  s    |du r|n||ƒ}t | tƒrrt |tƒrrt|ƒdkrr| jrrtt| ¡ ƒƒ}| j|t	||pZg |p`i d�d�I dH }n*t	|pzg ||p‚i d�}	| j||	d�I dH }|S )z%Run a chain asynchronously on inputs.NrI   rá   r   ©rÐ   rÞ   rß   )
r`   r+   ra   rœ   r±   r�   rž   rŸ   rã   r   ©
r�   r’   rÞ   rÐ   rª   rß   Zinputs_ÚvalrA   Úrunnable_configr8   r8   r9   Ú_arun_chain  s2    
ÿþ
ýüýþ	ýrí   )rª   r   z'Union[dict, str, LLMResult, ChatResult])rµ   r   r~   rª   r=   c          	   
   Ã  sÖ   t |tƒrdnd}d}zvt |tƒrRt|| jp.i |d |d || d¡d�I dH }n4|ƒ }t|| jpdi |d |d || d¡d�I dH }|}W nD tyÐ } z,t d|| j	| j|¡ t
|d	�}W Y d}~n
d}~0 0 |S )
a[  Asynchronously run the Chain or language model.

    Args:
        example: The example to run.
        config: The configuration for the run.
        llm_or_chain_factory: The Chain or language model constructor to run.
        input_mapper: Optional function to map the input to the expected format.

    Returns:
        A list of outputs.
    ÚLLMr+   NrÐ   rÞ   rß   rÝ   z*%s failed for example %s with inputs %s
%s©rW   )r`   r   rç   r’   r_   rí   Ú	Exceptionr�   rÇ   Úidrl   )	rµ   r   r~   rª   Úchain_or_llmrj   rA   r�   re   r8   r8   r9   Ú_arun_llm_or_chain+  sB    ÿ
ú	úû ró   )r¼   r’   rÞ   rÐ   rª   rß   r=   c                C  sÞ   |durn||ƒ}t |tƒs6t |tƒrXtdd„ |D ƒƒrX| j|t||pFg |pLi d�d�}qÚd|› d�}t|ƒ‚nlz,t|ƒ}	| j|	t||pˆg |pŽi d�d�}W n> tyØ   t|ƒ}
| jf i |
¤dt||pÈi d	�i¤Ž}Y n0 |S )
a  Run the language model on the example.

    Args:
        llm: The language model to run.
        inputs: The input dictionary.
        callbacks: The callbacks to use during the run.
        tags: Optional tags to add to the run.
        input_mapper: function to map to the inputs dictionary from an Example
        metadata: Optional metadata to add to the run.

    Returns:
        The LLMResult or ChatResult.

    Raises:
        ValueError: If the LLM type is unsupported.
        InputFormatError: If the input format is invalid.
    Nc                 s  s   | ]}t |tƒV  qd S r€   r«   r¬   r8   r8   r9   r™   „  rT   z_run_llm.<locals>.<genexpr>rá   r   z'Input mapper returned invalid format:  râ   r   )rÞ   rß   )	r`   rt   r›   rG   Úinvoker   r3   r    r§   )r¼   r’   rÞ   rÐ   rª   rß   rä   rå   rf   Zllm_promptsræ   r8   r8   r9   Ú_run_llme  sL    
ÿþýþ
ÿÿ
ýþ
ÿþrõ   c          
      C  s”   |du r|n||ƒ}t | tƒrlt |tƒrlt|ƒdkrl| jrltt| ¡ ƒƒ}| j|t	||pZg |p`i d�d�}n$t	|ptg ||p|i d�}	| j||	d�}|S )zRun a chain on inputs.NrI   rá   r   ré   )
r`   r+   ra   rœ   r±   r�   rž   rŸ   rô   r   rê   r8   r8   r9   Ú
_run_chain©  s2    
ÿþ
ýüýþ	ýrö   c          
      C  sÖ   t |tƒrdnd}d}zjt |tƒrLt|| jp.i |d |d || d¡d�}n.|ƒ }t|| jp^i |d |d || d¡d�}|}W nP tyÐ } z8t|ƒj}	t	 
d|| j| j|	|¡ t|d	�}W Y d}~n
d}~0 0 |S )
aµ  Run the Chain or language model synchronously.

    Args:
        example: The example to run.
        config: The configuration for the run.
        llm_or_chain_factory: The Chain or language model constructor to run.
        input_mapper: Optional function to map the input to the expected format.

    Returns:
        Union[List[dict], List[str], List[LLMResult], List[ChatResult]]:
          The outputs of the model or chain.
    rî   r+   NrÞ   rÐ   rß   rè   zC%s failed for example %s with inputs %s
Error Type: %s, Message: %srï   )r`   r   rõ   r’   r_   rö   rð   rš   r4   r�   rÇ   rñ   rl   )
rµ   r   r~   rª   rò   rj   rA   r�   re   Z
error_typer8   r8   r9   Ú_run_llm_or_chainÍ  sF    ÿ
ú	ú
ú r÷   r   zOptional[Union[str, datetime]]z1tuple[MCF, TracerSession, Dataset, list[Example]])Úclientr   r~   Úproject_nameÚproject_metadatarÐ   Údataset_versionr=   c              
   C  sx  t ||ƒ}| j|d�}t| j|j|d�ƒ}	|	sBd|› d�}
t|
ƒ‚dd„ |	D ƒ}|r\t|ƒnd }|rl| ¡ nd }zN|pxi }tƒ }|r”i |¥d|i¥}||d< | j	||j|r²d	|ini |d
�}W nt t
ttf�y2 } zTdt|ƒvrä‚ t ¡ }d|› d|› d|› d�}d|› d|› �}
t|
ƒ|‚W Y d }~n
d }~0 0 |jd|j› � }td|› d|› d|› d|j› �dd� ||||	fS )N)r   )Z
dataset_idZas_ofzDataset z has no example rows.c                 S  s   g | ]}|j r|j ‘qS r8   )Úmodified_at)rD   Úexr8   r8   r9   rF     rT   z%_prepare_eval_run.<locals>.<listcomp>Úgitrû   rÐ   )Zreference_dataset_idZproject_extrarß   zalready exists z+
run_on_dataset(
    ...
    project_name="z - z", # Update since z already exists
)
zTest project z/ already exists. Please use a different name:

z/compare?selectedSessions=z)View the evaluation results for project 'z' at:
z

View all tests for Dataset z at:
T)Úflush)r‘   Zread_datasetr›   Zlist_examplesrñ   rˆ   ÚmaxÚ	isoformatr   Zcreate_projectr)   r(   rt   ÚuuidÚuuid4ÚurlÚprint)rø   r   r~   rù   rú   rÐ   rû   Úwrapped_modelÚdatasetrº   rf   rü   Zmax_modified_atZinferred_versionZgit_infoÚprojectre   ÚuidZexample_msgZcomparison_urlr8   r8   r9   Ú_prepare_eval_run  sj    	
ÿþü
ýýýÿÿ ÿþþür
  c                   @  s*   e Zd ZU dZded< ded< ded< dS )	Ú
_RowResultz5A dictionary of the results for a single example row.z Optional[list[EvaluationResult]]rO   zOptional[float]rX   rÅ   rY   Nr£   r8   r8   r8   r9   r  F  s   
r  F)Útotalc                   @  sÄ   e Zd ZU dZded< ded< ded< ded	< d
ed< dZded< ddddœdd„Zdddœdd„Zddœdd„Zddd œd!d"„Z	d5dd$dd%œd&d'„Z
ed6dd)d*d+d,d-d.d/d0d+d1d d2œd3d4„ƒZdS )7Ú_DatasetRunContainerz3A container to help manage the state of a eval run.r   rø   r'   r  r}   r  r¸   rº   zlist[RunnableConfig]ÚconfigsNz6Optional[list[smith_eval_config.BATCH_EVALUATOR_LIKE]]Úbatch_evaluatorsr›   údict[str, _RowResult]ra   )Úbatch_resultsÚall_eval_resultsr=   c                 C  s°   i }t | j|ƒD ]š\}}td| t|jƒi ¡ƒ}|j| dg ¡| d¡| d¡dœ|t|jƒ< t|tƒr~|j	|t|jƒ d< n||t|jƒ d< |j
r|j
|t|jƒ d< q|S )	Nr  rO   rX   rY   )rB   rO   rX   rY   rW   rA   r@   )Úziprº   r
   r_   rt   rñ   r’   r`   rl   rW   r¾   )rL   r  r  rN   rµ   rA   Z
row_resultr8   r8   r9   Ú_merge_test_outputsY  s    
ü
z(_DatasetRunContainer._merge_test_outputszdict[str, Run]z
list[dict])Úrunsr=   c              
     sØ   | j }|sg S ‡ fdd„| jD ƒ}g }tj ¡ �”}|D ]~}zV||| jƒ}t|tƒrZ| ¡ }| t	d|ƒ¡ |j
| jjfi |¤d | jjdœ¤Ž W q6 ty²   t dt|ƒ¡ Y q60 q6W d   ƒ n1 sÊ0    Y  |S )Nc                   s   g | ]}ˆ t |jƒ ‘qS r8   )rt   rñ   ©rD   rµ   ©r  r8   r9   rF   s  rT   z>_DatasetRunContainer._run_batch_evaluators.<locals>.<listcomp>ra   )rY   Z
project_idz Error running batch evaluator %s)r  rº   Ú
concurrentZfuturesZThreadPoolExecutorr`   r   ra   rc   r
   Zsubmitrø   Zcreate_feedbackr  rñ   rð   r�   Ú	exceptionÚrepr)rL   r  rÚ   Z	runs_listÚaggregate_feedbackÚexecutorZ	evaluatorrj   r8   r  r9   Ú_run_batch_evaluatorso  s2    
ÿþüÿ*z*_DatasetRunContainer._run_batch_evaluatorsz,tuple[dict[str, _RowResult], dict[str, Run]]r<   c                 C  sæ   i }i }| j D ]È}td|d ƒD ]´}t|tƒrf|j}| ¡ D ]&\\}}}| t|ƒi ¡ d|i¡ q<q t|t	ƒr |j
}	|	r�|	jr�|	j|	j  ¡ nd }
|	r¢t|	jƒnd }| t|jƒi ¡ |
||	dœ¡ |	|t|jƒ< q qtd|ƒ|fS )Nr›   rÞ   rO   )rX   rY   Úrunr  )r  r
   r`   r   Zlogged_eval_resultsr^   Ú
setdefaultrt   rb   r   Z
latest_runÚend_timeÚ
start_timeÚtotal_secondsrñ   ri   )rL   r  Úall_runsÚcÚcallbackZeval_resultsÚ_ri   rR   r  rX   rY   r8   r8   r9   Ú_collect_metricsˆ  s6    

ÿ
ÿÿýýÿz%_DatasetRunContainer._collect_metricsz-list[Union[dict, str, LLMResult, ChatResult]]r:   )r  r=   c                 C  sX   t  d¡ tƒ  |  ¡ \}}d }| jr:t  d¡ |  |¡}|  ||¡}t| jj	||d�S )Nz#Waiting for evaluators to complete.zRunning session evaluators.)rù   rN   Zaggregate_metrics)
r�   rŽ   r   r'  r  r  r  r:   r  ru   )rL   r  r  r#  r  rN   r8   r8   r9   Ú_collect_test_results¥  s    


ýz*_DatasetRunContainer._collect_test_resultsFÚbool)r  Úverboser=   c              
   C  s²   |   |¡}|rXz| ¡ }t|ƒ W n4 tyV } ztjd|dd� W Y d }~n
d }~0 0 z | jj| jj	t
 tj¡d� W n4 ty¬ } ztjd|dd� W Y d }~n
d }~0 0 |S )Nz&Failed to print aggregate feedback: %sT)Úexc_info)r   zFailed to close project: %s)r(  rM   Ú_display_aggregate_resultsrð   r�   r¶   rø   Zupdate_projectr  rñ   r   Únowr   Úutc)rL   r  r*  rN   Zagg_feedbackre   r8   r8   r9   Úfinish·  s    
&
þ
&z_DatasetRunContainer.finishé   rt   r|   rÅ   r¹   rÄ   r¨   Úintrà   úOptional[Union[datetime, str]])rø   r   r~   rù   r0   rÐ   rª   Úconcurrency_levelrú   Úrevision_idrû   r=   c              	     s  |p
t  ¡ }|
r&|	si }	|	 d|
i¡ tˆ ||||	ˆ|d�\}‰}}ˆpJg ‰ˆj d¡pZi  ¡ D ]\}}ˆ d|› d|› �¡ q`dˆjd i‰|
rš|
ˆd< t|ƒ}t	||||j
p´tjƒ‰t|d ||ƒ t t|ƒ¡‰‡ ‡‡‡‡‡‡fdd	„|D ƒ}| ˆ ˆ||||�r|jnd d
�S )Nr4  )rú   rÐ   rû   rþ   zgit:ú=rû   r   c              
     sB   g | ]:}t tˆjˆ |jd �tˆp"g ˆ |jdd�ˆgˆˆˆd�‘qS ))rù   rø   ri   r   )rÚ   rø   ri   Úmax_concurrency)rÞ   rÐ   r6  rß   )r   r   ru   rñ   r   r  ©rø   r3  Úprogress_barr  rÂ   Zrun_metadatarÐ   r8   r9   rF   ù  s&   íýüôïz0_DatasetRunContainer.prepare.<locals>.<listcomp>)rø   r  r  rº   r  r  )r1   Zrandom_namerb   r
  rß   r_   r^   rc   r‘   rÃ   r»   r$   Úkvr·   r2   ZProgressBarCallbackrœ   r  )Úclsrø   r   r~   rù   r0   rÐ   rª   r3  rú   r4  rû   r  r  rº   rQ   rR   r  r8   r7  r9   ÚprepareÍ  sP    ù	
üìúz_DatasetRunContainer.prepare)F)NNNr0  NNN)r4   r5   r6   r7   r¤   r  r  r  r'  r(  r/  Úclassmethodr;  r8   r8   r8   r9   r  N  s,   
 ý       ôr  r)  r<   c                  C  sH   z.ddl m}  | ƒ }| ƒ d uo,dtt|ƒƒv W S  tyB   Y dS 0 d S )Nr   )Úget_ipythonZzmqshellF)ZIPython.core.getipythonr=  rt   rš   r]   )r=  Úresr8   r8   r9   Ú_is_jupyter_environment  s    r?  r;   )Úaggregate_resultsr=   c                 C  sR   t ƒ r,ddlm}m} ||dƒƒ || ƒ n"| jdd„ dd�}tdƒ t|ƒ d S )	Nr   )ÚHTMLÚdisplayz<h3>Experiment Results:</h3>c                 S  s   | d›S )Nz.2fr8   )Úxr8   r8   r9   r‚   +  rT   z,_display_aggregate_results.<locals>.<lambda>Úright)Zfloat_formatZjustifyz
 Experiment Results:)r?  ZIPython.displayrA  rB  Z	to_stringr  )r@  rA  rB  Zformatted_stringr8   r8   r9   r,  #  s    
þr,  aè  The input_mapper argument is deprecated and will be removed in a future release. Please add a  RunnableLambda to your chain to map inputs to the expected format instead. Example:
def construct_chain():
    my_chain = ...
    input_mapper = {'other_key': 'MyOtherInput', 'my_input_key': x}
    return input_mapper | my_chain
run_on_dataset(..., llm_or_chain_factory=construct_chain)
(See https://api.python.langchain.com/en/latest/schema/langchain.schema.runnable.base.RunnableLambda.html)r0  )r0   rû   r3  rù   rú   r*  r4  zOptional[Client]r2  r1  r   )rø   r   r~   r0   rû   r3  rù   rú   r*  r4  ro   r=   c                Ë  sä   |
  dd¡}|rtdtdd� |	du r2tƒ  d¡}	|
  dd¡}|rPtdd	dd� |
rntdd
|
 ¡ › d�dd� | pvtƒ } tj| |||||||||	|d�}t	j
|jd  d¡gttjt|j|d�|j|jƒ¢R Ž I dH }|j||d�S )a+  Run on dataset.

    Run the Chain or language model on a dataset and store traces
    to the specified project name.

    For the (usually faster) async version of this function,
    see :func:`arun_on_dataset`.

    Args:
        dataset_name: Name of the dataset to run the chain on.
        llm_or_chain_factory: Language model or Chain constructor to run
            over the dataset. The Chain constructor is used to permit
            independent calls on each example without carrying over state.
        evaluation: Configuration for evaluators to run on the
            results of the chain.
        dataset_version: Optional version of the dataset.
        concurrency_level: The number of async tasks to run concurrently.
        project_name: Name of the project to store the traces in.
            Defaults to {dataset_name}-{chain class name}-{datetime}.
        project_metadata: Optional metadata to add to the project.
            Useful for storing information the test variant.
            (prompt version, model version, etc.)
        client: LangSmith client to use to access the dataset and to
            log feedback and run traces.
        verbose: Whether to print progress.
        revision_id: Optional revision identifier to assign this test run to
            track the performance of different versions of your system.
        **kwargs: Should not be used, but is provided for backwards compatibility.

    Returns:
        A dictionary containing the run's project name and the resulting model outputs.

    Examples:

    .. code-block:: python

        from langsmith import Client
        from langchain_openai import ChatOpenAI
        from langchain.chains import LLMChain
        from langchain.smith import smith_eval.RunEvalConfig, run_on_dataset

        # Chains may have memory. Passing in a constructor function lets the
        # evaluation framework avoid cross-contamination between runs.
        def construct_chain():
            llm = ChatOpenAI(temperature=0)
            chain = LLMChain.from_string(
                llm,
                "What's the answer to {your_input_key}"
            )
            return chain

        # Load off-the-shelf evaluators via config or the EvaluatorType (string or enum)
        evaluation_config = smith_eval.RunEvalConfig(
            evaluators=[
                "qa",  # "Correctness" against a reference answer
                "embedding_distance",
                smith_eval.RunEvalConfig.Criteria("helpfulness"),
                smith_eval.RunEvalConfig.Criteria({
                    "fifth-grader-score": "Do you have to be smarter than a fifth "
                    "grader to answer this question?"
                }),
            ]
        )

        client = Client()
        await arun_on_dataset(
            client,
            dataset_name="<my_dataset_name>",
            llm_or_chain_factory=construct_chain,
            evaluation=evaluation_config,
        )

    You can also create custom evaluators by subclassing the
    :class:`StringEvaluator <langchain.evaluation.schema.StringEvaluator>`
    or LangSmith's `RunEvaluator` classes.

    .. code-block:: python

        from typing import Optional
        from langchain.evaluation import StringEvaluator


        class MyStringEvaluator(StringEvaluator):
            @property
            def requires_input(self) -> bool:
                return False

            @property
            def requires_reference(self) -> bool:
                return True

            @property
            def evaluation_name(self) -> str:
                return "exact_match"

            def _evaluate_strings(
                self, prediction, reference=None, input=None, **kwargs
            ) -> dict:
                return {"score": prediction == reference}


        evaluation_config = smith_eval.RunEvalConfig(
            custom_evaluators=[MyStringEvaluator()],
        )

        await arun_on_dataset(
            client,
            dataset_name="<my_dataset_name>",
            llm_or_chain_factory=construct_chain,
            evaluation=evaluation_config,
        )

    rª   Nú0.0.305T©ÚmessageÚpendingr4  rÐ   ú0.1.9úqThe tags argument is deprecated and will be removed in a future release. Please specify project_metadata instead.úPThe following arguments are deprecated and will be removed in a future release: r®   ©rG  Zremoval©rú   r4  rû   r   r6  ©r~   rª   ©r*  )r¦   r   Ú_INPUT_MAPPER_DEP_WARNINGr   r_   r³   r   r  r;  Úrunnable_utilsZgather_with_concurrencyr  ÚmapÚ	functoolsÚpartialró   r  rº   r/  )rø   r   r~   r0   rû   r3  rù   rú   r*  r4  ro   rª   rÐ   Ú	containerr  r8   r8   r9   Úarun_on_datasetC  s`    üþû
õÿýùþrV  c                  s$  |
  dd¡‰ˆrtdtdd� |
  dd¡}|r<tdddd� |	du rPtƒ  d	¡}	|
rntdd
|
 ¡ › d�dd� | pvtƒ } tj| |||||ˆ|||	|d�‰ |dkrÀ‡ ‡fdd„t	ˆ j
ˆ jƒD ƒ}nVt ˆ jd ¡�4}t| tjtˆ jˆd�ˆ j
ˆ j¡ƒ}W d  ƒ n1 �s0    Y  ˆ j||d�S )a  Run on dataset.

    Run the Chain or language model on a dataset and store traces
    to the specified project name.

    For the (usually faster) async version of this function,
    see :func:`arun_on_dataset`.

    Args:
        dataset_name: Name of the dataset to run the chain on.
        llm_or_chain_factory: Language model or Chain constructor to run
            over the dataset. The Chain constructor is used to permit
            independent calls on each example without carrying over state.
        evaluation: Configuration for evaluators to run on the
            results of the chain.
        dataset_version: Optional version of the dataset.
        concurrency_level: The number of async tasks to run concurrently.
        project_name: Name of the project to store the traces in.
            Defaults to {dataset_name}-{chain class name}-{datetime}.
        project_metadata: Optional metadata to add to the project.
            Useful for storing information the test variant.
            (prompt version, model version, etc.)
        client: LangSmith client to use to access the dataset and to
            log feedback and run traces.
        verbose: Whether to print progress.
        revision_id: Optional revision identifier to assign this test run to
            track the performance of different versions of your system.
        **kwargs: Should not be used, but is provided for backwards compatibility.

    Returns:
        A dictionary containing the run's project name and the resulting model outputs.

    Examples:

    .. code-block:: python

        from langsmith import Client
        from langchain_openai import ChatOpenAI
        from langchain.chains import LLMChain
        from langchain.smith import smith_eval.RunEvalConfig, run_on_dataset

        # Chains may have memory. Passing in a constructor function lets the
        # evaluation framework avoid cross-contamination between runs.
        def construct_chain():
            llm = ChatOpenAI(temperature=0)
            chain = LLMChain.from_string(
                llm,
                "What's the answer to {your_input_key}"
            )
            return chain

        # Load off-the-shelf evaluators via config or the EvaluatorType (string or enum)
        evaluation_config = smith_eval.RunEvalConfig(
            evaluators=[
                "qa",  # "Correctness" against a reference answer
                "embedding_distance",
                smith_eval.RunEvalConfig.Criteria("helpfulness"),
                smith_eval.RunEvalConfig.Criteria({
                    "fifth-grader-score": "Do you have to be smarter than a fifth "
                    "grader to answer this question?"
                }),
            ]
        )

        client = Client()
        run_on_dataset(
            client,
            dataset_name="<my_dataset_name>",
            llm_or_chain_factory=construct_chain,
            evaluation=evaluation_config,
        )

    You can also create custom evaluators by subclassing the
    :class:`StringEvaluator <langchain.evaluation.schema.StringEvaluator>`
    or LangSmith's `RunEvaluator` classes.

    .. code-block:: python

        from typing import Optional
        from langchain.evaluation import StringEvaluator


        class MyStringEvaluator(StringEvaluator):
            @property
            def requires_input(self) -> bool:
                return False

            @property
            def requires_reference(self) -> bool:
                return True

            @property
            def evaluation_name(self) -> str:
                return "exact_match"

            def _evaluate_strings(
                self, prediction, reference=None, input=None, **kwargs
            ) -> dict:
                return {"score": prediction == reference}


        evaluation_config = smith_eval.RunEvalConfig(
            custom_evaluators=[MyStringEvaluator()],
        )

        run_on_dataset(
            client,
            dataset_name="<my_dataset_name>",
            llm_or_chain_factory=construct_chain,
            evaluation=evaluation_config,
        )

    rª   NrE  TrF  rÐ   rI  rJ  r4  rK  r®   rL  rM  r   c                   s"   g | ]\}}t ||ˆ jˆd �‘qS )rN  )r÷   r  )rD   rµ   r   ©rU  rª   r8   r9   rF   ™  s   úüz"run_on_dataset.<locals>.<listcomp>rN  rO  )r¦   r   rP  r   r_   r³   r   r  r;  r  rº   r  rì   Zget_executor_for_configr›   rR  rS  rT  r÷   r  r/  )rø   r   r~   r0   rû   r3  rù   rú   r*  r4  ro   rÐ   r  r  r8   rW  r9   Úrun_on_datasetõ  sf    üþû
õù
ýùÿ$rX  )r{   )NNN)}r7   Ú
__future__r   Úconcurrent.futuresr  ZdataclassesrS  r‹   Úloggingr  r   r   Útypingr   r   r   r   r	   r
   Zlangchain_core._apir   Zlangchain_core.callbacksr   Zlangchain_core.language_modelsr   Zlangchain_core.messagesr   r   Zlangchain_core.outputsr   r   Zlangchain_core.runnablesr   r   r   r   rì   r   rQ  Z!langchain_core.tracers.evaluationr   r   Z langchain_core.tracers.langchainr   Zlangsmith.clientr   Zlangsmith.envr   r   Zlangsmith.evaluationr   r   r    rÓ   Zlangsmith.run_helpersr!   r"   Zlangsmith.schemasr#   r$   r%   r&   r'   Zlangsmith.utilsr(   Úrequestsr)   Ztyping_extensionsr*   Zlangchain.chains.baser+   Zlangchain.evaluation.loadingr,   Zlangchain.evaluation.schemar-   r.   r/   Zlangchain.smithr0   rÔ   Zlangchain.smith.evaluationrÒ   r1   r2   r\   rd   Ú	getLoggerr4   r�   ra   r|   r}   rð   r3   r:   rl   r‘   r    r¡   r§   r¯   r´   r·   rÃ   rÈ   rÊ   rÍ   rØ   rÙ   r½   rç   rí   ró   rõ   rö   r÷   r
  r  Z	dataclassr  r?  r,  rP  rV  rX  r8   r8   r8   r9   Ú<module>   sä    	
üÿH! þ@2
+#(EOùKù)û?ùIù)û?   ù? K
ÿõ( 8õ