a
    •˜ëjeH  ã                   @  s`  d Z ddlmZ ddlZddlZddlmZ ddlmZm	Z	m
Z
mZ ddlmZmZ ddlmZ ddlmZ dd	lmZ dd
lmZmZmZ ddlmZmZ ddlmZmZmZ ddl m!Z! ddl"m#Z# ddl$m%Z% ddl&m'Z' e (e)¡Z*dddœdd„Z+G dd„ deƒZ,G dd„ de,ƒZ-G dd„ de,ƒZ.G dd„ de,ƒZ/G dd„ deƒZ0G d d!„ d!e#eƒZ1dS )"z,Run evaluator wrapper for string evaluators.é    )ÚannotationsN)Úabstractmethod)ÚAnyÚOptionalÚUnionÚcast)ÚAsyncCallbackManagerForChainRunÚCallbackManagerForChainRun)Údumpd)Úload)ÚSerializable)ÚBaseMessageÚget_buffer_stringÚmessages_from_dict)ÚEvaluationResultÚRunEvaluator)ÚDataTypeÚExampleÚRun)Úoverride)ÚChain)ÚStringEvaluator)ÚRUN_KEYú
list[dict]zlist[BaseMessage]©ÚmessagesÚreturnc                 C  s.   | sg S | d }d|v r&dd„ | D ƒS t | ƒS )Nr   Úlcc                 S  s   g | ]}t t|ƒd d�‘qS )r   )Zallowed_objects)r   r
   )Ú.0Úmessage© r    ún/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/langchain/smith/evaluation/string_run_evaluator.pyÚ
<listcomp>"   s   z/_get_messages_from_run_dict.<locals>.<listcomp>)r   )r   Zfirst_messager    r    r!   Ú_get_messages_from_run_dict   s    ÿr#   c                   @  sF   e Zd ZdZeddœdd„ƒZedddœd	d
„ƒZdddœdd„ZdS )ÚStringRunMapperú.Extract items to evaluate from the run object.ú	list[str]©r   c                 C  s   ddgS )ú!The keys to extract from the run.Ú
predictionÚinputr    ©Úselfr    r    r!   Úoutput_keys+   s    zStringRunMapper.output_keysr   údict[str, str]©Úrunr   c                 C  s   dS )úMaps the Run to a dictionary.Nr    )r,   r0   r    r    r!   Úmap0   s    zStringRunMapper.mapc                 C  s&   |j sd|j› d�}t|ƒ‚|  |¡S )r1   úRun ú has no outputs to evaluate.©ÚoutputsÚidÚ
ValueErrorr2   ©r,   r0   Úmsgr    r    r!   Ú__call__4   s    zStringRunMapper.__call__N)	Ú__name__Ú
__module__Ú__qualname__Ú__doc__Úpropertyr-   r   r2   r;   r    r    r    r!   r$   (   s   r$   c                   @  sP   e Zd ZdZdddœdd„Zdddœd	d
„Zdddœdd„Zdddœdd„ZdS )ÚLLMStringRunMapperr%   z#Union[list[dict], list[list[dict]]]Ústrr   c                 C  sx   t |tƒrb|rbt |d tƒr,ttd|ƒƒ}n.t |d tƒrHt|d ƒ}nd|› �}t|ƒ‚t|ƒS d|› �}t|ƒ‚dS )ú(Extract the input messages from the run.r   r   z'Could not extract messages to evaluate N)Ú
isinstanceÚlistÚdictr#   r   r8   r   )r,   r   Úchat_messagesr:   r    r    r!   Úserialize_chat_messages?   s    ÿ

z*LLMStringRunMapper.serialize_chat_messagesrF   ©Úinputsr   c                 C  sR   d|v rd  |d ¡}n6d|v r*|d }n$d|v rB|  |d ¡}nd}t|ƒ‚|S )a7  Serialize inputs.

        Args:
            inputs: The inputs from the run, expected to contain prompts or messages.

        Returns:
            The serialized input text from the prompts or messages.

        Raises:
            ValueError: If neither prompts nor messages are found in the inputs.
        Zpromptsz

Úpromptr   z7LLM Run must have either messages or prompts as inputs.)ÚjoinrH   r8   )r,   rJ   Úinput_r:   r    r    r!   Úserialize_inputsR   s    
z#LLMStringRunMapper.serialize_inputs)r6   r   c                 C  sn   |  d¡sd}t|ƒ‚|d }|s.d}t|ƒ‚|d }t|tƒrH|d }d|v rb|  |d g¡}n|d }|S )aP  Serialize outputs.

        Args:
            outputs: The outputs from the run, expected to contain generations.

        Returns:
            The serialized output text from the first generation.

        Raises:
            ValueError: If no generations are found in the outputs,
            or if the generations are empty.
        Úgenerationsz,Cannot evaluate LLM Run without generations.z/Cannot evaluate LLM run with empty generations.r   r   Útext)Úgetr8   rD   rE   rH   )r,   r6   r:   rO   Zfirst_generationÚoutput_r    r    r!   Úserialize_outputsi   s    

z$LLMStringRunMapper.serialize_outputsr   r.   r/   c              
   C  sö   |j dkrd}t|ƒ‚|jsT|jr>d|j› d|j› �}t|ƒ‚d|j› d�}t|ƒ‚z|  |j¡}W n: tyž } z"d|j› �}t|ƒ|‚W Y d}~n
d}~0 0 z|  |j¡}W n: tyê } z"d	|j› �}t|ƒ|‚W Y d}~n
d}~0 0 ||d
œS )r1   Úllmz%LLM RunMapper only supports LLM runs.z Cannot evaluate errored LLM run ú: r3   z* has no outputs. Cannot evaluate this run.z)Could not parse LM input from run inputs Nz/Could not parse LM prediction from run outputs ©r*   r)   )	Úrun_typer8   r6   Úerrorr7   rN   rJ   Ú	ExceptionrS   )r,   r0   r:   rJ   ÚerR   r    r    r!   r2   ˆ   s(    
  zLLMStringRunMapper.mapN)r<   r=   r>   r?   rH   rN   rS   r2   r    r    r    r!   rA   <   s
   rA   c                   @  sN   e Zd ZU dZdZded< dZded< dddddœd	d
„Zdddœdd„ZdS )ÚChainStringRunMapperz;Extract items to evaluate from the run object from a chain.NúOptional[str]Ú	input_keyÚprediction_keyrF   rB   )ÚsourceÚkeyÚwhichr   c                 C  sP   |d ur|| S t |ƒdkr,tt| ¡ ƒƒS d|› d|› d|› d�}t|ƒ‚d S )Né   zCould not map run z with multiple keys: z
Please manually specify a Ú_key)ÚlenÚnextÚiterÚvaluesr8   )r,   r_   r`   ra   r:   r    r    r!   Ú_get_key¬   s    ÿÿÿzChainStringRunMapper._get_keyr   r.   r/   c              	   C  sÜ   |j sd|j› d�}t|ƒ‚| jdurd| j|jvrdd|j› d| j› d|j ¡ › d| j› d�	}t|ƒ‚| jdur®| j|j vr®d |j  ¡ ¡}d|j› d	| j› d
|› d�}t|ƒ‚|  |j| jd¡}|  |j | jd¡}||dœS )r1   zRun with ID zI lacks outputs required for evaluation. Ensure the Run has valid outputs.Nz$ is missing the expected input key 'z*'.
Available input keys in this Run  are: zK.
Adjust the evaluator's input_key or ensure your input data includes key 'z'.z, z+ doesn't have the expected prediction key 'z.'. Available prediction keys in this Run are: z\. Adjust the evaluator's prediction_key or ensure the Run object's outputs the expected key.r*   r)   rV   )	r6   r7   r8   r]   rJ   Úkeysr^   rL   rh   )r,   r0   r:   Zavailable_keysrM   r)   r    r    r!   r2   ·   s8    ÿ
ÿþüÿ
ÿþÿþzChainStringRunMapper.map)	r<   r=   r>   r?   r]   Ú__annotations__r^   rh   r2   r    r    r    r!   r[       s
   
r[   c                   @  s$   e Zd ZdZedddœdd„ƒZdS )ÚToolStringRunMapperzMap an input to the tool.r   r.   r/   c                 C  s2   |j sd|j› d�}t|ƒ‚|jd |j d dœS )Nr3   r4   r*   ÚoutputrV   )r6   r7   r8   rJ   r9   r    r    r!   r2   Ý   s    zToolStringRunMapper.mapN)r<   r=   r>   r?   r   r2   r    r    r    r!   rk   Ú   s   rk   c                   @  s`   e Zd ZU dZdZded< eddœdd„ƒZd	d
dœdd„Zdddœdd„Z	dddœdd„Z
dS )ÚStringExampleMapperzFMap an example, or row in the dataset, to the inputs of an evaluation.Nr\   Úreference_keyr&   r'   c                 C  s   dgS )r(   Ú	referencer    r+   r    r    r!   r-   ê   s    zStringExampleMapper.output_keysr   rB   r   c                 C  s   t |ƒ}t|ƒS )rC   )r#   r   )r,   r   rG   r    r    r!   rH   ï   s    z+StringExampleMapper.serialize_chat_messagesr   r.   )Úexampler   c                 C  sÈ   |j sd|j› d�}t|ƒ‚| jdu r^t|j ƒdkrJd|j› d�}t|ƒ‚tt|j  ¡ ƒƒ}n8| j|j vrŠd|j› d| j› d�}t|ƒ‚n|j | j }dt|t	ƒrÂ| 
d	¡rÂ| 
d
¡rÂ|  |g¡n|iS )z1Maps the Example, or dataset row to a dictionary.úExample z& has no outputs to use as a reference.Nrb   z; has multiple outputs, so you must specify a reference_key.z does not have reference key Ú.ro   ÚtypeÚdata)r6   r7   r8   rn   rd   re   rf   rg   rD   rF   rQ   rH   )r,   rp   r:   rl   r    r    r!   r2   ô   s2    
ÿ
ÿÿ
ÿÿÿýzStringExampleMapper.mapc                 C  s&   |j sd|j› d�}t|ƒ‚|  |¡S )z)Maps the Run and Example to a dictionary.rq   z+ has no outputs to use as areference label.r5   )r,   rp   r:   r    r    r!   r;     s    zStringExampleMapper.__call__)r<   r=   r>   r?   rn   rj   r@   r-   rH   r2   r;   r    r    r    r!   rm   å   s   
rm   c                   @  s  e Zd ZU dZded< dZded< ded< d	ed
< eeddœdd„ƒƒZeeddœdd„ƒƒZ	dddœdd„Z
dddœdd„Zd1ddddœdd„Zd2ddddœdd„Zdd dœd!d"„Zed3d#d$d%d d&œd'd(„ƒZed4d#d$d%d d&œd)d*„ƒZed5d	dd+d,d,d,d-d d.œd/d0„ƒZdS )6ÚStringRunEvaluatorChainz#Evaluate Run and optional examples.r$   Ú
run_mapperNzOptional[StringExampleMapper]Úexample_mapperrB   Únamer   Ústring_evaluatorr&   r'   c                 C  s   ddgS )Nr0   rp   r    r+   r    r    r!   Ú
input_keys$  s    z"StringRunEvaluatorChain.input_keysc                 C  s   dgS ©NÚfeedbackr    r+   r    r    r!   r-   )  s    z#StringRunEvaluatorChain.output_keyszdict[str, Any]r.   rI   c                 C  s~   |d }|  d¡}|  |¡}| jjs0| dd ¡ |rT| jrT| jjrT| |  |¡¡ n&| jjrzd| j› d|j	› d�}t
|ƒ‚|S )Nr0   rp   r*   ú
Evaluator zO requires an reference example from the dataset, but none was provided for run rr   )rQ   rv   ry   Zrequires_inputÚpoprw   Úrequires_referenceÚupdaterx   r7   r8   )r,   rJ   r0   rp   Úevaluate_strings_inputsr:   r    r    r!   Ú_prepare_input.  s    


þÿz&StringRunEvaluatorChain._prepare_input)rl   r   c                 C  s:   t f | j| d¡dœ|¤Ž}t|v r2|t |jt< d|iS )NZ	reasoning©r`   Úcommentr|   )r   rx   rQ   r   Úevaluator_info)r,   rl   Zevaluation_resultr    r    r!   Ú_prepare_output@  s    þýz'StringRunEvaluatorChain._prepare_outputz$Optional[CallbackManagerForChainRun])rJ   Úrun_managerr   c                 C  sD   |   |¡}|pt ¡ }| ¡ }| jjf i |¤|ddœ¤Ž}|  |¡S )úCall the evaluation chain.T©Ú	callbacksÚinclude_run_info)r‚   r	   Úget_noop_managerÚ	get_childry   Zevaluate_stringsr†   ©r,   rJ   r‡   r�   Z_run_managerrŠ   Zchain_outputr    r    r!   Ú_callK  s    

ÿý
zStringRunEvaluatorChain._callz)Optional[AsyncCallbackManagerForChainRun]c                 Ã  sJ   |   |¡}|pt ¡ }| ¡ }| jjf i |¤|ddœ¤ŽI dH }|  |¡S )rˆ   Tr‰   N)r‚   r   rŒ   r�   ry   Zaevaluate_stringsr†   rŽ   r    r    r!   Ú_acall[  s    

ÿýzStringRunEvaluatorChain._acallr   c                 C  s$   |d }t |jvr |t  |jt < |S r{   )r   r…   )r,   rl   r|   r    r    r!   Ú_prepare_evaluator_outputk  s    
z1StringRunEvaluatorChain._prepare_evaluator_outputr   zOptional[Example]zOptional[uuid.UUID])r0   rp   Úevaluator_run_idr   c              
   C  st   z| ||dœdd�}|   |¡W S  tyn } z8t d|j¡ t| jjd|j› d|› �d�W  Y d}~S d}~0 0 dS )	úEvaluate an example.©r0   rp   T©r‹   úError evaluating run %súError evaluating run rU   rƒ   N)r‘   rY   Ú_loggerÚ	exceptionr7   r   ry   Úevaluation_name©r,   r0   rp   r’   ÚresultrZ   r    r    r!   Úevaluate_runq  s    þz$StringRunEvaluatorChain.evaluate_runc              
   Ã  s|   z&| j ||dœdd�I dH }|  |¡W S  tyv } z8t d|j¡ t| jjd|j› d|› �d�W  Y d}~S d}~0 0 dS )	r“   r”   Tr•   Nr–   r—   rU   rƒ   )	Zacallr‘   rY   r˜   r™   r7   r   ry   rš   r›   r    r    r!   Úaevaluate_run„  s    þþz%StringRunEvaluatorChain.aevaluate_runr   r\   zOptional[list[str]])Ú	evaluatorrW   Ú	data_typer]   r^   rn   Útagsr   c                 C  sš   |dkrt ƒ }n*|dkr&t||d�}nd|› d�}	t|	ƒ‚|dusX|tjtjfv sX|jrdt|d�}
n"|jr‚d|j› d	�}	t|	ƒ‚nd}
| |j||
||d
�S )a\  Create a StringRunEvaluatorChain.

        Create a StringRunEvaluatorChain from an evaluator and the run and dataset
        types.

        This method provides an easy way to instantiate a StringRunEvaluatorChain, by
        taking an evaluator and information about the type of run and the data.
        The method supports LLM and chain runs.

        Args:
            evaluator: The string evaluator to use.
            run_type: The type of run being evaluated.
                Supported types are LLM and Chain.
            data_type: The type of dataset used in the run.
            input_key: The key used to map the input from the run.
            prediction_key: The key used to map the prediction from the run.
            reference_key: The key used to map the reference from the dataset.
            tags: List of tags to attach to the evaluation chain.

        Returns:
            The instantiated evaluation chain.

        Raises:
            If the run type is not supported, or if the evaluator requires a
            reference from the dataset but the reference key is not provided.

        rT   Úchain)r]   r^   zUnsupported run type z#. Expected one of 'llm' or 'chain'.N)rn   r}   zw requires a reference example from the dataset. Please specify the reference key from amongst the dataset outputs keys.)rx   rv   rw   ry   r¡   )	rA   r[   r8   r   rT   Zchatr   rm   rš   )ÚclsrŸ   rW   r    r]   r^   rn   r¡   rv   r:   rw   r    r    r!   Úfrom_run_and_data_type™  s8    'þÿþýÿ
ûz.StringRunEvaluatorChain.from_run_and_data_type)N)N)NN)NN)NNNN)r<   r=   r>   r?   rj   rw   r@   r   rz   r-   r‚   r†   r�   r�   r‘   r�   rž   Úclassmethodr¤   r    r    r    r!   ru     s@   
 ý ý  ü  ü    øru   )2r?   Ú
__future__r   ÚloggingÚuuidÚabcr   Útypingr   r   r   r   Z langchain_core.callbacks.managerr   r	   Zlangchain_core.load.dumpr
   Zlangchain_core.load.loadr   Z langchain_core.load.serializabler   Zlangchain_core.messagesr   r   r   Z	langsmithr   r   Zlangsmith.schemasr   r   r   Ztyping_extensionsr   Zlangchain.chains.baser   Zlangchain.evaluation.schemar   Zlangchain.schemar   Ú	getLoggerr<   r˜   r#   r$   rA   r[   rk   rm   ru   r    r    r    r!   Ú<module>   s0   
d:2