a
    •˜ëj8+  ã                   @  s  d Z ddlmZ ddlZddlZddlmZ ddlmZm	Z	 ddl
mZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZmZ ddlmZmZ ddlmZ dddœdd„Zdddœdd„Z G dd„ deeeƒZ!G dd„ deeeƒZ"G dd„ de"ƒZ#dS )z-LLM Chains for evaluating question answering.é    )ÚannotationsN)ÚSequence)ÚAnyÚOptional)Ú	Callbacks)ÚBaseLanguageModel)ÚPromptTemplate)Ú
ConfigDict)Úoverride)ÚLLMChain)ÚCONTEXT_PROMPTÚ
COT_PROMPTÚPROMPT)ÚLLMEvalChainÚStringEvaluator)ÚRUN_KEYÚstrzOptional[tuple[str, int]])ÚtextÚreturnc                 C  sì   t  d|  ¡ t j¡}|rD| d¡ ¡ dkr.dS | d¡ ¡ dkrDdS z�|  ¡  ¡ d  t 	ddt
j¡¡}| ¡ dkrzW dS | ¡ dkrŒW dS |  ¡  ¡ d	  t 	ddt
j¡¡}| ¡ dkrÀW dS | ¡ dkrÒW dS W n tyæ   Y n0 d S )
Nzgrade:\s*(correct|incorrect)é   ÚCORRECT)r   r   Ú	INCORRECT)r   r   r   Ú éÿÿÿÿ)ÚreÚsearchÚstripÚ
IGNORECASEÚgroupÚupperÚsplitÚ	translater   Ú	maketransÚstringÚpunctuationÚ
IndexError)r   ÚmatchZ
first_wordÚ	last_word© r(   úa/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/langchain/evaluation/qa/eval_chain.pyÚ
_get_score   s4     ÿ
ÿþÿ
r*   Údictc                 C  s6   |   ¡ }t|ƒ}|du r"d\}}n|\}}|||dœS )z‚Parse the output text.

    Args:
        text (str): The output text to parse.

    Returns:
        Any: The parsed output.
    N)NN)Ú	reasoningÚvalueÚscore)r   r*   )r   r,   Zparsed_scoresr-   r.   r(   r(   r)   Ú_parse_string_eval_output3   s    	
ýr/   c                
   @  s   e Zd ZU dZdZded< edd�Zee	ddœd	d
„ƒƒZ
ee	ddœdd„ƒƒZee	ddœdd„ƒƒZee	ddœdd„ƒƒZed.dddd dœdd„ƒZd/ddœddddddddœd d!„Zd"d"d#œd$d%„Ze	dddd&d'œdd(d(dddd"d)œd*d+„ƒZe	dddd&d'œdd(d(dddd"d)œd,d-„ƒZdS )0ÚQAEvalChainz,LLM Chain for evaluating question answering.Úresultsr   Ú
output_keyÚignore©ÚextraÚbool©r   c                 C  s   dS ©NFr(   ©Úclsr(   r(   r)   Úis_lc_serializableR   s    zQAEvalChain.is_lc_serializablec                 C  s   dS )NZcorrectnessr(   ©Úselfr(   r(   r)   Úevaluation_nameW   s    zQAEvalChain.evaluation_namec                 C  s   dS ©NTr(   r<   r(   r(   r)   Úrequires_reference\   s    zQAEvalChain.requires_referencec                 C  s   dS r?   r(   r<   r(   r(   r)   Úrequires_inputa   s    zQAEvalChain.requires_inputNr   úOptional[PromptTemplate]r   ©ÚllmÚpromptÚkwargsr   c                 K  sL   |pt }h d£}|t|jƒkr8d|› d|j› �}t|ƒ‚| f ||dœ|¤ŽS )aÒ  Load QA Eval Chain from LLM.

        Args:
            llm (BaseLanguageModel): the base language model to use.

            prompt (PromptTemplate): A prompt template containing the input_variables:
            'input', 'answer' and 'result' that will be used as the prompt
            for evaluation.
            Defaults to PROMPT.

            **kwargs: additional keyword arguments.

        Returns:
            QAEvalChain: the loaded QA eval chain.
        >   ÚresultÚanswerÚqueryúInput variables should be ú
, but got ©rD   rE   )r   ÚsetÚinput_variablesÚ
ValueError)r:   rD   rE   rF   Úexpected_input_varsÚmsgr(   r(   r)   Úfrom_llmf   s    ÿÿzQAEvalChain.from_llmrI   rH   rG   ©Ú	callbackszSequence[dict]r   ú
list[dict])ÚexamplesÚpredictionsÚquestion_keyÚ
answer_keyÚprediction_keyrT   r   c                  s*   ‡ ‡‡‡fdd„t |ƒD ƒ}| j||d�S )ú5Evaluate question answering examples and predictions.c                   s,   g | ]$\}}|ˆ |ˆ  ˆ| ˆ d œ‘qS )©rI   rH   rG   r(   ©Ú.0ÚiZexample©rY   rZ   rW   rX   r(   r)   Ú
<listcomp>‘   s
   ü
ýz(QAEvalChain.evaluate.<locals>.<listcomp>rS   ©Ú	enumerateÚapply)r=   rV   rW   rX   rY   rZ   rT   Úinputsr(   r`   r)   Úevaluate†   s    ú	zQAEvalChain.evaluater+   ©rG   r   c                 C  s&   t || j ƒ}t|v r"|t |t< |S ©N©r/   r2   r   ©r=   rG   Úparsed_resultr(   r(   r)   Ú_prepare_outputœ   s    zQAEvalChain._prepare_outputF©Ú	referenceÚinputrT   Úinclude_run_infoúOptional[str]©Ú
predictionrn   ro   rT   rp   rF   r   c                K  s    | |||dœ||d�}|   |¡S )aÏ  Evaluate Chain or LLM output, based on optional input and label.

        Args:
            prediction (str): the LLM or chain prediction to evaluate.
            reference (Optional[str], optional): the reference label
                to evaluate against.
            input (Optional[str], optional): the input to consider during evaluation
            callbacks (Callbacks, optional): the callbacks to use for tracing.
            include_run_info (bool, optional): whether to include run info in the
                returned results.
            **kwargs: additional keyword arguments, including callbacks, tags, etc.

        Returns:
            dict: The evaluation results containing the score or value.
        r\   ©rT   rp   ©rl   ©r=   rs   rn   ro   rT   rp   rF   rG   r(   r(   r)   Ú_evaluate_strings¢   s    ýù	zQAEvalChain._evaluate_stringsc                Ë  s(   | j |||dœ||d�I d H }|  |¡S )Nr\   ©re   rT   rp   ©Zacallrl   rv   r(   r(   r)   Ú_aevaluate_stringsÇ   s    
ýzQAEvalChain._aevaluate_strings)N)rI   rH   rG   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r2   Ú__annotations__r	   Úmodel_configÚclassmethodr
   r;   Úpropertyr>   r@   rA   rR   rf   rl   rw   rz   r(   r(   r(   r)   r0   I   sN   
ÿ ý#   úøù $ùr0   c                
   @  s  e Zd ZdZeeddœdd„ƒƒZeddœdd„ƒZeddœdd	„ƒZ	e
d
d�Zedddœdd„ƒZeeddœdd„ƒƒZed0dddd dœdd„ƒZd1ddœdddddd dd!œd"d#„Zd$d$d%œd&d'„Zedddd(d)œdd*d*d ddd$d+œd,d-„ƒZedddd(d)œdd*d*d ddd$d+œd.d/„ƒZdS )2ÚContextQAEvalChainz4LLM Chain for evaluating QA w/o GT based on context.r6   r7   c                 C  s   dS r8   r(   r9   r(   r(   r)   r;   Ý   s    z%ContextQAEvalChain.is_lc_serializablec                 C  s   dS )z.Whether the chain requires a reference string.Tr(   r<   r(   r(   r)   r@   â   s    z%ContextQAEvalChain.requires_referencec                 C  s   dS )z+Whether the chain requires an input string.Tr(   r<   r(   r(   r)   rA   ç   s    z!ContextQAEvalChain.requires_inputr3   r4   r   ÚNone)rE   r   c                 C  s4   h d£}|t |jƒkr0d|› d|j› �}t|ƒ‚d S )N>   rG   rI   ÚcontextrJ   rK   )rM   rN   rO   )r:   rE   rP   rQ   r(   r(   r)   Ú_validate_input_varsð   s    ÿÿz'ContextQAEvalChain._validate_input_varsr   c                 C  s   dS )NzContextual Accuracyr(   r<   r(   r(   r)   r>   ú   s    z"ContextQAEvalChain.evaluation_nameNr   rB   r   rC   c                 K  s&   |pt }|  |¡ | f ||dœ|¤ŽS )aÚ  Load QA Eval Chain from LLM.

        Args:
            llm (BaseLanguageModel): the base language model to use.

            prompt (PromptTemplate): A prompt template containing the input_variables:
            'query', 'context' and 'result' that will be used as the prompt
            for evaluation.
            Defaults to PROMPT.

            **kwargs: additional keyword arguments.

        Returns:
            ContextQAEvalChain: the loaded QA eval chain.
        rL   )r   r†   ©r:   rD   rE   rF   r(   r(   r)   rR   ÿ   s    
zContextQAEvalChain.from_llmrI   r…   rG   rS   rU   r   )rV   rW   rX   Úcontext_keyrZ   rT   r   c                  s*   ‡ ‡‡‡fdd„t |ƒD ƒ}| j||d�S )r[   c                   s,   g | ]$\}}|ˆ |ˆ  ˆ| ˆ d œ‘qS )©rI   r…   rG   r(   r]   ©rˆ   rZ   rW   rX   r(   r)   ra   $  s
   ü
ýz/ContextQAEvalChain.evaluate.<locals>.<listcomp>rS   rb   )r=   rV   rW   rX   rˆ   rZ   rT   re   r(   rŠ   r)   rf     s    ú	zContextQAEvalChain.evaluater+   rg   c                 C  s&   t || j ƒ}t|v r"|t |t< |S rh   ri   rj   r(   r(   r)   rl   /  s    z"ContextQAEvalChain._prepare_outputFrm   rq   rr   c                K  s    | |||dœ||d�}|   |¡S )Nr‰   rt   ru   rv   r(   r(   r)   rw   5  s    ýù	z$ContextQAEvalChain._evaluate_stringsc                Ë  s(   | j |||dœ||d�I d H }|  |¡S )Nr‰   rx   ry   rv   r(   r(   r)   rz   K  s    
ýz%ContextQAEvalChain._aevaluate_strings)N)rI   r…   rG   )r{   r|   r}   r~   r�   r
   r;   r‚   r@   rA   r	   r€   r†   r>   rR   rf   rl   rw   rz   r(   r(   r(   r)   rƒ   Ú   sL   ÿ	 ý   úøù ùrƒ   c                   @  sV   e Zd ZdZeeddœdd„ƒƒZeeddœdd„ƒƒZedd
ddd dœdd„ƒZ	d	S )ÚCotQAEvalChainz=LLM Chain for evaluating QA using chain of thought reasoning.r6   r7   c                 C  s   dS r8   r(   r9   r(   r(   r)   r;   a  s    z!CotQAEvalChain.is_lc_serializabler   c                 C  s   dS )NzCOT Contextual Accuracyr(   r<   r(   r(   r)   r>   f  s    zCotQAEvalChain.evaluation_nameNr   rB   r   rC   c                 K  s&   |pt }|  |¡ | f ||dœ|¤ŽS )zLoad QA Eval Chain from LLM.rL   )r   r†   r‡   r(   r(   r)   rR   k  s    
zCotQAEvalChain.from_llm)N)
r{   r|   r}   r~   r�   r
   r;   r‚   r>   rR   r(   r(   r(   r)   r‹   ^  s    ýr‹   )$r~   Ú
__future__r   r   r#   Úcollections.abcr   Útypingr   r   Zlangchain_core.callbacksr   Zlangchain_core.language_modelsr   Zlangchain_core.promptsr   Zpydanticr	   Ztyping_extensionsr
   Zlangchain.chains.llmr   Z#langchain.evaluation.qa.eval_promptr   r   r   Zlangchain.evaluation.schemar   r   Zlangchain.schemar   r*   r/   r0   rƒ   r‹   r(   r(   r(   r)   Ú<module>   s*     