a
    •˜ëj_7  ã                   @   s  d Z ddlZddlmZ ddlmZmZmZmZm	Z	 ddl
mZ ddlmZ ddlmZmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZmZ ddlmZ ddl m!Z! ddl"m#Z#m$Z$ ddl%m&Z&m'Z' dZ(G dd„ deƒZ)G dd„ deƒZ*G dd„ de&e'ƒZ+dS )zóA chain for evaluating ReAct style agents.

This chain is used to evaluate ReAct style agents by reasoning about
the sequence of actions taken and their outcomes. It uses a language model
chain (LLMChain) to generate the reasoning and scores.
é    N)ÚSequence)ÚAnyÚOptionalÚ	TypedDictÚUnionÚcast)ÚAgentAction)Ú	Callbacks)ÚAsyncCallbackManagerForChainRunÚCallbackManagerForChainRun)ÚOutputParserException)ÚBaseLanguageModel)ÚBaseChatModel)ÚBaseOutputParser)ÚBaseTool)Ú
ConfigDictÚField)Úoverride)ÚLLMChain)ÚEVAL_CHAT_PROMPTÚTOOL_FREE_EVAL_CHAT_PROMPT)ÚAgentTrajectoryEvaluatorÚLLMEvalChainé   c                   @   s"   e Zd ZU dZeed< eed< dS )ÚTrajectoryEvalzBA named tuple containing the score and reasoning for a trajectory.ÚscoreÚ	reasoningN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚfloatÚ__annotations__Ústr© r$   r$   úp/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/langchain/evaluation/agents/trajectory_eval_chain.pyr   *   s   
r   c                   @   s2   e Zd ZdZeedœdd„ƒZeedœdd„ZdS )	ÚTrajectoryOutputParserzTrajectory output parser.©Úreturnc                 C   s   dS )NÚagent_trajectoryr$   ©Úselfr$   r$   r%   Ú_type6   s    zTrajectoryOutputParser._type)Útextr(   c                 C   sÄ   d|vrd|› �}t |ƒ‚|jddd�\}}| ¡ | ¡  }}t d|¡}|du s`d| d¡v rrd	|› �}t |ƒ‚t| d¡ƒ}d|  kr”tks¨n d
|› �}t |ƒ‚|d td  }t||d�S )a—  Parse the output text and extract the score and reasoning.

        Args:
            text (str): The output text to parse.

        Returns:
            TrajectoryEval: A named tuple containing the normalized score and reasoning.

        Raises:
            OutputParserException: If the score is not found in the output text or
                if the LLM's score is not a digit in the range 1-5.
        zScore:z+Could not find score in model eval output: zScore: é   )Úmaxsplitz(\d+(\.\d+)?)NÚ.z0Score is not an integer digit in the range 1-5: z'Score is not a digit in the range 1-5: )r   r   )	r   ÚsplitÚstripÚreÚsearchÚgroupÚintÚ
_MAX_SCOREr   )r+   r-   Úmsgr   Z	score_strZ_scorer   Znormalized_scorer$   r$   r%   Úparse:   s    


zTrajectoryOutputParser.parseN)	r   r   r   r    Úpropertyr#   r,   r   r9   r$   r$   r$   r%   r&   3   s   r&   c                       s:  e Zd ZU dZdZeee  ed< e	ed< e
ed�Zeed< dZeed< ed	d
�Zeedœdd„ƒZeedœdd„ƒZeeeeeeef  f edœdd„ƒZeee edœdd„ƒZed+eeee  ee ed dœdd„ƒZeee dœdd„ƒZ eee dœdd„ƒZ!ee"eef ef e"eef dœ‡ fdd„Z#d,e"eef ee$ e"eef d œd!d"„Z%d-e"eef ee& e"eef d œd#d$„Z'e(dddddd%œeeeeeef  ee e)eee  ee"eef  eee"d&œ
d'd(„ƒZ*e(dddddd%œeeeeeef  ee e)eee  ee"eef  eee"d&œ
d)d*„ƒZ+‡  Z,S ).ÚTrajectoryEvalChaina.  A chain for evaluating ReAct style agents.

    This chain is used to evaluate ReAct style agents by reasoning about
    the sequence of actions taken and their outcomes.
    Based on the paper "ReAct: Synergizing Reasoning and Acting in Language Models"
    (https://arxiv.org/abs/2210.03629)

    Example:

    .. code-block:: python

        from langchain.agents import AgentType, initialize_agent
        from langchain_community.chat_models import ChatOpenAI
        from langchain.evaluation import TrajectoryEvalChain
        from langchain.tools import tool

        @tool
        def geography_answers(country: str, question: str) -> str:
            """Very helpful answers to geography questions."""
            return f"{country}? IDK - We may never know {question}."

        llm = ChatOpenAI(model="gpt-3.5-turbo", temperature=0)
        agent = initialize_agent(
            tools=[geography_answers],
            llm=llm,
            agent=AgentType.OPENAI_FUNCTIONS,
            return_intermediate_steps=True,
        )

        question = "How many dwell in the largest minor region in Argentina?"
        response = agent(question)

        eval_chain = TrajectoryEvalChain.from_llm(
            llm=llm, agent_tools=[geography_answers], return_reasoning=True
        )

        result = eval_chain.evaluate_agent_trajectory(
            input=question,
            agent_trajectory=response["intermediate_steps"],
            prediction=response["output"],
            reference="Paris",
        )
        print(result["score"])  # noqa: T201
        # 0

    NÚagent_toolsÚ
eval_chain)Údefault_factoryÚoutput_parserFÚreturn_reasoningÚignore)Úextrar'   c                 C   s   dS )z2Whether this evaluator requires a reference label.Fr$   r*   r$   r$   r%   Úrequires_reference¡   s    z&TrajectoryEvalChain.requires_referencec                 C   s*   | j du rdS d dd„ t| j dƒD ƒ¡S )zwGet the description of the agent tools.

        Returns:
            str: The description of the agent tools.
        NÚ ú

c                 S   s*   g | ]"\}}d |› d|j › d|j› �‘qS )zTool z: z
Description: )ÚnameÚdescription)Ú.0ÚiÚtoolr$   r$   r%   Ú
<listcomp>°   s   þÿz:TrajectoryEvalChain._tools_description.<locals>.<listcomp>r.   )r<   ÚjoinÚ	enumerater*   r$   r$   r%   Ú_tools_description¦   s    

ýÿz&TrajectoryEvalChain._tools_description)Ústepsr(   c                 C   s(   t | tƒr| S d dd„ t| dƒD ƒ¡S )zßGet the agent trajectory as a formatted string.

        Args:
            steps (Union[str, List[Tuple[AgentAction, str]]]): The agent trajectory.

        Returns:
            str: The formatted agent trajectory.
        rE   c              
   S   s4   g | ],\}\}}d |› d|j › d|j› d|› �‘qS )zStep z:
Tool used: z
Tool input: z
Tool output: )rJ   Z
tool_input)rH   rI   ÚactionÚoutputr$   r$   r%   rK   Ç   s   
üÿþýz<TrajectoryEvalChain.get_agent_trajectory.<locals>.<listcomp>r.   )Ú
isinstancer#   rL   rM   )rO   r$   r$   r%   Úget_agent_trajectory·   s    
ûÿz(TrajectoryEvalChain.get_agent_trajectory)Ú	referencer(   c                 C   s   | sdS d| › d�S )z¤Format the reference text.

        Args:
            reference (str): The reference text.

        Returns:
            str: The formatted reference text.
        rD   zX

The following is the expected answer. Use this to measure correctness:
[GROUND_TRUTH]
z
[END_GROUND_TRUTH]
r$   )rT   r$   r$   r%   Ú_format_referenceÐ   s
    
üz%TrajectoryEvalChain._format_reference)Úllmr<   r?   Úkwargsr(   c                 K   sJ   t |tƒsd}t|ƒ‚|rtnt}t||d�}| f |||p>tƒ dœ|¤ŽS )a¶  Create a TrajectoryEvalChain object from a language model chain.

        Args:
            llm: The language model chain.
            agent_tools: A list of tools available to the agent.
            output_parser : The output parser used to parse the chain output into a
                score.
            **kwargs: Additional keyword arguments.

        Returns:
            TrajectoryEvalChain: The TrajectoryEvalChain object.
        z9Only chat models supported by the current trajectory eval)rV   Úprompt)r<   r=   r?   )rR   r   ÚNotImplementedErrorr   r   r   r&   )ÚclsrV   r<   r?   rW   r8   rX   r=   r$   r$   r%   Úfrom_llmä   s    
ýüzTrajectoryEvalChain.from_llmc                 C   s   g d¢S )zcGet the input keys for the chain.

        Returns:
            List[str]: The input keys.
        ©Zquestionr)   ZanswerrT   r$   r*   r$   r$   r%   Ú
input_keys  s    zTrajectoryEvalChain.input_keysc                 C   s   ddgS )zeGet the output keys for the chain.

        Returns:
            List[str]: The output keys.
        r   r   r$   r*   r$   r$   r%   Úoutput_keys  s    zTrajectoryEvalChain.output_keys)Úinputsr(   c                    s    |   | d¡¡|d< tƒ  |¡S )zValidate and prep inputs.rT   )rU   ÚgetÚsuperÚprep_inputs)r+   r_   ©Ú	__class__r$   r%   rb     s    zTrajectoryEvalChain.prep_inputs)r_   Úrun_managerr(   c                 C   sJ   i |¥}| j r| j|d< |p"t ¡ }| jj|| ¡ d�}td| j 	|¡ƒS )áI  Run the chain and generate the output.

        Args:
            inputs (Dict[str, str]): The input values for the chain.
            run_manager (Optional[CallbackManagerForChainRun]): The callback
                manager for the chain run.

        Returns:
            Dict[str, Any]: The output values of the chain.
        Útool_descriptions©Ú	callbacksÚdict)
r<   rN   r   Úget_noop_managerr=   ÚrunÚ	get_childr   r?   r9   ©r+   r_   re   Zchain_inputZ_run_managerZ
raw_outputr$   r$   r%   Ú_call  s    
þzTrajectoryEvalChain._callc                 Ã   sP   i |¥}| j r| j|d< |p"t ¡ }| jj|| ¡ d�I dH }td| j 	|¡ƒS )rf   rg   rh   Nrj   )
r<   rN   r
   rk   r=   Zarunrm   r   r?   r9   rn   r$   r$   r%   Ú_acall4  s    
þzTrajectoryEvalChain._acall)rT   ri   ÚtagsÚmetadataÚinclude_run_info)
Ú
predictionÚinputr)   rT   ri   rq   rr   rs   rW   r(   c                K   s*   ||   |¡||dœ}
| j|
||||dd�S )a¦  Evaluate a trajectory.

        Args:
            prediction: The final predicted response.
            input: The input to the agent.
            agent_trajectory: The intermediate steps forming the agent trajectory.
            reference: The reference answer.
            callbacks: Callbacks to use for this chain run.
            tags: The tags to apply.
            metadata: The metadata to use.
            include_run_info: Whether to include run info in the output.
            **kwargs: Additional keyword arguments.

        Returns:
            The evaluation result, which includes the score and optionally
                the reasoning for reaching that.
        r\   T©r_   ri   rq   rr   rs   Zreturn_only_outputs)rS   Ú__call__©r+   rt   ru   r)   rT   ri   rq   rr   rs   rW   r_   r$   r$   r%   Ú_evaluate_agent_trajectoryM  s     üúz.TrajectoryEvalChain._evaluate_agent_trajectoryc                Ë   s0   ||   |¡||dœ}
| j|
||||dd�I dH S )aµ  Asynchronously evaluate a trajectory.

        Args:
            prediction: The final predicted response.
            input: The input to the agent.
            agent_trajectory: The intermediate steps forming the agent trajectory.
            reference: The reference answer.
            callbacks: Callbacks to use for this chain run.
            tags: The tags to apply.
            metadata: The metadata to use.
            include_run_info: Whether to include run info in the output.
            **kwargs: Additional keyword arguments.

        Returns:
            The evaluation result, which includes the score and optionally
                the reasoning for reaching that.
        r\   Trv   N)rS   Zacallrx   r$   r$   r%   Ú_aevaluate_agent_trajectory{  s     üúz/TrajectoryEvalChain._aevaluate_agent_trajectory)NN)N)N)-r   r   r   r    r<   r   Úlistr   r"   r   r   r&   r?   r@   Úboolr   Zmodel_configr:   rC   r#   rN   Ústaticmethodr   r   Útupler   rS   rU   Úclassmethodr   r   r[   r]   r^   rj   rb   r   ro   r
   rp   r   r	   ry   rz   Ú__classcell__r$   r$   rc   r%   r;   b   s¦   
/ÿÿþ  ü
ú, ý

ü ý

üö
ô-ö
ôr;   ),r    r3   Úcollections.abcr   Útypingr   r   r   r   r   Zlangchain_core.agentsr   Zlangchain_core.callbacksr	   Z langchain_core.callbacks.managerr
   r   Zlangchain_core.exceptionsr   Zlangchain_core.language_modelsr   Z*langchain_core.language_models.chat_modelsr   Zlangchain_core.output_parsersr   Zlangchain_core.toolsr   Zpydanticr   r   Ztyping_extensionsr   Zlangchain.chains.llmr   Z2langchain.evaluation.agents.trajectory_eval_promptr   r   Zlangchain.evaluation.schemar   r   r7   r   r&   r;   r$   r$   r$   r%   Ú<module>   s(   	/