a
    ”˜ëjv*  ã                   @  s’   d dl mZ d dlmZmZmZmZmZmZ d dl	m
Z
 d dlmZ d dlmZmZ erpd dlmZ d dl	mZ G dd	„ d	eƒZG d
d„ dƒZdS )é    )Úannotations)ÚTYPE_CHECKINGÚAnyÚCallableÚOptionalÚ	TypedDictÚUnion)ÚDynamicRunEvaluator)Ú	traceable)ÚExampleÚRun©ÚStringEvaluator)ÚRunEvaluatorc                   @  s*   e Zd ZU dZded< ded< ded< dS )	ÚSingleEvaluatorInputz!The input to a `StringEvaluator`.ÚstrÚ
predictionzOptional[Any]Ú	referencezOptional[str]ÚinputN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú__annotations__© r   r   úk/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/langsmith/evaluation/integrations/_langchain.pyr      s
   
r   c                   @  s8   e Zd ZdZdddœddddœdd	„Zd
dœdd„ZdS )ÚLangChainStringEvaluatora¾  A class for wrapping a LangChain StringEvaluator.

    Requires the `langchain` package to be installed.

    Attributes:
        evaluator (StringEvaluator): The underlying StringEvaluator OR the name
            of the evaluator to load.

    Methods:
        as_run_evaluator() -> RunEvaluator:
            Convert the LangChainStringEvaluator to a RunEvaluator.

    Examples:
        Creating a simple LangChainStringEvaluator:

        >>> evaluator = LangChainStringEvaluator("exact_match")

        Converting a LangChainStringEvaluator to a RunEvaluator:

        >>> from langsmith.evaluation import LangChainStringEvaluator
        >>> from langchain_openai import ChatOpenAI
        >>> evaluator = LangChainStringEvaluator(
        ...     "criteria",
        ...     config={
        ...         "criteria": {
        ...             "usefulness": "The prediction is useful if"
        ...             " it is correct and/or asks a useful followup question."
        ...         },
        ...         "llm": ChatOpenAI(model="gpt-4o"),
        ...     },
        ... )
        >>> run_evaluator = evaluator.as_run_evaluator()
        >>> run_evaluator  # doctest: +ELLIPSIS
        <DynamicRunEvaluator ...>

        Customizing the LLM model used by the evaluator:

        >>> from langsmith.evaluation import LangChainStringEvaluator
        >>> from langchain_anthropic import ChatAnthropic
        >>> evaluator = LangChainStringEvaluator(
        ...     "criteria",
        ...     config={
        ...         "criteria": {
        ...             "usefulness": "The prediction is useful if"
        ...             " it is correct and/or asks a useful followup question."
        ...         },
        ...         "llm": ChatAnthropic(model="claude-3-opus-20240229"),
        ...     },
        ... )
        >>> run_evaluator = evaluator.as_run_evaluator()
        >>> run_evaluator  # doctest: +ELLIPSIS
        <DynamicRunEvaluator ...>

        Using the `evaluate` API with different evaluators:
        >>> def prepare_data(run: Run, example: Example):
        ...     # Convert the evaluation data into the format expected by the evaluator
        ...     # Only required for datasets with multiple inputs/output keys
        ...     return {
        ...         "prediction": run.outputs["prediction"],
        ...         "reference": example.outputs["answer"],
        ...         "input": str(example.inputs),
        ...     }
        >>> import re
        >>> from langchain_anthropic import ChatAnthropic
        >>> import langsmith
        >>> from langsmith.evaluation import LangChainStringEvaluator, evaluate
        >>> criteria_evaluator = LangChainStringEvaluator(
        ...     "criteria",
        ...     config={
        ...         "criteria": {
        ...             "usefulness": "The prediction is useful if it is correct"
        ...             " and/or asks a useful followup question."
        ...         },
        ...         "llm": ChatAnthropic(model="claude-3-opus-20240229"),
        ...     },
        ...     prepare_data=prepare_data,
        ... )
        >>> embedding_evaluator = LangChainStringEvaluator("embedding_distance")
        >>> exact_match_evaluator = LangChainStringEvaluator("exact_match")
        >>> regex_match_evaluator = LangChainStringEvaluator(
        ...     "regex_match", config={"flags": re.IGNORECASE}, prepare_data=prepare_data
        ... )
        >>> scoring_evaluator = LangChainStringEvaluator(
        ...     "labeled_score_string",
        ...     config={
        ...         "criteria": {
        ...             "accuracy": "Score 1: Completely inaccurate\nScore 5: Somewhat accurate\nScore 10: Completely accurate"
        ...         },
        ...         "normalize_by": 10,
        ...         "llm": ChatAnthropic(model="claude-3-opus-20240229"),
        ...     },
        ...     prepare_data=prepare_data,
        ... )
        >>> string_distance_evaluator = LangChainStringEvaluator(
        ...     "string_distance",
        ...     config={"distance_metric": "levenshtein"},
        ...     prepare_data=prepare_data,
        ... )
        >>> from langsmith import Client
        >>> client = Client()
        >>> results = evaluate(
        ...     lambda inputs: {"prediction": "foo"},
        ...     data=client.list_examples(dataset_name="Evaluate Examples", limit=1),
        ...     evaluators=[
        ...         embedding_evaluator,
        ...         criteria_evaluator,
        ...         exact_match_evaluator,
        ...         regex_match_evaluator,
        ...         scoring_evaluator,
        ...         string_distance_evaluator,
        ...     ],
        ... )  # doctest: +ELLIPSIS
        View the evaluation results for experiment:...
    N)ÚconfigÚprepare_datazUnion[StringEvaluator, str]zOptional[dict]zBOptional[Callable[[Run, Optional[Example]], SingleEvaluatorInput]])Ú	evaluatorr   r   c                C  sh   ddl m} t||ƒr|| _n@t|tƒrLddlm} ||fi |pBi ¤Ž| _ntdt|ƒ› �ƒ‚|| _	dS )a(  Initialize a LangChainStringEvaluator.

        See: https://api.python.langchain.com/en/latest/evaluation/langchain.evaluation.schema.StringEvaluator.html#langchain-evaluation-schema-stringevaluator

        Args:
            evaluator (StringEvaluator): The underlying StringEvaluator.
        r   r   )Úload_evaluatorzUnsupported evaluator type: N)
Úlangchain.evaluation.schemar   Ú
isinstancer   r   Zlangchain.evaluationr    ÚNotImplementedErrorÚtypeÚ_prepare_data)Úselfr   r   r   r   r    r   r   r   Ú__init__’   s    

z!LangChainStringEvaluator.__init__r   )Úreturnc                   s¨   ˆj jrdnd}ˆj jrdnd}d|› |› d�‰ tdddd	d
œ‡ ‡fdd„ƒ‰tˆj jd�ddddd
œ‡‡fdd„ƒ}tˆj jd�ddddd
œ‡‡fdd„ƒ}t||ƒS )zÊConvert the LangChainStringEvaluator to a RunEvaluator.

        This is the object used in the LangSmith `evaluate` API.

        Returns:
            RunEvaluator: The converted RunEvaluator.
        z)
       "input": example.inputs['input'],Ú z0
       "reference": example.outputs['expected']z]
def prepare_data(run, example):
    return {
        "prediction": run.outputs['my_output'],zL
    }
evaluator = LangChainStringEvaluator(..., prepare_data=prepare_data)
Nr   zOptional[Example]r   )ÚrunÚexampler(   c                   sú   | j r*t| j ƒdkr*tdˆj› dˆ › �ƒ‚ˆjjr`|r`|j r`t|j ƒdkr`tdˆj› dˆ › �ƒ‚ˆjjr–|r–|jr–t|jƒdkr–tdˆj› dˆ › �ƒ‚ttt	| j  
¡ ƒƒˆjjrÌ|rÌ|j rÌtt	|j  
¡ ƒƒnd ˆjjrò|rò|jròtt	|j 
¡ ƒƒnd d�S )Né   z
Evaluator z{ only supports a single prediction key. Please ensure that the run has a single output. Or initialize with a prepare_data:
z� nly supports a single reference key. Please ensure that the example has a single output. Or create a custom evaluator yourself:
zy only supports a single input key. Please ensure that the example has a single input. Or initialize with a prepare_data:
)r   r   r   )ÚoutputsÚlenÚ
ValueErrorr   Úrequires_referenceÚrequires_inputÚinputsr   ÚnextÚiterÚvalues)r*   r+   )Úcustomization_error_strr&   r   r   Úprepare_evaluator_inputsÍ   sb    
ýÿÿþýü
ýÿÿþýü
ýÿþýüÿÿÿòzKLangChainStringEvaluator.as_run_evaluator.<locals>.prepare_evaluator_inputs)ÚnameÚdictc                   sB   ˆj d u rˆ | |ƒn
ˆ  | |¡}ˆjjf i |¤Ž}dˆjji|¥S ©NÚkey)r%   r   Zevaluate_stringsÚevaluation_name©r*   r+   Zeval_inputsÚresults©r7   r&   r   r   Úevaluate  s    ÿ
ýz;LangChainStringEvaluator.as_run_evaluator.<locals>.evaluatec                 “  sH   ˆj d u rˆ | |ƒn
ˆ  | |¡}ˆjjf i |¤ŽI d H }dˆjji|¥S r:   )r%   r   Zaevaluate_stringsr<   r=   r?   r   r   Ú	aevaluate  s    ÿ
ýz<LangChainStringEvaluator.as_run_evaluator.<locals>.aevaluate)N)N)N)r   r1   r0   r
   r<   r	   )r&   Z	input_strZreference_strr@   rA   r   )r6   r7   r&   r   Úas_run_evaluator±   s*    ÿýÿýýý ÿ5		z)LangChainStringEvaluator.as_run_evaluator)r   r   r   r   r'   rB   r   r   r   r   r      s
   wùr   N)Ú
__future__r   Útypingr   r   r   r   r   r   Zlangsmith.evaluation.evaluatorr	   Zlangsmith.run_helpersr
   Zlangsmith.schemasr   r   r!   r   r   r   r   r   r   r   r   Ú<module>   s    