a
    ”˜ëj{:  ã                   @  sÐ   d Z ddlmZ ddlZddlZddlmZmZmZm	Z	m
Z
mZmZ ddlmZ ddlmZ ddlmZ ddlmZ er�dd	lmZ dd
lmZ G dd„ dƒZeƒ ZG dd„ dƒZG dd„ dƒZeƒ ZdgZdS )aÃ  Make approximate assertions as "expectations" on test results.

This module is designed to be used within test cases decorated with the
`@pytest.mark.decorator` decorator
It allows you to log scores about a test case and optionally make assertions that log as
"expectation" feedback to LangSmith.

Example usage:
    .. code-block:: python

        import pytest
        from langsmith import expect


        @pytest.mark.langsmith
        def test_output_semantically_close():
            response = oai_client.chat.completions.create(
                model="gpt-3.5-turbo",
                messages=[
                    {"role": "system", "content": "You are a helpful assistant."},
                    {"role": "user", "content": "Say hello!"},
                ],
            )
            response_txt = response.choices[0].message.content
            # Intended usage
            expect.embedding_distance(
                prediction=response_txt,
                reference="Hello!",
            ).to_be_less_than(0.9)

            # Score the test case
            matcher = expect.edit_distance(
                prediction=response_txt,
                reference="Hello!",
            )
            # Apply an assertion and log 'expectation' feedback to LangSmith
            matcher.to_be_less_than(1)

            # You can also directly make assertions on values directly
            expect.value(response_txt).to_contain("Hello!")
            # Or using a custom check
            expect.value(response_txt).against(lambda x: "Hello" in x)

            # You can even use this for basic metric logging within tests

            expect.score(0.8)
            expect.score(0.7, key="similarity").to_be_greater_than(0.7)
é    )ÚannotationsN)ÚTYPE_CHECKINGÚAnyÚCallableÚLiteralÚOptionalÚUnionÚoverload©Úclient)Úrun_helpers)Ú	run_trees)Úutils)ÚEditDistanceConfig)ÚEmbeddingConfigc                   @  s,   e Zd ZdZddœdd„Zddœdd„Zd	S )
Ú_NULL_SENTRYzŸA sentinel singleton class used to distinguish omitted keyword arguments
    from those passed in with the value None (which may have different behavior).
    zLiteral[False]©Úreturnc                 C  s   dS )NF© ©Úselfr   r   úP/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/langsmith/_expect.pyÚ__bool__P   s    z_NULL_SENTRY.__bool__Ústrc                 C  s   dS )NÚ	NOT_GIVENr   r   r   r   r   Ú__repr__S   s    z_NULL_SENTRY.__repr__N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r   r   r   r   r   K   s   r   c                   @  sÔ   e Zd ZdZd,ddddddœd	d
„Zd-ddddœdd„Zdddddœdd„Zdddœdd„Zdddœdd„Zddddœdd„Z	d.ddddœdd „Z
dddœd!d"„Zdd#œd$d%„Zdddœd&d'„Zd(dd)œd*d+„ZdS )/Ú_Matcherz4A class for making assertions on expectation values.NúOptional[ls_client.Client]r   r   z,Optional[ls_utils.ContextThreadPoolExecutor]úOptional[str])r   ÚkeyÚvalueÚ	_executorÚrun_idc                 C  s@   || _ || _|| _|p tjdd�| _t ¡ }|r6|jn|| _	d S )Né   ©Úmax_workers)
Ú_clientr#   r$   Úls_utilsÚContextThreadPoolExecutorr%   ÚrhÚget_current_run_treeÚtrace_idÚ_run_id)r   r   r#   r$   r%   r&   Úrtr   r   r   Ú__init__]   s    z_Matcher.__init__ÚintÚNone)ÚscoreÚmessager   c                 C  s8   t  ¡ s4| jst ¡ | _| jj| jj| jd||d� d S )NZexpectation)r&   r#   r5   Úcomment)	r+   Útest_tracking_is_disabledr*   r1   Úget_cached_clientr%   ÚsubmitÚcreate_feedbackr0   )r   r5   r6   r   r   r   Ú_submit_feedbackl   s    
ûz_Matcher._submit_feedbackÚbool)Ú	conditionr6   Úmethod_namer   c              
   C  sl   z,|sJ |ƒ‚| j dd| j› d|› �d� W n: tyf } z"|   dt|ƒ¡ |d ‚W Y d }~n
d }~0 0 d S )Né   z	Success: Ú.)r6   r   )r<   r#   ÚAssertionErrorÚrepr)r   r>   r6   r?   Úer   r   r   Ú_assertx   s     z_Matcher._assertÚfloat©r$   r   c              	   C  s.   |   | j|k d| j› d|› d| j› �d¡ dS )zðAssert that the expectation value is less than the given value.

        Args:
            value: The value to compare against.

        Raises:
            AssertionError: If the expectation value is not less than the given value.
        ú	Expected z to be less than ú
, but got Úto_be_less_thanN©rE   r$   r#   ©r   r$   r   r   r   rJ   €   s
    	ýz_Matcher.to_be_less_thanc              	   C  s.   |   | j|kd| j› d|› d| j› �d¡ dS )a  Assert that the expectation value is greater than the given value.

        Args:
            value: The value to compare against.

        Raises:
            AssertionError: If the expectation value is not
            greater than the given value.
        rH   z to be greater than rI   Úto_be_greater_thanNrK   rL   r   r   r   rM   �   s
    
ýz_Matcher.to_be_greater_than)Ú	min_valueÚ	max_valuer   c                 C  sD   |   || j  k o|k n  d| j› d|› d|› d| j› �d¡ dS )aJ  Assert that the expectation value is between the given min and max values.

        Args:
            min_value: The minimum value (exclusive).
            max_value: The maximum value (exclusive).

        Raises:
            AssertionError: If the expectation value
                is not between the given min and max.
        rH   z to be between z and rI   Úto_be_betweenNrK   )r   rN   rO   r   r   r   rP   Ÿ   s    ÿüz_Matcher.to_be_betweené   )r$   Ú	precisionr   c              	   C  s:   |   t| j|ƒt||ƒkd| j› d|› d| j› �d¡ dS )ak  Assert that the expectation value is approximately equal to the given value.

        Args:
            value: The value to compare against.
            precision: The number of decimal places to round to for comparison.

        Raises:
            AssertionError: If the rounded expectation value
                does not equal the rounded given value.
        rH   z to be approximately rI   Úto_be_approximatelyN)rE   Úroundr$   r#   )r   r$   rR   r   r   r   rS   ±   s
    ýz_Matcher.to_be_approximatelyc              	   C  s.   |   | j|kd| j› d|› d| j› �d¡ dS )a   Assert that the expectation value equals the given value.

        Args:
            value: The value to compare against.

        Raises:
            AssertionError: If the expectation value does
                not exactly equal the given value.
        rH   z to be equal to rI   Úto_equalNrK   rL   r   r   r   rU   Â   s
    
ýz_Matcher.to_equalr   c                 C  s(   |   | jdu d| j› d| j› �d¡ dS )z†Assert that the expectation value is None.

        Raises:
            AssertionError: If the expectation value is not None.
        NrH   z to be None, but got Ú
to_be_nonerK   r   r   r   r   rV   Ò   s
    ýz_Matcher.to_be_nonec                 C  s(   |   || jv d| j› d|› d�d¡ dS )zòAssert that the expectation value contains the given value.

        Args:
            value: The value to check for containment.

        Raises:
            AssertionError: If the expectation value does not contain the given value.
        rH   z to contain z, but it does notÚ
to_containNrK   rL   r   r   r   rW   Þ   s
    	ýz_Matcher.to_containr   )Úfuncr   c                C  s0   t  |¡}|  || jƒd|› d| j› �d¡ dS )zñAssert the expectation value against a custom function.

        Args:
            func: A custom function that takes the expectation value as input.

        Raises:
            AssertionError: If the custom function returns False.
        z
Assertion z failed for ÚagainstN)ÚinspectÚ	signaturerE   r$   r#   )r   rX   Zfunc_signaturer   r   r   rY   î   s    	
ýz_Matcher.against)NN)N)rQ   )r   r   r   r   r2   r<   rE   rJ   rM   rP   rS   rU   rV   rW   rY   r   r   r   r   r    Z   s     úr    c                   @  sØ   e Zd ZdZddœddœdd„Zddœddd	d
dœdd„Zddœdddd
dœdd„Zdd
dœdd„Zddddœddddd
dœdd„Ze	dd
dœdd„ƒZ
e	dd d œd!d„ƒZ
edfdd"d#d$œd%d„Z
dd&d'œd(d)„ZdS )*Ú_Expectz1A class for setting expectations on test results.Nr
   r!   c                C  s*   || _ tjdd�| _tj| jjdd� d S )Nr'   r(   T)Úwait)r*   r+   r,   ÚexecutorÚatexitÚregisterÚshutdown©r   r   r   r   r   r2     s    z_Expect.__init__©Úconfigr   zOptional[EmbeddingConfig]r    )Ú
predictionÚ	referencerd   r   c          	   	   C  s€   ddl m} |pi }| d¡r"dnd}||d�}|j||d�}||jdœ}|  d	||d
|› d|j› �dœ¡ t| jd	|| jd�S )a  Compute the embedding distance between the prediction and reference.

        This logs the embedding distance to LangSmith and returns a `_Matcher` instance
        for making assertions on the distance value.

        By default, this uses the OpenAI API for computing embeddings.

        Args:
            prediction: The predicted string to compare.
            reference: The reference string to compare against.
            config: Optional configuration for the embedding distance evaluator.
                Supported options:
                - `encoder`: A custom encoder function to encode the list of input
                     strings to embeddings. Defaults to the OpenAI API.
                - `metric`: The distance metric to use for comparison.
                    Supported values: "cosine", "euclidean", "manhattan",
                    "chebyshev", "hamming".

        Returns:
            A `_Matcher` instance for the embedding distance value.


        Examples:
            >>> expect.embedding_distance(
            ...     prediction="hello",
            ...     reference="hi",
            ... ).to_be_less_than(1.0)
        r   )ÚEmbeddingDistanceÚencoderZcustomZopenairc   ©re   rf   )rh   ÚmetricÚembedding_distanceúUsing z
, Metric: ©r5   Úsource_infor7   ©r%   )	Ú'langsmith._internal._embedding_distancerg   ÚgetÚevaluateZdistancer<   r    r*   r^   )	r   re   rf   rd   rg   Zencoder_funcÚ	evaluatorr5   Úsrc_infor   r   r   rk     s     #
ýþÿz_Expect.embedding_distancezOptional[EditDistanceConfig]c          
   	   C  s„   ddl m} |pi }| d¡p d}| dd¡}||d�}|j||d�}||d	œ}	|  d
||	d|› d|› �dœ¡ t| jd
|| jd�S )a  Compute the string distance between the prediction and reference.

        This logs the string distance (Damerau-Levenshtein) to LangSmith and returns
        a `_Matcher` instance for making assertions on the distance value.

        This depends on the `rapidfuzz` package for string distance computation.

        Args:
            prediction: The predicted string to compare.
            reference: The reference string to compare against.
            config: Optional configuration for the string distance evaluator.
                Supported options:
                - `metric`: The distance metric to use for comparison.
                    Supported values: "damerau_levenshtein", "levenshtein",
                    "jaro", "jaro_winkler", "hamming", "indel".
                - `normalize_score`: Whether to normalize the score between 0 and 1.

        Returns:
            A `_Matcher` instance for the string distance value.

        Examples:
            >>> expect.edit_distance("hello", "helo").to_be_less_than(1)
        r   )ÚEditDistancerj   Zdamerau_levenshteinZnormalize_scoreTrc   ri   )rj   Ú	normalizeÚedit_distancerl   z, Normalize: rm   ro   )Ú"langsmith._internal._edit_distanceru   rq   rr   r<   r    r*   r^   )
r   re   rf   rd   ru   rj   rv   rs   r5   rt   r   r   r   rw   =  s(    

ýþüz_Expect.edit_distancer   rG   c                 C  s   t | jd|| jd�S )a"  Create a `_Matcher` instance for making assertions on the given value.

        Args:
            value: The value to make assertions on.

        Returns:
            A `_Matcher` instance for the given value.

        Examples:
           >>> expect.value(10).to_be_less_than(20)
        r$   ro   )r    r*   r^   rL   r   r   r   r$   r  s    z_Expect.valuer5   )r#   Úsource_run_idr7   zUnion[float, int, bool]zOptional[ls_client.ID_TYPE]r"   )r5   r#   ry   r7   r   c                C  s.   |   ||ddi||dœ¡ t| j||| jd�S )a  Log a numeric score to LangSmith.

        Args:
            score: The score value to log.
            key: The key to use for logging the score. Defaults to "score".

        Examples:
            >>> expect.score(0.8)  # doctest: +ELLIPSIS
            <langsmith._expect._Matcher object at ...>

            >>> expect.score(0.8, key="similarity").to_be_greater_than(0.7)
        Úmethodzexpect.score)r5   rn   ry   r7   ro   )r<   r    r*   r^   )r   r5   r#   ry   r7   r   r   r   r5   €  s    üþ	z_Expect.scorec                C  s   d S ©Nr   rL   r   r   r   Ú__call__¡  s    z_Expect.__call__zls_client.Client)r   r   c               C  s   d S r{   r   rb   r   r   r   r|   ¤  s    zOptional[Any]zUnion[_Expect, _Matcher])r   r$   r   c                C  s    t |d�}|tur| |¡S |S )Nr
   )r\   r   r$   )r   r$   r   Úexpectedr   r   r   r|   §  s    

Údict)r#   Úresultsc                 C  sP   t  ¡ }|r|jnd }t ¡ sL| js.t ¡ | _| jj	| jj
f||dœ|¤Ž d S )N)r&   r#   )r-   r.   r/   r+   r8   r*   r1   r9   r^   r:   r;   )r   r#   r   Zcurrent_runr&   r   r   r   r<   ²  s    
ÿÿÿz_Expect._submit_feedback)r   r   r   r   r2   rk   rw   r$   r5   r	   r|   r   r<   r   r   r   r   r\   ÿ   s&   
û;û5ú!ür\   Úexpect) r   Ú
__future__r   r_   rZ   Útypingr   r   r   r   r   r   r	   Z	langsmithr   Z	ls_clientr   r-   r   r1   r   r+   rx   r   rp   r   r   r   r    r\   r€   Ú__all__r   r   r   r   Ú<module>   s&   1$
 & ?