a
    ”˜ëj   ã                
   @   sZ  d Z ddlZddlZddlZddlZddlmZ ddlmZm	Z	 ddl
mZ ddlmZ ddlmZ ddlmZ ddlmZ eedœd	d
„Zejeee dœdd„Zedddddœeej eee ee eeejdœdd„ƒZeeeej dœdd„Z e	dƒZ!e	dƒZ"ee! ee" ee#e!e"f  dœdd„Z$edddœeeee% ee ddœdd„ƒZ&dS ) zfBeta utility functions to assist in common eval workflows.

These functions may change in the future.
é    N)ÚSequence)ÚOptionalÚTypeVar)Ú
evaluation)Ú	warn_beta)ÚClient)Úrun_dictÚid_mapc                 C   sf   | d }|  ¡ D ]\}}| t|ƒt|ƒ¡}q|| d< |  d¡rP|| d  | d< |  d¡sbi | d< | S )a  Convert the IDs in the run dictionary using the provided ID map.

    Parameters:
    - run_dict (dict): The dictionary representing a run.
    - id_map (dict): The dictionary mapping old IDs to new IDs.

    Returns:
    - dict: The updated run dictionary.
    Údotted_orderÚparent_run_idÚextra)ÚitemsÚreplaceÚstrÚget)r   r	   ZdoÚkÚv© r   úT/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/langsmith/beta/_evals.pyÚ_convert_ids   s    


r   )ÚrootÚrun_to_example_mapÚreturnc                    sº   | g}t  ¡ }| j|i‰ g }|r’| ¡ }|jh d£d�}ˆ  |d t  ¡ ¡ˆ |d < ˆ |d  |d< ˆ |d  |d< |jr†| |j¡ | |¡ q‡ fdd„|D ƒ}|| j	 |d d< |S )	a&  Convert the root run and its child runs to a list of dictionaries.

    Parameters:
    - root (ls_schemas.Run): The root run to convert.
    - run_to_example_map (dict): The dictionary mapping run IDs to example IDs.

    Returns:
    - List[dict]: The list of converted run dictionaries.
    >   Zchild_run_idsZ
session_idZparent_run_ids)ÚexcludeÚidÚtrace_idc                    s   g | ]}t |ˆ ƒ‘qS r   )r   ©Ú.0Úr©r	   r   r   Ú
<listcomp>A   ó    z%_convert_root_run.<locals>.<listcomp>r   Zreference_example_id)
ÚuuidÚuuid4r   ÚpopÚdictr   Ú
child_runsÚextendÚappendr   )r   r   Zruns_r   ÚresultsÚsrcZsrc_dictÚresultr   r   r   Ú_convert_root_run*   s     

r,   F)Útest_project_nameÚclientÚload_child_runsÚinclude_outputs)ÚrunsÚdataset_namer-   r.   r/   r0   r   c                   sv  | st d| › �ƒ‚ˆ pt ¡ ‰ ˆ j|d�}|r<dd„ | D ƒnd}ˆ jdd„ | D ƒ|dd„ | D ƒ|jd� ˆsr| }n‡ ‡fd	d„| D ƒ}|p d
t ¡ jdd… › �}t	ˆ j
|d�ƒ}	dd„ |	D ƒ‰|	d jrÔ|	d jn|	d j}
‡fdd„|D ƒ}ˆ j||jd|
 ¡ dœd�}|D ]T}|d |d  }tjjtjjd�|d< |d | |d< ˆ jf i |¤d|i¤Ž �qˆ  |j¡}|S )aƒ  Convert the following runs to a dataset + test.

    This makes it easy to sample prod runs into a new regression testing
    workflow and compare against a candidate system.

    Internally, this function does the following:
        1. Create a dataset from the provided production run inputs.
        2. Create a new test project.
        3. Clone the production runs and re-upload against the dataset.

    Parameters:
    - runs (Sequence[ls_schemas.Run]): A sequence of runs to be executed as a test.
    - dataset_name (str): The name of the dataset to associate with the test runs.
    - client (Optional[Client]): An optional LangSmith client instance. If not provided,
        a new client will be created.
    - load_child_runs (bool): Whether to load child runs when copying runs.
        Defaults to False.

    Returns:
    - ls_schemas.TracerSession: The project containing the cloned runs.

    Examples:
    --------
    .. code-block:: python

        import langsmith
        import random

        client = langsmith.Client()

        # Randomly sample 100 runs from a prod project
        runs = list(client.list_runs(project_name="My Project", execution_order=1))
        sampled_runs = random.sample(runs, min(len(runs), 100))

        runs_as_test(runs, dataset_name="Random Runs")

        # Select runs named "extractor" whose root traces received good feedback
        runs = client.list_runs(
            project_name="<your_project>",
            filter='eq(name, "extractor")',
            trace_filter='and(eq(feedback_key, "user_score"), eq(feedback_score, 1))',
        )
        runs_as_test(runs, dataset_name="Extraction Good")
    z1Expected a non-empty sequence of runs. Received: )r2   c                 S   s   g | ]
}|j ‘qS r   )Úoutputsr   r   r   r   r    €   r!   z(convert_runs_to_test.<locals>.<listcomp>Nc                 S   s   g | ]
}|j ‘qS r   )Úinputsr   r   r   r   r    ‚   r!   c                 S   s   g | ]
}|j ‘qS r   )r   r   r   r   r   r    „   r!   )r4   r3   Zsource_run_idsZ
dataset_idc                    s   g | ]}ˆ j |jˆd �‘qS ))r/   )Zread_runr   r   )r.   r/   r   r   r    ‹   s   zprod-baseline-é   c                 S   s   i | ]}|j |j“qS r   )Zsource_run_idr   )r   Úer   r   r   Ú
<dictcomp>’   r!   z(convert_runs_to_test.<locals>.<dictcomp>r   c                    s    g | ]}t |ˆ ƒD ]}|‘qqS r   )r,   )r   Zroot_runr   )r   r   r   r    —   s   þzprod-baseline)ÚwhichÚdataset_version)Úproject_nameZreference_dataset_idÚmetadataZend_timeÚ
start_time)Útzr:   )Ú
ValueErrorÚrtÚget_cached_clientZcreate_datasetZcreate_examplesr   r"   r#   ÚhexÚlistZlist_examplesZmodified_atZ
created_atZcreate_projectÚ	isoformatÚdatetimeÚnowÚtimezoneÚutcZ
create_runZupdate_project)r1   r2   r-   r.   r/   r0   Zdsr3   Zruns_to_copyZexamplesr9   Z	to_createÚprojectZnew_runZlatencyÚ_r   )r.   r/   r   r   Úconvert_runs_to_testF   sP    6üÿÿ
þþý	ÿrJ   )r:   r.   r   c           	      C   s„   |j | d�}t t¡}g }i }|D ]4}|jd urB||j  |¡ n
| |¡ |||j< q"| ¡ D ]\}}t|dd„ d�|| _	q`|S )N)r:   c                 S   s   | j S ©N)r
   )r   r   r   r   Ú<lambda>À   r!   z%_load_nested_traces.<locals>.<lambda>)Úkey)
Z	list_runsÚcollectionsÚdefaultdictrB   r   r(   r   r   Úsortedr&   )	r:   r.   r1   Ztreemapr)   Zall_runsÚrunZrun_idr&   r   r   r   Ú_load_nested_traces²   s    ÿ

rR   ÚTÚU)Úlist1Úlist2r   c                 C   s   t t | |¡ƒS rK   )rB   Ú	itertoolsÚproduct)rU   rV   r   r   r   Ú_outer_productÈ   s    rY   é
   )Úmax_concurrencyr.   )r:   Ú
evaluatorsr[   r.   r   c                C   sÊ   ddl m} g }|D ]H}t|tjƒr0| |¡ qt|ƒrJ| t |¡¡ qtdt	|ƒ› �ƒ‚q|pht
 ¡ }t| |ƒ}||d��.}|j|jgtt||ƒŽ ¢R Ž }	W d  ƒ n1 s²0    Y  |	D ]}
qÀdS )a  Compute test metrics for a given test name using a list of evaluators.

    Args:
        project_name (str): The name of the test project to evaluate.
        evaluators (list): A list of evaluators to compute metrics with.
        max_concurrency (Optional[int], optional): The maximum number of concurrent
            evaluations. Defaults to 10.
        client (Optional[Client], optional): The client to use for evaluations.
            Defaults to None.

    Returns:
        None: This function does not return any value.
    r   )ÚContextThreadPoolExecutorz5Evaluation not yet implemented for evaluator of type )Úmax_workersN)Ú	langsmithr]   Ú
isinstanceÚls_evalZRunEvaluatorr(   ÚcallableZrun_evaluatorÚNotImplementedErrorÚtyper?   r@   rR   ÚmapZevaluate_runÚziprY   )r:   r\   r[   r.   r]   Zevaluators_ÚfuncZtracesÚexecutorr)   rI   r   r   r   Úcompute_test_metricsÌ   s(    ÿ
ÿÿ&ri   )'Ú__doc__rN   rD   rW   r"   Úcollections.abcr   Útypingr   r   Zlangsmith.run_treesZ	run_treesr?   Zlangsmith.schemasZschemasZ
ls_schemasr_   r   ra   Z#langsmith._internal._beta_decoratorr   Zlangsmith.clientr   r%   r   ZRunrB   r,   r   ÚboolZTracerSessionrJ   rR   rS   rT   ÚtuplerY   Úintri   r   r   r   r   Ú<module>   sR   ùøk&ûú