a
    •˜ëj«*  ã                   @   s  d Z ddlZddlmZ ddlmZmZmZ ddlm	Z	 ddl
mZmZmZmZmZ ddlmZmZ ddlmZ dd	lmZmZ dd
lmZ ddlmZmZ ddlmZmZ ddl m!Z! ddl"m#Z# edƒZ$eded�Z%ee$ ee$ge%f ee$ dœdd„Z&G dd„ deƒZ'dS )z„Ensemble Retriever.

Ensemble retriever that ensemble the results of
multiple retrievers by using weighted  Reciprocal Rank Fusion.
é    N)Údefaultdict)ÚHashableÚIterableÚIterator)Úchain)ÚAnyÚCallableÚOptionalÚTypeVarÚcast)Ú#AsyncCallbackManagerForRetrieverRunÚCallbackManagerForRetrieverRun)ÚDocument)ÚBaseRetrieverÚRetrieverLike)ÚRunnableConfig)Úensure_configÚpatch_config)ÚConfigurableFieldSpecÚget_unique_config_specs)Úmodel_validator)ÚoverrideÚTÚH)Úbound)ÚiterableÚkeyÚreturnc                 c   s4   t ƒ }| D ]$}||ƒ }|vr
| |¡ |V  q
dS )a  Yield unique elements of an iterable based on a key function.

    Args:
        iterable: The iterable to filter.
        key: A function that returns a hashable key for each element.

    Yields:
        Unique elements of the iterable based on the key function.
    N)ÚsetÚadd)r   r   ÚseenÚeÚk© r#   ú\/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/langchain/retrievers/ensemble.pyÚunique_by_key&   s
    

r%   c                   @   sR  e Zd ZU dZee ed< ee ed< dZe	ed< dZ
ee ed< eee dœd	d
„ƒZedd�eeeef edœdd„ƒƒZed#eee eee dœdd„ƒZed$eee eee dœdd„ƒZeeee dœdd„Zeeee dœdd„Zddœeeee ee dœdd„Zddœeeee ee dœdd„Zeee  ee d œd!d"„Z dS )%ÚEnsembleRetrieverae  Retriever that ensembles the multiple retrievers.

    It uses a rank fusion.

    Args:
        retrievers: A list of retrievers to ensemble.
        weights: A list of weights corresponding to the retrievers. Defaults to equal
            weighting for all retrievers.
        c: A constant added to the rank, controlling the balance between the importance
            of high-ranked items and the consideration given to lower-ranked items.
            Default is 60.
        id_key: The key in the document's metadata used to determine unique documents.
            If not specified, page_content is used.
    Ú
retrieversÚweightsé<   ÚcNÚid_key)r   c                 C   s   t dd„ | jD ƒƒS )z+List configurable fields for this runnable.c                 s   s   | ]}|j D ]
}|V  qqd S ©N)Úconfig_specs)Ú.0Ú	retrieverÚspecr#   r#   r$   Ú	<genexpr>O   s   z1EnsembleRetriever.config_specs.<locals>.<genexpr>)r   r'   ©Úselfr#   r#   r$   r-   L   s    ÿzEnsembleRetriever.config_specsÚbefore)Úmode)Úvaluesr   c                 C   s,   |  d¡s(t|d ƒ}d| g| |d< |S )Nr(   r'   é   )ÚgetÚlen)Úclsr6   Zn_retrieversr#   r#   r$   Ú_set_weightsS   s    
zEnsembleRetriever._set_weights)ÚinputÚconfigÚkwargsr   c           	   
   K   sÔ   ddl m} t|ƒ}|j| d¡d | dd¡| dg ¡| j| di ¡| jd�}|jd |fd	| d
¡pj|  ¡ i|¤Ž}z| j	|||d�}W n0 t
y¸ } z| |¡ ‚ W Y d }~n d }~0 0 |j|fi |¤Ž |S d S )Nr   )ÚCallbackManagerÚ	callbacksÚverboseFÚtagsÚmetadata©rA   Zinheritable_tagsZ
local_tagsZinheritable_metadataZlocal_metadataÚnameÚrun_name©Úrun_managerr=   )Úlangchain_core.callbacksr?   r   Ú	configurer8   rB   rC   Úon_retriever_startÚget_nameÚrank_fusionÚ	ExceptionÚon_retriever_errorÚon_retriever_end)	r3   r<   r=   r>   r?   Úcallback_managerrH   Úresultr!   r#   r#   r$   Úinvoke[   s<    


ù	þýü
ÿþzEnsembleRetriever.invokec           	   
   Ë   sì   ddl m} t|ƒ}|j| d¡d | dd¡| dg ¡| j| di ¡| jd�}|jd |fd	| d
¡pj|  ¡ i|¤ŽI d H }z| j	|||d�I d H }W n6 t
yÊ } z| |¡I d H  ‚ W Y d }~n&d }~0 0 |j|fi |¤ŽI d H  |S d S )Nr   )ÚAsyncCallbackManagerr@   rA   FrB   rC   rD   rE   rF   rG   )rI   rT   r   rJ   r8   rB   rC   rK   rL   Úarank_fusionrN   rO   rP   )	r3   r<   r=   r>   rT   rQ   rH   rR   r!   r#   r#   r$   Úainvoke€   sD    


ù	þýüýÿþzEnsembleRetriever.ainvoke)ÚqueryrH   r   c                C   s   |   ||¡S )zãGet the relevant documents for a given query.

        Args:
            query: The query to search for.
            run_manager: The callback handler to use.

        Returns:
            A list of reranked documents.
        )rM   ©r3   rW   rH   r#   r#   r$   Ú_get_relevant_documents©   s    z)EnsembleRetriever._get_relevant_documentsc                Ã   s   |   ||¡I dH S )zòAsynchronously get the relevant documents for a given query.

        Args:
            query: The query to search for.
            run_manager: The callback handler to use.

        Returns:
            A list of reranked documents.
        N)rU   rX   r#   r#   r$   Ú_aget_relevant_documents»   s    z*EnsembleRetriever._aget_relevant_documents)r=   )rW   rH   r=   r   c                   sN   ‡ ‡‡fdd„t | jƒD ƒ}tt|ƒƒD ]}dd„ || D ƒ||< q(|  |¡S )ak  Rank fusion.

        Retrieve the results of the retrievers and use rank_fusion_func to get
        the final result.

        Args:
            query: The query to search for.
            run_manager: The callback handler to use.
            config: Optional configuration for the retrievers.

        Returns:
            A list of reranked documents.
        c                    s6   g | ].\}}|  ˆtˆ ˆjd |d › �d�d�¡‘qS ©Z
retriever_r7   )Útag)r@   )rS   r   Ú	get_child©r.   Úir/   ©r=   rW   rH   r#   r$   Ú
<listcomp>â   s   ùþþz1EnsembleRetriever.rank_fusion.<locals>.<listcomp>c                 S   s*   g | ]"}t |tƒr"ttd |ƒd�n|‘qS )Ústr©Úpage_content)Ú
isinstancerb   r   r   ©r.   Údocr#   r#   r$   ra   ï   s   ÿ)Ú	enumerater'   Úranger9   Úweighted_reciprocal_rank©r3   rW   rH   r=   Zretriever_docsr_   r#   r`   r$   rM   Í   s    øþzEnsembleRetriever.rank_fusionc                ƒ   sZ   t j‡ ‡‡fdd„t| jƒD ƒŽ I dH }tt|ƒƒD ]}dd„ || D ƒ||< q4|  |¡S )az  Rank fusion.

        Asynchronously retrieve the results of the retrievers
        and use rank_fusion_func to get the final result.

        Args:
            query: The query to search for.
            run_manager: The callback handler to use.
            config: Optional configuration for the retrievers.

        Returns:
            A list of reranked documents.
        c                    s6   g | ].\}}|  ˆtˆ ˆjd |d › �d�d�¡‘qS r[   )rV   r   r]   r^   r`   r#   r$   ra     s   ùþþz2EnsembleRetriever.arank_fusion.<locals>.<listcomp>Nc                 S   s$   g | ]}t |tƒst|d �n|‘qS )rc   )re   r   rf   r#   r#   r$   ra     s   ÿ)ÚasyncioZgatherrh   r'   ri   r9   rj   rk   r#   r`   r$   rU   ÷   s    øÿ
þzEnsembleRetriever.arank_fusion)Ú	doc_listsr   c                    s´   t |ƒt ˆjƒkrd}t|ƒ‚ttƒ‰ t|ˆjƒD ]P\}}t|dd�D ]:\}}ˆ ˆjdu r`|jn
|j	ˆj   ||ˆj
  7  < qFq2t |¡}tt|‡fdd„ƒd‡ ‡fdd„d	�S )
aº  Perform weighted Reciprocal Rank Fusion on multiple rank lists.

        You can find more details about RRF here:
        https://plg.uwaterloo.ca/~gvcormac/cormacksigir09-rrf.pdf.

        Args:
            doc_lists: A list of rank lists, where each rank list contains unique items.

        Returns:
            list: The final aggregated list of items sorted by their weighted RRF
                    scores in descending order.
        z<Number of rank lists must be equal to the number of weights.r7   )ÚstartNc                    s   ˆ j d u r| jS | jˆ j  S r,   ©r+   rd   rC   ©rg   r2   r#   r$   Ú<lambda>I  s    ÿz<EnsembleRetriever.weighted_reciprocal_rank.<locals>.<lambda>Tc                    s    ˆ ˆj d u r| jn
| jˆj   S r,   ro   rp   ©Z	rrf_scorer3   r#   r$   rq   P  s   ÿ)Úreverser   )r9   r(   Ú
ValueErrorr   ÚfloatÚziprh   r+   rd   rC   r*   r   Úfrom_iterableÚsortedr%   )r3   rm   ÚmsgZdoc_listZweightZrankrg   Zall_docsr#   rr   r$   rj   #  s,    ÿ
üú
	

þöz*EnsembleRetriever.weighted_reciprocal_rank)N)N)!Ú__name__Ú
__module__Ú__qualname__Ú__doc__Úlistr   Ú__annotations__ru   r*   Úintr+   r	   rb   Úpropertyr   r-   r   ÚclassmethodÚdictr   r;   r   r   r   rS   rV   r   rY   r   rZ   rM   rU   rj   r#   r#   r#   r$   r&   7   sf   
 ýû$ ýû*ûûûú/ûú.
ýr&   )(r}   rl   Úcollectionsr   Úcollections.abcr   r   r   Ú	itertoolsr   Útypingr   r   r	   r
   r   rI   r   r   Zlangchain_core.documentsr   Zlangchain_core.retrieversr   r   Zlangchain_core.runnablesr   Zlangchain_core.runnables.configr   r   Zlangchain_core.runnables.utilsr   r   Zpydanticr   Ztyping_extensionsr   r   r   r%   r&   r#   r#   r#   r$   Ú<module>   s"   $