a
    ”˜ëj]–  ã                   @  sZ  U d Z ddlmZ ddlZddlZddlZddlZddlmZm	Z	m
Z
mZmZ ddlmZ ddlmZmZmZmZmZmZmZmZ ddlmZ ddlmZ dd	lmZ dd
lmZm Z  ddl!m"Z" ej#dd�Z$edƒZ%dddœdd„Z&da'de(d< ddœdd„Z)ddddœdd„Z*dddd œd!d"„Z+d#d$d%d&œd'd(„Z,d#d)d*d&œd+d,„Z-d-d.d/œd0d1„Z.d2d3d4œd5d6„Z/G d7d8„ d8eƒZ0dddd9œd:d;„Z1d<d=d<d>œd?d@„Z2G dAdB„ dBƒZ3dCdDddEœdFdG„Z4G dHdI„ dIeƒZ5dJdddKddLddMœdNdOdCd#dPd-d#ddQdRdIdSœdTdU„Z6d$dVdWœdXdY„Z7dCdDddEœdZd[„Z8dJdddKddLddMœd\dOdCd#dPd-d#ddQdRdIdSœd]d^„Z9dS )_z@Module contains logic for indexing documents into vector stores.é    )ÚannotationsN)ÚAsyncIterableÚAsyncIteratorÚIterableÚIteratorÚSequence)Úislice)ÚAnyÚCallableÚLiteralÚOptionalÚ	TypedDictÚTypeVarÚUnionÚcast)Ú
BaseLoader)ÚDocument)ÚLangChainException)ÚDocumentIndexÚRecordManager)ÚVectorStoreiÀ  )ÚintÚTÚstr)Úinput_stringÚreturnc                 C  s(   t j|  d¡dd� ¡ }tt t|¡ƒS )z3Hashes a string and returns the corresponding UUID.úutf-8F©Úusedforsecurity)ÚhashlibÚsha1ÚencodeÚ	hexdigestr   ÚuuidÚuuid5ÚNAMESPACE_UUID)r   Ú
hash_value© r'   úZ/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/langchain_core/indexing/api.pyÚ_hash_string_to_uuid%   s    
ÿ
r)   FÚboolÚ_WARNED_ABOUT_SHA1ÚNone)r   c                   C  s   t stjdtdd� da dS )z9Emit a one-time warning about SHA-1 collision weaknesses.aU  Using SHA-1 for document hashing. SHA-1 is *not* collision-resistant; a motivated attacker can construct distinct inputs that map to the same fingerprint. If this matters in your threat model, switch to a stronger algorithm such as 'blake2b', 'sha256', or 'sha512' by specifying  `key_encoder` parameter in the `index` or `aindex` function. é   )ÚcategoryÚ
stacklevelTN)r+   ÚwarningsÚwarnÚUserWarningr'   r'   r'   r(   Ú_warn_about_sha10   s    ø
r3   z.Literal['sha1', 'sha256', 'sha512', 'blake2b']z	uuid.UUID)r   Ú	algorithmr   c                C  s$   |dkrt ƒ  t| |ƒ}t t|¡S )zKHash *input_string* to a deterministic UUID using the configured algorithm.r    )r3   Ú_calculate_hashr#   r$   r%   )r   r4   r&   r'   r'   r(   Ú_hash_stringB   s    
r6   zdict[Any, Any])Údatar4   r   c                C  s   t j| dd�}t||d�S )zBHash a nested dictionary to a UUID using the configured algorithm.T©Ú	sort_keys©r4   )ÚjsonÚdumpsr6   )r7   r4   Zserialized_datar'   r'   r(   Ú_hash_nested_dictL   s    r=   r   zIterable[T]zIterator[list[T]])ÚsizeÚiterabler   c                 c  s*   t |ƒ}tt|| ƒƒ}|sdS |V  qdS ©zUtility batching function.N)ÚiterÚlistr   )r>   r?   ÚitÚchunkr'   r'   r(   Ú_batchT   s
    rE   zAsyncIterable[T]zAsyncIterator[list[T]]c                 C sR   g }|2 z83 dH W }t |ƒ| k r*| |¡ t |ƒ| kr|V  g }q6 |rN|V  dS r@   )ÚlenÚappend)r>   r?   ÚbatchÚelementr'   r'   r(   Ú_abatch^   s    
rJ   z+Union[str, Callable[[Document], str], None]z&Callable[[Document], Union[str, None]])Úsource_id_keyr   c                   sT   ˆ du rdd„ S t ˆ tƒr&‡ fdd„S tˆ ƒr2ˆ S dˆ › dtˆ ƒ› d�}t|ƒ‚dS )z$Get the source id from the document.Nc                 S  s   d S ©Nr'   )Z_docr'   r'   r(   Ú<lambda>r   ó    z)_get_source_id_assigner.<locals>.<lambda>c                   s
   | j ˆ  S rL   )Úmetadata)Údoc©rK   r'   r(   rM   t   rN   zAsource_id_key should be either None, a string or a callable. Got z	 of type Ú.)Ú
isinstancer   ÚcallableÚtypeÚ
ValueError)rK   Úmsgr'   rQ   r(   Ú_get_source_id_assignerm   s    
ÿÿÿrX   zIterable[Document]zIterator[Document])Úhashed_documentsr   c                 c  s6   t ƒ }| D ]&}|j|vr
| td|jƒ¡ |V  q
dS )z>Deduplicate a list of hashed documents while preserving order.r   N)ÚsetÚidÚaddr   )rY   ÚseenÚ
hashed_docr'   r'   r(   Ú_deduplicate_in_order~   s
    
r_   c                   @  s   e Zd ZdZdS )ÚIndexingExceptionz(Raised when an indexing operation fails.N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r'   r'   r'   r(   r`   Œ   s   r`   )Útextr4   r   c                 C  sš   |dkr0t j|  d¡dd� ¡ }tt t|¡ƒS |dkrLt  |  d¡¡ ¡ S |dkrht  	|  d¡¡ ¡ S |dkr„t  
|  d¡¡ ¡ S d|› �}t|ƒ‚d	S )
z8Return a hexadecimal digest of *text* using *algorithm*.r    r   Fr   Úblake2bÚsha256Úsha512zUnsupported hashing algorithm: N)r   r    r!   r"   r   r#   r$   r%   rf   rg   rh   rV   )re   r4   ÚdigestrW   r'   r'   r(   r5   �   s    
r5   r   zPUnion[Callable[[Document], str], Literal['sha1', 'sha256', 'sha512', 'blake2b']])ÚdocumentÚkey_encoderr   c          	   
   C  sª   t | jp
i ƒ}t|ƒr || ƒ}nxt| j|d�}ztj|dd�}W n: tyz } z"d|› d�}t|ƒ|‚W Y d}~n
d}~0 0 t||d�}t|| |d�}t	|| j| jd�S )a
  Calculate a hash of the document, and assign it to the uid.

    When using one of the predefined hashing algorithms, the hash is calculated
    by hashing the content and the metadata of the document.

    Args:
        document: Document to hash.
        key_encoder: Hashing algorithm to use for hashing the document.
            If not provided, a default encoder using SHA-1 will be used.
            SHA-1 is not collision-resistant, and a motivated attacker
            could craft two different texts that hash to the
            same cache key.

            New applications should use one of the alternative encoders
            or provide a custom and strong key encoder function to avoid this risk.

            When changing the key encoder, you must change the
            index as well to avoid duplicated documents in the cache.

    Raises:
        ValueError: If the metadata cannot be serialized using json.

    Returns:
        Document with a unique identifier based on the hash of the content and metadata.
    r:   Tr8   zFailed to hash metadata: z6. Please use a dict that can be serialized using json.N)r[   Úpage_contentrO   )
ÚdictrO   rT   r5   rl   r;   r<   Ú	ExceptionrV   r   )	rj   rk   rO   Úhash_Zcontent_hashZserialized_metaÚerW   Zmetadata_hashr'   r'   r(   Ú_get_document_with_hash¢   s"     

ÿ ürq   c                   @  s   e Zd Zddddœdd„ZdS )Ú_HashedDocumentr	   r,   )ÚargsÚkwargsr   c                 O  s   d}t |ƒ‚dS )z-Raise an error if this class is instantiated.zî_HashedDocument is an internal abstraction that was deprecated in  langchain-core 0.3.63. This abstraction is marked as private and  should not have been used directly. If you are seeing this error, please  update your code appropriately.N)ÚNotImplementedError)Úselfrs   rt   rW   r'   r'   r(   Ú__init__à   s    ÿz_HashedDocument.__init__N)ra   rb   rc   rw   r'   r'   r'   r(   rr   ß   s   rr   z!Union[VectorStore, DocumentIndex]z	list[str])Úvector_storeÚidsr   c                 C  s„   t | tƒr2|  |¡}|d ur€|du r€d}t|ƒ‚nNt | tƒrh|  |¡}d|v r€|d dkr€d}t|ƒ‚ndt| ƒ› d�}t|ƒ‚d S ©NFz+The delete operation to VectorStore failed.Z
num_failedr   z-The delete operation to DocumentIndex failed.úCVectorstore should be either a VectorStore or a DocumentIndex. Got rR   )rS   r   Údeleter`   r   rU   Ú	TypeError©rx   ry   Z	delete_okrW   Zdelete_responser'   r'   r(   Ú_deleteë   s    





ÿÿr   c                   @  s2   e Zd ZU dZded< ded< ded< ded< dS )ÚIndexingResultzFReturn a detailed a breakdown of the result of the indexing operation.r   Ú	num_addedÚnum_updatedÚnum_deletedÚnum_skippedN)ra   rb   rc   rd   Ú__annotations__r'   r'   r'   r(   r€     s   
r€   éd   iè  r    )Ú
batch_sizeÚcleanuprK   Úcleanup_batch_sizeÚforce_updaterk   Úupsert_kwargsz%Union[BaseLoader, Iterable[Document]]r   z7Optional[Literal['incremental', 'full', 'scoped_full']]zPUnion[Literal['sha1', 'sha256', 'sha512', 'blake2b'], Callable[[Document], str]]zOptional[dict[str, Any]])Údocs_sourceÚrecord_managerrx   r‡   rˆ   rK   r‰   rŠ   rk   r‹   r   c          %        s  ˆ dkrt ƒ  |dvr*d|› d�}
t|
ƒ‚|dv rF|du rFd}
t|
ƒ‚|}t|tƒr¦dd	g}|D ]&}t||ƒs`d
|› d|› �}
t|
ƒ‚q`t|ƒjtjkrÊd}
t|
ƒ‚n$t|tƒr²ndt|ƒ› d�}
t|
ƒ‚t| t	ƒ�rz|  
¡ }W n  t�y   t|  ¡ ƒ}Y n0 nt| ƒ}t|ƒ‰| ¡ }d}d}d}d}tƒ }t||ƒD �]^}t|ƒ}tt‡ fdd„|D ƒƒƒ}||t|ƒ 7 }‡fdd„|D ƒ}|dv �rît||ƒD ]F\}}|du �rÌd|jdd… › d�}
t|
ƒ‚|dk�rœ| |¡ �qœtd|ƒ}| tddd„ |D ƒƒ¡}g }g }g }tƒ }t||ƒD ]P\}} td|jƒ}!| �r^|�rP| |!¡ n| |!¡ �q$| |!¡ | |¡ �q$|�r–|j||d� |t|ƒ7 }|�rt|tƒ�rÈ|j|f||dœ|	�pÀi ¤Ž n$t|tƒ�rì|j|fi |	�pæi ¤Ž |t|ƒt|ƒ 7 }|t|ƒ7 }|jtddd„ |D ƒƒ||d� |dk�r>|D ]}|du �r:d}
t|
ƒ‚�q:td|ƒ}"|j |"||d� }#�r>t!||#ƒ | "|#¡ |t|#ƒ7 }�qb�q>|d k�sº|dk�r
|�r
d}$|dk�rÐt|ƒ}$|j |$||d� }#�r
t!||#ƒ | "|#¡ |t|#ƒ7 }�qÐ||||d!œS )"a|  Index data from the loader into the vector store.

    Indexing functionality uses a manager to keep track of which documents
    are in the vector store.

    This allows us to keep track of which documents were updated, and which
    documents were deleted, which documents should be skipped.

    For the time being, documents are indexed using their hashes, and users
    are not able to specify the uid of the document.

    .. versionchanged:: 0.3.25
        Added ``scoped_full`` cleanup mode.

    .. important::

        * In full mode, the loader should be returning
          the entire dataset, and not just a subset of the dataset.
          Otherwise, the auto_cleanup will remove documents that it is not
          supposed to.
        * In incremental mode, if documents associated with a particular
          source id appear across different batches, the indexing API
          will do some redundant work. This will still result in the
          correct end state of the index, but will unfortunately not be
          100% efficient. For example, if a given document is split into 15
          chunks, and we index them using a batch size of 5, we'll have 3 batches
          all with the same source id. In general, to avoid doing too much
          redundant work select as big a batch size as possible.
        * The ``scoped_full`` mode is suitable if determining an appropriate batch size
          is challenging or if your data loader cannot return the entire dataset at
          once. This mode keeps track of source IDs in memory, which should be fine
          for most use cases. If your dataset is large (10M+ docs), you will likely
          need to parallelize the indexing process regardless.

    Args:
        docs_source: Data loader or iterable of documents to index.
        record_manager: Timestamped set to keep track of which documents were
            updated.
        vector_store: VectorStore or DocumentIndex to index the documents into.
        batch_size: Batch size to use when indexing. Default is 100.
        cleanup: How to handle clean up of documents. Default is None.

            - incremental: Cleans up all documents that haven't been updated AND
              that are associated with source ids that were seen during indexing.
              Clean up is done continuously during indexing helping to minimize the
              probability of users seeing duplicated content.
            - full: Delete all documents that have not been returned by the loader
              during this run of indexing.
              Clean up runs after all documents have been indexed.
              This means that users may see duplicated content during indexing.
            - scoped_full: Similar to Full, but only deletes all documents
              that haven't been updated AND that are associated with
              source ids that were seen during indexing.
            - None: Do not delete any documents.
        source_id_key: Optional key that helps identify the original source
            of the document. Default is None.
        cleanup_batch_size: Batch size to use when cleaning up documents.
            Default is 1_000.
        force_update: Force update documents even if they are present in the
            record manager. Useful if you are re-indexing with updated embeddings.
            Default is False.
        key_encoder: Hashing algorithm to use for hashing the document content and
            metadata. Default is "sha1".
            Other options include "blake2b", "sha256", and "sha512".

            .. versionadded:: 0.3.66

        key_encoder: Hashing algorithm to use for hashing the document.
            If not provided, a default encoder using SHA-1 will be used.
            SHA-1 is not collision-resistant, and a motivated attacker
            could craft two different texts that hash to the
            same cache key.

            New applications should use one of the alternative encoders
            or provide a custom and strong key encoder function to avoid this risk.

            When changing the key encoder, you must change the
            index as well to avoid duplicated documents in the cache.
        upsert_kwargs: Additional keyword arguments to pass to the add_documents
            method of the VectorStore or the upsert method of the DocumentIndex.
            For example, you can use this to specify a custom vector_field:
            upsert_kwargs={"vector_field": "embedding"}
            .. versionadded:: 0.3.10

    Returns:
        Indexing result which contains information about how many documents
        were added, updated, deleted, or skipped.

    Raises:
        ValueError: If cleanup mode is not one of 'incremental', 'full' or None
        ValueError: If cleanup mode is incremental and source_id_key is None.
        ValueError: If vectorstore does not have
            "delete" and "add_documents" required methods.
        ValueError: If source_id_key is not None, but is not a string or callable.
        TypeError: If ``vectorstore`` is not a VectorStore or a DocumentIndex.
        AssertionError: If ``source_id`` is None when cleanup mode is incremental.
            (should be unreachable code).
    r    >   Úscoped_fullÚfullNÚincrementalúKcleanup should be one of 'incremental', 'full', 'scoped_full' or None. Got rR   >   rŽ   r�   NúJSource id key is required when cleanup mode is incremental or scoped_full.r|   Úadd_documentsúVectorstore ú does not have required method z1Vectorstore has not implemented the delete methodr{   r   c                   s   g | ]}t |ˆ d �‘qS ©©rk   ©rq   ©Ú.0rP   r—   r'   r(   Ú
<listcomp>È  s   ÿzindex.<locals>.<listcomp>c                   s   g | ]}ˆ |ƒ‘qS r'   r'   )rš   r^   ©Úsource_id_assignerr'   r(   r›   Ñ  s   úlSource ids are required when cleanup mode is incremental or scoped_full. Document that starts with content: r†   ú was not assigned as source id.rŽ   úSequence[str]c                 S  s   g | ]
}|j ‘qS r'   ©r[   r™   r'   r'   r(   r›   ç  rN   r   ©Útime_at_least©ry   r‡   c                 S  s   g | ]
}|j ‘qS r'   r¡   r™   r'   r'   r(   r›     rN   ©Ú	group_idsr£   r�   úAsource_id cannot be None at this point. Reached unreachable code.©r¦   ÚbeforeÚlimitr�   ©r�   r‚   r„   rƒ   )#r3   rV   rS   r   ÚhasattrrU   r|   r   r}   r   Z	lazy_loadru   rA   ÚloadrX   Zget_timerZ   rE   rF   rB   r_   Úziprl   r\   r   Úexistsr[   rG   Úupdater“   ZupsertÚAssertionErrorZ	list_keysr   Zdelete_keys)%rŒ   r�   rx   r‡   rˆ   rK   r‰   rŠ   rk   r‹   rW   ÚdestinationÚmethodsÚmethodZdoc_iteratorÚindex_start_dtr�   r„   r‚   rƒ   Úscoped_full_cleanup_source_idsÚ	doc_batchÚoriginal_batch_sizeÚhashed_docsÚ
source_idsÚ	source_idr^   Úexists_batchÚuidsÚdocs_to_indexÚuids_to_refreshÚ	seen_docsÚ
doc_existsÚ	hashed_idÚsource_ids_Úuids_to_deleteÚdelete_group_idsr'   ©rk   r�   r(   Úindex  s   sÿÿÿ

ÿ


ÿÿ
þÿÿ	
ÿ

ýÿ

ÿ

ÿýüÿþý

ÿ
ÿ


ÿÿ
ÿ

ürÇ   zAsyncIterator[T])Úiteratorr   c                 C s   | D ]
}|V  qdS )z)Convert an iterable to an async iterator.Nr'   )rÈ   Úitemr'   r'   r(   Ú_to_async_iteratorK  s    rÊ   c                 Ã  s�   t | tƒr8|  |¡I d H }|d urŒ|du rŒd}t|ƒ‚nTt | tƒrt|  |¡I d H }d|v rŒ|d dkrŒd}t|ƒ‚ndt| ƒ› d�}t|ƒ‚d S rz   )rS   r   Úadeleter`   r   rU   r}   r~   r'   r'   r(   Ú_adeleteQ  s    



ÿÿrÌ   z>Union[BaseLoader, Iterable[Document], AsyncIterator[Document]]c          %      ƒ  sŒ  ˆ dkrt ƒ  |dvr*d|› d�}
t|
ƒ‚|dv rF|du rFd}
t|
ƒ‚|}t|tƒr¶dd	g}|D ]&}t||ƒs`d
|› d|› �}
t|
ƒ‚q`t|ƒjtjkrÚt|ƒjtjkrÚd}
t|
ƒ‚n$t|tƒrÂndt|ƒ› d�}
t	|
ƒ‚t| t
ƒ�rz|  ¡ }W n  t�y   t|  ¡ ƒ}Y n0 nt| dƒ�r(| }nt| ƒ}t|ƒ‰| ¡ I dH }d}d}d}d}tƒ }t||ƒ2 �z–3 dH W }t|ƒ}tt‡ fdd„|D ƒƒƒ}||t|ƒ 7 }‡fdd„|D ƒ}|dv �rt||ƒD ]F\}}|du �rüd|jdd… › d�}
t|
ƒ‚|dk�rÌ| |¡ �qÌtd|ƒ}| tddd„ |D ƒƒ¡I dH }g }g }g }tƒ }t||ƒD ]P\}} td|jƒ}!| �r”|�r†| |!¡ n| |!¡ �qZ| |!¡ | |¡ �qZ|�rÒ|j||d�I dH  |t|ƒ7 }|�rTt|tƒ�r
|j|f||dœ|	�püi ¤ŽI dH  n*t|tƒ�r4|j|fi |	�p(i ¤ŽI dH  |t|ƒt|ƒ 7 }|t|ƒ7 }|jtddd„ |D ƒƒ||d�I dH  |dk�rf|D ]}|du �rˆd}
t |
ƒ‚�qˆtd|ƒ}"|j!|"||d �I dH  }#�rft"||#ƒI dH  | #|#¡I dH  |t|#ƒ7 }�q°�qf6 |d!k�s|dk�r~|�r~d}$|dk�r2t|ƒ}$|j!|$||d �I dH  }#�r~t"||#ƒI dH  | #|#¡I dH  |t|#ƒ7 }�q2||||d"œS )#a”  Async index data from the loader into the vector store.

    Indexing functionality uses a manager to keep track of which documents
    are in the vector store.

    This allows us to keep track of which documents were updated, and which
    documents were deleted, which documents should be skipped.

    For the time being, documents are indexed using their hashes, and users
    are not able to specify the uid of the document.

    .. versionchanged:: 0.3.25
        Added ``scoped_full`` cleanup mode.

    .. important::

        * In full mode, the loader should be returning
          the entire dataset, and not just a subset of the dataset.
          Otherwise, the auto_cleanup will remove documents that it is not
          supposed to.
        * In incremental mode, if documents associated with a particular
          source id appear across different batches, the indexing API
          will do some redundant work. This will still result in the
          correct end state of the index, but will unfortunately not be
          100% efficient. For example, if a given document is split into 15
          chunks, and we index them using a batch size of 5, we'll have 3 batches
          all with the same source id. In general, to avoid doing too much
          redundant work select as big a batch size as possible.
        * The ``scoped_full`` mode is suitable if determining an appropriate batch size
          is challenging or if your data loader cannot return the entire dataset at
          once. This mode keeps track of source IDs in memory, which should be fine
          for most use cases. If your dataset is large (10M+ docs), you will likely
          need to parallelize the indexing process regardless.

    Args:
        docs_source: Data loader or iterable of documents to index.
        record_manager: Timestamped set to keep track of which documents were
            updated.
        vector_store: VectorStore or DocumentIndex to index the documents into.
        batch_size: Batch size to use when indexing. Default is 100.
        cleanup: How to handle clean up of documents. Default is None.

            - incremental: Cleans up all documents that haven't been updated AND
              that are associated with source ids that were seen during indexing.
              Clean up is done continuously during indexing helping to minimize the
              probability of users seeing duplicated content.
            - full: Delete all documents that have not been returned by the loader
              during this run of indexing.
              Clean up runs after all documents have been indexed.
              This means that users may see duplicated content during indexing.
            - scoped_full: Similar to Full, but only deletes all documents
              that haven't been updated AND that are associated with
              source ids that were seen during indexing.
            - None: Do not delete any documents.
        source_id_key: Optional key that helps identify the original source
            of the document. Default is None.
        cleanup_batch_size: Batch size to use when cleaning up documents.
            Default is 1_000.
        force_update: Force update documents even if they are present in the
            record manager. Useful if you are re-indexing with updated embeddings.
            Default is False.
        key_encoder: Hashing algorithm to use for hashing the document content and
            metadata. Default is "sha1".
            Other options include "blake2b", "sha256", and "sha512".

            .. versionadded:: 0.3.66

        key_encoder: Hashing algorithm to use for hashing the document.
            If not provided, a default encoder using SHA-1 will be used.
            SHA-1 is not collision-resistant, and a motivated attacker
            could craft two different texts that hash to the
            same cache key.

            New applications should use one of the alternative encoders
            or provide a custom and strong key encoder function to avoid this risk.

            When changing the key encoder, you must change the
            index as well to avoid duplicated documents in the cache.
        upsert_kwargs: Additional keyword arguments to pass to the add_documents
            method of the VectorStore or the upsert method of the DocumentIndex.
            For example, you can use this to specify a custom vector_field:
            upsert_kwargs={"vector_field": "embedding"}
            .. versionadded:: 0.3.10

    Returns:
        Indexing result which contains information about how many documents
        were added, updated, deleted, or skipped.

    Raises:
        ValueError: If cleanup mode is not one of 'incremental', 'full' or None
        ValueError: If cleanup mode is incremental and source_id_key is None.
        ValueError: If vectorstore does not have
            "adelete" and "aadd_documents" required methods.
        ValueError: If source_id_key is not None, but is not a string or callable.
        TypeError: If ``vector_store`` is not a VectorStore or DocumentIndex.
        AssertionError: If ``source_id_key`` is None when cleanup mode is
            incremental or ``scoped_full`` (should be unreachable).
    r    >   rŽ   r�   Nr�   r‘   rR   >   rŽ   r�   Nr’   rË   Úaadd_documentsr”   r•   z<Vectorstore has not implemented the adelete or delete methodr{   Ú	__aiter__r   c                   s   g | ]}t |ˆ d �‘qS r–   r˜   r™   r—   r'   r(   r›   )  s   ÿzaindex.<locals>.<listcomp>c                   s   g | ]}ˆ |ƒ‘qS r'   r'   r™   rœ   r'   r(   r›   2  s   rž   r†   rŸ   rŽ   r    c                 S  s   g | ]
}|j ‘qS r'   r¡   r™   r'   r'   r(   r›   H  rN   r   r¢   r¤   c                 S  s   g | ]
}|j ‘qS r'   r¡   r™   r'   r'   r(   r›   v  rN   r¥   r�   r§   r¨   r�   r«   )$r3   rV   rS   r   r¬   rU   rË   r|   r   r}   r   Z
alazy_loadru   rÊ   r­   rX   Z	aget_timerZ   rJ   rF   rB   r_   r®   rl   r\   r   Zaexistsr[   rG   ZaupdaterÍ   Zaupsertr±   Z
alist_keysrÌ   Zadelete_keys)%rŒ   r�   rx   r‡   rˆ   rK   r‰   rŠ   rk   r‹   rW   r²   r³   r´   Zasync_doc_iteratorrµ   r�   r„   r‚   rƒ   r¶   r·   r¸   r¹   rº   r»   r^   r¼   r½   r¾   r¿   rÀ   rÁ   rÂ   rÃ   rÄ   rÅ   r'   rÆ   r(   Úaindexg  s$   sÿÿÿ

ÿ
ÿþ

ÿÿ
þÿÿ	
ÿ

ýÿ

ÿ


ÿýüÿþý

ÿ
ÿ
ÿÿ
ÿürÏ   ):rd   Ú
__future__r   r   r;   r#   r0   Úcollections.abcr   r   r   r   r   Ú	itertoolsr   Útypingr	   r
   r   r   r   r   r   r   Z$langchain_core.document_loaders.baser   Zlangchain_core.documentsr   Zlangchain_core.exceptionsr   Zlangchain_core.indexing.baser   r   Zlangchain_core.vectorstoresr   ÚUUIDr%   r   r)   r+   r…   r3   r6   r=   rE   rJ   rX   r_   r`   r5   rq   rr   r   r€   rÇ   rÊ   rÌ   rÏ   r'   r'   r'   r(   Ú<module>   sd   (

=ó&  <ó