a
    •˜ëj  ã                   @  sH   d dl mZ d dlmZmZmZ d dlmZmZm	Z	 G dd„ deƒZ
dS )é    )Úannotations)ÚAnyÚOptionalÚcast)ÚTextSplitterÚ	TokenizerÚsplit_text_on_tokensc                      s~   e Zd ZU dZdddddd	d
œ‡ fdd„Zdd	dœdd„Zdddœdd„Zdddœdd„ZdZde	d< dddœdd„Z
‡  ZS )Ú%SentenceTransformersTokenTextSplitterz8Splitting text to tokens using sentence model tokenizer.é2   ú'sentence-transformers/all-mpnet-base-v2NÚintÚstrzOptional[int]r   ÚNone)Úchunk_overlapÚ
model_nameÚtokens_per_chunkÚkwargsÚreturnc              
     sŠ   t ƒ jf i |¤d|i¤Ž zddlm} W n2 ty\ } zd}t|ƒ|‚W Y d}~n
d}~0 0 || _|| jƒ| _| jj| _| j|d� dS )zCreate a new TextSplitter.r   r   )ÚSentenceTransformerz¸Could not import sentence_transformers python package. This is needed in order to for SentenceTransformersTokenTextSplitter. Please install it with `pip install sentence-transformers`.N)r   )	ÚsuperÚ__init__Zsentence_transformersr   ÚImportErrorr   Ú_modelÚ	tokenizerÚ_initialize_chunk_configuration)Úselfr   r   r   r   r   ÚerrÚmsg©Ú	__class__© úm/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/langchain_text_splitters/sentence_transformers.pyr      s    ÿ 
z.SentenceTransformersTokenTextSplitter.__init__)r   r   c                C  sX   | j j| _|d u r| j| _n|| _| j| jkrTd| j› d| j› d| j› d�}t|ƒ‚d S )NzThe token limit of the models 'z' is: z. Argument tokens_per_chunk=z > maximum token limit.)r   Zmax_seq_lengthZmaximum_tokens_per_chunkr   r   Ú
ValueError)r   r   r   r    r    r!   r   $   s    


ÿþÿzESentenceTransformersTokenTextSplitter._initialize_chunk_configurationz	list[str]©Útextr   c                   s8   dddœ‡ fdd„}t ˆ jˆ jˆ jj|d�}t||d�S )aú  Splits the input text into smaller components by splitting text on tokens.

        This method encodes the input text using a private `_encode` method, then
        strips the start and stop token IDs from the encoded result. It returns the
        processed segments as a list of strings.

        Args:
            text (str): The input text to be split.

        Returns:
            List[str]: A list of string components derived from the input text after
            encoding and processing.
        r   ú	list[int]r#   c                   s   ˆ   | ¡dd… S )Né   éÿÿÿÿ)Ú_encode)r$   ©r   r    r!   Ú%encode_strip_start_and_stop_token_idsF   s    z_SentenceTransformersTokenTextSplitter.split_text.<locals>.encode_strip_start_and_stop_token_ids)r   r   ÚdecodeÚencode)r$   r   )r   Z_chunk_overlapr   r   r+   r   )r   r$   r*   r   r    r)   r!   Ú
split_text7   s    üz0SentenceTransformersTokenTextSplitter.split_textc                C  s   t |  |¡ƒS )ay  Counts the number of tokens in the given text.

        This method encodes the input text using a private `_encode` method and
        calculates the total number of tokens in the encoded result.

        Args:
            text (str): The input text for which the token count is calculated.

        Returns:
            int: The number of tokens in the encoded text.
        )Úlenr(   )r   r$   r    r    r!   Úcount_tokensR   s    z2SentenceTransformersTokenTextSplitter.count_tokensl        Ú _max_length_equal_32_bit_integerr%   c                 C  s   | j j|| jdd�}td|ƒS )NZdo_not_truncate)Ú
max_lengthZ
truncationr%   )r   r,   r0   r   )r   r$   Z&token_ids_with_start_and_end_token_idsr    r    r!   r(   b   s    ýz-SentenceTransformersTokenTextSplitter._encode)r
   r   N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r-   r/   r0   Ú__annotations__r(   Ú__classcell__r    r    r   r!   r	      s   
   ür	   N)Ú
__future__r   Útypingr   r   r   Zlangchain_text_splitters.baser   r   r   r	   r    r    r    r!   Ú<module>   s   