a
    •˜ëj2  ã                   @  s   d dl mZ d dlZd dlZd dlmZmZ d dlmZ d dl	m
Z
 d dlmZmZmZmZmZmZmZ d dlmZmZ d dlmZ er¨d d	lmZmZmZ d d
lmZ e e¡Z eddd�Z!G dd„ deeƒZ"G dd„ de"ƒZ#G dd„ de$e
ƒZ%edd�G dd„ dƒƒZ&ddddœdd„Z'dS )é    )ÚannotationsN)ÚABCÚabstractmethod)Ú	dataclass)ÚEnum)ÚTYPE_CHECKINGÚAnyÚCallableÚLiteralÚOptionalÚTypeVarÚUnion)ÚBaseDocumentTransformerÚDocument)ÚSelf)Ú
CollectionÚIterableÚSequence)ÚSetÚTSÚTextSplitter)Úboundc                	   @  sä   e Zd ZdZddedddfddddd	d	d
dœdd„Zedddœdd„ƒZd5ddddœdd„Zdddœdd„Z	ddddœdd „Z
d!ddd"œd#d$„Zed%d%d d&œd'd(„ƒZed)deƒ d*fddd+d,d%d-d.œd/d0„ƒZd1d%d1d2œd3d4„ZdS )6r   z)Interface for splitting text into chunks.i   éÈ   FTÚintzCallable[[str], int]z$Union[bool, Literal['start', 'end']]ÚboolÚNone)Ú
chunk_sizeÚchunk_overlapÚlength_functionÚkeep_separatorÚadd_start_indexÚstrip_whitespaceÚreturnc                 C  s~   |dkrd|› �}t |ƒ‚|dk r4d|› �}t |ƒ‚||krVd|› d|› d�}t |ƒ‚|| _|| _|| _|| _|| _|| _dS )ad  Create a new TextSplitter.

        Args:
            chunk_size: Maximum size of chunks to return
            chunk_overlap: Overlap in characters between chunks
            length_function: Function that measures the length of given chunks
            keep_separator: Whether to keep the separator and where to place it
                            in each corresponding chunk (True='start')
            add_start_index: If `True`, includes chunk's start index in metadata
            strip_whitespace: If `True`, strips whitespace from the start and end of
                              every document
        r   zchunk_size must be > 0, got z chunk_overlap must be >= 0, got zGot a larger chunk overlap (z) than chunk size (z), should be smaller.N)Ú
ValueErrorÚ_chunk_sizeÚ_chunk_overlapÚ_length_functionZ_keep_separatorÚ_add_start_indexÚ_strip_whitespace)Úselfr   r   r   r   r    r!   Úmsg© r+   ú\/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/langchain_text_splitters/base.pyÚ__init__!   s$    

ÿÿzTextSplitter.__init__Ústrú	list[str]©Útextr"   c                 C  s   dS )z$Split text into multiple components.Nr+   )r)   r1   r+   r+   r,   Ú
split_textI   s    zTextSplitter.split_textNzOptional[list[dict[Any, Any]]]zlist[Document])ÚtextsÚ	metadatasr"   c                 C  sž   |pi gt |ƒ }g }t|ƒD ]z\}}d}d}|  |¡D ]^}	t || ¡}
| jr€|| | j }| |	td|ƒ¡}||
d< t |	ƒ}t	|	|
d�}| 
|¡ q8q|S )z&Create documents from a list of texts.r   Ústart_index)Úpage_contentÚmetadata)ÚlenÚ	enumerater2   ÚcopyÚdeepcopyr'   r%   ÚfindÚmaxr   Úappend)r)   r3   r4   Z
_metadatasÚ	documentsÚir1   ÚindexZprevious_chunk_lenÚchunkr7   ÚoffsetZnew_docr+   r+   r,   Úcreate_documentsM   s    zTextSplitter.create_documentszIterable[Document])r?   r"   c                 C  s:   g g  }}|D ]}|  |j¡ |  |j¡ q| j||d�S )zSplit documents.)r4   )r>   r6   r7   rD   )r)   r?   r3   r4   Údocr+   r+   r,   Úsplit_documentsa   s
    
zTextSplitter.split_documentsúOptional[str])ÚdocsÚ	separatorr"   c                 C  s(   |  |¡}| jr| ¡ }|dkr$d S |S )NÚ )Újoinr(   Ústrip)r)   rH   rI   r1   r+   r+   r,   Ú
_join_docsi   s    
zTextSplitter._join_docszIterable[str])ÚsplitsrI   r"   c           
      C  sV  |   |¡}g }g }d}|D �]}|   |¡}|| t|ƒdkr@|nd | jk�r|| jkrpt d|› d| j› �¡ t|ƒdk�r|  ||¡}	|	d urœ| |	¡ || jksÔ|| t|ƒdkr¼|nd | jk�r|dk�r||   |d ¡t|ƒdkrò|nd 8 }|dd … }qœ| |¡ ||t|ƒdk�r(|nd 7 }q|  ||¡}	|	d u�rR| |	¡ |S )Nr   zCreated a chunk of size z%, which is longer than the specified é   )r&   r8   r$   ÚloggerÚwarningrM   r>   r%   )
r)   rN   rI   Zseparator_lenrH   Zcurrent_docÚtotalÚdÚ_lenrE   r+   r+   r,   Ú_merge_splitsq   sJ    


ÿÿ
ÿÿ

ÿÿýÿ
 

zTextSplitter._merge_splitsr   )Ú	tokenizerÚkwargsr"   c              
     s€   z:ddl m} tˆ |ƒs$d}t|ƒ‚dddœ‡ fdd„}W n2 tyl } zd	}t|ƒ|‚W Y d
}~n
d
}~0 0 | f d|i|¤ŽS )z>Text splitter that uses HuggingFace tokenizer to count length.r   )ÚPreTrainedTokenizerBasezATokenizer received was not an instance of PreTrainedTokenizerBaser.   r   r0   c                   s   t ˆ  | ¡ƒS )N)r8   Útokenize©r1   ©rV   r+   r,   Ú_huggingface_tokenizer_length§   s    zNTextSplitter.from_huggingface_tokenizer.<locals>._huggingface_tokenizer_lengthz`Could not import transformers python package. Please install it with `pip install transformers`.Nr   )Z$transformers.tokenization_utils_baserX   Ú
isinstancer#   ÚImportError)ÚclsrV   rW   rX   r*   r\   Úerrr+   r[   r,   Úfrom_huggingface_tokenizer›   s    
ÿÿ z'TextSplitter.from_huggingface_tokenizerÚgpt2Úallú'Union[Literal['all'], AbstractSet[str]]ú&Union[Literal['all'], Collection[str]]r   ©Úencoding_nameÚ
model_nameÚallowed_specialÚdisallowed_specialrW   r"   c              
     s¬   zddl }W n2 ty> } zd}t|ƒ|‚W Y d}~n
d}~0 0 |durT| |¡‰n
| |¡‰dddœ‡ ‡‡fdd„}	t| tƒrš||ˆ ˆd	œ}
i |¥|
¥}| f d
|	i|¤ŽS )z9Text splitter that uses tiktoken encoder to count length.r   Nz”Could not import tiktoken python package. This is needed in order to calculate max_tokens_for_prompt. Please install it with `pip install tiktoken`.r.   r   r0   c                   s   t ˆj| ˆ ˆd�ƒS ©N)ri   rj   )r8   ÚencoderZ   ©ri   rj   Úencr+   r,   Ú_tiktoken_encoderË   s    ýÿz=TextSplitter.from_tiktoken_encoder.<locals>._tiktoken_encoder)rg   rh   ri   rj   r   )Útiktokenr^   Úencoding_for_modelÚget_encodingÚ
issubclassÚTokenTextSplitter)r_   rg   rh   ri   rj   rW   rp   r`   r*   ro   Úextra_kwargsr+   rm   r,   Úfrom_tiktoken_encoder²   s$    
ÿ 
	
üz"TextSplitter.from_tiktoken_encoderzSequence[Document])r?   rW   r"   c                 K  s   |   t|ƒ¡S )z2Transform sequence of documents by splitting them.)rF   Úlist)r)   r?   rW   r+   r+   r,   Útransform_documentsß   s    z TextSplitter.transform_documents)N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r8   r-   r   r2   rD   rF   rM   rU   Úclassmethodra   Úsetrv   rx   r+   r+   r+   r,   r      s0   ù( ÿ*û,c                      sL   e Zd ZdZddeƒ dfddddd	d
dœ‡ fdd„Zdddœdd„Z‡  ZS )rt   z/Splitting text to tokens using model tokenizer.rb   Nrc   r.   rG   rd   re   r   r   rf   c           
   
     s†   t ƒ jf i |¤Ž zddl}W n2 tyP } zd}t|ƒ|‚W Y d}~n
d}~0 0 |durf| |¡}	n
| |¡}	|	| _|| _|| _dS )zCreate a new TextSplitter.r   NzŠCould not import tiktoken python package. This is needed in order to for TokenTextSplitter. Please install it with `pip install tiktoken`.)	Úsuperr-   rp   r^   rq   rr   Ú
_tokenizerÚ_allowed_specialÚ_disallowed_special)
r)   rg   rh   ri   rj   rW   rp   r`   r*   rn   ©Ú	__class__r+   r,   r-   é   s    	ÿ 
zTokenTextSplitter.__init__r/   r0   c                   s8   dddœ‡ fdd„}t ˆ jˆ jˆ jj|d�}t||d�S )a  Splits the input text into smaller chunks based on tokenization.

        This method uses a custom tokenizer configuration to encode the input text
        into tokens, processes the tokens in chunks of a specified size with overlap,
        and decodes them back into text chunks. The splitting is performed using the
        `split_text_on_tokens` function.

        Args:
            text (str): The input text to be split into smaller chunks.

        Returns:
            List[str]: A list of text chunks, where each chunk is derived from a portion
            of the input text based on the tokenization and chunking rules.
        r.   z	list[int])Ú_textr"   c                   s   ˆ j j| ˆ jˆ jd�S rk   )r€   rl   r�   r‚   )r…   ©r)   r+   r,   Ú_encode  s
    ýz-TokenTextSplitter.split_text.<locals>._encode)r   Útokens_per_chunkÚdecoderl   )r1   rV   )Ú	Tokenizerr%   r$   r€   r‰   Úsplit_text_on_tokens)r)   r1   r‡   rV   r+   r†   r,   r2     s    üzTokenTextSplitter.split_text)ry   rz   r{   r|   r~   r-   r2   Ú__classcell__r+   r+   rƒ   r,   rt   æ   s   ûrt   c                   @  s|   e Zd ZdZdZdZdZdZdZdZ	dZ
d	Zd
ZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdS )ÚLanguagez"Enum of the programming languages.ÚcppZgoÚjavaZkotlinZjsÚtsZphpÚprotoÚpythonZrstÚrubyZrustZscalaZswiftZmarkdownZlatexÚhtmlZsolZcsharpZcobolÚcZluaZperlZhaskellZelixirZ
powershellZvisualbasic6N)ry   rz   r{   r|   ÚCPPZGOZJAVAZKOTLINZJSr   ZPHPÚPROTOÚPYTHONZRSTZRUBYZRUSTZSCALAZSWIFTZMARKDOWNZLATEXZHTMLZSOLZCSHARPZCOBOLÚCZLUAZPERLZHASKELLZELIXIRZ
POWERSHELLZVISUALBASIC6r+   r+   r+   r,   r�   &  s8   r�   T)Úfrozenc                   @  s2   e Zd ZU dZded< ded< ded< ded< d	S )
rŠ   zTokenizer data class.r   r   rˆ   zCallable[[list[int]], str]r‰   zCallable[[str], list[int]]rl   N)ry   rz   r{   r|   Ú__annotations__r+   r+   r+   r,   rŠ   F  s   
rŠ   r.   r/   )r1   rV   r"   c                 C  s’   g }|  | ¡}d}t||j t|ƒƒ}|||… }|t|ƒk rŽ| | |¡¡ |t|ƒkr\qŽ||j|j 7 }t||j t|ƒƒ}|||… }q2|S )z6Split incoming text and return chunks using tokenizer.r   )rl   Úminrˆ   r8   r>   r‰   r   )r1   rV   rN   Z	input_idsZ	start_idxZcur_idxZ	chunk_idsr+   r+   r,   r‹   T  s    
r‹   )(Ú
__future__r   r:   ÚloggingÚabcr   r   Zdataclassesr   Úenumr   Útypingr   r   r	   r
   r   r   r   Zlangchain_core.documentsr   r   Ztyping_extensionsr   Úcollections.abcr   r   r   r   ÚAbstractSetÚ	getLoggerry   rP   r   r   rt   r.   r�   rŠ   r‹   r+   r+   r+   r,   Ú<module>   s(   $

 I@ 