a
    •˜ëj‚  ã                   @  s8   d dl mZ d dlmZ d dlmZ G dd„ deƒZdS )é    )Úannotations)ÚAny)ÚTextSplitterc                      sF   e Zd ZdZdddœddddd	d
œ‡ fdd„Zdddœdd„Z‡  ZS )ÚNLTKTextSplitterz"Splitting text using NLTK package.ú

ÚenglishF)Úuse_span_tokenizeÚstrÚboolr   ÚNone)Ú	separatorÚlanguager   ÚkwargsÚreturnc             
     s¦   t ƒ jf i |¤Ž || _|| _|| _| jr@| jdkr@d}t|ƒ‚z.ddl}| jrb|j | j¡| _	n
|jj
| _	W n2 ty  } zd}t|ƒ|‚W Y d}~n
d}~0 0 dS )zInitialize the NLTK splitter.Ú z6When use_span_tokenize is True, separator should be ''r   NzANLTK is not installed, please install it with `pip install nltk`.)ÚsuperÚ__init__Ú
_separatorÚ	_languageÚ_use_span_tokenizeÚ
ValueErrorÚnltkÚtokenizeZ_get_punkt_tokenizerÚ
_tokenizerZsent_tokenizeÚImportError)Úselfr   r   r   r   Úmsgr   Úerr©Ú	__class__© ú\/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/langchain_text_splitters/nltk.pyr      s    	zNLTKTextSplitter.__init__z	list[str])Útextr   c           	      C  s˜   | j rzt| j |¡ƒ}g }t|ƒD ]T\}\}}|dkr`||d  d }|||… |||…  }n|||… }| |¡ q"n| j|| jd�}|  || j¡S )z&Split incoming text and return chunks.r   é   )r   )	r   Úlistr   Zspan_tokenizeÚ	enumerateÚappendr   Z_merge_splitsr   )	r   r"   ZspansÚsplitsÚiÚstartÚendZprev_endZsentencer    r    r!   Ú
split_text&   s    zNLTKTextSplitter.split_text)r   r   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r+   Ú__classcell__r    r    r   r!   r      s     ýûr   N)Ú
__future__r   Útypingr   Zlangchain_text_splitters.baser   r   r    r    r    r!   Ú<module>   s   