a
    •˜ëj›  ã                   @  sP   d dl mZ d dlmZ d dlmZ G dd„ deƒZddœdd	d
dœdd„ZdS )é    )Úannotations)ÚAny)ÚTextSplitterc                	      sH   e Zd ZdZdddœdddd	d
ddœ‡ fdd„Zdddœdd„Z‡  ZS )ÚSpacyTextSplitteraQ  Splitting text using Spacy package.

    Per default, Spacy's `en_core_web_sm` model is used and
    its default max_length is 1000000 (it is the length of maximum character
    this model takes which can be increased for large files). For a faster, but
    potentially less accurate splitting, you can use `pipeline='sentencizer'`.
    ú

Úen_core_web_smé@B T)Ústrip_whitespaceÚstrÚintÚboolr   ÚNone)Ú	separatorÚpipelineÚ
max_lengthr	   ÚkwargsÚreturnc                  s0   t ƒ jf i |¤Ž t||d�| _|| _|| _dS )z#Initialize the spacy text splitter.©r   N)ÚsuperÚ__init__Ú"_make_spacy_pipeline_for_splittingÚ
_tokenizerÚ
_separatorÚ_strip_whitespace)Úselfr   r   r   r	   r   ©Ú	__class__© ú]/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/langchain_text_splitters/spacy.pyr      s    
ÿzSpacyTextSplitter.__init__z	list[str])Útextr   c                   s(   ‡ fdd„ˆ   |¡jD ƒ}ˆ  |ˆ j¡S )z&Split incoming text and return chunks.c                 3  s    | ]}ˆ j r|jn|jV  qd S )N)r   r   Ztext_with_ws)Ú.0Ús©r   r   r   Ú	<genexpr>$   s   ÿz/SpacyTextSplitter.split_text.<locals>.<genexpr>)r   ZsentsZ_merge_splitsr   )r   r   Úsplitsr   r"   r   Ú
split_text"   s    

þzSpacyTextSplitter.split_text)r   r   r   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r%   Ú__classcell__r   r   r   r   r      s   
   üú r   r   r   r
   r   r   )r   r   r   c             
   C  s‚   zdd l }W n2 ty> } zd}t|ƒ|‚W Y d }~n
d }~0 0 | dkrfddlm} |ƒ }| d¡ n|j| ddgd�}||_|S )Nr   zCSpacy is not installed, please install it with `pip install spacy`.Úsentencizer)ÚEnglishZnerZtagger)Úexclude)ÚspacyÚImportErrorZspacy.lang.enr,   Zadd_pipeÚloadr   )r   r   r.   ÚerrÚmsgr,   r+   r   r   r   r   +   s     r   N)Ú
__future__r   Útypingr   Zlangchain_text_splitters.baser   r   r   r   r   r   r   Ú<module>   s
   $ÿ