a
    ¡˜ëj		  ã                   @  sô   U d dl mZ d dlZd dlmZ d dlmZmZmZ d dl	Z	d dl	m
Z d dl	mZ d dl	mZ dZd	ed
< ddddœdd„Zdd„ Ze dd¡ ¡ dkr¢eƒ  eed�dddœdd„ƒZeed�dddœdd„ƒZeed�dddœdd„ƒZ
dS )é    )ÚannotationsN)Ú	lru_cache)ÚFinalÚListÚTuple)Úpos_tag)Úsent_tokenize)Úword_tokenizeé€   z
Final[int]ÚCACHE_MAX_SIZEÚstrÚbool)Úpackage_nameÚpackage_categoryÚreturnc              	   C  sp   g }t jjD ]&}| d¡s(tj |d¡}| |¡ qzt j|› d| › �|d� W dS  tt	fyj   Y dS 0 dS )z@Checks to see if the specified NLTK package exists on the image.Z	nltk_dataú/)ÚpathsTFN)
ÚnltkÚdataÚpathÚendswithÚosÚjoinÚappendÚfindÚLookupErrorÚOSError)r   r   r   r   © r   úX/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/unstructured/nlp/tokenize.pyÚcheck_for_nltk_package   s    
r   c                  C  s@   t ddd�} t ddd�}|r | s<tjddd� tjddd� dS )	z;If required NLTK packages are not available, download them.ZtaggersZaveraged_perceptron_tagger_eng)r   r   Z
tokenizersZ	punkt_tabT)ÚquietN)r   r   Údownload)Ztagger_availableZtokenizer_availabler   r   r   Údownload_nltk_packages   s    þÿr"   ZAUTO_DOWNLOAD_NLTKÚTrueÚtrue)Úmaxsizez	List[str])Útextr   c                 C  s   t | ƒS )zFA wrapper around the NLTK sentence tokenizer with LRU caching enabled.)Ú_sent_tokenize©r&   r   r   r   r   3   s    r   c                 C  s   t | ƒS )zBA wrapper around the NLTK word tokenizer with LRU caching enabled.)Ú_word_tokenizer(   r   r   r   r	   9   s    r	   zList[Tuple[str, str]]c                 C  s0   t | ƒ}g }|D ]}t|ƒ}| t|ƒ¡ q|S )z>A wrapper around the NLTK POS tagger with LRU caching enabled.)r'   r)   ÚextendÚ_pos_tag)r&   Ú	sentencesZparts_of_speechZsentenceÚtokensr   r   r   r   ?   s    r   )Ú
__future__r   r   Ú	functoolsr   Útypingr   r   r   r   r   r+   r   r'   r	   r)   r   Ú__annotations__r   r"   ÚgetenvÚlowerr   r   r   r   Ú<module>   s$   