a
     ˜ëj2  ã                   @   sÈ   d dl Z d dlmZ d dlmZ d dlmZ eddgd�Ze jed�e  	¡ d	d
„ ƒƒZ
e
 d¡e jddddd�e jdddddd�e jddddd�e jddddd�e jdddd d�d!d"„ ƒƒƒƒƒƒZdS )#é    N)Útqdm)Úword_tokenize)Úparallelize_preprocessz-hz--help)Zhelp_option_names)Zcontext_settingsc                   C   s   d S )N© r   r   r   úG/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/nltk/cli.pyÚcli   s    r   Útokenizez
--languagez-lÚenz1The language for the Punkt sentence tokenization.)ÚdefaultÚhelpz--preserve-lineTzIAn option to keep the preserve the sentence and not sentence tokenize it.)r
   Zis_flagr   z--processesz-jé   zNo. of processes.z
--encodingz-eÚutf8zSpecify encoding of file.z--delimiterz-dú z%Specify delimiter to join the tokens.c           	   	   C   sÂ   t jd|d�� }t jd|d��r}|dkrTt| ¡ ƒD ]}t| t|ƒ¡d|d� q4n.tt| ¡ |dd�D ]}t| |¡d|d� qhW d	  ƒ n1 s–0    Y  W d	  ƒ n1 s´0    Y  d	S )
z;This command tokenizes text stream using nltk.word_tokenizeÚstdin)ÚencodingÚstdoutr   Ú
)ÚendÚfileT)Úprogress_barN)ÚclickZget_text_streamr   Ú	readlinesÚprintÚjoinr   r   )	ÚlanguageZpreserve_lineZ	processesr   Ú	delimiterZfinZfoutÚlineÚoutliner   r   r   Útokenize_file   s    ÿ
r   )r   r   Znltkr   Z	nltk.utilr   ÚdictZCONTEXT_SETTINGSÚgroupZversion_optionr   ÚcommandÚoptionr   r   r   r   r   Ú<module>   s6   
üûÿ