a
     ˜ëj]%  ã                   @   sp   d dl Z d dlZd dlZd dlZd dlmZ d dlmZmZm	Z	m
Z
mZmZ d dlmZ dZG dd„ deƒZdS )é    N)ÚPIPE)Ú_java_optionsÚconfig_javaÚfind_dirÚ	find_fileÚfind_jarÚjava)Ú
TokenizerIz!https://nlp.stanford.edu/softwarec                       sX   e Zd ZdZdZddd	„Zd
d„ Z‡ fdd„Zdd„ Zdd„ Z	dd„ Z
ddd„Z‡  ZS )ÚStanfordSegmenteru[  Interface to the Stanford Segmenter

    If stanford-segmenter version is older than 2016-10-31, then path_to_slf4j
    should be provieded, for example::

        seg = StanfordSegmenter(path_to_slf4j='/YOUR_PATH/slf4j-api.jar')

    >>> from nltk.tokenize.stanford_segmenter import StanfordSegmenter
    >>> seg = StanfordSegmenter() # doctest: +SKIP
    >>> seg.default_config('zh') # doctest: +SKIP
    >>> sent = u'è¿™æ˜¯æ–¯å�¦ç¦�ä¸­æ–‡åˆ†è¯�å™¨æµ‹è¯•'
    >>> print(seg.segment(sent)) # doctest: +SKIP
    è¿™ æ˜¯ æ–¯å�¦ç¦� ä¸­æ–‡ åˆ†è¯�å™¨ æµ‹è¯•
    <BLANKLINE>
    >>> seg.default_config('ar') # doctest: +SKIP
    >>> sent = u'Ù‡Ø°Ø§ Ù‡Ùˆ ØªØµÙ†ÙŠÙ� Ø³ØªØ§Ù†Ù�ÙˆØ±Ø¯ Ø§Ù„Ø¹Ø±Ø¨ÙŠ Ù„Ù„ÙƒÙ„Ù…Ø§Øª'
    >>> print(seg.segment(sent.split())) # doctest: +SKIP
    Ù‡Ø°Ø§ Ù‡Ùˆ ØªØµÙ†ÙŠÙ� Ø³ØªØ§Ù†Ù�ÙˆØ±Ø¯ Ø§Ù„Ø¹Ø±Ø¨ÙŠ Ù„ Ø§Ù„ÙƒÙ„Ù…Ø§Øª
    <BLANKLINE>
    zstanford-segmenter.jarNÚfalseúUTF-8Fú-mx2gc                 C   sÞ   t  dt¡ t jtdƒtdd� t  dt¡ t| j|ddt|d�}|d ur`td	|d
dt|d�}nd }tj	 
dd„ ||fD ƒ¡| _|| _|| _|| _|| _|| _|| _|	| _|| _|
d u r¼i n|
}
d 
dd„ |
 ¡ D ƒ¡| _d S )NÚalwaysz}
The StanfordTokenizer will be deprecated in version 3.2.5.
Please use [91mnltk.parse.corenlp.CoreNLPTokenizer[0m instead.'é   )Ú
stacklevelÚignore©ÚSTANFORD_SEGMENTER© )Úenv_varsÚ
searchpathÚurlÚverbosezslf4j-api.jar)ZSLF4Jr   c                 s   s   | ]}|d ur|V  qd S ©Nr   )Ú.0Ú_r   r   ú_/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/nltk/tokenize/stanford_segmenter.pyÚ	<genexpr>j   s   z-StanfordSegmenter.__init__.<locals>.<genexpr>ú,c                 s   s&   | ]\}}|› d t  |¡› �V  qdS )ú=N)ÚjsonÚdumps)r   ÚkeyÚvalr   r   r   r   x   s   )ÚwarningsÚsimplefilterÚDeprecationWarningÚwarnÚstrr   Ú_JARÚ_stanford_urlÚosÚpathsepÚjoinÚ_stanford_jarÚ_java_classÚ_modelÚ_sihan_corpora_dictÚ_sihan_post_processingÚ_keep_whitespacesÚ_dictÚ	_encodingÚjava_optionsÚitemsÚ_options_cmd)ÚselfZpath_to_jarZpath_to_slf4jZ
java_classZpath_to_modelÚpath_to_dictZpath_to_sihan_corpora_dictZsihan_post_processingZkeep_whitespacesÚencodingÚoptionsr   r6   Zstanford_segmenterZslf4jr   r   r   Ú__init__8   sT    ÿù	úú	ÿ

ÿzStanfordSegmenter.__init__c              
   C   sx  d}t j d¡r(t j t j d¡d¡h}d| _d| _d| _|dkrNd| _d}nØ|d	k�rd
| _d}d| _d}zt	||t
ddd�| _W n2 ty¶ } ztd| ƒ|‚W Y d}~n
d}~0 0 d}z$t|t
ddd�}t j ||¡| _W n4 t�y } ztd| ƒ|‚W Y d}~n
d}~0 0 ntd|› �ƒ‚zt	||t
ddd�| _W n4 t�yr } ztd| ƒ|‚W Y d}~n
d}~0 0 dS )z©
        Attempt to initialize Stanford Word Segmenter for the specified language
        using the STANFORD_SEGMENTER and STANFORD_MODELS environment variables
        r   r   ÚdataNr   Úarz=edu.stanford.nlp.international.arabic.process.ArabicSegmenterz'arabic-segmenter-atb+bn+arztrain.ser.gzÚzhz%edu.stanford.nlp.ie.crf.CRFClassifierzpku.gzÚtruezdict-chris6.ser.gzF)ÚSTANFORD_MODELS)r   r   r   r   z_Could not find '%s' (tried using env. variables STANFORD_MODELS and <STANFORD_SEGMENTER>/data/)z./data/r   )r   r   r   zMCould not find '%s' (tried using the STANFORD_SEGMENTER environment variable)zUnsupported language )rB   r   )r+   ÚenvironÚgetÚpathr-   r4   r1   r2   r/   r   r*   ÚLookupErrorr   r0   )r9   ÚlangÚsearch_pathÚmodelr:   ÚeZ	sihan_dirZpath_to_sihan_dirr   r   r   Údefault_config|   s‚    ÿ
ûþÿüüÿÿýûÿÿýz StanfordSegmenter.default_configc                    s   t ƒ  |¡ d S r   )ÚsuperÚtokenize)r9   Ús©Ú	__class__r   r   rM   Å   s    zStanfordSegmenter.tokenizec                 C   sL   | j d| jd| jd|g}| jdur>| d| jd| jd| jg¡ |  |¡}|S )ú ú-loadClassifierú-keepAllWhitespacesú	-textFileNú-serDictionaryú-sighanCorporaDictú-sighanPostProcessing)r/   r0   r3   r1   Úextendr4   r2   Ú_execute)r9   Zinput_file_pathÚcmdÚstdoutr   r   r   Úsegment_fileÈ   s(    ù	
úÿ
zStanfordSegmenter.segment_filec                 C   s   |   |g¡S r   )Úsegment_sents)r9   Útokensr   r   r   Úsegmentã   s    zStanfordSegmenter.segmentc                 C   s¼   | j }tjdd�\}| _t |d¡}d dd„ |D ƒ¡}t|tƒrP|rP| 	|¡}| 
|¡ | ¡  | jd| jd| jd	| jg}| jd
ur¢| d| jd| jd| jg¡ |  |¡}t | j¡ |S )rQ   T)ÚtextÚwbÚ
c                 s   s   | ]}d   |¡V  qdS )rQ   N)r-   )r   Úxr   r   r   r   î   ó    z2StanfordSegmenter.segment_sents.<locals>.<genexpr>rR   rS   rT   NrU   rV   rW   )r5   ÚtempfileÚmkstempZ_input_file_pathr+   Úfdopenr-   Ú
isinstancer(   ÚencodeÚwriteÚcloser/   r0   r3   r1   rX   r4   r2   rY   Úunlink)r9   Ú	sentencesr;   Z	_input_fhÚ_inputrZ   r[   r   r   r   r]   æ   s:    

ù	
úÿ
zStanfordSegmenter.segment_sentsc                 C   sv   | j }| d|g¡ | j}|r.| d| jg¡ d t¡}t| j|d� t|| jt	t	d�\}}| 
|¡}t|dd� |S )Nz-inputEncodingz-optionsrQ   )r<   r   )Z	classpathr[   ÚstderrF)r5   rX   r8   r-   r   r   r6   r   r.   r   Údecode)r9   rZ   r   r;   r8   Zdefault_optionsr[   Ú_stderrr   r   r   rY     s    

ÿ

zStanfordSegmenter._execute)NNNNNNr   r   r   NFr   )F)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r)   r=   rK   rM   r\   r_   r]   rY   Ú__classcell__r   r   rO   r   r
       s*               ó
DI*r
   )r    r+   re   r$   Ú
subprocessr   Znltk.internalsr   r   r   r   r   r   Znltk.tokenize.apir	   r*   r
   r   r   r   r   Ú<module>   s    