a
     ˜ëj°  ã                   @   st   d dl Z d dlZd dlZd dlZd dlmZ d dlmZmZm	Z	m
Z
 d dlmZ d dlmZ dZG dd„ deƒZdS )	é    N)ÚPIPE)Ú_java_optionsÚconfig_javaÚfind_jarÚjava)ÚCoreNLPParser)Ú
TokenizerIz1https://nlp.stanford.edu/software/tokenizer.shtmlc                   @   s<   e Zd ZdZdZddd„Zed	d
„ ƒZdd„ Zddd„Z	dS )ÚStanfordTokenizeraF  
    Interface to the Stanford Tokenizer

    >>> from nltk.tokenize.stanford import StanfordTokenizer
    >>> s = "Good muffins cost $3.88\nin New York.  Please buy me\ntwo of them.\nThanks."
    >>> StanfordTokenizer().tokenize(s) # doctest: +SKIP
    ['Good', 'muffins', 'cost', '$', '3.88', 'in', 'New', 'York', '.', 'Please', 'buy', 'me', 'two', 'of', 'them', '.', 'Thanks', '.']
    >>> s = "The colour of the wall is blue."
    >>> StanfordTokenizer(options={"americanize": True}).tokenize(s) # doctest: +SKIP
    ['The', 'color', 'of', 'the', 'wall', 'is', 'blue', '.']
    zstanford-postagger.jarNÚutf8Fú-mx1000mc                 C   sf   t jtdƒtdd� t| j|ddt|d�| _|| _|| _	|d u rDi n|}d 
dd	„ | ¡ D ƒ¡| _d S )
Nzz
The StanfordTokenizer will be deprecated in version 3.2.5.
Please use [91mnltk.parse.corenlp.CoreNLPParser[0m instead.'é   )Ú
stacklevel)ZSTANFORD_POSTAGGER© )Zenv_varsZ
searchpathÚurlÚverboseú,c                 s   s    | ]\}}|› d |› �V  qdS )ú=Nr   )Ú.0ÚkeyÚvalr   r   úU/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/nltk/tokenize/stanford.pyÚ	<genexpr>E   ó    z-StanfordTokenizer.__init__.<locals>.<genexpr>)ÚwarningsÚwarnÚstrÚDeprecationWarningr   Ú_JARÚ_stanford_urlÚ_stanford_jarÚ	_encodingÚjava_optionsÚjoinÚitemsÚ_options_cmd)ÚselfZpath_to_jarÚencodingÚoptionsr   r!   r   r   r   Ú__init__%   s&    	ÿù
ú	zStanfordTokenizer.__init__c                 C   s   |   ¡ S )N)Ú
splitlines)Úsr   r   r   Ú_parse_tokenized_outputG   s    z)StanfordTokenizer._parse_tokenized_outputc                 C   s   dg}|   |  ||¡¡S )zW
        Use stanford tokenizer's PTBTokenizer to tokenize multiple sentences.
        z%edu.stanford.nlp.process.PTBTokenizer)r+   Ú_execute)r%   r*   Úcmdr   r   r   ÚtokenizeK   s    zStanfordTokenizer.tokenizec           
      C   sæ   | j }| d|g¡ | j}|r.| d| jg¡ d t¡}t| j|d� tjddd��f}t	|t
ƒrn|rn| |¡}| |¡ | ¡  | |j¡ t|| jttd�\}}	| |¡}W d   ƒ n1 sÀ0    Y  t |j¡ t|dd� |S )	Nz-charsetz-optionsú )r'   r   ÚwbF)ÚmodeÚdelete)Z	classpathÚstdoutÚstderr)r    Úextendr$   r"   r   r   r!   ÚtempfileÚNamedTemporaryFileÚ
isinstancer   ÚencodeÚwriteÚflushÚappendÚnamer   r   r   ÚdecodeÚosÚunlink)
r%   r-   Zinput_r   r&   r$   Zdefault_optionsZ
input_filer3   r4   r   r   r   r,   R   s(    



ÿ
(zStanfordTokenizer._execute)Nr
   NFr   )F)
Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r(   Ústaticmethodr+   r.   r,   r   r   r   r   r	      s        ú
"
r	   )Újsonr?   r6   r   Ú
subprocessr   Znltk.internalsr   r   r   r   Znltk.parse.corenlpr   Znltk.tokenize.apir   r   r	   r   r   r   r   Ú<module>	   s   