a
     ˜ëj-  ã                   @   s:  d Z ddlZddlZddlmZ ddlmZmZ ddlm	Z	 ddl
mZ ddlmZ ddlmZmZ dd	lmZmZmZmZmZmZmZ dd
lmZ ddlmZmZ ddlmZm Z m!Z!m"Z" ddl#m$Z$ ddl%m&Z& ddl'm(Z( ddl)m*Z* ddl+m,Z,m-Z- ddl.m/Z/m0Z0 ej1ddd„ƒZ2ddd„Z3e	ƒ Z4ddd„Z5dS )a@	  
NLTK Tokenizer Package

Tokenizers divide strings into lists of substrings.  For example,
tokenizers can be used to find the words and punctuation in a string:

    >>> from nltk.tokenize import word_tokenize
    >>> s = '''Good muffins cost $3.88\nin New York.  Please buy me
    ... two of them.\n\nThanks.'''
    >>> word_tokenize(s) # doctest: +NORMALIZE_WHITESPACE
    ['Good', 'muffins', 'cost', '$', '3.88', 'in', 'New', 'York', '.',
    'Please', 'buy', 'me', 'two', 'of', 'them', '.', 'Thanks', '.']

This particular tokenizer requires the Punkt sentence tokenization
models to be installed. NLTK also provides a simpler,
regular-expression based tokenizer, which splits text on whitespace
and punctuation:

    >>> from nltk.tokenize import wordpunct_tokenize
    >>> wordpunct_tokenize(s) # doctest: +NORMALIZE_WHITESPACE
    ['Good', 'muffins', 'cost', '$', '3', '.', '88', 'in', 'New', 'York', '.',
    'Please', 'buy', 'me', 'two', 'of', 'them', '.', 'Thanks', '.']

We can also operate at the level of sentences, using the sentence
tokenizer directly as follows:

    >>> from nltk.tokenize import sent_tokenize, word_tokenize
    >>> sent_tokenize(s)
    ['Good muffins cost $3.88\nin New York.', 'Please buy me\ntwo of them.', 'Thanks.']
    >>> [word_tokenize(t) for t in sent_tokenize(s)] # doctest: +NORMALIZE_WHITESPACE
    [['Good', 'muffins', 'cost', '$', '3.88', 'in', 'New', 'York', '.'],
    ['Please', 'buy', 'me', 'two', 'of', 'them', '.'], ['Thanks', '.']]

Caution: when tokenizing a Unicode string, make sure you are not
using an encoded version of the string (it may be necessary to
decode it first, e.g. with ``s.decode("utf8")``.

NLTK tokenizers can produce token-spans, represented as tuples of integers
having the same semantics as string slices, to support efficient comparison
of tokenizers.  (These methods are implemented as generators.)

    >>> from nltk.tokenize import WhitespaceTokenizer
    >>> list(WhitespaceTokenizer().span_tokenize(s)) # doctest: +NORMALIZE_WHITESPACE
    [(0, 4), (5, 12), (13, 17), (18, 23), (24, 26), (27, 30), (31, 36), (38, 44),
    (45, 48), (49, 51), (52, 55), (56, 58), (59, 64), (66, 73)]

There are numerous ways to tokenize text.  If you need more control over
tokenization, see the other methods provided in this package.

For further information, please see Chapter 3 of the NLTK book.
é    N)Úload)ÚTweetTokenizerÚcasual_tokenize)ÚNLTKWordTokenizer)ÚLegalitySyllableTokenizer)ÚMWETokenizer)ÚPunktSentenceTokenizerÚPunktTokenizer)ÚBlanklineTokenizerÚRegexpTokenizerÚWhitespaceTokenizerÚWordPunctTokenizerÚblankline_tokenizeÚregexp_tokenizeÚwordpunct_tokenize)ÚReppTokenizer)ÚSExprTokenizerÚsexpr_tokenize)ÚLineTokenizerÚSpaceTokenizerÚTabTokenizerÚline_tokenize)ÚSyllableTokenizer)ÚStanfordSegmenter)ÚTextTilingTokenizer)ÚToktokTokenizer)ÚTreebankWordDetokenizerÚTreebankWordTokenizer)Úregexp_span_tokenizeÚstring_span_tokenizeÚenglishc                 C   s   t | ƒS )z®
    A constructor for the PunktTokenizer that utilizes
    a lru cache for performance.

    :param language: the model name in the Punkt corpus
    :type language: str
    )r	   )Úlanguage© r"   úU/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/nltk/tokenize/__init__.pyÚ_get_punkt_tokenizer`   s    	r$   c                 C   s   t |ƒ}| | ¡S )a  
    Return a sentence-tokenized copy of *text*,
    using NLTK's recommended sentence tokenizer
    (currently :class:`.PunktSentenceTokenizer`
    for the specified language).

    :param text: text to split into sentences
    :param language: the model name in the Punkt corpus
    )r$   Útokenize)Útextr!   Ú	tokenizerr"   r"   r#   Úsent_tokenizem   s    
r(   Fc                 C   s"   |r
| gnt | |ƒ}dd„ |D ƒS )aê  
    Return a tokenized copy of *text*,
    using NLTK's recommended word tokenizer
    (currently an improved :class:`.TreebankWordTokenizer`
    along with :class:`.PunktSentenceTokenizer`
    for the specified language).

    :param text: text to split into words
    :type text: str
    :param language: the model name in the Punkt corpus
    :type language: str
    :param preserve_line: A flag to decide whether to sentence tokenize the text or not.
    :type preserve_line: bool
    c                 S   s    g | ]}t  |¡D ]}|‘qqS r"   )Ú_treebank_word_tokenizerr%   )Ú.0ÚsentÚtokenr"   r"   r#   Ú
<listcomp>�   s   z!word_tokenize.<locals>.<listcomp>)r(   )r&   r!   Zpreserve_lineÚ	sentencesr"   r"   r#   Úword_tokenize   s    ÿr/   )r    )r    )r    F)6Ú__doc__Ú	functoolsÚreZ	nltk.datar   Znltk.tokenize.casualr   r   Znltk.tokenize.destructiver   Z nltk.tokenize.legality_principler   Znltk.tokenize.mwer   Znltk.tokenize.punktr   r	   Znltk.tokenize.regexpr
   r   r   r   r   r   r   Znltk.tokenize.reppr   Znltk.tokenize.sexprr   r   Znltk.tokenize.simpler   r   r   r   Z!nltk.tokenize.sonority_sequencingr   Z nltk.tokenize.stanford_segmenterr   Znltk.tokenize.texttilingr   Znltk.tokenize.toktokr   Znltk.tokenize.treebankr   r   Znltk.tokenize.utilr   r   Ú	lru_cacher$   r(   r)   r/   r"   r"   r"   r#   Ú<module>
   s.   4$	
