a
     ˜ëj=  ã                
   @   sè   d Z ddlZddlmZ ddlZddlmZ dZdZdZ	dZ
eed	d
dddde	df
Zed e
gedd… ¢R Ze d¡Ze eejejB ejB ¡Ze d¡Ze d¡Zd%dd„Zd&dd„ZG dd„ deƒZdd„ Zd d!„ Zd'd#d$„ZdS )(a�  
Twitter-aware tokenizer, designed to be flexible and easy to adapt to new
domains and tasks. The basic logic is this:

1. The tuple REGEXPS defines a list of regular expression
   strings.

2. The REGEXPS strings are put, in order, into a compiled
   regular expression object called WORD_RE, under the TweetTokenizer
   class.

3. The tokenization is done by WORD_RE.findall(s), where s is the
   user-supplied string, inside the tokenize() method of the class
   TweetTokenizer.

4. When instantiating Tokenizer objects, there are several options:
    * preserve_case. By default, it is set to True. If it is set to
      False, then the tokenizer will downcase everything except for
      emoticons.
    * reduce_len. By default, it is set to False. It specifies whether
      to replace repeated character sequences of length 3 or greater
      with sequences of length 3.
    * strip_handles. By default, it is set to False. It specifies
      whether to remove Twitter handles of text used in the
      `tokenize` method.
    * match_phone_numbers. By default, it is set to True. It indicates
      whether the `tokenize` method should look for phone numbers.
é    N)ÚList)Ú
TokenizerIac  
    (?:
      [<>]?
      [:;=8]                     # eyes
      [\-o\*\']?                 # optional nose
      [\)\]\(\[dDpP/\:\}\{@\|\\] # mouth
      |
      [\)\]\(\[dDpP/\:\}\{@\|\\] # mouth
      [\-o\*\']?                 # optional nose
      [:;=8]                     # eyes
      [<>]?
      |
      </?3                       # heart
    )u  			# Capture 1: entire matched URL
  (?:
  https?:				# URL protocol and colon
    (?:
      /{1,3}				# 1-3 slashes
      |					#   or
      [a-z0-9%]				# Single letter or digit or '%'
                                       # (Trying not to match e.g. "URI::Escape")
    )
    |					#   or
                                       # looks like domain name followed by a slash:
    [a-z0-9.\-]+[.]
    (?:[a-z]{2,13})
    /
  )
  (?:					# One or more:
    [^\s()<>{}\[\]]+			# Run of non-space, non-()<>{}[]
    |					#   or
    \([^\s()]*?\([^\s()]+\)[^\s()]*?\) # balanced parens, one level deep: (...(...)...)
    |
    \([^\s]+?\)				# balanced parens, non-recursive: (...)
  )+
  (?:					# End with:
    \([^\s()]*?\([^\s()]+\)[^\s()]*?\) # balanced parens, one level deep: (...(...)...)
    |
    \([^\s]+?\)				# balanced parens, non-recursive: (...)
    |					#   or
    [^\s`!()\[\]{};:'".,<>?Â«Â»â€œâ€�â€˜â€™]	# not a space or one of these punct chars
  )
  |					# OR, the following to match naked domains:
  (?:
  	(?<!@)			        # not preceded by a @, avoid matching foo@_gmail.com_
    [a-z0-9]+
    (?:[.\-][a-z0-9]+)*
    [.]
    (?:[a-z]{2,13})
    \b
    /?
    (?!@)			        # not succeeded by a @,
                            # avoid matching "foo.na" in "foo.na@example.com"
  )
uÏ  
  (?:
    [\U0001F1E6-\U0001F1FF]{2}  # all enclosed letter pairs
    |
    # English flag
    \U0001F3F4\U000E0067\U000E0062\U000E0065\U000E006e\U000E0067\U000E007F
    |
    # Scottish flag
    \U0001F3F4\U000E0067\U000E0062\U000E0073\U000E0063\U000E0074\U000E007F
    |
    # For Wales? Why Richard, it profit a man nothing to give his soul for the whole world â€¦ but for Wales!
    \U0001F3F4\U000E0067\U000E0062\U000E0077\U000E006C\U000E0073\U000E007F
  )
a	  
    (?:
      (?:            # (international)
        \+?[01]
        [ *\-.\)]*
      )?
      (?:            # (area code)
        [\(]?
        \d{3}
        [ *\-.\)]*
      )?
      \d{3}          # exchange
      [ *\-.\)]*
      \d{4}          # base
    )z	<[^>\s]+>z[\-]+>|<[\-]+z(?:@[\w_]+)z(?:\#+[\w_]+[\w\'_\-]*[\w_]+)z#[\w.+-]+@[\w-]+\.(?:[\w-]\.?)+[\w-]uR   .(?:
        [ðŸ�»-ðŸ�¿]?(?:â€�.[ðŸ�»-ðŸ�¿]?)+
        |
        [ðŸ�»-ðŸ�¿]
    )a…  
    (?:[^\W\d_](?:[^\W\d_]|['\-_])+[^\W\d_]) # Words with apostrophes or dashes.
    |
    (?:[+\-]?\d+[,/.:-]\d+[+\-]?)  # Numbers, including fractions, decimals.
    |
    (?:[\w_]+)                     # Words without apostrophes or dashes.
    |
    (?:\.(?:\s*\.){1,})            # Ellipsis dots.
    |
    (?:\S)                         # Everything else that isn't whitespace.
    é   z([^a-zA-Z0-9])\1{3,}z&(#?(x?))([^&;\s]+);zZ(?<![A-Za-z0-9_!@#\$%&*])@(([A-Za-z0-9_]){15}(?!@)|([A-Za-z0-9_]){1,14}(?![A-Za-z0-9_]*@))Ústrictc                 C   s&   |d u rd}t | tƒr"|  ||¡S | S )Núutf-8)Ú
isinstanceÚbytesÚdecode)ÚtextÚencodingÚerrors© r   úS/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/nltk/tokenize/casual.pyÚ_str_to_unicodeì   s
    
r   r   Tr   c                    s    ‡ ‡fdd„}t  |t| |ƒ¡S )u·  
    Remove entities from text by converting them to their
    corresponding unicode character.

    :param text: a unicode string or a byte string encoded in the given
    `encoding` (which defaults to 'utf-8').

    :param list keep:  list of entity names which should not be replaced.    This supports both numeric entities (``&#nnnn;`` and ``&#hhhh;``)
    and named entities (such as ``&nbsp;`` or ``&gt;``).

    :param bool remove_illegal: If `True`, entities that can't be converted are    removed. Otherwise, entities that can't be converted are kept "as
    is".

    :returns: A unicode string with the entities removed.

    See https://github.com/scrapy/w3lib/blob/master/w3lib/html.py

        >>> from nltk.tokenize.casual import _replace_html_entities
        >>> _replace_html_entities(b'Price: &pound;100')
        'Price: \xa3100'
        >>> print(_replace_html_entities(b'Price: &pound;100'))
        Price: Â£100
        >>>
    c              	      sØ   |   d¡}|   d¡r|zN|   d¡r,t|dƒ}n
t|dƒ}d|  krJdkr`n nt|fƒ d¡W S W qœ tyx   d }Y qœ0 n |ˆ v rŽ|   d	¡S tjj |¡}|d urÆz
t	|ƒW S  tt
fyÄ   Y n0 ˆrÎd
S |   d	¡S )Né   r   é   é   é
   é€   éŸ   Úcp1252r   Ú )ÚgroupÚintr   r	   Ú
ValueErrorÚhtmlÚentitiesÚname2codepointÚgetÚchrÚOverflowError)ÚmatchZentity_bodyÚnumber©ÚkeepÚremove_illegalr   r   Ú_convert_entity  s&    





z/_replace_html_entities.<locals>._convert_entity)ÚENT_REÚsubr   )r
   r$   r%   r   r&   r   r#   r   Ú_replace_html_entitiesô   s    r)   c                   @   sZ   e Zd ZdZdZdZddd„Zeee dœdd	„Z	e
d
dœdd„ƒZe
d
dœdd„ƒZdS )ÚTweetTokenizeraÚ  
    Tokenizer for tweets.

        >>> from nltk.tokenize import TweetTokenizer
        >>> tknzr = TweetTokenizer()
        >>> s0 = "This is a cooool #dummysmiley: :-) :-P <3 and some arrows < > -> <--"
        >>> tknzr.tokenize(s0) # doctest: +NORMALIZE_WHITESPACE
        ['This', 'is', 'a', 'cooool', '#dummysmiley', ':', ':-)', ':-P', '<3', 'and', 'some', 'arrows', '<', '>', '->',
         '<--']

    Examples using `strip_handles` and `reduce_len parameters`:

        >>> tknzr = TweetTokenizer(strip_handles=True, reduce_len=True)
        >>> s1 = '@remy: This is waaaaayyyy too much for you!!!!!!'
        >>> tknzr.tokenize(s1)
        [':', 'This', 'is', 'waaayyy', 'too', 'much', 'for', 'you', '!', '!', '!']
    NTFc                 C   s   || _ || _|| _|| _dS )ae  
        Create a `TweetTokenizer` instance with settings for use in the `tokenize` method.

        :param preserve_case: Flag indicating whether to preserve the casing (capitalisation)
            of text used in the `tokenize` method. Defaults to True.
        :type preserve_case: bool
        :param reduce_len: Flag indicating whether to replace repeated character sequences
            of length 3 or greater with sequences of length 3. Defaults to False.
        :type reduce_len: bool
        :param strip_handles: Flag indicating whether to remove Twitter handles of text used
            in the `tokenize` method. Defaults to False.
        :type strip_handles: bool
        :param match_phone_numbers: Flag indicating whether the `tokenize` method should look
            for phone numbers. Defaults to True.
        :type match_phone_numbers: bool
        N©Úpreserve_caseÚ
reduce_lenÚstrip_handlesÚmatch_phone_numbers)Úselfr,   r-   r.   r/   r   r   r   Ú__init__L  s    zTweetTokenizer.__init__)r
   Úreturnc                 C   sl   t |ƒ}| jrt|ƒ}| jr$t|ƒ}t d|¡}| jrD| j 	|¡}n| j
 	|¡}| jshttdd„ |ƒƒ}|S )zÒTokenize the input text.

        :param text: str
        :rtype: list(str)
        :return: a tokenized list of strings; joining this list returns        the original string if `preserve_case=False`.
        ú\1\1\1c                 S   s   t  | ¡r| S |  ¡ S )N)ÚEMOTICON_REÚsearchÚlower)Úxr   r   r   Ú<lambda>‚  ó    z)TweetTokenizer.tokenize.<locals>.<lambda>)r)   r.   Úremove_handlesr-   Úreduce_lengtheningÚHANG_REr(   r/   ÚPHONE_WORD_REÚfindallÚWORD_REr,   ÚlistÚmap)r0   r
   Z	safe_textÚwordsr   r   r   Útokenizeh  s    	ÿzTweetTokenizer.tokenizezregex.Pattern)r2   c                 C   sB   t | ƒjs8t dd t¡› d�tjtjB tjB ¡t | ƒ_t | ƒjS )zCore TweetTokenizer regexú(ú|ú))	ÚtypeÚ_WORD_REÚregexÚcompileÚjoinÚREGEXPSÚVERBOSEÚIÚUNICODE©r0   r   r   r   r?   †  s    
þ
zTweetTokenizer.WORD_REc                 C   sB   t | ƒjs8t dd t¡› d�tjtjB tjB ¡t | ƒ_t | ƒjS )z#Secondary core TweetTokenizer regexrD   rE   rF   )	rG   Ú_PHONE_WORD_RErI   rJ   rK   ÚREGEXPS_PHONErM   rN   rO   rP   r   r   r   r=   ‘  s    
þ
zTweetTokenizer.PHONE_WORD_RE)TFFT)Ú__name__Ú
__module__Ú__qualname__Ú__doc__rH   rQ   r1   Ústrr   rC   Úpropertyr?   r=   r   r   r   r   r*   2  s       û

r*   c                 C   s   t  d¡}| d| ¡S )ze
    Replace repeated character sequences of length 3 or greater with sequences
    of length 3.
    z	(.)\1{2,}r3   )rI   rJ   r(   )r
   Úpatternr   r   r   r;   ¢  s    
r;   c                 C   s   t  d| ¡S )z4
    Remove Twitter username handles from text.
    ú )Ú
HANDLES_REr(   )r
   r   r   r   r:   «  s    r:   Fc                 C   s   t ||||d� | ¡S )z:
    Convenience function for wrapping the tokenizer.
    r+   )r*   rC   )r
   r,   r-   r.   r/   r   r   r   Úcasual_tokenize¸  s    
üûr\   )Nr   )r   Tr   )TFFT)rV   r   Útypingr   rI   Znltk.tokenize.apir   Z	EMOTICONSZURLSÚFLAGSZPHONE_REGEXrL   rR   rJ   r<   rM   rN   rO   r4   r'   r[   r   r)   r*   r;   r:   r\   r   r   r   r   Ú<module>   sH    3é%

ÿ

>p	    û