a
     ˜ëjHy  ã                   @   sŒ   d Z ddlmZmZ ddlZddlmZmZmZm	Z	m
Z
mZmZmZ ddlmZ dd„ Zejjeƒ  dd	�ZG d
d„ dƒZG dd„ dƒZdS )zC
Unit tests for nltk.tokenize.
See also nltk/test/tokenize.doctest
é    )ÚListÚTupleN)ÚLegalitySyllableTokenizerÚStanfordSegmenterÚSyllableTokenizerÚTreebankWordTokenizerÚTweetTokenizerÚpunktÚsent_tokenizeÚword_tokenize)ÚCharTokenizerc                  C   s:   z t ƒ } |  d¡ |  d¡ W dS  ty4   Y dS 0 d S )NÚarÚzhTF)r   Údefault_configÚLookupError)Úseg© r   ú[/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/nltk/test/unit/test_tokenize.pyÚload_stanford_segmenter   s    

r   z/NLTK was unable to find stanford-segmenter.jar.)Úreasonc                   @   s*  e Zd Zdd„ Zej ddg d¢g d¢ffdg d¢g d	¢ffd
g d¢g d¢ffdg d¢g d¢ffdg d¢g d¢ffdg d¢g d¢ffdg d¢g d¢ffdg d¢g d¢ffg¡eee	e e	e f dœdd„ƒZ
dd„ Zdd „ Zd!d"„ Zed#d$„ ƒZed%d&„ ƒZd'd(„ Zd)d*„ Zd+d,„ Zd-d.„ Zd/d0„ Zd1d2„ Zd3d4„ Zd5d6„ Zd7d8„ Zd9d:„ Zd;d<„ Zd=d>„ Zej d?g d@¢¡dedBdC„ƒZdDdE„ Zej dFdGg dH¢fdIg dJ¢fdKdKgfdLdLgfdMdNdOgfdPdQdRgfdSg dT¢fdUdVdOgfdWdXdOgfdYg dZ¢fd[g dZ¢fd\d]d^gfg¡ee	e d_œd`da„ƒZdAdbœdcdd„Z dAS )fÚTestTokenizec                 C   s2   t ddd�}d}| |¡}g d¢}||ks.J ‚dS )zW
        Test TweetTokenizer using words with special and accented characters.
        T)Ústrip_handlesÚ
reduce_lenuA   @myke: Let's test these words: resumÃ© EspaÃ±a MÃ¼nchen franÃ§ais)
ú:zLet'sÚtestZtheseÚwordsr   u   resumÃ©u   EspaÃ±au   MÃ¼nchenu	   franÃ§aisN©r   Útokenize)ÚselfÚ	tokenizerZs9ÚtokensÚexpectedr   r   r   Útest_tweet_tokenizer(   s
    
z!TestTokenize.test_tweet_tokenizerztest_input, expectedsz#My text 0106404243030 is great text)ÚMyÚtextÚ01064042430Z30ÚisÚgreatr$   )r#   r$   Z0106404243030r&   r'   r$   zMy ticket id is 1234543124123)r#   ÚticketÚidr&   Z12345431241Z23)r#   r(   r)   r&   Z1234543124123z<@remy: This is waaaaayyyy too much for you!!!!!! 01064042430)r   ÚThisr&   ZwaaayyyZtooZmuchÚforÚyouú!r-   r-   r%   z*My number is 06-46124080, except it's not.)	r#   Únumberr&   z06-46124080ú,Úexceptúit'sÚnotÚ.z+My number is 601-984-4813, except it's not.)	r#   r.   r&   z601-984-4813r/   r0   r1   r2   r3   )
r#   r.   r&   z601-984-Z4813r/   r0   r1   r2   r3   z/My number is (393)  928 -3010, except it's not.)	r#   r.   r&   ú(393)  928 -3010r/   r0   r1   r2   r3   )r#   r.   r&   ú(Ú393ú)Z928ú-Z3010r/   r0   r1   r2   r3   z1The product identification number is 48103284512.)ÚTheÚproductÚidentificationr.   r&   Z
4810328451Ú2r3   )r9   r:   r;   r.   r&   Z48103284512r3   z(My favourite substraction is 240 - 1353.)r#   Ú	favouriteÚsubstractionr&   z
240 - 1353r3   )r#   r=   r>   r&   Z240r8   Z1353r3   )Ú
test_inputÚ	expectedsc                 C   s@   t ddg|ƒD ],\}}tdd|d�}| |¡}||ksJ ‚qdS )a¦  
        Test `match_phone_numbers` in TweetTokenizer.

        Note that TweetTokenizer is also passed the following for these tests:
            * strip_handles=True
            * reduce_len=True

        :param test_input: The input string to tokenize using TweetTokenizer.
        :type test_input: str
        :param expecteds: A 2-tuple of tokenized sentences. The first of the two
            tokenized is the expected output of tokenization with `match_phone_numbers=True`.
            The second of the two tokenized lists is the expected output of tokenization
            with `match_phone_numbers=False`.
        :type expecteds: Tuple[List[str], List[str]]
        TF)r   r   Úmatch_phone_numbersN)Úzipr   r   )r   r?   r@   rA   r!   r   Z	predictedr   r   r   Útest_tweet_tokenizer_expanded>   s     Eý
z*TestTokenize.test_tweet_tokenizer_expandedc                 C   s$   t ƒ }| d¡}|g d¢ks J ‚dS )ú3
        Test SyllableTokenizer tokenizer.
        Zjustification)ZjusZtiÚfiÚcaZtionN©r   r   )r   r   r    r   r   r   Ú+test_sonority_sequencing_syllable_tokenizer  s    
z8TestTokenize.test_sonority_sequencing_syllable_tokenizerc                 C   s*   t ƒ }dd }| |¡}||gks&J ‚dS )rD   Ú9i'  NrG   )r   r   r$   r    r   r   r   Útest_syllable_tokenizer_numbers  s    
z,TestTokenize.test_syllable_tokenizer_numbersc                 C   s:   ddl m} d}t| ¡ ƒ}| |¡}|g d¢ks6J ‚dS )z;
        Test LegalitySyllableTokenizer tokenizer.
        r   )r   Z	wonderful)ZwonZderZfulN)Znltk.corpusr   r   r   )r   r   Z	test_wordr   r    r   r   r   Ú*test_legality_principle_syllable_tokenizer  s
    
z7TestTokenize.test_legality_principle_syllable_tokenizerc                 C   s:   t ƒ }| d¡ d}| | ¡ ¡}| ¡ g d¢ks6J ‚dS )zN
        Test the Stanford Word Segmenter for Arabic (default config)
        r   un   ÙŠØ¨Ø­Ø« Ø¹Ù„Ù… Ø§Ù„Ø­Ø§Ø³ÙˆØ¨ Ø§Ø³ØªØ®Ø¯Ø§Ù… Ø§Ù„Ø­ÙˆØ³Ø¨Ø© Ø¨Ø¬Ù…ÙŠØ¹ Ø§Ø´ÙƒØ§Ù„Ù‡Ø§ Ù„Ø­Ù„ Ø§Ù„Ù…Ø´ÙƒÙ„Ø§Øª)u   ÙŠØ¨Ø­Ø«u   Ø¹Ù„Ù…u   Ø§Ù„Ø­Ø§Ø³ÙˆØ¨u   Ø§Ø³ØªØ®Ø¯Ø§Ù…u   Ø§Ù„Ø­ÙˆØ³Ø¨Ø©u   Ø¨u   Ø¬Ù…ÙŠØ¹u
   Ø§Ø´ÙƒØ§Ù„u   Ù‡Ø§u   Ù„u   Ø­Ù„u   Ø§Ù„Ù…Ø´ÙƒÙ„Ø§ØªN©r   r   ÚsegmentÚsplit©r   r   ÚsentZsegmented_sentr   r   r   Útest_stanford_segmenter_arabic'  s
    
z+TestTokenize.test_stanford_segmenter_arabicc                 C   s:   t ƒ }| d¡ d}| | ¡ ¡}| ¡ g d¢ks6J ‚dS )zO
        Test the Stanford Word Segmenter for Chinese (default config)
        r   u$   è¿™æ˜¯æ–¯å�¦ç¦�ä¸­æ–‡åˆ†è¯�å™¨æµ‹è¯•)u   è¿™u   æ˜¯u	   æ–¯å�¦ç¦�u   ä¸­æ–‡u	   åˆ†è¯�å™¨u   æµ‹è¯•NrL   rO   r   r   r   Útest_stanford_segmenter_chinese?  s
    
z,TestTokenize.test_stanford_segmenter_chinesec                 C   sL   t ƒ }d}dg}| |¡}||ks&J ‚d}g d¢}| |¡}||ksHJ ‚dS )zT
        Test a string that resembles a phone number but contains a newline
        r4   z(393)
928 -3010)r5   r6   r7   z	928 -3010Nr   )r   r   Útest1r!   ÚresultÚtest2r   r   r   Útest_phone_tokenizerQ  s    

z!TestTokenize.test_phone_tokenizerc           
      C   sÒ   t ƒ }d}dg}| |¡}||ks&J ‚d}dg}| |¡}||ksFJ ‚d}g d¢}| |¡}||kshJ ‚d}g d¢}| |¡}||ksŠJ ‚d}g d¢}| |¡}||ks¬J ‚d	}	g d
¢}| |	¡}||ksÎJ ‚dS )zX
        Test a string that contains Emoji ZWJ Sequences and skin tone modifier
        u   ðŸ‘¨â€�ðŸ‘©â€�ðŸ‘§â€�ðŸ‘§u   ðŸ‘¨ðŸ�¿uŒ   ðŸ¤” ðŸ™ˆ me asÃ­, seðŸ˜Œ ds ðŸ’•ðŸ‘­ðŸ‘™ hello ðŸ‘©ðŸ�¾â€�ðŸŽ“ emoji hello ðŸ‘¨â€�ðŸ‘©â€�ðŸ‘¦â€�ðŸ‘¦ how are ðŸ˜Š you todayðŸ™…ðŸ�½ðŸ™…ðŸ�½)u   ðŸ¤”u   ðŸ™ˆÚmeu   asÃ­r/   Úseu   ðŸ˜ŒZdsu   ðŸ’•u   ðŸ‘­u   ðŸ‘™Úhellou   ðŸ‘©ðŸ�¾â€�ðŸŽ“ÚemojirY   u   ðŸ‘¨â€�ðŸ‘©â€�ðŸ‘¦â€�ðŸ‘¦ÚhowZareu   ðŸ˜Šr,   Útodayõ   ðŸ™…ðŸ�½r]   u   ðŸ‡¦ðŸ‡µðŸ‡µðŸ‡±ðŸ‡ª)u   ðŸ‡¦ðŸ‡µõ   ðŸ‡µðŸ‡±u   ðŸ‡ªu   Hi ðŸ‡¨ðŸ‡¦, ðŸ˜�!!)ZHiõ   ðŸ‡¨ðŸ‡¦r/   u   ðŸ˜�r-   r-   u   <3 ðŸ‡¨ðŸ‡¦ ðŸ¤� ðŸ‡µðŸ‡± <3)ú<3r_   u   ðŸ¤�r^   r`   Nr   )
r   r   rS   r!   rT   rU   Útest3Útest4Útest5Útest6r   r   r   Útest_emoji_tokenizerd  s2    





z!TestTokenize.test_emoji_tokenizerc                 C   s    d}g d¢}t |ƒ|ksJ ‚dS )zA
        Test padding of asterisk for word tokenization.
        z1This is a, *weird sentence with *asterisks in it.)r*   r&   Úar/   Ú*ZweirdÚsentenceÚwithrg   Z	asterisksÚinÚitr3   N©r   ©r   r$   r!   r   r   r   Útest_pad_asterisk¥  s    zTestTokenize.test_pad_asteriskc                 C   s    d}g d¢}t |ƒ|ksJ ‚dS )z@
        Test padding of dotdot* for word tokenization.
        zPWhy did dotdot.. not get tokenized but dotdotdot... did? How about manydots.....)ZWhyÚdidÚdotdotz..r2   ÚgetZ	tokenizedÚbutZ	dotdotdotz...ro   ú?ZHowZaboutZmanydotsz.....Nrl   rm   r   r   r   Útest_pad_dotdot»  s    zTestTokenize.test_pad_dotdotc                 C   sü   t dd�}d}g d¢}| |¡}||ks,J ‚d}g d¢}| |¡}||ksNJ ‚d}g d¢}| |¡}||kspJ ‚d	}g d
¢}| |¡}||ks’J ‚d}g d¢}| |¡}||ks´J ‚d}	g d¢}| |	¡}||ksÖJ ‚d}
g d¢}| |
¡}||ksøJ ‚dS )zW
        Test remove_handle() from casual.py with specially crafted edge cases
        T)r   z-@twitter hello @twi_tter_. hi @12345 @123news)rY   r3   Úhiu]   @n`@n~@n(@n)@n-@n=@n+@n\@n|@n[@n]@n{@n}@n;@n:@n'@n"@n/@n?@n.@n,@n<@n>@n @n
@n Ã±@n.Ã¼@n.Ã§@n.)ú`ú~r5   r7   r8   ú=ú+ú\ú|ú[ú]Ú{Ú}ú;r   ú'ú"ú/rs   r3   r/   ú<ú>õ   Ã±r3   õ   Ã¼r3   õ   Ã§r3   zKa@n j@n z@n A@n L@n Z@n 1@n 4@n 7@n 9@n 0@n _@n !@n @@n #@n $@n %@n &@n *@n)&rf   ú@nÚjr‰   Úzr‰   ÚAr‰   ÚLr‰   ÚZr‰   Ú1r‰   Ú4r‰   Ú7r‰   rI   r‰   Ú0r‰   Ú_r‰   r-   r‰   ú@r‰   ú#r‰   ú$r‰   ú%r‰   ú&r‰   rg   r‰   z@n!a @n#a @n$a @n%a @n&a @n*a)r-   rf   r•   rf   r–   rf   r—   rf   r˜   rf   rg   rf   zD@n!@n @n#@n @n$@n @n%@n @n&@n @n*@n @n@n @@n @n@@n @n_@n @n7@n @nj@n)r-   r‰   r•   r‰   r–   r‰   r—   r‰   r˜   r‰   rg   r‰   r‰   r‰   r”   r‰   r‰   r”   r‰   z@n_r‰   z@n7r‰   z@njr‰   z^@abcdefghijklmnopqrstuvwxyz @abcdefghijklmno1234 @abcdefghijklmno_ @abcdefghijklmnoendofhandle)ZpqrstuvwxyzÚ1234r“   Zendofhandlez^@abcdefghijklmnop@abcde @abcdefghijklmno@abcde @abcdefghijklmno_@abcde @abcdefghijklmno5@abcde)Úpú@abcdez@abcdefghijklmnor›   r“   r›   Ú5r›   Nr   )r   r   rS   r!   rT   rU   ra   rb   rc   rd   Ztest7r   r   r   Útest_remove_handleÔ  s:    


(





zTestTokenize.test_remove_handlec                 C   s|   t ƒ }d}g d¢}t| |¡ƒ}||ks,J ‚d}g d¢}t| |¡ƒ}||ksRJ ‚d}g d¢}t| |¡ƒ}||ksxJ ‚dS )zC
        Test TreebankWordTokenizer.span_tokenize function
        zNGood muffins cost $3.88
in New (York).  Please (buy) me
two of them.
(Thanks).))r   é   )é   é   )é   é   )é   é   )r¤   é   )é   é   )é   é   )é   é    )r«   é$   )r¬   é%   )r­   é&   )é(   é.   )é/   é0   )r²   é3   )r³   é4   )é5   é7   )é8   é;   )é<   é>   )é?   éD   )éE   éF   )r¾   éL   )r¿   éM   )rÀ   éN   zmThe DUP is similar to the "religious right" in the United States and takes a hardline stance on social issues)©r   é   ©rž   é   ©é   é
   ©é   r£   ©r¤   é   ©é   é   ©r§   r¨   ©r¨   r¬   ©r­   é*   ©rÓ   é+   ©é,   r°   ©r±   é2   ©r³   é9   ©é:   é@   ©éA   r¼   ©r½   éJ   ©éK   r¿   )rÀ   éU   )éV   é\   )é]   é_   )é`   éf   )ég   ém   zqThe DUP is similar to the "religious right" in the United States and takes a ``hardline'' stance on social issues)rÂ   rÄ   rÆ   rÉ   rË   rÍ   rÐ   rÑ   rÒ   rÔ   rÖ   rØ   rÚ   rÜ   rß   rá   rã   )rÀ   éO   )rî   éW   )rï   éY   )éZ   rê   )éa   éc   )éd   éj   )ék   éq   N)r   ÚlistÚspan_tokenize)r   r   rS   r!   rT   rU   ra   r   r   r   Útest_treebank_span_tokenizerm  s    z)TestTokenize.test_treebank_span_tokenizerc                 C   s<   d}g d¢}t |ƒ|ksJ ‚d}g d¢}t |ƒ|ks8J ‚dS )z-
        Test word_tokenize function
        z0The 'v', I've been fooled but I'll seek revenge.)r9   r�   Úvr�   r/   ÚIz'veZbeenZfooledrr   rü   z'llÚseekZrevenger3   z'v' 're')r�   rû   r�   z'rer�   Nrl   )r   rh   r!   r   r   r   Útest_word_tokenizeÎ  s    zTestTokenize.test_word_tokenizec                 C   sT   dddgfdg d¢fdg d¢fg}|D ](\}}dd	„ t  |¡D ƒ}||ks&J ‚q&d S )
NZ12©r�   r<   )r<   NZ123)rÿ   ©r<   Ú3)r  Nr™   )rÿ   r   )r  r�   )r�   Nc                 S   s   g | ]}|‘qS r   r   )Ú.0Úxr   r   r   Ú
<listcomp>ó  ó    z5TestTokenize.test_punkt_pair_iter.<locals>.<listcomp>)r	   Ú
_pair_iter)r   Z
test_casesr?   Zexpected_outputZactual_outputr   r   r   Útest_punkt_pair_iterë  s    


ýz!TestTokenize.test_punkt_pair_iterc                 C   s   t g ƒ}t |¡}t|ƒ d S ©N)Úiterr	   r  rø   )r   rk   Úgenr   r   r   Ú5test_punkt_pair_iter_handles_stop_iteration_exception÷  s    
zBTestTokenize.test_punkt_pair_iter_handles_stop_iteration_exceptionc                 C   s0   t  ¡ }G dd„ dƒ}|ƒ |_t| d¡ƒ d S )Nc                   @   s   e Zd Zdd„ ZdS )zkTestTokenize.test_punkt_tokenize_words_handles_stop_iteration_exception.<locals>.TestPunktTokenizeWordsMockc                 S   s   t g ƒS r  )r	  )r   Úsr   r   r   r     s    zyTestTokenize.test_punkt_tokenize_words_handles_stop_iteration_exception.<locals>.TestPunktTokenizeWordsMock.word_tokenizeN)Ú__name__Ú
__module__Ú__qualname__r   r   r   r   r   ÚTestPunktTokenizeWordsMock  s   r  r   )r	   ZPunktBaseClassÚ
_lang_varsrø   Z_tokenize_words)r   Úobjr  r   r   r   Ú:test_punkt_tokenize_words_handles_stop_iteration_exceptionÿ  s    zGTestTokenize.test_punkt_tokenize_words_handles_stop_iteration_exceptionc                 C   sB   G dd„ dt jƒ}t j|ƒ d�}d}g d¢}| |¡|ks>J ‚d S )Nc                   @   s   e Zd ZdZdS )zNTestTokenize.test_punkt_tokenize_custom_lang_vars.<locals>.BengaliLanguageVars)r3   rs   r-   u   à¥¤N©r  r  r  Zsent_end_charsr   r   r   r   ÚBengaliLanguageVars  s   r  )Ú	lang_varsõc  à¦‰à¦ªà¦°à¦¾à¦·à§�à¦Ÿà§�à¦°à¦ªà¦¤à¦¿ à¦¶à§�à¦°à§€ à¦�à¦® à¦­à§‡à¦™à§�à¦•à¦¾à¦‡à§Ÿà¦¾ à¦¨à¦¾à¦‡à¦¡à§� à¦¸à§‹à¦®à¦¬à¦¾à¦° à¦†à¦‡ à¦†à¦‡ à¦Ÿà¦¿ à¦¦à¦¿à¦²à§�à¦²à¦¿à¦° à¦¹à§€à¦°à¦• à¦œà§Ÿà¦¨à§�à¦¤à§€ à¦‰à¦¦à¦¯à¦¾à¦ªà¦¨à§‡à¦° à¦‰à¦¦à§�à¦¬à§‹à¦§à¦¨ à¦•à¦°à§‡à¦›à§‡à¦¨à¥¤ à¦…à¦¨à¦²à¦¾à¦‡à¦¨à§‡à¦° à¦®à¦¾à¦§à§�à¦¯à¦®à§‡ à¦�à¦‡ à¦…à¦¨à§�à¦·à§�à¦ à¦¾à¦¨à§‡ à¦•à§‡à¦¨à§�à¦¦à§�à¦°à§€à§Ÿ à¦®à¦¾à¦¨à¦¬ à¦¸à¦®à§�à¦ªà¦¦ à¦‰à¦¨à§�à¦¨à§Ÿà¦¨à¦®à¦¨à§�à¦¤à§�à¦°à§€ à¦¶à§�à¦°à§€ à¦°à¦®à§‡à¦¶ à¦ªà§‹à¦–à¦°à¦¿à§Ÿà¦¾à¦² â€˜à¦¨à¦¿à¦¶à¦¾à¦™à§�à¦•â€™  à¦‰à¦ªà¦¸à§�à¦¥à¦¿à¦¤ à¦›à¦¿à¦²à§‡à¦¨à¥¤ à¦�à¦‡ à¦‰à¦ªà¦²à¦•à§�à¦·à§�à¦¯à§‡ à¦‰à¦ªà¦°à¦¾à¦·à§�à¦Ÿà§�à¦°à¦ªà¦¤à¦¿ à¦¹à§€à¦°à¦•à¦œà§Ÿà¦¨à§�à¦¤à§€à¦° à¦²à§‹à¦—à§‹ à¦�à¦¬à¦‚ à§¨à§¦à§©à§¦-à¦�à¦° à¦œà¦¨à§�à¦¯ à¦ªà§�à¦°à¦¤à¦¿à¦·à§�à¦ à¦¾à¦¨à§‡à¦° à¦²à¦•à§�à¦·à§�à¦¯ à¦“ à¦ªà¦°à¦¿à¦•à¦²à§�à¦ªà¦¨à¦¾à¦° à¦¨à¦¥à¦¿ à¦ªà§�à¦°à¦•à¦¾à¦¶ à¦•à¦°à§‡à¦›à§‡à¦¨à¥¤)u  à¦‰à¦ªà¦°à¦¾à¦·à§�à¦Ÿà§�à¦°à¦ªà¦¤à¦¿ à¦¶à§�à¦°à§€ à¦�à¦® à¦­à§‡à¦™à§�à¦•à¦¾à¦‡à§Ÿà¦¾ à¦¨à¦¾à¦‡à¦¡à§� à¦¸à§‹à¦®à¦¬à¦¾à¦° à¦†à¦‡ à¦†à¦‡ à¦Ÿà¦¿ à¦¦à¦¿à¦²à§�à¦²à¦¿à¦° à¦¹à§€à¦°à¦• à¦œà§Ÿà¦¨à§�à¦¤à§€ à¦‰à¦¦à¦¯à¦¾à¦ªà¦¨à§‡à¦° à¦‰à¦¦à§�à¦¬à§‹à¦§à¦¨ à¦•à¦°à§‡à¦›à§‡à¦¨à¥¤u+  à¦…à¦¨à¦²à¦¾à¦‡à¦¨à§‡à¦° à¦®à¦¾à¦§à§�à¦¯à¦®à§‡ à¦�à¦‡ à¦…à¦¨à§�à¦·à§�à¦ à¦¾à¦¨à§‡ à¦•à§‡à¦¨à§�à¦¦à§�à¦°à§€à§Ÿ à¦®à¦¾à¦¨à¦¬ à¦¸à¦®à§�à¦ªà¦¦ à¦‰à¦¨à§�à¦¨à§Ÿà¦¨à¦®à¦¨à§�à¦¤à§�à¦°à§€ à¦¶à§�à¦°à§€ à¦°à¦®à§‡à¦¶ à¦ªà§‹à¦–à¦°à¦¿à§Ÿà¦¾à¦² â€˜à¦¨à¦¿à¦¶à¦¾à¦™à§�à¦•â€™  à¦‰à¦ªà¦¸à§�à¦¥à¦¿à¦¤ à¦›à¦¿à¦²à§‡à¦¨à¥¤u/  à¦�à¦‡ à¦‰à¦ªà¦²à¦•à§�à¦·à§�à¦¯à§‡ à¦‰à¦ªà¦°à¦¾à¦·à§�à¦Ÿà§�à¦°à¦ªà¦¤à¦¿ à¦¹à§€à¦°à¦•à¦œà§Ÿà¦¨à§�à¦¤à§€à¦° à¦²à§‹à¦—à§‹ à¦�à¦¬à¦‚ à§¨à§¦à§©à§¦-à¦�à¦° à¦œà¦¨à§�à¦¯ à¦ªà§�à¦°à¦¤à¦¿à¦·à§�à¦ à¦¾à¦¨à§‡à¦° à¦²à¦•à§�à¦·à§�à¦¯ à¦“ à¦ªà¦°à¦¿à¦•à¦²à§�à¦ªà¦¨à¦¾à¦° à¦¨à¦¥à¦¿ à¦ªà§�à¦°à¦•à¦¾à¦¶ à¦•à¦°à§‡à¦›à§‡à¦¨à¥¤)r	   ÚPunktLanguageVarsÚPunktSentenceTokenizerr   )r   r  r  Ú	sentencesr!   r   r   r   Ú$test_punkt_tokenize_custom_lang_vars
  s
    z1TestTokenize.test_punkt_tokenize_custom_lang_varsc                 C   s(   t  ¡ }d}dg}| |¡|ks$J ‚d S )Nr  )r	   r  r   )r   r  r  r!   r   r   r   Ú'test_punkt_tokenize_no_custom_lang_vars  s
    ÿz4TestTokenize.test_punkt_tokenize_no_custom_lang_varsz%input_text,n_sents,n_splits,lang_vars))z4Subject: Some subject. Attachments: Some attachmentsé   é   )z4Subject: Some subject! Attachments: Some attachmentsr  r  )z4This is just a normal sentence, just like any other.r  r   Nc                 C   sJ   t  ¡ }|d kr||_t| |¡ƒ|ks,J ‚tt| |¡ƒƒ|ksFJ ‚d S r  )r	   r  r  Úlenr   rø   Zdebug_decisions)r   Z
input_textÚn_sentsÚn_splitsr  r   r   r   r   Úpunkt_debug_decisions&  s
    z"TestTokenize.punkt_debug_decisionsc                 C   s*   G dd„ dt jƒ}| jddd|ƒ d� d S )Nc                   @   s   e Zd ZdZdS )zGTestTokenize.test_punkt_debug_decisions_custom_end.<locals>.ExtLangVars)r3   rs   r-   ú^Nr  r   r   r   r   ÚExtLangVarsB  s   r$  z4Subject: Some subject^ Attachments: Some attachmentsr  r  )r   r!  r  )r	   r  r"  )r   r$  r   r   r   Ú%test_punkt_debug_decisions_custom_end?  s    üz2TestTokenize.test_punkt_debug_decisions_custom_endzsentences, expectedzthis is a test. . new sentence.)zthis is a test.r3   znew sentence.zThis. . . That)úThis.r3   r3   ÚThatzThis..... ThatzThis... ThatzThis.. . ThatzThis.. .r'  zThis. .. Thatr&  z.. ThatzThis. ,. That)r&  z,.r'  zThis!!! ThatzThis!!!z
This! ThatzThis!z+1. This is R .
2. This is A .
3. That's all)z1.zThis is R .z2.zThis is A .z3.z
That's allz+1. This is R .	2. This is A .	3. That's allzHello.	TherezHello.ZThere)r  r!   c                 C   s   t |ƒ|ksJ ‚d S r  )r
   )r   r  r!   r   r   r   Útest_sent_tokenizeN  s    zTestTokenize.test_sent_tokenize©Úreturnc                 C   s>   d}t ƒ }| |¡t|ƒks J ‚t| |¡ƒg d¢ks:J ‚d S )NzHello there))r   r  )r  r  )r  rÃ   )rÃ   rž   )rž   rŸ   )rŸ   é   )r+  rÅ   )rÅ   rÇ   )rÇ   é	   )r,  rÈ   )rÈ   rÊ   )r   r   rø   rù   )r   rh   r   r   r   r   Útest_string_tokenizerk  s    z"TestTokenize.test_string_tokenizer)N)!r  r  r  r"   ÚpytestÚmarkZparametrizeÚstrr   r   rC   rH   rJ   rK   Úcheck_stanford_segmenterrQ   rR   rV   re   rn   rt   r�   rú   rþ   r  r  r  r  r  r"  r%  r(  r-  r   r   r   r   r   '   sÆ   þþþþñþ&ôýôýôý#
õýþý€ Úþ 4ÿ	

A aþþ





þþ
ëþr   c                   @   s6   e Zd Zddœdd„Zddœdd„Zddœdd„ZdS )	ÚTestPunktTrainerNr)  c                 C   s   t  ¡ }| d¡ d S )NzThis is a test.©r	   ZPunktTrainerZtrain©r   Ztrainerr   r   r   Útest_punkt_train  s    z!TestPunktTrainer.test_punkt_trainc                 C   s   t  ¡ }| d¡ d S )Nr&  r3  r4  r   r   r   Útest_punkt_train_single_wordƒ  s    z-TestPunktTrainer.test_punkt_train_single_wordc                 C   s   t  ¡ }| d¡ d S )NzThis is a testr3  r4  r   r   r   Útest_punkt_train_no_punc‡  s    z)TestPunktTrainer.test_punkt_train_no_punc)r  r  r  r5  r6  r7  r   r   r   r   r2  ~  s   r2  )Ú__doc__Útypingr   r   r.  Znltk.tokenizer   r   r   r   r   r	   r
   r   Znltk.tokenize.simpler   r   r/  Zskipifr1  r   r2  r   r   r   r   Ú<module>   s"   (

þ      ]