a
     ˜ëj.  ã                   @   s`   d dl Z d dlmZ d dlmZ d dlmZ d dlmZ G dd„ de j	ƒZ
G dd	„ d	e j	ƒZdS )
é    N)Úclosing)Údata)ÚPorterStemmer)ÚSnowballStemmerc                   @   s4   e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zd	d
„ ZdS )ÚSnowballTestc                 C   s\  t ddƒ}| d¡dksJ ‚| d¡dks.J ‚| d¡dks@J ‚| d¡d	ksRJ ‚| d
¡d	ksdJ ‚| d¡d	ksvJ ‚| d¡d	ksˆJ ‚| d¡dksšJ ‚| d¡dks¬J ‚t ddƒ}| d¡dksÈJ ‚| d¡d	ksÚJ ‚| d¡dksìJ ‚t dƒ}| d¡dk�sJ ‚| d¡dk�sJ ‚| d¡dk�s0J ‚| d¡d	k�sDJ ‚| d¡dk�sXJ ‚dS )z„
        this unit testing for test the snowball arabic light stemmer
        this stemmer deals with prefixes and suffixes
        ÚarabicTu&   Ø§Ù„Ù’Ø¹ÙŽØ±ÙŽØ¨Ù�Ù€Ù€Ù€Ù€Ù€Ù€ÙŠÙ‘ÙŽØ©u   Ø¹Ø±Ø¨u   Ø§Ù„Ø¹Ø±Ø¨ÙŠØ©u   Ù�Ù‚Ø§Ù„ÙˆØ§u   Ù‚Ø§Ù„u   Ø§Ù„Ø·Ø§Ù„Ø¨Ø§Øªu   Ø·Ø§Ù„Ø¨u   Ù�Ø§Ù„Ø·Ø§Ù„Ø¨Ø§Øªu   ÙˆØ§Ù„Ø·Ø§Ù„Ø¨Ø§Øªu   Ø§Ù„Ø·Ø§Ù„Ø¨ÙˆÙ†u   Ø§Ù„Ù„Ø°Ø§Ù†u   Ù…Ù†Fu   Ø§Ù„Ù„Ø°u   Ø§Ù„ÙƒÙ„Ù…Ø§Øªu   ÙƒÙ„Ù…N©r   Ústem)ÚselfZ
ar_stemmer© r   úW/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/nltk/test/unit/test_stem.pyÚtest_arabic
   s(    

zSnowballTest.test_arabicc                 C   s   t dƒ}| d¡dksJ ‚d S )NÚrussianu   Ð°Ð²Ð°Ð½Ñ‚Ð½ÐµÐ½ÑŒÐºÐ°Ñ�u   Ð°Ð²Ð°Ð½Ñ‚Ð½ÐµÐ½ÑŒÐºr   )r
   Zstemmer_russianr   r   r   Útest_russian'   s    zSnowballTest.test_russianc                 C   s`   t dƒ}t ddd�}| d¡dks&J ‚| d¡dks8J ‚| d¡dksJJ ‚| d¡dks\J ‚d S )NÚgermanT)Zignore_stopwordsu	   SchrÃ¤nkeZschrankZkeinenZkeinr   )r
   Zstemmer_germanZstemmer_german2r   r   r   Útest_german+   s    zSnowballTest.test_germanc                 C   s0   t dƒ}| d¡dksJ ‚| d¡dks,J ‚d S )NÚspanishZ	VisionadoZvisionZalgueZalgur   ©r
   Ústemmerr   r   r   Útest_spanish5   s    zSnowballTest.test_spanishc                 C   s   t dƒ}| d¡dksJ ‚d S )NÚenglishzy'sÚyr   r   r   r   r   Útest_short_strings_bug=   s    z#SnowballTest.test_short_strings_bugN)Ú__name__Ú
__module__Ú__qualname__r   r   r   r   r   r   r   r   r   r   	   s
   
r   c                   @   sD   e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zd	d
„ Zdd„ Zdd„ Z	dS )Ú
PorterTestc                 C   sF   t t d¡jdd�ƒ�}| ¡  ¡ W  d   ƒ S 1 s80    Y  d S )Nz*stemmers/porter_test/porter_vocabulary.txtúutf-8©Úencoding)r   r   ÚfindÚopenÚreadÚ
splitlines©r
   Úfpr   r   r   Ú_vocabularyC   s    
ÿÿzPorterTest._vocabularyc                 C   sL   t |d�}t|  ¡ |ƒD ].\}}| |¡}||ksJ d ||||¡ƒ‚qd S )N)Úmodez*{} should stem to {} in {} mode but got {})r   Úzipr&   r	   Úformat)r
   Zstemmer_modeZexpected_stemsr   ÚwordZ	true_stemZour_stemr   r   r   Ú_test_against_expected_outputK   s    

ÿüþz(PorterTest._test_against_expected_outputc                 C   sP   t t d¡jdd�ƒ�&}|  tj| ¡  ¡ ¡ W d  ƒ n1 sB0    Y  dS )az  Tests all words from the test vocabulary provided by M Porter

        The sample vocabulary and output were sourced from
        https://tartarus.org/martin/PorterStemmer/voc.txt and
        https://tartarus.org/martin/PorterStemmer/output.txt
        and are linked to from the Porter Stemmer algorithm's homepage
        at https://tartarus.org/martin/PorterStemmer/
        z-stemmers/porter_test/porter_martin_output.txtr   r   N)	r   r   r    r!   r+   r   ZMARTIN_EXTENSIONSr"   r#   r$   r   r   r   Útest_vocabulary_martin_modeX   s    	
ÿÿÿz&PorterTest.test_vocabulary_martin_modec                 C   sP   t t d¡jdd�ƒ�&}|  tj| ¡  ¡ ¡ W d   ƒ n1 sB0    Y  d S )Nz+stemmers/porter_test/porter_nltk_output.txtr   r   )	r   r   r    r!   r+   r   ZNLTK_EXTENSIONSr"   r#   r$   r   r   r   Útest_vocabulary_nltk_modej   s    
ÿÿÿz$PorterTest.test_vocabulary_nltk_modec                 C   st   t t d¡jdd�ƒ�&}|  tj| ¡  ¡ ¡ W d   ƒ n1 sB0    Y  |  tjt d¡jdd� ¡  ¡ ¡ d S )Nz/stemmers/porter_test/porter_original_output.txtr   r   )	r   r   r    r!   r+   r   ZORIGINAL_ALGORITHMr"   r#   r$   r   r   r   Útest_vocabulary_original_modet   s    
ÿÿÿ"
ÿþz(PorterTest.test_vocabulary_original_modec                 C   s   t ƒ  d¡dksJ ‚dS )z„Test for bug https://github.com/nltk/nltk/issues/1581

        Ensures that 'oed' can be stemmed without throwing an error.
        ZoedÚoN©r   r	   )r
   r   r   r   Útest_oed_bugŒ   s    zPorterTest.test_oed_bugc                 C   sl   t ƒ }| d¡dksJ ‚| d¡dks*J ‚|jddd�dks@J ‚| d¡dksRJ ‚|jddd�dkshJ ‚d	S )
z�Test for improvement on https://github.com/nltk/nltk/issues/2507

        Ensures that stems are lowercased when `to_lowercase=True`
        ZOnÚonÚIÚiF)Zto_lowercaseZGithubZgithubNr0   )r
   Zporterr   r   r   Útest_lowercase_option“   s    z PorterTest.test_lowercase_optionN)
r   r   r   r&   r+   r,   r-   r.   r1   r5   r   r   r   r   r   B   s   
r   )ZunittestÚ
contextlibr   Znltkr   Znltk.stem.porterr   Znltk.stem.snowballr   ZTestCaser   r   r   r   r   r   Ú<module>   s   9