a
     ˜ëj�  ã                   @   sB   d dl Z d dlmZ d dlmZ d dlmZ G dd„ de jƒZdS )é    N)ÚCounter)Útimeit)Ú
Vocabularyc                   @   sÈ   e Zd ZdZedd„ ƒZdd„ Zdd„ Zdd	„ Zd
d„ Z	dd„ Z
dd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zd d!„ Zd"d#„ Zd$d%„ Zd&d'„ Zd(d)„ Zejd*d+�d,d-„ ƒZd.S )/ÚNgramModelVocabularyTestsztests Vocabulary Classc                 C   s   t g d¢dd�| _d S ©N)ÚzÚaÚbÚcÚfÚdÚeÚgr   r   r	   r   Úwé   ©Ú
unk_cutoff)r   Úvocab)Úcls© r   ú`/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/nltk/test/unit/lm/test_vocabulary.pyÚ
setUpClass   s    þz$NgramModelVocabularyTests.setUpClassc                 C   s   |   | j¡ d S ©N)Ú
assertTruer   ©Úselfr   r   r   Útest_truthiness   s    z)NgramModelVocabularyTests.test_truthinessc                 C   s   |   | jjd¡ d S )Nr   )ÚassertEqualr   Úcutoffr   r   r   r   Útest_cutoff_value_set_correctly   s    z9NgramModelVocabularyTests.test_cutoff_value_set_correctlyc                 C   s6   |   t¡� d| j_W d   ƒ n1 s(0    Y  d S )Né   )ÚassertRaisesÚAttributeErrorr   r   r   r   r   r   Útest_unable_to_change_cutoff   s    z6NgramModelVocabularyTests.test_unable_to_change_cutoffc                 C   sP   |   t¡�}tddd� W d   ƒ n1 s,0    Y  d}|  |t|jƒ¡ d S )NÚabcr   r   z*Cutoff value cannot be less than 1. Got: 0)r!   Ú
ValueErrorr   r   ÚstrÚ	exception)r   Úexc_infoZexpected_error_msgr   r   r   Útest_cutoff_setter_checks_value#   s    *z9NgramModelVocabularyTests.test_cutoff_setter_checks_valuec                 C   s@   |   | jjd d¡ |   | jjd d¡ |   | jjd d¡ d S )Nr   r   r	   r
   é   )r   r   Úcountsr   r   r   r   Útest_counts_set_correctly)   s    z3NgramModelVocabularyTests.test_counts_set_correctlyc                 C   s4   |   d| jv ¡ |  d| jv ¡ |  d| jv ¡ d S )Nr   r
   r   )r   r   ÚassertFalser   r   r   r   Ú%test_membership_check_respects_cutoff.   s    z?NgramModelVocabularyTests.test_membership_check_respects_cutoffc                 C   s   |   dt| jƒ¡ d S )Né   )r   Úlenr   r   r   r   r   Útest_vocab_len_respects_cutoff6   s    z8NgramModelVocabularyTests.test_vocab_len_respects_cutoffc                 C   s>   g d¢}g d¢}|   |t| jj ¡ ƒ¡ |   |t| jƒ¡ d S )N)	r   r	   r
   r   r   r   r   r   r   )r   r	   r   r   ú<UNK>)ÚassertCountEqualÚlistr   r+   Úkeys)r   Zvocab_countsZvocab_itemsr   r   r   Útest_vocab_iter_respects_cutoff;   s    z9NgramModelVocabularyTests.test_vocab_iter_respects_cutoffc                 C   sR   t dd�}|  t|ƒd¡ |  |¡ |  |j|¡ | tdƒ¡ |  |j|¡ d S )Nr   r   r   Úabcde)r   r   r0   r-   ZassertInÚ	unk_labelÚupdater4   )r   Úemptyr   r   r   Útest_update_empty_vocabB   s    

z1NgramModelVocabularyTests.test_update_empty_vocabc                 C   s,   |   | j d¡d¡ |   | j d¡d¡ d S )Nr   r
   r2   ©r   r   Úlookupr   r   r   r   Útest_lookupK   s    z%NgramModelVocabularyTests.test_lookupc                 C   sb   |   | j ddg¡d¡ |   | j d¡d¡ |   | j d¡d¡ |   | j tttdƒƒ¡d¡ d S )Nr   r	   ©r   r	   )r   r
   ©r   r2   r    )r2   r2   r2   )r   r   r=   Úmapr&   Úranger   r   r   r   Útest_lookup_iterablesO   s    ÿz/NgramModelVocabularyTests.test_lookup_iterablesc                 C   sh   |   | j d¡d¡ |   | j g ¡d¡ |   | j tg ƒ¡d¡ |   | j dd„ tddƒD ƒ¡d¡ d S )Nr   c                 s   s   | ]
}|V  qd S r   r   )Ú.0Únr   r   r   Ú	<genexpr>[   ó    zHNgramModelVocabularyTests.test_lookup_empty_iterables.<locals>.<genexpr>r   )r   r   r=   ÚiterrB   r   r   r   r   Útest_lookup_empty_iterablesW   s    z5NgramModelVocabularyTests.test_lookup_empty_iterablesc                 C   s`   |   | j ddgddgg¡d¡ |   | j ddgdg¡d¡ |   | j ddggggg¡d¡ d S )Nr   r	   r
   )r?   r@   )r?   r2   ))))r?   r<   r   r   r   r   Útest_lookup_recursive]   s
    ÿz/NgramModelVocabularyTests.test_lookup_recursivec                 C   sx   |   t¡� | j d ¡ W d   ƒ n1 s,0    Y  |   t¡�$ t| j d d g¡ƒ W d   ƒ n1 sj0    Y  d S r   ©r!   Ú	TypeErrorr   r=   r4   r   r   r   r   Útest_lookup_Noned   s    *z*NgramModelVocabularyTests.test_lookup_Nonec                 C   sx   |   t¡� | j d¡ W d   ƒ n1 s,0    Y  |   t¡�$ t| j ddg¡ƒ W d   ƒ n1 sj0    Y  d S )Nr*   r   rK   r   r   r   r   Útest_lookup_intj   s    *z)NgramModelVocabularyTests.test_lookup_intc                 C   s   |   | j d¡d¡ d S )NÚ r2   r<   r   r   r   r   Útest_lookup_empty_strp   s    z/NgramModelVocabularyTests.test_lookup_empty_strc                 C   sj   t g d¢dd�}t g d¢dd�}t g d¢ddd�}t ddgdd�}|  ||¡ |  ||¡ |  ||¡ d S )N)r   r	   r
   r*   r   Zblah)r   r8   r   r	   )r   r   ZassertNotEqual)r   Zv1Zv2Zv3Zv4r   r   r   Útest_eqalitys   s    z&NgramModelVocabularyTests.test_eqalityc                 C   s   |   t| jƒd¡ d S )Nz8<Vocabulary with cutoff=2 unk_label='<UNK>' and 5 items>)r   r&   r   r   r   r   r   Útest_str}   s    
ÿz"NgramModelVocabularyTests.test_strc                 C   s"   |   | jttg d¢ƒdd�¡ d S r   )r   r   r   r   r   r   r   r   Útest_creation_with_counter‚   s    ÿüþz4NgramModelVocabularyTests.test_creation_with_counterz?Test is known to be flaky as it compares (runtime) performance.)Úreasonc                 C   sP   t dƒ}ddlm} t | ¡ ƒ}tdtƒ d�}tdtƒ d�}| j||dd� d S )	Nr7   r   )Úenglishzlen(small_vocab))Úglobalszlen(large_vocab)r*   )Zplaces)r   Znltk.corpus.europarl_rawrU   Úwordsr   ÚlocalsZassertAlmostEqual)r   Zsmall_vocabrU   Zlarge_vocabZsmall_vocab_len_timeZlarge_vocab_len_timer   r   r   Útest_len_is_constant�   s    z.NgramModelVocabularyTests.test_len_is_constantN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__Úclassmethodr   r   r   r#   r)   r,   r.   r1   r6   r;   r>   rC   rI   rJ   rM   rN   rP   rQ   rR   rS   ÚunittestÚskiprY   r   r   r   r   r      s4   
	
ÿr   )r_   Úcollectionsr   r   Znltk.lmr   ZTestCaser   r   r   r   r   Ú<module>   s   