a
     ˜ëj{L  ã                   @   sä  d dl Z d dl mZ d dlmZ d dlZd dlmZmZm	Z	m
Z
mZmZmZmZ d dlmZ ejdd�dd	„ ƒZejdd�d
d„ ƒZejdd�dd„ ƒZejdd�dd„ ƒZejdd„ ƒZej dddgdfddgd fdddg¡dd„ ƒZdd„ Zdd„ Zd d!„ Zd"d#„ Zejd$d%„ ƒZej dd&ddgdfd'dd(g¡d)d*„ ƒZejd+d,„ ƒZ ej dddgd-fd.d/d0g¡d1d2„ ƒZ!d3d4„ Z"ejd5d6„ ƒZ#ej dddgd-fddgd7fdd8dgd-fdd8dgd7fg¡d9d:„ ƒZ$ejd;d<„ ƒZ%ej dddgd=fd>d?d@g¡dAdB„ ƒZ&dCdD„ Z'dEdF„ Z(ejdGdH„ ƒZ)ej ddIdJd(dd8gdKfddLd8gdMfddNd8gdKfg¡dOdP„ ƒZ*ejdQdR„ ƒZ+ej ddSdJddd8gdTfddLd8gdUfddNd8gdTfg¡dVdW„ ƒZ,ejdXdY„ ƒZ-ej ddIdJd(dd8gdZfddLd8gd[fddNd8gdZfg¡d\d]„ ƒZ.ejd^d_„ ƒZ/ej ddIdJd(dd8gd`fddLd8gdafddNd8gdbfg¡dcdd„ ƒZ0ejdd�dedf„ ƒZ1ej dgdd%d,d<dHdYdfej2d_ejj3dhdi�dj�g¡ejjdkg dl¢ed ƒdm�dndo„ ƒƒZ4dpdq„ Z5drds„ Z6dtdu„ Z7dvdw„ Z8dxdy„ Z9dzd{„ Z:d|d}„ Z;dS )~é    N)Úfsum)Ú
itemgetter)ÚMLEÚAbsoluteDiscountingInterpolatedÚKneserNeyInterpolatedÚLaplaceÚLidstoneÚStupidBackoffÚ
VocabularyÚWittenBellInterpolated)Úpadded_everygramsÚsession)Zscopec                   C   s   t g d¢dd�S )N)ÚaÚbÚcÚdÚzú<s>ú</s>é   )Z
unk_cutoff)r
   © r   r   ú\/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/nltk/test/unit/lm/test_models.pyÚ
vocabulary   s    r   c                   C   s   g d¢g d¢gS )N)r   r   r   r   )ÚeÚgr   r   r   r   r   r   r   r   r   Útraining_data   s    r   c                 C   s   dd„ | D ƒS )Nc                 S   s   g | ]}t td |ƒƒ‘qS )é   ©Úlistr   ©Ú.0Úsentr   r   r   Ú
<listcomp>&   ó    z(bigram_training_data.<locals>.<listcomp>r   ©r   r   r   r   Úbigram_training_data$   s    r%   c                 C   s   dd„ | D ƒS )Nc                 S   s   g | ]}t td |ƒƒ‘qS )é   r   r   r   r   r   r"   +   r#   z)trigram_training_data.<locals>.<listcomp>r   r$   r   r   r   Útrigram_training_data)   s    r'   c                 C   s   t d| d�}| |¡ |S ©Nr   ©r   ©r   Úfit)r   r%   Úmodelr   r   r   Úmle_bigram_model.   s    
r-   zword, context, expected_scorer   r   r   r   )r   Nr   )r   Ng’$I’$IÂ?)ÚyNgÛ¶mÛ¶mË?c                 C   s    t  |  ||¡d¡|ksJ ‚d S ©Nç-Cëâ6?©ÚpytestÚapproxÚscore)r-   ÚwordÚcontextÚexpected_scorer   r   r   Útest_mle_bigram_scores5   s    r8   c                 C   s   t  |  ddg¡¡sJ ‚d S )Nr   r   )ÚmathÚisinfZlogscore)r-   r   r   r   Ú'test_mle_bigram_logscore_for_zero_scoreH   s    r;   c                 C   sH   g d¢}d}d}t  |  |¡d¡|ks*J ‚t  |  |¡d¡|ksDJ ‚d S )N)©r   r   )r   r   )r   ú<UNK>)r=   r   )r   r   ©r   r   g�Âõ(\�ñ?g®Ø_vO@r0   ©r2   r3   ÚentropyÚ
perplexity)r-   ZtrainedÚHrA   r   r   r   Ú'test_mle_bigram_entropy_perplexity_seenL   s
    rC   c                 C   s4   g d¢}t  |  |¡¡sJ ‚t  |  |¡¡s0J ‚d S )N)r<   ©r   r   )r   r   r>   )r9   r:   r@   rA   )r-   Z	untrainedr   r   r   Ú)test_mle_bigram_entropy_perplexity_unseene   s    rE   c                 C   sH   d}d}g d¢}t  |  |¡d¡|ks*J ‚t  |  |¡d¡|ksDJ ‚d S )Ngú~j¼t@g³êsµ @)©r   ©r   ©r   )ú-©r   rH   )r   r0   r?   )r-   rB   rA   Útextr   r   r   Ú+test_mle_bigram_entropy_perplexity_unigramsm   s
    rL   c                 C   s   t d|d�}| | ¡ |S ©Nr&   ©Úorderr   r*   ©r'   r   r,   r   r   r   Úmle_trigram_model�   s    
rQ   )r   )r   r   r   )r   NgÇqÇq¼?)r.   NçUUUUUUÅ?c                 C   s    t  |  ||¡d¡|ksJ ‚d S r/   r1   )rQ   r5   r6   r7   r   r   r   Útest_mle_trigram_scoresˆ   s    rS   c                 C   s   t dd|d�}| | ¡ |S )Nçš™™™™™¹?r   rN   ©r   r+   ©r%   r   r,   r   r   r   Úlidstone_bigram_model�   s    
rW   gä8Žã8Žã?)r   Ng˜"Ÿuƒ)Â?)r   NgÐºÁù¬{?)r.   Ngù¬L‘ÏÊ?c                 C   s    t  |  ||¡d¡|ksJ ‚d S r/   r1   )rW   r5   r6   r7   r   r   r   Útest_lidstone_bigram_score¤   s    ÿÿrX   c                 C   sH   g d¢}d}d}t  |  |¡d¡|ks*J ‚t  |  |¡d¡|ksDJ ‚d S )N©r<   rD   )r   r=   )r=   r   )r   r   )r   r   g=,Ôšæ]@gé·¯ç1@r0   r?   )rW   rK   rB   rA   r   r   r   Ú test_lidstone_entropy_perplexityÃ   s
    rZ   c                 C   s   t dd|d�}| | ¡ |S )NrT   r&   rN   rU   rP   r   r   r   Úlidstone_trigram_modelÜ   s    
r[   gÇqÇq¬?r   c                 C   s    t  |  ||¡d¡|ksJ ‚d S r/   r1   )r[   r5   r6   r7   r   r   r   Útest_lidstone_trigram_scoreã   s    ÿÿr\   c                 C   s   t d|d�}| | ¡ |S r(   )r   r+   rV   r   r   r   Úlaplace_bigram_modelö   s    
r]   gÇqÇqÌ?)r   NgtÑE]tÁ?)r   NgF]tÑE§?)r.   NgF]tÑEÇ?c                 C   s    t  |  ||¡d¡|ksJ ‚d S r/   r1   )r]   r5   r6   r7   r   r   r   Útest_laplace_bigram_scoreý   s    ÿr^   c                 C   sH   g d¢}d}d}t  |  |¡d¡|ks*J ‚t  |  |¡d¡|ksDJ ‚d S )NrY   g…ëQ¸	@g˜Ý“‡…z!@r0   r?   )r]   rK   rB   rA   r   r   r   Ú&test_laplace_bigram_entropy_perplexity  s
    r_   c                 C   s   | j dksJ ‚d S )Nr   )Úgamma)r]   r   r   r   Útest_laplace_gamma5  s    ra   c                 C   s   t d|d�}| | ¡ |S )Nr&   r)   )r   r+   rP   r   r   r   Úwittenbell_trigram_model9  s    
rb   )r   NgÇqÇq¬?)r   Ng        grÇqÇÑ?r   gÇqÇqä?r   c                 C   s    t  |  ||¡d¡|ksJ ‚d S r/   r1   )rb   r5   r6   r7   r   r   r   Útest_wittenbell_trigram_score@  s    "ÿÿrc   c                 C   s   t dd|d�}| | ¡ |S )Nr&   g      è?)rO   Zdiscountr   ©r   r+   rP   r   r   r   Úkneserney_trigram_modelr  s    
re   )r   Ng’$I’$I²?g·mÛ¶mÛÆ?g$I’$I’Ø?c                 C   s    t  |  ||¡d¡|ksJ ‚d S r/   r1   )re   r5   r6   r7   r   r   r   Útest_kneserney_trigram_scorey  s    (ÿÿrf   c                 C   s   t d|d�}| | ¡ |S rM   )r   r+   rP   r   r   r   Ú"absolute_discounting_trigram_model¦  s    
rg   rR   g      Ø?c                 C   s    t  |  ||¡d¡|ksJ ‚d S r/   r1   )rg   r5   r6   r7   r   r   r   Ú'test_absolute_discounting_trigram_score­  s    %ÿÿrh   c                 C   s   t d|d�}| | ¡ |S rM   )r	   r+   rP   r   r   r   Ústupid_backoff_trigram_model×  s    
ri   g      à?ç      ð?gš™™™™™É?c                 C   s    t  |  ||¡d¡|ksJ ‚d S r/   r1   )ri   r5   r6   r7   r   r   r   Ú!test_stupid_backoff_trigram_scoreÞ  s    ÿÿrk   c                 C   s   t d|d�}| | ¡ |S )Nr   rN   rd   rV   r   r   r   Úkneserney_bigram_model  s    
rl   Úmodel_fixturez*Stupid Backoff is not a valid distribution)Úreason)Zmarksr6   )	rG   rH   rF   )r   ©r=   rJ   )r   )Úr)Úw)Úidsc                    s<   |  | ¡‰t‡ ‡fdd„ˆjD ƒƒ}t |d¡dks8J ‚d S )Nc                 3   s   | ]}ˆ  |ˆ ¡V  qd S ©N)r4   )r    rq   ©r6   r,   r   r   Ú	<genexpr>!  r#   z!test_sums_to_1.<locals>.<genexpr>gH¯¼šò×z>rj   )ZgetfixturevalueÚsumZvocabr2   r3   )rm   r6   ÚrequestZscores_for_contextr   rt   r   Útest_sums_to_1  s    
rx   c                 C   s   | j dd�dksJ ‚d S )Nr&   ©Úrandom_seedr=   ©Úgenerate©rQ   r   r   r   Útest_generate_one_no_context*  s    r~   c                 C   sJ   | j dgd�dksJ ‚| j ddgd�dks.J ‚| j ddgd�dksFJ ‚d S )Nr   ©Ú	text_seedr   r   r   r{   r}   r   r   r   Ú'test_generate_one_from_limiting_context.  s    r�   c                 C   s   | j ddd�dksJ ‚d S )N)r   r   r   ©r€   rz   r   r{   r}   r   r   r   Ú%test_generate_one_from_varied_context5  s    rƒ   c                 C   s<   t | jtdƒƒg}|  |¡ | jdddd�g d¢ks8J ‚d S )NZbdbdbdé   )r   r   é   r‚   )r   r   r   r   r   r   r   )r   rO   r   r+   r|   )rQ   Zmore_training_textr   r   r   Útest_generate_cycle:  s    
r†   c                 C   s    | j dddd�g d¢ksJ ‚d S )Nr…   )r   r   r&   r‚   )r=   r   r   r   r=   r{   r}   r   r   r   Útest_generate_with_text_seedK  s    r‡   c                 C   s$   | j ddd�| j ddd�ks J ‚d S )N)Zaliensr&   r‚   ro   r{   r}   r   r   r   Útest_generate_oov_text_seedU  s
    ÿþrˆ   c                 C   sX   t  t¡� | jdd� W d   ƒ n1 s,0    Y  | jd dd�| jdd�ksTJ ‚d S )Nrs   r   r&   r‚   ry   )r2   ZraisesÚ	TypeErrorr|   r}   r   r   r   Útest_generate_None_text_seed[  s    *ÿ
þrŠ   )<r9   r   rv   Úoperatorr   r2   Znltk.lmr   r   r   r   r   r	   r
   r   Znltk.lm.preprocessingr   Zfixturer   r   r%   r'   r-   ÚmarkZparametrizer8   r;   rC   rE   rL   rQ   rS   rW   rX   rZ   r[   r\   r]   r^   r_   ra   rb   rc   re   rf   rg   rh   ri   rk   rl   ÚparamZxfailrx   r~   r�   rƒ   r†   r‡   rˆ   rŠ   r   r   r   r   Ú<module>   sB  (











öþ


ôþ


íþ



ùþ


ìþ


çþ

	
áþ$
	

äþ!
	

ðþ


ÿþøþý
