a
     ˜ëjP/  ã                   @   s’   d Z ddlZddlZddlmZ ddd„Zdd„ Zdd	„ Zd
d„ Z	G dd„ dƒZ
dd„ Zdd„ Zefdd„Zefdd„Zi addd„Zdd„ ZdS )z0
Utility functions and classes for classifiers.
é    N)ÚLazyMapc                    sF   |du r|ot |d ttfƒ}|r8‡ fdd„}t||ƒS tˆ |ƒS dS )aÖ  
    Use the ``LazyMap`` class to construct a lazy list-like
    object that is analogous to ``map(feature_func, toks)``.  In
    particular, if ``labeled=False``, then the returned list-like
    object's values are equal to::

        [feature_func(tok) for tok in toks]

    If ``labeled=True``, then the returned list-like object's values
    are equal to::

        [(feature_func(tok), label) for (tok, label) in toks]

    The primary purpose of this function is to avoid the memory
    overhead involved in storing all the featuresets for every token
    in a corpus.  Instead, these featuresets are constructed lazily,
    as-needed.  The reduction in memory overhead can be especially
    significant when the underlying list of tokens is itself lazy (as
    is the case with many corpus readers).

    :param feature_func: The function that will be applied to each
        token.  It should return a featureset -- i.e., a dict
        mapping feature names to feature values.
    :param toks: The list of tokens to which ``feature_func`` should be
        applied.  If ``labeled=True``, then the list elements will be
        passed directly to ``feature_func()``.  If ``labeled=False``,
        then the list elements should be tuples ``(tok,label)``, and
        ``tok`` will be passed to ``feature_func()``.
    :param labeled: If true, then ``toks`` contains labeled tokens --
        i.e., tuples of the form ``(tok, label)``.  (Default:
        auto-detect based on types.)
    Nr   c                    s   ˆ | d ƒ| d fS )Nr   é   © )Zlabeled_token©Úfeature_funcr   úQ/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/nltk/classify/util.pyÚ	lazy_funcA   s    z!apply_features.<locals>.lazy_func)Ú
isinstanceÚtupleÚlistr   )r   ÚtoksZlabeledr   r   r   r   Úapply_features   s    !
r   c                 C   s   t dd„ | D ƒƒS )a!  
    :return: A list of all labels that are attested in the given list
        of tokens.
    :rtype: list of (immutable)
    :param tokens: The list of classified tokens from which to extract
        labels.  A classified token has the form ``(token, label)``.
    :type tokens: list
    c                 S   s   h | ]\}}|’qS r   r   )Ú.0ÚtokÚlabelr   r   r   Ú	<setcomp>R   ó    z"attested_labels.<locals>.<setcomp>)r
   )Útokensr   r   r   Úattested_labelsI   s    	r   c                 C   s>   |   dd„ |D ƒ¡}dd„ t||ƒD ƒ}t t|ƒt|ƒ ¡S )Nc                 S   s   g | ]\}}|‘qS r   r   ©r   ÚfsÚlr   r   r   Ú
<listcomp>V   r   z"log_likelihood.<locals>.<listcomp>c                 S   s   g | ]\\}}}|  |¡‘qS r   )Úprob)r   r   r   Úpdistr   r   r   r   W   r   )Úprob_classify_manyÚzipÚmathÚlogÚsumÚlen)Ú
classifierÚgoldÚresultsÚllr   r   r   Úlog_likelihoodU   s    r%   c                 C   sD   |   dd„ |D ƒ¡}dd„ t||ƒD ƒ}|r<t|ƒt|ƒ S dS d S )Nc                 S   s   g | ]\}}|‘qS r   r   r   r   r   r   r   \   r   zaccuracy.<locals>.<listcomp>c                 S   s   g | ]\\}}}||k‘qS r   r   )r   r   r   Úrr   r   r   r   ]   r   r   )Zclassify_manyr   r   r    )r!   r"   r#   Úcorrectr   r   r   Úaccuracy[   s
    r(   c                   @   s    e Zd ZdZdd„ Zdd„ ZdS )ÚCutoffCheckerzÉ
    A helper class that implements cutoff checks based on number of
    iterations and log likelihood.

    Accuracy cutoffs are also implemented, but they're almost never
    a good idea to use.
    c                 C   sR   |  ¡ | _d|v r$t|d ƒ |d< d|v r<t|d ƒ|d< d | _d | _d| _d S )NÚmin_llÚmin_lldeltar   )ÚcopyÚcutoffsÚabsr$   ÚaccÚiter)Úselfr-   r   r   r   Ú__init__m   s    
zCutoffChecker.__init__c                 C   s  | j }|  jd7  _d|v r.| j|d kr.dS tjj ||¡}t |¡rLdS d|v s\d|v r¢d|v rt||d krtdS d|v rœ| jrœ|| j t	|d ƒkrœdS || _d|v s´d|v �rtjj ||¡}d|v rÜ||d krÜdS d|v �r
| j
�r
|| j
 t	|d ƒk�r
dS || _
dS d S )	Nr   Zmax_iterTr*   r+   Zmax_accZmin_accdeltaF)r-   r0   ÚnltkZclassifyÚutilr%   r   Úisnanr$   r.   r/   )r1   r!   Z
train_toksr-   Znew_llZnew_accr   r   r   Úcheckw   s>    
ÿþýÿþýzCutoffChecker.checkN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r2   r6   r   r   r   r   r)   d   s   
r)   c                 C   sd   i }d|d< | d   ¡ |d< | d   ¡ |d< dD ].}|   ¡  |¡|d| < ||   ¡ v |d	| < q0|S )
NTÚalwaysonr   Ú
startswithéÿÿÿÿÚendswithÚabcdefghijklmnopqrstuvwxyzú	count(%s)úhas(%s)©ÚlowerÚcount©ÚnameÚfeaturesÚletterr   r   r   Únames_demo_features    s    rI   c                 C   sœ   i }d|d< | d   ¡ dv |d< | d   ¡ dv |d< dD ]^}|   ¡  |¡|d	| < ||   ¡ v |d
| < || d   ¡ k|d| < || d   ¡ k|d| < q8|S )NTr;   r   Zaeiouyzstartswith(vowel)r=   zendswith(vowel)r?   r@   rA   zstartswith(%s)zendswith(%s)rB   rE   r   r   r   Úbinary_names_demo_features«   s    rJ   c                    s~  dd l }ddlm} dd„ | d¡D ƒdd„ | d¡D ƒ }| d¡ | |¡ |d d	… }|d	d
… }tdƒ | ‡ fdd„|D ƒƒ}tdƒ t|‡ fdd„|D ƒƒ}td| ƒ z²‡ fdd„|D ƒ}	| |	¡}
dd„ t	||
ƒD ƒ}tdt
|ƒt|ƒ  ƒ tƒ  tdƒ tt	||
ƒƒd d… D ]@\\}}}|dk�r<d}nd}t||| d¡| d¡f ƒ �q W n t�yx   Y n0 |S )Nr   ©Únamesc                 S   s   g | ]}|d f‘qS )Úmaler   ©r   rF   r   r   r   r   ¾   r   znames_demo.<locals>.<listcomp>úmale.txtc                 S   s   g | ]}|d f‘qS )Úfemaler   rN   r   r   r   r   ¾   s   ú
female.txté@â iˆ  i|  úTraining classifier...c                    s   g | ]\}}ˆ |ƒ|f‘qS r   r   ©r   ÚnÚg©rG   r   r   r   Ê   r   úTesting classifier...c                    s   g | ]\}}ˆ |ƒ|f‘qS r   r   rT   rW   r   r   r   Î   r   úAccuracy: %6.4fc                    s   g | ]\}}ˆ |ƒ‘qS r   r   rT   rW   r   r   r   Ô   r   c                 S   s   g | ]\\}}}|  |¡‘qS r   ©Zlogprob©r   rF   r"   r   r   r   r   r   Ö   r   úAvg. log likelihood: %6.4fúMUnseen Names      P(Male)  P(Female)
----------------------------------------é   rM   ú  %-15s *%6.4f   %6.4fú  %-15s  %6.4f  *%6.4frP   )ÚrandomÚnltk.corpusrL   ÚwordsÚseedÚshuffleÚprintr(   r   r   r   r    r   r   ÚNotImplementedError)ÚtrainerrG   ra   rL   ÚnamelistÚtrainÚtestr!   r/   Útest_featuresetsÚpdistsr$   rF   Zgenderr   Úfmtr   rW   r   Ú
names_demo¸   s:    ÿ


"
&ro   c                    s²  dd l }ddlm} | d¡}| d¡}| d¡ | |¡ | |¡ tˆ |d d… ƒ}tˆ |dd… |d d…  ƒ}d	d
„ |dd… D ƒdd
„ |dd… D ƒ }| |¡ tdƒ | ||ƒ}	tdƒ t|	‡ fdd
„|D ƒƒ}
td|
 ƒ z®‡ fdd
„|D ƒ}|	 	|¡}dd
„ t
||ƒD ƒ}tdt|ƒt|ƒ  ƒ tƒ  tdƒ t
||ƒd d… D ]@\\}}}|dk�rpd}nd}t||| d¡| d¡f ƒ �qTW n t�y¬   Y n0 |	S )Nr   rK   rO   rQ   iñû	 iÐ  iÄ	  iô  c                 S   s   g | ]}|d f‘qS )Tr   rN   r   r   r   r   ú   r   z&partial_names_demo.<locals>.<listcomp>i¾
  c                 S   s   g | ]}|d f‘qS )Fr   rN   r   r   r   r   ú   s   iî  rS   rX   c                    s   g | ]\}}ˆ |ƒ|f‘qS r   r   ©r   rU   ÚmrW   r   r   r     r   rY   c                    s   g | ]\}}ˆ |ƒ‘qS r   r   rp   rW   r   r   r     r   c                 S   s   g | ]\\}}}|  |¡‘qS r   rZ   r[   r   r   r   r     r   r\   r]   r^   Tr_   r`   F)ra   rb   rL   rc   rd   re   Úmaprf   r(   r   r   r   r    r   rg   )rh   rG   ra   rL   Z
male_namesZfemale_namesZpositiveZ	unlabeledrk   r!   r/   rl   rm   r$   rF   Zis_maler   rn   r   rW   r   Úpartial_names_demoç   sB    





ÿ



&rs   éè  c                    sx  dd l }ddlm} tdƒ |tvr<dd„ | |¡D ƒt|< t| d d … }|t|ƒkr`t|ƒ}tdd„ |D ƒƒ}tdd	 |¡ ƒ td
ƒ | 	d¡ | 
|¡ |d td| ƒ… }|td| ƒ|… }	tdƒ | ‡ fdd„|D ƒƒ}
tdƒ t|
‡ fdd„|	D ƒƒ}td| ƒ zL‡ fdd„|	D ƒ}|
 |¡}dd„ t|	|ƒD ƒ}tdt|ƒt|	ƒ  ƒ W n t�yr   Y n0 |
S )Nr   )ÚsensevalzReading data...c                 S   s   g | ]}||j d  f‘qS )r   )Úsenses)r   Úir   r   r   r   +  r   zwsd_demo.<locals>.<listcomp>c                 S   s   h | ]\}}|’qS r   r   ©r   rw   r   r   r   r   r   /  r   zwsd_demo.<locals>.<setcomp>z
  Senses: ú zSplitting into test & train...rR   gš™™™™™é?rS   c                    s   g | ]\}}ˆ |ƒ|f‘qS r   r   rx   rW   r   r   r   ;  r   rX   c                    s   g | ]\}}ˆ |ƒ|f‘qS r   r   rx   rW   r   r   r   ?  r   rY   c                    s   g | ]\}}ˆ |ƒ‘qS r   r   )r   rw   rU   rW   r   r   r   E  r   c                 S   s   g | ]\\}}}|  |¡‘qS r   rZ   r[   r   r   r   r   G  r   r\   )ra   rb   ru   rf   Ú_inst_cacheÚ	instancesr    r   Újoinrd   re   Úintr(   r   r   r   rg   )rh   ÚwordrG   rU   ra   ru   r{   rv   rj   rk   r!   r/   rl   rm   r$   r   rW   r   Úwsd_demo"  s8    


r   c               
   C   sD   zt  W n6 ty> }  ztdƒ}t|ƒ| ‚W Y d} ~ n
d} ~ 0 0 dS )z8
    Checks whether the MEGAM binary is configured.
    z\Please configure your megam binary first, e.g.
>>> nltk.config_megam('/usr/bin/local/megam')N)Z
_megam_binÚ	NameErrorÚstr)ÚeÚerr_msgr   r   r   Úcheck_megam_configP  s    ÿr„   )N)rt   )r:   r   Znltk.classify.utilr3   Z	nltk.utilr   r   r   r%   r(   r)   rI   rJ   ro   rs   rz   r   r„   r   r   r   r   Ú<module>	   s   
-	</8
.