a
     ˜ëjÝ>  ã                   @   sÎ   d Z ddlZddlmZmZ ddlmZ dd„ Zej	Z
dd„ ZdZzdd	lmZ W n eyl   d
d„ ZY n0 dZdZdZG dd„ ded�ZG dd„ deƒZG dd„ deƒZG dd„ deƒZG dd„ dƒZdS )zÌ
Provides scoring functions for a number of association measures through a
generic, abstract implementation in ``NgramAssocMeasures``, and n-specific
``BigramAssocMeasures`` and ``TrigramAssocMeasures``.
é    N)ÚABCMetaÚabstractmethod©Úreducec                 C   s
   t  | ¡S ©N)Ú_mathÚlog2)Úx© r
   úW/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/nltk/metrics/association.pyÚ<lambda>   ó    r   c                 C   s   t dd„ | ƒS )Nc                 S   s   | | S r   r
   )r	   Úyr
   r
   r   r      r   z<lambda>.<locals>.<lambda>r   )Úsr
   r
   r   r      r   g#B’¡œÇ;)Úfisher_exactc                  O   s   t ‚d S r   ©ÚNotImplementedError)Ú_argsÚ_kwargsr
   r
   r   r      s    r   éþÿÿÿéÿÿÿÿc                   @   s    e Zd ZdZdZeedd„ ƒƒZeedd„ ƒƒZe	dd„ ƒZ
ed	d
„ ƒZe	dd„ ƒZe	dd„ ƒZedd„ ƒZe	dd„ ƒZe	dd„ ƒZe	dd„ ƒZe	dd„ ƒZdS )ÚNgramAssocMeasuresa¿  
    An abstract class defining a collection of generic association measures.
    Each public method returns a score, taking the following arguments::

        score_fn(count_of_ngram,
                 (count_of_n-1gram_1, ..., count_of_n-1gram_j),
                 (count_of_n-2gram_1, ..., count_of_n-2gram_k),
                 ...,
                 (count_of_1gram_1, ..., count_of_1gram_n),
                 count_of_total_words)

    See ``BigramAssocMeasures`` and ``TrigramAssocMeasures``

    Inheriting classes should define a property _n, and a method _contingency
    which calculates contingency values from marginals in order for all
    association measures defined here to be usable.
    r   c                  G   s   t dƒ‚dS )z>Calculates values of a contingency table from marginal values.ú?The contingency table is not availablein the general ngram caseNr   ©Ú	marginalsr
   r
   r   Ú_contingencyB   s    ÿzNgramAssocMeasures._contingencyc                  G   s   t dƒ‚dS )úACalculates values of contingency table marginals from its values.r   Nr   ©Úcontingencyr
   r
   r   Ú
_marginalsJ   s    ÿzNgramAssocMeasures._marginalsc                 #   s\   t ˆƒ}dd„ tˆ jƒD ƒ}ttˆƒƒD ].‰t‡ ‡‡fdd„|D ƒƒ|ˆ jd   V  q(dS )ú3Calculates expected values for a contingency table.c                 S   s   g | ]}d |> ‘qS )é   r
   )Ú.0Úir
   r
   r   Ú
<listcomp>V   r   z7NgramAssocMeasures._expected_values.<locals>.<listcomp>c                 3   s2   | ]*‰ t ‡‡‡ fd d„tdˆj ƒD ƒƒV  qdS )c                 3   s&   | ]}|ˆ@ ˆˆ@ krˆ | V  qd S r   r
   )r"   r	   )Úcontr#   Újr
   r   Ú	<genexpr>]   r   z@NgramAssocMeasures._expected_values.<locals>.<genexpr>.<genexpr>é   N)ÚsumÚrangeÚ_n)r"   ©Úclsr%   r#   )r&   r   r'   \   s   ÿz6NgramAssocMeasures._expected_values.<locals>.<genexpr>r!   N)r)   r*   r+   ÚlenÚ_product)r-   r%   Ún_allÚbitsr
   r,   r   Ú_expected_valuesR   s    þüÿz#NgramAssocMeasures._expected_valuesc                  G   s   | t  | t  S )z Scores ngrams by their frequency)ÚNGRAMÚTOTALr   r
   r
   r   Úraw_freqc   s    zNgramAssocMeasures.raw_freqc                 G   s6   |t  t|t ƒ|t | jd    |t  t d  S )z�Scores ngrams using Student's t test with independence hypothesis
        for unigrams, as in Manning and Schutze 5.3.1.
        r!   g      à?)r3   r/   ÚUNIGRAMSr4   r+   Ú_SMALL©r-   r   r
   r
   r   Ú	student_th   s
    ÿýzNgramAssocMeasures.student_tc                 G   s,   | j |Ž }|  |¡}tdd„ t||ƒD ƒƒS )zZScores ngrams using Pearson's chi-square as in Manning and Schutze
        5.3.3.
        c                 s   s&   | ]\}}|| d  |t   V  qdS )r(   N)r7   ©r"   ZobsÚexpr
   r
   r   r'   y   r   z,NgramAssocMeasures.chi_sq.<locals>.<genexpr>)r   r2   r)   Úzip)r-   r   r%   Zexpsr
   r
   r   Úchi_sqr   s    

zNgramAssocMeasures.chi_sqc                  O   s    | t  | dd¡ t| t ƒ S )zÂScores ngrams using a variant of mutual information. The keyword
        argument power sets an exponent (default 3) for the numerator. No
        logarithm of the result is calculated.
        Úpoweré   )r3   Úgetr/   r6   )r   Úkwargsr
   r
   r   Úmi_like{   s    ÿzNgramAssocMeasures.mi_likec                 G   s.   t |t |t | jd   ƒt t|t ƒƒ S )z^Scores ngrams by pointwise mutual information, as in Manning and
        Schutze 5.4.
        r!   )Ú_log2r3   r4   r+   r/   r6   r8   r
   r
   r   Úpmi…   s    
ÿzNgramAssocMeasures.pmic                 G   s,   | j |Ž }dtdd„ t||  |¡ƒD ƒƒ S )zFScores ngrams using likelihood ratios as in Manning and Schutze 5.3.4.r(   c                 s   s*   | ]"\}}|t ||t  t ƒ V  qd S r   )Ú_lnr7   r:   r
   r
   r   r'   ’   s   ÿz6NgramAssocMeasures.likelihood_ratio.<locals>.<genexpr>)r   r)   r<   r2   ©r-   r   r%   r
   r
   r   Úlikelihood_ratioŽ   s    

þz#NgramAssocMeasures.likelihood_ratioc                 G   s:   t |t ƒ|t | jd   }|t t|t | ƒd  S )z1Scores ngrams using the Poisson-Stirling measure.r!   )r/   r6   r4   r+   r3   rC   )r-   r   r;   r
   r
   r   Úpoisson_stirling—   s    z#NgramAssocMeasures.poisson_stirlingc                 G   s"   | j |Ž }|d t|dd… ƒ S )z&Scores ngrams using the Jaccard index.r   Nr   )r   r)   rF   r
   r
   r   Újaccard�   s    
zNgramAssocMeasures.jaccardN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r+   Ústaticmethodr   r   r   Úclassmethodr2   r5   r9   r=   rB   rD   rG   rH   rI   r
   r
   r
   r   r   -   s4   


	

	


r   )Ú	metaclassc                   @   sh   e Zd ZdZdZedd„ ƒZedd„ ƒZedd„ ƒZe	d	d
„ ƒZ
e	dd„ ƒZe	dd„ ƒZedd„ ƒZdS )ÚBigramAssocMeasuresaˆ  
    A collection of bigram association measures. Each association measure
    is provided as a function with three arguments::

        bigram_score_fn(n_ii, (n_ix, n_xi), n_xx)

    The arguments constitute the marginals of a contingency table, counting
    the occurrences of particular events in a corpus. The letter i in the
    suffix refers to the appearance of the word in question, while x indicates
    the appearance of any word. Thus, for example:

    - n_ii counts ``(w1, w2)``, i.e. the bigram being scored
    - n_ix counts ``(w1, *)``
    - n_xi counts ``(*, w2)``
    - n_xx counts ``(*, *)``, i.e. any bigram

    This may be shown with respect to a contingency table::

                w1    ~w1
             ------ ------
         w2 | n_ii | n_oi | = n_xi
             ------ ------
        ~w2 | n_io | n_oo |
             ------ ------
             = n_ix        TOTAL = n_xx
    r(   c                 C   s0   |\}}||  }||  }| ||||  | | fS )zECalculates values of a bigram contingency table from marginal values.r
   )Ún_iiÚn_ix_xi_tupleÚn_xxÚn_ixÚn_xiÚn_oiÚn_ior
   r
   r   r   Â   s    z BigramAssocMeasures._contingencyc                 C   s"   | ||  ||  f|| | |  fS )r   r
   )rR   rW   rX   Ún_oor
   r
   r   r   Ê   s    zBigramAssocMeasures._marginalsc                 c   sH   t | ƒ}tdƒD ]2}| | | |dA   | | | |dA    | V  qdS )r    é   r!   r(   N)r)   r*   )r%   rT   r#   r
   r
   r   r2   Ï   s    z$BigramAssocMeasures._expected_valuesc                 G   sF   | j |Ž \}}}}|| ||  d || ||  ||  ||   S )zdScores bigrams using phi-square, the square of the Pearson correlation
        coefficient.
        r(   )r   )r-   r   rR   rX   rW   rY   r
   r
   r   Úphi_sq×   s    ÿzBigramAssocMeasures.phi_sqc                 C   s   |\}}||   |||f|¡ S )zƒScores bigrams using chi-square, i.e. phi-sq multiplied by the number
        of bigrams, as in Manning and Schutze 5.3.3.
        )r[   )r-   rR   rS   rT   rU   rV   r
   r
   r   r=   â   s    zBigramAssocMeasures.chi_sqc                 G   s2   | j |Ž \}}}}t||g||ggdd�\}}|S )zºScores bigrams using Fisher's Exact Test (Pedersen 1996).  Less
        sensitive to small counts than PMI or Chi Sq, but also more expensive
        to compute. Requires scipy.
        Zless)Úalternative)r   r   )r-   r   rR   rX   rW   rY   ZoddsZpvaluer
   r
   r   Úfisherê   s    zBigramAssocMeasures.fisherc                 C   s   |\}}d|  ||  S )z(Scores bigrams using Dice's coefficient.r(   r
   )rR   rS   rT   rU   rV   r
   r
   r   Údiceö   s    zBigramAssocMeasures.diceN)rJ   rK   rL   rM   r+   rN   r   r   r2   rO   r[   r=   r]   r^   r
   r
   r
   r   rQ   ¤   s    






rQ   c                   @   s,   e Zd ZdZdZedd„ ƒZedd„ ƒZdS )ÚTrigramAssocMeasuresa×  
    A collection of trigram association measures. Each association measure
    is provided as a function with four arguments::

        trigram_score_fn(n_iii,
                         (n_iix, n_ixi, n_xii),
                         (n_ixx, n_xix, n_xxi),
                         n_xxx)

    The arguments constitute the marginals of a contingency table, counting
    the occurrences of particular events in a corpus. The letter i in the
    suffix refers to the appearance of the word in question, while x indicates
    the appearance of any word. Thus, for example:

    - n_iii counts ``(w1, w2, w3)``, i.e. the trigram being scored
    - n_ixx counts ``(w1, *, *)``
    - n_xxx counts ``(*, *, *)``, i.e. any trigram
    r?   c                 C   s�   |\}}}|\}}}	||  }
||  }||  }|	|  |
 | }||  |
 | }||  | | }||  |
 | | | | | }| |
||||||fS )zÔCalculates values of a trigram contingency table (or cube) from
        marginal values.
        >>> TrigramAssocMeasures._contingency(1, (1, 1, 1), (1, 73, 1), 2000)
        (1, 0, 0, 0, 0, 72, 0, 1927)
        r
   )Ún_iiiZn_iix_tupleZn_ixx_tupleZn_xxxZn_iixZn_ixiZn_xiiZn_ixxZn_xixZn_xxiÚn_oiiÚn_ioiÚn_iioÚn_ooiÚn_oioÚn_iooÚn_ooor
   r
   r   r     s    

 z!TrigramAssocMeasures._contingencyc            	      G   s`   | \}}}}}}}}||| || || f|| | | || | | || | | ft | ƒfS )z»Calculates values of contingency table marginals from its values.
        >>> TrigramAssocMeasures._marginals(1, 0, 0, 0, 0, 72, 0, 1927)
        (1, (1, 1, 1), (1, 73, 1), 2000)
        ©r)   )	r   r`   ra   rb   rd   rc   re   rf   rg   r
   r
   r   r   &  s    ýøzTrigramAssocMeasures._marginalsN©rJ   rK   rL   rM   r+   rN   r   r   r
   r
   r
   r   r_   ý   s   
r_   c                   @   s,   e Zd ZdZdZedd„ ƒZedd„ ƒZdS )ÚQuadgramAssocMeasuresaF  
    A collection of quadgram association measures. Each association measure
    is provided as a function with five arguments::

        trigram_score_fn(n_iiii,
                        (n_iiix, n_iixi, n_ixii, n_xiii),
                        (n_iixx, n_ixix, n_ixxi, n_xixi, n_xxii, n_xiix),
                        (n_ixxx, n_xixx, n_xxix, n_xxxi),
                        n_all)

    The arguments constitute the marginals of a contingency table, counting
    the occurrences of particular events in a corpus. The letter i in the
    suffix refers to the appearance of the word in question, while x indicates
    the appearance of any word. Thus, for example:

    - n_iiii counts ``(w1, w2, w3, w4)``, i.e. the quadgram being scored
    - n_ixxi counts ``(w1, *, *, w4)``
    - n_xxxx counts ``(*, *, *, *)``, i.e. any quadgram
    rZ   c           "      C   sŒ  |\}}}}|\}	}
}}}}|\}}}}||  }||  }||  }||  | | }||  | | }||  | | }||  | | | | | | }||  }||  | | }|
|  | | }||  | | | | | | }|	|  | | }||  | | | | | | }||  | | | | | | } ||  | | | | | | | | | | | | | |  }!| |||||||||||||| |!fS )zXCalculates values of a quadgram contingency table from
        marginal values.
        r
   )"Ún_iiiiZn_iiix_tupleZn_iixx_tupleZn_ixxx_tupleZn_xxxxÚn_iiixÚn_iixiÚn_ixiiÚn_xiiiÚn_iixxÚn_ixixÚn_ixxiÚn_xixiÚn_xxiiÚn_xiixÚn_ixxxÚn_xixxÚn_xxixÚn_xxxiÚn_oiiiÚn_ioiiÚn_iioiÚn_ooiiÚn_oioiÚn_iooiÚn_oooiÚn_iiioÚn_oiioÚn_ioioÚn_ooioÚn_iiooÚn_oiooÚn_ioooÚn_oooor
   r
   r   r   P  s„        ÿþýüûúùø	÷
öõôóòñÿðz"QuadgramAssocMeasures._contingencyc                   G   sV  | \}}}}}}}}}	}
}}}}}}||	 }|| }|| }|| }|| |	 | }|| |	 | }|| | | }|| | | }|| | | }|| |	 |
 }|| | |	 | | | | }|| | |	 | |
 | | }|| | |	 | | |
 | }|| | | | | | | }t | ƒ}|||||f||||||f||||f|fS )a  Calculates values of contingency table marginals from its values.
        QuadgramAssocMeasures._marginals(1, 0, 2, 46, 552, 825, 2577, 34967, 1, 0, 2, 48, 7250, 9031, 28585, 356653)
        (1, (2, 553, 3, 1), (7804, 6, 3132, 1378, 49, 2), (38970, 17660, 100, 38970), 440540)
        rh   ) r   rk   rz   r{   r}   r|   r~   r   r€   r�   r‚   rƒ   r„   r…   r†   r‡   rˆ   rl   rm   rn   ro   rp   rq   rr   rs   rt   ru   rv   rw   rx   ry   r0   r
   r
   r   r   Œ  sN    ï    

ûz QuadgramAssocMeasures._marginalsNri   r
   r
   r
   r   rj   9  s   
;rj   c                   @   s$   e Zd ZdZdd„ Zedd„ ƒZdS )ÚContingencyMeasuresz�Wraps NgramAssocMeasures classes such that the arguments of association
    measures are contingency table values rather than marginals.
    c                 C   sZ   d|j j | j _t|ƒD ]<}| d¡r(qt||ƒ}| d¡sH|  ||¡}t| ||ƒ qdS )zAConstructs a ContingencyMeasures given a NgramAssocMeasures classZContingencyÚ__Ú_N)Ú	__class__rJ   ÚdirÚ
startswithÚgetattrÚ_make_contingency_fnÚsetattr)ÚselfÚmeasuresÚkÚvr
   r
   r   Ú__init__Æ  s    


zContingencyMeasures.__init__c                    s"   ‡ ‡fdd„}ˆj |_ ˆj|_|S )z‡From an association measure function, produces a new function which
        accepts contingency table values as its arguments.
        c                     s   ˆˆ j | Ž Ž S r   )r   r   ©r“   Úold_fnr
   r   Úres×  s    z5ContingencyMeasures._make_contingency_fn.<locals>.res)rM   rJ   )r“   r˜   r™   r
   r—   r   r�   Ñ  s    z(ContingencyMeasures._make_contingency_fnN)rJ   rK   rL   rM   r–   rN   r�   r
   r
   r
   r   r‰   Á  s   r‰   )rM   Úmathr   Úabcr   r   Ú	functoolsr   rC   ÚlogrE   r/   r7   Zscipy.statsr   ÚImportErrorr3   r6   r4   r   rQ   r_   rj   r‰   r
   r
   r
   r   Ú<module>   s(   wY< 	