a
     ˜ëjÀn  ã                   @   s.  d Z ddlZddlZddlZddlmZmZmZ ddlm	Z	 ddl
mZ ddlmZ ddlmZ ddlmZ dd	lmZmZ dd
lmZ ddlmZ ddlmZ ddlmZmZmZ edg d¢ƒZG dd„ dƒZ G dd„ dƒZ!G dd„ dƒZ"G dd„ dƒZ#G dd„ de#ƒZ$dd„ Z%e&dk�r"e%ƒ  g d¢Z'dS )a  
This module brings together a variety of NLTK functionality for
text analysis, and provides simple, interactive interfaces.
Functionality includes: concordancing, collocation discovery,
regular expression search over tokenized strings, and
distributional similarity.
é    N)ÚCounterÚdefaultdictÚ
namedtuple)Úreduce)Úlog)ÚBigramCollocationFinder)ÚMLE)Úpadded_everygram_pipeline)ÚBigramAssocMeasuresÚ	f_measure)ÚConditionalFreqDist)ÚFreqDist)Úsent_tokenize)ÚLazyConcatenationÚ
cut_stringÚ	tokenwrapÚConcordanceLine)ÚleftÚqueryÚrightÚoffsetÚ
left_printÚright_printÚlinec                   @   sT   e Zd ZdZedd„ ƒZdddd„ fdd„Zd	d
„ Zdd„ Zddd„Z	ddd„Z
dS )ÚContextIndexa  
    A bidirectional index between words and their 'contexts' in a text.
    The context of a word is usually defined to be the words that occur
    in a fixed window around the word; but other definitions may also
    be used by providing a custom context function.
    c                 C   sH   |dkr| |d    ¡ nd}|t| ƒd kr<| |d    ¡ nd}||fS )z;One left token and one right token, normalized to lowercaser   é   ú*START*ú*END*)ÚlowerÚlen)ÚtokensÚir   r   © r"   úH/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/nltk/text.pyÚ_default_context/   s    $zContextIndex._default_contextNc                 C   s   | S ©Nr"   ©Úxr"   r"   r#   Ú<lambda>6   ó    zContextIndex.<lambda>c                    sv   |ˆ_ ˆˆ_|r|ˆ_nˆjˆ_ˆ r6‡ fdd„ˆD ƒ‰t‡‡fdd„tˆƒD ƒƒˆ_t‡‡fdd„tˆƒD ƒƒˆ_d S )Nc                    s   g | ]}ˆ |ƒr|‘qS r"   r"   )Ú.0Út)Úfilterr"   r#   Ú
<listcomp>>   r)   z)ContextIndex.__init__.<locals>.<listcomp>c                 3   s(   | ] \}}ˆ   |¡ˆ  ˆ|¡fV  qd S r%   )Ú_keyÚ_context_func©r*   r!   Úw©Úselfr    r"   r#   Ú	<genexpr>?   s   z(ContextIndex.__init__.<locals>.<genexpr>c                 3   s(   | ] \}}ˆ   ˆ|¡ˆ  |¡fV  qd S r%   )r/   r.   r0   r2   r"   r#   r4   B   s   )r.   Ú_tokensr/   r$   ÚCFDÚ	enumerateÚ_word_to_contextsÚ_context_to_words)r3   r    Zcontext_funcr,   Úkeyr"   )r,   r3   r    r#   Ú__init__6   s    ÿ
ÿzContextIndex.__init__c                 C   s   | j S )zw
        :rtype: list(str)
        :return: The document that this context index was
            created from.
        ©r5   ©r3   r"   r"   r#   r    F   s    zContextIndex.tokensc                 C   sF   |   |¡}t| j| ƒ}i }| j ¡ D ]\}}t|t|ƒƒ||< q&|S )z 
        Return a dictionary mapping from words to 'similarity scores,'
        indicating how often these two words occur in the same
        context.
        )r.   Úsetr8   Úitemsr   )r3   ÚwordZword_contextsÚscoresr1   Z
w_contextsr"   r"   r#   Úword_similarity_dictN   s    
z!ContextIndex.word_similarity_dicté   c                 C   sv   t tƒ}| j|  |¡ D ]D}| j| D ]4}||kr&||  | j| | | j| |  7  < q&qt||jdd�d |… S )NT)r:   Úreverse)r   Úintr8   r.   r9   ÚsortedÚget)r3   r@   ÚnrA   Úcr1   r"   r"   r#   Úsimilar_words]   s    ÿ
zContextIndex.similar_wordsFc                    s�   ‡fdd„ˆD ƒ‰‡fdd„ˆD ƒ‰‡‡fdd„t tˆƒƒD ƒ}ttjˆƒ‰ |rf|rftdd ˆ¡ƒ‚n&ˆ sptƒ S t‡ ‡fdd„ˆD ƒƒ}|S d	S )
a§  
        Find contexts where the specified words can all appear; and
        return a frequency distribution mapping each context to the
        number of times that context was used.

        :param words: The words used to seed the similarity search
        :type words: str
        :param fail_on_unknown: If true, then raise a value error if
            any of the given words do not occur at all in the index.
        c                    s   g | ]}ˆ   |¡‘qS r"   )r.   ©r*   r1   r=   r"   r#   r-   r   r)   z0ContextIndex.common_contexts.<locals>.<listcomp>c                    s   g | ]}t ˆ j| ƒ‘qS r"   )r>   r8   rK   r=   r"   r#   r-   s   r)   c                    s   g | ]}ˆ | sˆ| ‘qS r"   r"   )r*   r!   )ÚcontextsÚwordsr"   r#   r-   t   r)   z%The following word(s) were not found:ú c                 3   s*   | ]"}ˆj | D ]}|ˆ v r|V  qqd S r%   )r8   ©r*   r1   rI   )Úcommonr3   r"   r#   r4   |   s   z/ContextIndex.common_contexts.<locals>.<genexpr>N)Úranger   r   r>   ÚintersectionÚ
ValueErrorÚjoinr   )r3   rM   Zfail_on_unknownÚemptyÚfdr"   )rP   rL   r3   rM   r#   Úcommon_contextsg   s    ÿzContextIndex.common_contexts)rC   )F)Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ústaticmethodr$   r;   r    rB   rJ   rW   r"   r"   r"   r#   r   '   s   


r   c                   @   sL   e Zd ZdZdd„ fdd„Zdd„ Zdd	„ Zd
d„ Zddd„Zddd„Z	dS )ÚConcordanceIndexzs
    An index that can be used to look up the offset locations at which
    a given word occurs in a document.
    c                 C   s   | S r%   r"   r&   r"   r"   r#   r(   ˆ   r)   zConcordanceIndex.<lambda>c                 C   sF   || _ || _ttƒ| _t|ƒD ]"\}}|  |¡}| j|  |¡ qdS )aé  
        Construct a new concordance index.

        :param tokens: The document (list of tokens) that this
            concordance index was created from.  This list can be used
            to access the context of a given word occurrence.
        :param key: A function that maps each token to a normalized
            version that will be used as a key in the index.  E.g., if
            you use ``key=lambda s:s.lower()``, then the index will be
            case-insensitive.
        N)r5   r.   r   ÚlistÚ_offsetsr7   Úappend)r3   r    r:   Úindexr@   r"   r"   r#   r;   ˆ   s    

zConcordanceIndex.__init__c                 C   s   | j S )z{
        :rtype: list(str)
        :return: The document that this concordance index was
            created from.
        r<   r=   r"   r"   r#   r    ¢   s    zConcordanceIndex.tokensc                 C   s   |   |¡}| j| S )zä
        :rtype: list(int)
        :return: A list of the offset positions at which the given
            word occurs.  If a key function was specified for the
            index, then given word's key will be looked up.
        )r.   r_   ©r3   r@   r"   r"   r#   Úoffsetsª   s    
zConcordanceIndex.offsetsc                 C   s   dt | jƒt | jƒf S )Nz+<ConcordanceIndex for %d tokens (%d types)>)r   r5   r_   r=   r"   r"   r#   Ú__repr__´   s    þzConcordanceIndex.__repr__éP   c              	      sT  t |tƒr|}n|g}d |¡}tdd„ |D ƒƒ}|| d d }|d }g }|  |d ¡}	t|dd… ƒD ].\‰ }‡ fd	d
„|  |¡D ƒ}
t|
 |	¡ƒ}	ql|	�rP|	D ]¨‰ d | jˆ ˆ t	|ƒ … ¡}| jt
dˆ | ƒˆ … }| jˆ t	|ƒ ˆ | … }td |¡| ƒ |¡}td |¡|ƒ}d |||g¡}t|||ˆ |||ƒ}| |¡ q¦|S )z‹
        Find all concordance lines given the query word.

        Provided with a list of words, these will be found as a phrase.
        rN   c                 s   s   | ]}t  |¡sd V  qdS )r   N)ÚunicodedataÚ	combining)r*   Úcharr"   r"   r#   r4   Æ   r)   z4ConcordanceIndex.find_concordance.<locals>.<genexpr>é   é   r   r   Nc                    s   h | ]}|ˆ  d  ’qS )r   r"   )r*   r   ©r!   r"   r#   Ú	<setcomp>Î   r)   z4ConcordanceIndex.find_concordance.<locals>.<setcomp>)Ú
isinstancer^   rT   Úsumrc   r7   rF   rR   r5   r   Úmaxr   Úrjustr   r`   )r3   r@   ÚwidthÚphraseZ
phrase_strZ
phrase_lenZ
half_widthÚcontextÚconcordance_listrc   Zword_offsetsZ
query_wordZleft_contextZright_contextr   r   Z
line_printÚconcordance_liner"   rk   r#   Úfind_concordanceº   sB    

ÿù	z!ConcordanceIndex.find_concordanceé   c                 C   sl   | j ||d�}|stdƒ nLt|t|ƒƒ}td|› dt|ƒ› d�ƒ t|d|… ƒD ]\}}t|jƒ qTdS )a±  
        Print concordance lines given the query word.
        :param word: The target word or phrase (a list of strings)
        :type word: str or list
        :param lines: The number of lines to display (default=25)
        :type lines: int
        :param width: The width of each line, in characters (default=80)
        :type width: int
        :param save: The option to save the concordance.
        :type save: bool
        )rq   z
no matcheszDisplaying z of z	 matches:N)rv   ÚprintÚminr   r7   r   )r3   r@   rq   Úlinesrt   r!   ru   r"   r"   r#   Úprint_concordanceê   s    
z"ConcordanceIndex.print_concordanceN)re   )re   rw   )
rX   rY   rZ   r[   r;   r    rc   rd   rv   r{   r"   r"   r"   r#   r]   ‚   s   

0r]   c                   @   s    e Zd ZdZdd„ Zdd„ ZdS )ÚTokenSearcheraâ  
    A class that makes it easier to use regular expressions to search
    over tokenized strings.  The tokenized string is converted to a
    string where tokens are marked with angle brackets -- e.g.,
    ``'<the><window><is><still><open>'``.  The regular expression
    passed to the ``findall()`` method is modified to treat angle
    brackets as non-capturing parentheses, in addition to matching the
    token boundaries; and to have ``'.'`` not match the angle brackets.
    c                 C   s   d  dd„ |D ƒ¡| _d S )NÚ c                 s   s   | ]}d | d V  qdS )ú<ú>Nr"   rK   r"   r"   r#   r4     r)   z)TokenSearcher.__init__.<locals>.<genexpr>)rT   Ú_rawr2   r"   r"   r#   r;     s    zTokenSearcher.__init__c                 C   s~   t  dd|¡}t  dd|¡}t  dd|¡}t  dd|¡}t  || j¡}|D ] }| d¡sJ| d¡rJtd	ƒ‚qJd
d„ |D ƒ}|S )a  
        Find instances of the regular expression in the text.
        The text is a list of tokens, and a regexp pattern to match
        a single token must be surrounded by angle brackets.  E.g.

        >>> from nltk.text import TokenSearcher
        >>> from nltk.book import text1, text5, text9
        >>> text5.findall("<.*><.*><bro>")
        you rule bro; telling you bro; u twizted bro
        >>> text1.findall("<a>(<.*>)<man>")
        monied; nervous; dangerous; white; white; white; pious; queer; good;
        mature; white; Cape; great; wise; wise; butterless; white; fiendish;
        pale; furious; better; certain; complete; dismasted; younger; brave;
        brave; brave; brave
        >>> text9.findall("<th.*>{3,}")
        thread through those; the thought that; that the thing; the thing
        that; that that thing; through these than through; them that the;
        through the thick; them that they; thought that the

        :param regexp: A regular expression
        :type regexp: str
        z\sr}   r~   z(?:<(?:r   z)>)z	(?<!\\)\.z[^>]z$Bad regexp for TokenSearcher.findallc                 S   s   g | ]}|d d…   d¡‘qS )r   éÿÿÿÿz><©Úsplit©r*   Úhr"   r"   r#   r-   5  r)   z)TokenSearcher.findall.<locals>.<listcomp>)ÚreÚsubÚfindallr€   Ú
startswithÚendswithrS   )r3   ÚregexpÚhitsr…   r"   r"   r#   rˆ     s    
zTokenSearcher.findallN)rX   rY   rZ   r[   r;   rˆ   r"   r"   r"   r#   r|     s   
r|   c                   @   sØ   e Zd ZdZdZd6dd„Zdd„ Zdd	„ Zd7dd„Zd8dd„Z	d9dd„Z
d:dd„Zdd„ Zdd„ Zdd„ Zd;dd„Zd<dd„Zd d!„ Zd=d#d$„Zd>d'd(„Zd)d*„ Zd+d,„ Zd-d.„ Ze d/¡Zd0d1„ Zd2d3„ Zd4d5„ ZdS )?ÚTextaÛ  
    A wrapper around a sequence of simple (string) tokens, which is
    intended to support initial exploration of texts (via the
    interactive console).  Its methods perform a variety of analyses
    on the text's contexts (e.g., counting, concordancing, collocation
    discovery), and display the results.  If you wish to write a
    program which makes use of these analyses, then you should bypass
    the ``Text`` class, and use the appropriate analysis function or
    class directly instead.

    A ``Text`` is typically initialized from a given document or
    corpus.  E.g.:

    >>> import nltk.corpus
    >>> from nltk.text import Text
    >>> moby = Text(nltk.corpus.gutenberg.words('melville-moby_dick.txt'))

    TNc                 C   sˆ   | j rt|ƒ}|| _|r || _ndd|dd… v rb|dd…  d¡}d dd„ |d|… D ƒ¡| _n"d dd„ |dd	… D ƒ¡d
 | _dS )zv
        Create a Text object.

        :param tokens: The source text.
        :type tokens: sequence of str
        ú]NrC   rN   c                 s   s   | ]}t |ƒV  qd S r%   ©Ústr©r*   Útokr"   r"   r#   r4   b  r)   z Text.__init__.<locals>.<genexpr>r   c                 s   s   | ]}t |ƒV  qd S r%   r�   r‘   r"   r"   r#   r4   d  r)   é   z...)Ú_COPY_TOKENSr^   r    Únamera   rT   )r3   r    r•   Úendr"   r"   r#   r;   S  s     zText.__init__c                 C   s
   | j | S r%   )r    )r3   r!   r"   r"   r#   Ú__getitem__j  s    zText.__getitem__c                 C   s
   t | jƒS r%   )r   r    r=   r"   r"   r#   Ú__len__m  s    zText.__len__éO   rw   c                 C   s.   d| j vrt| jdd„ d�| _| j |||¡S )aÌ  
        Prints a concordance for ``word`` with the specified context window.
        Word matching is not case-sensitive.

        :param word: The target word or phrase (a list of strings)
        :type word: str or list
        :param width: The width of each line, in characters (default=80)
        :type width: int
        :param lines: The number of lines to display (default=25)
        :type lines: int

        :seealso: ``ConcordanceIndex``
        Ú_concordance_indexc                 S   s   |   ¡ S r%   ©r   ©Úsr"   r"   r#   r(   „  r)   z"Text.concordance.<locals>.<lambda>©r:   )Ú__dict__r]   r    rš   r{   ©r3   r@   rq   rz   r"   r"   r#   Úconcordancet  s
    

ÿzText.concordancec                 C   s4   d| j vrt| jdd„ d�| _| j ||¡d|… S )aÎ  
        Generate a concordance for ``word`` with the specified context window.
        Word matching is not case-sensitive.

        :param word: The target word or phrase (a list of strings)
        :type word: str or list
        :param width: The width of each line, in characters (default=80)
        :type width: int
        :param lines: The number of lines to display (default=25)
        :type lines: int

        :seealso: ``ConcordanceIndex``
        rš   c                 S   s   |   ¡ S r%   r›   rœ   r"   r"   r#   r(   ™  r)   z'Text.concordance_list.<locals>.<lambda>rž   N)rŸ   r]   r    rš   rv   r    r"   r"   r#   rt   ‰  s
    

ÿzText.concordance_listrC   ri   c                    sŠ   d| j v r| j|kr| j|ks„|| _|| _ddlm} | d¡‰ t | j|¡}| 	d¡ | 
‡ fdd„¡ tƒ }t| |j|¡ƒ| _| jS )aÚ  
        Return collocations derived from the text, ignoring stopwords.

            >>> from nltk.book import text4
            >>> text4.collocation_list()[:2]
            [('United', 'States'), ('fellow', 'citizens')]

        :param num: The maximum number of collocations to return.
        :type num: int
        :param window_size: The number of tokens spanned by a collocation (default=2)
        :type window_size: int
        :rtype: list(tuple(str, str))
        Ú_collocationsr   )Ú	stopwordsÚenglishri   c                    s   t | ƒdk p|  ¡ ˆ v S )Né   )r   r   )r1   ©Zignored_wordsr"   r#   r(   ¹  r)   z'Text.collocation_list.<locals>.<lambda>)rŸ   Z_numZ_window_sizeÚnltk.corpusr£   rM   r   Z
from_wordsr    Zapply_freq_filterZapply_word_filterr
   r^   ZnbestZlikelihood_ratior¢   )r3   ÚnumÚwindow_sizer£   ÚfinderZbigram_measuresr"   r¦   r#   Úcollocation_list�  s$    ÿþý

ÿzText.collocation_listc                 C   s*   dd„ |   ||¡D ƒ}tt|dd�ƒ dS )aý  
        Print collocations derived from the text, ignoring stopwords.

            >>> from nltk.book import text4
            >>> text4.collocations() # doctest: +NORMALIZE_WHITESPACE
            United States; fellow citizens; years ago; four years; Federal
            Government; General Government; Vice President; American people; God
            bless; Chief Justice; one another; fellow Americans; Old World;
            Almighty God; Fellow citizens; Chief Magistrate; every citizen; Indian
            tribes; public debt; foreign nations


        :param num: The maximum number of collocations to print.
        :type num: int
        :param window_size: The number of tokens spanned by a collocation (default=2)
        :type window_size: int
        c                 S   s   g | ]\}}|d  | ‘qS ©rN   r"   ©r*   Zw1Zw2r"   r"   r#   r-   Ó  s   z%Text.collocations.<locals>.<listcomp>ú; )Ú	separatorN)r«   rx   r   )r3   r¨   r©   Zcollocation_stringsr"   r"   r#   ÚcollocationsÀ  s    
ÿzText.collocationsc                 C   s   | j  |¡S )zJ
        Count the number of times this word appears in the text.
        )r    Úcountrb   r"   r"   r#   r±   Ø  s    z
Text.countc                 C   s   | j  |¡S )zQ
        Find the index of the first occurrence of the word in the text.
        )r    ra   rb   r"   r"   r#   ra   Þ  s    z
Text.indexc                 C   s   t ‚d S r%   )ÚNotImplementedError)r3   Úmethodr"   r"   r#   Úreadabilityä  s    zText.readabilityc                    s˜   d| j vr$t| jdd„ dd„ d�| _ˆ ¡ ‰| jj‰ˆˆ ¡ v rŒtˆˆ ƒ‰ t‡ ‡‡fdd„ˆ ¡ D ƒƒ}dd	„ | 	|¡D ƒ}t
t|ƒƒ nt
d
ƒ dS )a~  
        Distributional similarity: find other words which appear in the
        same contexts as the specified word; list most similar words first.

        :param word: The word used to seed the similarity search
        :type word: str
        :param num: The number of words to generate (default=20)
        :type num: int
        :seealso: ContextIndex.similar_words()
        Ú_word_context_indexc                 S   s   |   ¡ S r%   )Úisalphar&   r"   r"   r#   r(   ö  r)   zText.similar.<locals>.<lambda>c                 S   s   |   ¡ S r%   r›   rœ   r"   r"   r#   r(   ö  r)   )r,   r:   c                 3   s0   | ](}ˆ| D ]}|ˆ v r|ˆks|V  qqd S r%   r"   rO   ©rL   Zwcir@   r"   r#   r4   ÿ  s   ýzText.similar.<locals>.<genexpr>c                 S   s   g | ]\}}|‘qS r"   r"   ©r*   r1   Ú_r"   r"   r#   r-     r)   z Text.similar.<locals>.<listcomp>z
No matchesN)rŸ   r   r    rµ   r   r8   Z
conditionsr>   r   Úmost_commonrx   r   )r3   r@   r¨   rV   rM   r"   r·   r#   Úsimilarè  s    
ÿþzText.similarc              
   C   sš   d| j vrt| jdd„ d�| _zJ| j |d¡}|s<tdƒ n*dd„ | |¡D ƒ}ttd	d
„ |D ƒƒƒ W n, ty” } zt|ƒ W Y d}~n
d}~0 0 dS )aY  
        Find contexts where the specified words appear; list
        most frequent common contexts first.

        :param words: The words used to seed the similarity search
        :type words: str
        :param num: The number of words to generate (default=20)
        :type num: int
        :seealso: ContextIndex.common_contexts()
        rµ   c                 S   s   |   ¡ S r%   r›   rœ   r"   r"   r#   r(     r)   z&Text.common_contexts.<locals>.<lambda>rž   TzNo common contexts were foundc                 S   s   g | ]\}}|‘qS r"   r"   r¸   r"   r"   r#   r-      r)   z(Text.common_contexts.<locals>.<listcomp>c                 s   s   | ]\}}|d  | V  qdS )r¹   Nr"   r­   r"   r"   r#   r4   !  r)   z'Text.common_contexts.<locals>.<genexpr>N)	rŸ   r   r    rµ   rW   rx   rº   r   rS   )r3   rM   r¨   rV   Zranked_contextsÚer"   r"   r#   rW   
  s    

ÿ
zText.common_contextsc                 C   s   ddl m} || |ƒ dS )zü
        Produce a plot showing the distribution of the words through the text.
        Requires pylab to be installed.

        :param words: The words to be plotted
        :type words: list(str)
        :seealso: nltk.draw.dispersion_plot()
        r   )Údispersion_plotN)Z	nltk.drawr½   )r3   rM   r½   r"   r"   r#   r½   &  s    	zText.dispersion_plotr¥   c                 C   s(   t ||ƒ\}}t|d�}| ||¡ |S )N)Úorder)r	   r   Zfit)r3   Ztokenized_sentsrH   Z
train_dataZpadded_sentsÚmodelr"   r"   r#   Ú_train_default_ngram_lm3  s    
zText._train_default_ngram_lméd   é*   c           	      C   sæ   dd„ t d | j¡ƒD ƒ| _t| dƒsFtdtjd� | j| jdd�| _	g }|d	ksZJ d
ƒ‚t
|ƒ|k r°t| j	j|||d�ƒD ](\}}|dkrŽq||dkrš q¦| |¡ q||d7 }qZ|rÂd |¡d nd}|t|d|… ƒ }t|ƒ |S )a  
        Print random text, generated using a trigram language model.
        See also `help(nltk.lm)`.

        :param length: The length of text to generate (default=100)
        :type length: int

        :param text_seed: Generation can be conditioned on preceding context.
        :type text_seed: list(str)

        :param random_seed: A random seed or an instance of `random.Random`. If provided,
            makes the random sampling part of generation reproducible. (default=42)
        :type random_seed: int
        c                 S   s   g | ]}|  d ¡‘qS r¬   r‚   )r*   Úsentr"   r"   r#   r-   I  s   z!Text.generate.<locals>.<listcomp>rN   Ú_trigram_modelzBuilding ngram index...)Úfiler¥   )rH   r   z!The `length` must be more than 0.)Ú	text_seedÚrandom_seedz<s>z</s>r   r}   N)r   rT   r    Z_tokenized_sentsÚhasattrrx   ÚsysÚstderrrÀ   rÄ   r   r7   Úgenerater`   r   )	r3   ÚlengthrÆ   rÇ   Zgenerated_tokensÚidxÚtokenÚprefixZ
output_strr"   r"   r#   rË   9  s4    ÿ
ÿÿÿ
zText.generatec                 G   s   |   ¡ j|Ž S )zc
        See documentation for FreqDist.plot()
        :seealso: nltk.prob.FreqDist.plot()
        )ÚvocabÚplot)r3   Úargsr"   r"   r#   rÑ   g  s    z	Text.plotc                 C   s   d| j vrt| ƒ| _| jS )z.
        :seealso: nltk.prob.FreqDist
        Ú_vocab)rŸ   r   rÓ   r=   r"   r"   r#   rÐ   n  s    

z
Text.vocabc                 C   s@   d| j vrt| ƒ| _| j |¡}dd„ |D ƒ}tt|dƒƒ dS )aÓ  
        Find instances of the regular expression in the text.
        The text is a list of tokens, and a regexp pattern to match
        a single token must be surrounded by angle brackets.  E.g.

        >>> from nltk.book import text1, text5, text9
        >>> text5.findall("<.*><.*><bro>")
        you rule bro; telling you bro; u twizted bro
        >>> text1.findall("<a>(<.*>)<man>")
        monied; nervous; dangerous; white; white; white; pious; queer; good;
        mature; white; Cape; great; wise; wise; butterless; white; fiendish;
        pale; furious; better; certain; complete; dismasted; younger; brave;
        brave; brave; brave
        >>> text9.findall("<th.*>{3,}")
        thread through those; the thought that; that the thing; the thing
        that; that that thing; through these than through; them that the;
        through the thick; them that they; thought that the

        :param regexp: A regular expression
        :type regexp: str
        Ú_token_searcherc                 S   s   g | ]}d   |¡‘qS r¬   )rT   r„   r"   r"   r#   r-   ’  r)   z Text.findall.<locals>.<listcomp>r®   N)rŸ   r|   rÔ   rˆ   rx   r   )r3   r‹   rŒ   r"   r"   r#   rˆ   w  s
    

zText.findallz\w+|[\.\!\?]c                 C   sŒ   |d }|dkr*| j  || ¡s*|d8 }q|dkr:|| nd}|d }|t|ƒk rl| j  || ¡sl|d7 }qF|t|ƒkr€|| nd}||fS )zÙ
        One left & one right token, both case-normalized.  Skip over
        non-sentence-final punctuation.  Used by the ``ContextIndex``
        that is created for ``similar()`` and ``common_contexts()``.
        r   r   r   r   )Ú_CONTEXT_REÚmatchr   )r3   r    r!   Újr   r   r"   r"   r#   Ú_context›  s    

zText._contextc                 C   s
   d| j  S ©Nz
<Text: %s>©r•   r=   r"   r"   r#   Ú__str__³  s    zText.__str__c                 C   s
   d| j  S rÙ   rÚ   r=   r"   r"   r#   rd   ¶  s    zText.__repr__)N)r™   rw   )r™   rw   )rC   ri   )rC   ri   )rC   )rC   )r¥   )rÁ   NrÂ   )rX   rY   rZ   r[   r”   r;   r—   r˜   r¡   rt   r«   r°   r±   ra   r´   r»   rW   r½   rÀ   rË   rÑ   rÐ   rˆ   r†   ÚcompilerÕ   rØ   rÛ   rd   r"   r"   r"   r#   r�   9  s0   



#

"


.	"
r�   c                   @   s0   e Zd ZdZdd„ Zdd„ Zdd„ Zdd	„ Zd
S )ÚTextCollectiona;  A collection of texts, which can be loaded with list of texts, or
    with a corpus consisting of one or more texts, and which supports
    counting, concordancing, collocation discovery, etc.  Initialize a
    TextCollection as follows:

    >>> import nltk.corpus
    >>> from nltk.text import TextCollection
    >>> from nltk.book import text1, text2, text3
    >>> gutenberg = TextCollection(nltk.corpus.gutenberg)
    >>> mytexts = TextCollection([text1, text2, text3])

    Iterating over a TextCollection produces all the tokens of all the
    texts in order.
    c                    s@   t ˆ dƒr ‡ fdd„ˆ  ¡ D ƒ‰ ˆ | _t | tˆ ƒ¡ i | _d S )NrM   c                    s   g | ]}ˆ   |¡‘qS r"   )rM   )r*   Úf©Úsourcer"   r#   r-   Í  r)   z+TextCollection.__init__.<locals>.<listcomp>)rÈ   ZfileidsÚ_textsr�   r;   r   Ú
_idf_cache)r3   rà   r"   rß   r#   r;   Ë  s
    
zTextCollection.__init__c                 C   s   |  |¡t|ƒ S )z"The frequency of the term in text.)r±   r   ©r3   ÚtermÚtextr"   r"   r#   ÚtfÓ  s    zTextCollection.tfc                    sj   | j  ˆ ¡}|du rft‡ fdd„| jD ƒƒ}t| jƒdkrBtdƒ‚|rXtt| jƒ| ƒnd}|| j ˆ < |S )z¤The number of texts in the corpus divided by the
        number of texts that the term appears in.
        If a term does not appear in the corpus, 0.0 is returned.Nc                    s   g | ]}ˆ |v rd ‘qS )Tr"   )r*   rå   ©rä   r"   r#   r-   Þ  r)   z&TextCollection.idf.<locals>.<listcomp>r   z+IDF undefined for empty document collectiong        )râ   rG   r   rá   rS   r   )r3   rä   ÚidfÚmatchesr"   rç   r#   rè   ×  s    
zTextCollection.idfc                 C   s   |   ||¡|  |¡ S r%   )ræ   rè   rã   r"   r"   r#   Útf_idfå  s    zTextCollection.tf_idfN)rX   rY   rZ   r[   r;   ræ   rè   rê   r"   r"   r"   r#   rÝ   »  s
   rÝ   c                  C   sâ   ddl m}  t| jdd�ƒ}t|ƒ tƒ  tdƒ | d¡ tƒ  tdƒ | d¡ tƒ  tdƒ | ¡  tƒ  tdƒ | g d	¢¡ tƒ  td
ƒ | 	d¡ tƒ  tdƒ td|d ƒ td|dd… ƒ td| 
¡ d ƒ d S )Nr   )ÚbrownÚnews)Ú
categorieszConcordance:zDistributionally similar words:zCollocations:zDispersion plot:)rì   ÚreportZsaidZ	announcedzVocabulary plot:é2   z	Indexing:ztext[3]:r¥   z
text[3:5]:é   ztext.vocab()['news']:)r§   rë   r�   rM   rx   r¡   r»   r°   r½   rÑ   rÐ   )rë   rå   r"   r"   r#   Údemoé  s.    


rñ   Ú__main__)r   r]   r|   r�   rÝ   )(r[   r†   rÉ   rf   Úcollectionsr   r   r   Ú	functoolsr   Úmathr   Znltk.collocationsr   Znltk.lmr   Znltk.lm.preprocessingr	   Znltk.metricsr
   r   Znltk.probabilityr   r6   r   Znltk.tokenizer   Z	nltk.utilr   r   r   r   r   r]   r|   r�   rÝ   rñ   rX   Ú__all__r"   r"   r"   r#   Ú<module>	   s<   þ[8   .
