a
     ˜ëjÚ   ã                   @   s¢   d Z ddlZddlZddlmZmZ ddlmZ ddlmZ ddl	m
Z
 ddlmZ ddlmZ G d	d
„ d
ed�Zdd„ Zdd„ Zddd„ZG dd„ ded�ZdS )zLanguage Model Interface.é    N)ÚABCMetaÚabstractmethod)Úbisect)Ú
accumulate)ÚNgramCounter)Ú	log_base2)Ú
Vocabularyc                   @   s0   e Zd ZdZdd„ Zedd„ ƒZedd„ ƒZdS )	Ú	SmoothingzìNgram Smoothing Interface

    Implements Chen & Goodman 1995's idea that all smoothing algorithms have
    certain features in common. This should ideally allow smoothing algorithms to
    work both with Backoff and Interpolation.
    c                 C   s   || _ || _dS )zä
        :param vocabulary: The Ngram vocabulary object.
        :type vocabulary: nltk.lm.vocab.Vocabulary
        :param counter: The counts of the vocabulary items.
        :type counter: nltk.lm.counter.NgramCounter
        N)ÚvocabÚcounts)ÚselfÚ
vocabularyÚcounter© r   úJ/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/nltk/lm/api.pyÚ__init__   s    zSmoothing.__init__c                 C   s
   t ƒ ‚d S ©N©ÚNotImplementedError)r   Úwordr   r   r   Úunigram_score&   s    zSmoothing.unigram_scorec                 C   s
   t ƒ ‚d S r   r   ©r   r   Úcontextr   r   r   Úalpha_gamma*   s    zSmoothing.alpha_gammaN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r   r   r   r   r   r   r	      s   

r	   )Ú	metaclassc                 C   s   t | ƒt| ƒ S )z0Return average (aka mean) for sequence of items.)ÚsumÚlen)Úitemsr   r   r   Ú_mean/   s    r"   c                 C   s   t | tjƒr| S t | ¡S r   )Ú
isinstanceÚrandomÚRandom)Zseed_or_generatorr   r   r   Ú_random_generator4   s    r&   c                 C   sR   | st dƒ‚t| ƒt|ƒkr$t dƒ‚tt|ƒƒ}|d }| ¡ }| t||| ƒ S )z`Like random.choice, but with weights.

    Heavily inspired by python 3.6 `random.choices`.
    z"Can't choose from empty populationz3The number of weights does not match the populationéÿÿÿÿ)Ú
ValueErrorr    Úlistr   r$   r   )Ú
populationÚweightsÚrandom_generatorÚcum_weightsÚtotalÚ	thresholdr   r   r   Ú_weighted_choice:   s    r0   c                   @   sh   e Zd ZdZddd„Zddd„Zddd„Zedd	d
„ƒZddd„Z	dd„ Z
dd„ Zdd„ Zddd„ZdS )ÚLanguageModelzKABC for Language Models.

    Cannot be directly instantiated itself.

    Nc                 C   sZ   || _ |r.t|tƒs.tjd| jj›d�dd� |du r<tƒ n|| _|du rPtƒ n|| _	dS )ap  Creates new LanguageModel.

        :param vocabulary: If provided, this vocabulary will be used instead
            of creating a new one when training.
        :type vocabulary: `nltk.lm.Vocabulary` or None
        :param counter: If provided, use this object to count ngrams.
        :type counter: `nltk.lm.NgramCounter` or None
        :param ngrams_fn: If given, defines how sentences in training text are turned to ngram
            sequences.
        :type ngrams_fn: function or None
        :param pad_fn: If given, defines how sentences in training text are padded.
        :type pad_fn: function or None
        z$The `vocabulary` argument passed to z- must be an instance of `nltk.lm.Vocabulary`.é   )Ú
stacklevelN)
Úorderr#   r   ÚwarningsÚwarnÚ	__class__r   r
   r   r   )r   r4   r   r   r   r   r   r   P   s    ýzLanguageModel.__init__c                    s@   ˆ j s"|du rtdƒ‚ˆ j  |¡ ˆ j ‡ fdd„|D ƒ¡ dS )zeTrains the model on a text.

        :param text: Training text as a sequence of sentences.

        Nz:Cannot fit without a vocabulary or text to create it from.c                 3   s   | ]}ˆ j  |¡V  qd S r   )r
   Úlookup)Ú.0Úsent©r   r   r   Ú	<genexpr>t   ó    z$LanguageModel.fit.<locals>.<genexpr>)r
   r(   Úupdater   )r   ÚtextZvocabulary_textr   r;   r   Úfith   s    ÿzLanguageModel.fitc                 C   s$   |   | j |¡|r| j |¡nd¡S )z©Masks out of vocab (OOV) words and computes their model score.

        For model-specific logic of calculating scores, see the `unmasked_score`
        method.
        N)Úunmasked_scorer
   r8   r   r   r   r   Úscorev   s    ÿzLanguageModel.scorec                 C   s
   t ƒ ‚dS )aÒ  Score a word given some optional context.

        Concrete models are expected to provide an implementation.
        Note that this method does not mask its arguments with the OOV label.
        Use the `score` method for that.

        :param str word: Word for which we want the score
        :param tuple(str) context: Context the word is in.
            If `None`, compute unigram score.
        :param context: tuple(str) or None
        :rtype: float
        Nr   r   r   r   r   rA   €   s    zLanguageModel.unmasked_scorec                 C   s   t |  ||¡ƒS )z‡Evaluate the log score of this word in this context.

        The arguments are the same as for `score` and `unmasked_score`.

        )r   rB   r   r   r   r   Úlogscore�   s    zLanguageModel.logscorec                 C   s"   |r| j t|ƒd  | S | j jS )z²Helper method for retrieving counts for a given context.

        Assumes context has been checked and oov words in it masked.
        :type context: tuple(str) or None

        é   )r   r    Zunigrams)r   r   r   r   r   Úcontext_counts˜   s     ÿzLanguageModel.context_countsc                    s   dt ‡ fdd„|D ƒƒ S )a?  Calculate cross-entropy of model for given evaluation text.

        This implementation is based on the Shannon-McMillan-Breiman theorem,
        as used and referenced by Dan Jurafsky and Jordan Boyd-Graber.

        :param Iterable(tuple(str)) text_ngrams: A sequence of ngram tuples.
        :rtype: float

        r'   c                    s$   g | ]}ˆ   |d  |dd … ¡‘qS )r'   N)rC   )r9   Zngramr;   r   r   Ú
<listcomp>®   r=   z)LanguageModel.entropy.<locals>.<listcomp>)r"   ©r   Ztext_ngramsr   r;   r   Úentropy£   s    
ÿzLanguageModel.entropyc                 C   s   t d|  |¡ƒS )zŽCalculates the perplexity of the given text.

        This is simply 2 ** cross-entropy for the text, so the arguments are the same.

        g       @)ÚpowrH   rG   r   r   r   Ú
perplexity±   s    zLanguageModel.perplexityrD   c                    sì   |du rg nt |ƒ}t|ƒ}|dkr¼t|ƒˆjkrF|ˆj d d… n|‰ ˆ ˆj ˆ ¡¡}ˆ r”|s”tˆ ƒdkr|ˆ dd… ng ‰ ˆ ˆj ˆ ¡¡}q\t|ƒ}t|t	‡ ‡fdd„|D ƒƒ|ƒS g }t
|ƒD ]}| ˆjd|| |d�¡ qÈ|S )aà  Generate words from the model.

        :param int num_words: How many words to generate. By default 1.
        :param text_seed: Generation can be conditioned on preceding context.
        :param random_seed: A random seed or an instance of `random.Random`. If provided,
            makes the random sampling part of generation reproducible.
        :return: One (str) word or a list of words generated from model.

        Examples:

        >>> from nltk.lm import MLE
        >>> lm = MLE(2)
        >>> lm.fit([[("a", "b"), ("b", "c")]], vocabulary_text=['a', 'b', 'c'])
        >>> lm.fit([[("a",), ("b",), ("c",)]])
        >>> lm.generate(random_seed=3)
        'a'
        >>> lm.generate(text_seed=['a'])
        'b'

        NrD   c                 3   s   | ]}ˆ  |ˆ ¡V  qd S r   )rB   )r9   Úw©r   r   r   r   r<   á   r=   z)LanguageModel.generate.<locals>.<genexpr>)Ú	num_wordsÚ	text_seedÚrandom_seed)r)   r&   r    r4   rE   r
   r8   Úsortedr0   ÚtupleÚrangeÚappendÚgenerate)r   rM   rN   rO   r,   ZsamplesÚ	generatedÚ_r   rL   r   rT   ¹   s6    ÿýýýÿzLanguageModel.generate)NN)N)N)N)N)rD   NN)r   r   r   r   r   r@   rB   r   rA   rC   rE   rH   rJ   rT   r   r   r   r   r1   I   s   




r1   )N)r   r$   r5   Úabcr   r   r   Ú	itertoolsr   Znltk.lm.counterr   Znltk.lm.utilr   Znltk.lm.vocabularyr   r	   r"   r&   r0   r1   r   r   r   r   Ú<module>   s   
