a
     ˜ëj  ã                   @   sZ   d dl mZ d dlmZ d dlmZmZ ejZeeddddd�Z	de	_
d	d
„ Zdd„ ZdS )é    )Úpartial)Úchain)Ú
everygramsÚpad_sequenceTz<s>z</s>)Zpad_leftZleft_pad_symbolZ	pad_rightZright_pad_symbolz‘Pads both ends of a sentence to length specified by ngram order.

    Following convention <s> pads the start of sentence </s> pads its end.
    c                 C   s   t tt|| d�ƒ| d�S )zpHelper with some useful defaults.

    Applies pad_both_ends to sentence and follows it up with everygrams.
    ©Ún©Úmax_len)r   ÚlistÚpad_both_ends)ÚorderZsentence© r   úT/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/nltk/lm/preprocessing.pyÚpadded_everygrams   s    r   c                    s.   t tˆ d�‰‡ ‡fdd„|D ƒttˆ|ƒƒfS )aú  Default preprocessing for a sequence of sentences.

    Creates two iterators:

    - sentences padded and turned into sequences of `nltk.util.everygrams`
    - sentences padded as above and chained together for a flat stream of words

    :param order: Largest ngram length produced by `everygrams`.
    :param text: Text to iterate over. Expected to be an iterable of sentences.
    :type text: Iterable[Iterable[str]]
    :return: iterator over text as ngrams, iterator over text as vocabulary data
    r   c                 3   s"   | ]}t tˆ|ƒƒˆ d �V  qdS )r   N)r   r
   )Ú.0Úsent©r   Z
padding_fnr   r   Ú	<genexpr>1   ó    z,padded_everygram_pipeline.<locals>.<genexpr>)r   r   ÚflattenÚmap)r   Útextr   r   r   Úpadded_everygram_pipeline"   s    þr   N)Ú	functoolsr   Ú	itertoolsr   Z	nltk.utilr   r   Úfrom_iterabler   r   Ú__doc__r   r   r   r   r   r   Ú<module>   s   û