a
     ˜ëjx  ã                   @   sT   d Z ddlZddlZddlmZ ddlmZmZmZ ddl	m
Z
 G dd„ deƒZdS )z{
A reader for corpora that consist of Tweets. It is assumed that the Tweets
have been serialised into line-delimited JSON.
é    N)ÚCorpusReader)ÚStreamBackedCorpusViewÚZipFilePathPointerÚconcat)ÚTweetTokenizerc                   @   sL   e Zd ZdZeZdeƒ dfdd„Zddd„Zddd	„Z	dd
d„Z
dd„ ZdS )ÚTwitterCorpusReadera7  
    Reader for corpora that consist of Tweets represented as a list of line-delimited JSON.

    Individual Tweets can be tokenized using the default tokenizer, or by a
    custom tokenizer specified as a parameter to the constructor.

    Construct a new Tweet corpus reader for a set of documents
    located at the given root directory.

    If you made your own tweet collection in a directory called
    `twitter-files`, then you can initialise the reader as::

        from nltk.corpus import TwitterCorpusReader
        reader = TwitterCorpusReader(root='/path/to/twitter-files', '.*\.json')

    However, the recommended approach is to set the relevant directory as the
    value of the environmental variable `TWITTER`, and then invoke the reader
    as follows::

       root = os.environ['TWITTER']
       reader = TwitterCorpusReader(root, '.*\.json')

    If you want to work directly with the raw Tweets, the `json` library can
    be used::

       import json
       for tweet in reader.docs():
           print(json.dumps(tweet, indent=1, sort_keys=True))

    NÚutf8c                 C   sX   t  | |||¡ |  | j¡D ]0}t|tƒr,qtj |¡dkrt	d|› d�ƒ‚q|| _
dS )a  
        :param root: The root directory for this corpus.
        :param fileids: A list or regexp specifying the fileids in this corpus.
        :param word_tokenizer: Tokenizer for breaking the text of Tweets into
            smaller units, including but not limited to words.
        r   zFile z	 is emptyN)r   Ú__init__ÚabspathsZ_fileidsÚ
isinstancer   ÚosÚpathÚgetsizeÚ
ValueErrorÚ_word_tokenizer)ÚselfÚrootÚfileidsZword_tokenizerÚencodingr   © r   úY/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/nltk/corpus/reader/twitter.pyr	   :   s    	
zTwitterCorpusReader.__init__c                    s    t ‡ fdd„ˆ  |dd¡D ƒƒS )a(  
        Returns the full Tweet objects, as specified by `Twitter
        documentation on Tweets
        <https://dev.twitter.com/docs/platform-objects/tweets>`_

        :return: the given file(s) as a list of dictionaries deserialised
            from JSON.
        :rtype: list(dict)
        c                    s$   g | ]\}}}ˆ j |ˆ j|d �‘qS ))r   )Ú
CorpusViewÚ_read_tweets)Ú.0r   ÚencZfileid©r   r   r   Ú
<listcomp>Y   s   ÿz,TwitterCorpusReader.docs.<locals>.<listcomp>T)r   r
   )r   r   r   r   r   ÚdocsN   s
    

þÿzTwitterCorpusReader.docsc              	   C   s\   |   |¡}g }|D ]D}z,|d }t|tƒr6| | j¡}| |¡ W q tyT   Y q0 q|S )z›
        Returns only the text content of Tweets in the file(s)

        :return: the given file(s) as a list of Tweets.
        :rtype: list(str)
        Útext)r   r   ÚbytesÚdecoder   ÚappendÚKeyError)r   r   Z
fulltweetsÚtweetsZjsonor   r   r   r   Ústrings_   s    

zTwitterCorpusReader.stringsc                    s"   |   |¡}| j‰ ‡ fdd„|D ƒS )zÎ
        :return: the given file(s) as a list of the text content of Tweets as
            as a list of words, screenanames, hashtags, URLs and punctuation symbols.

        :rtype: list(list(str))
        c                    s   g | ]}ˆ   |¡‘qS r   )Útokenize)r   Út©Ú	tokenizerr   r   r   {   ó    z1TwitterCorpusReader.tokenized.<locals>.<listcomp>)r$   r   )r   r   r#   r   r'   r   Ú	tokenizedr   s    
zTwitterCorpusReader.tokenizedc                 C   s>   g }t dƒD ],}| ¡ }|s$|  S t |¡}| |¡ q|S )zS
        Assumes that each line in ``stream`` is a JSON-serialised object.
        é
   )ÚrangeÚreadlineÚjsonÚloadsr!   )r   Ústreamr#   ÚiÚlineZtweetr   r   r   r   }   s    
z TwitterCorpusReader._read_tweets)N)N)N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r   r	   r   r$   r*   r   r   r   r   r   r      s   ÿ



r   )r6   r.   r   Znltk.corpus.reader.apir   Znltk.corpus.reader.utilr   r   r   Znltk.tokenizer   r   r   r   r   r   Ú<module>   s   