a
     ˜ëj¥6  ã                   @   s„   d Z ddlZddlZddlmZ ddlmZmZ ddlm	Z	 dd„ Z
G dd	„ d	e	ƒZG d
d„ dƒZG dd„ dƒZG dd„ deƒZdS )z9
A reader for corpora whose documents are in MTE format.
é    N)Úreduce)ÚTaggedCorpusReaderÚconcat)ÚXMLCorpusViewc                 C   s   |   ||¡S ©N)Úfindall)ÚrootÚpathÚns© r   úU/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/nltk/corpus/reader/mte.pyÚxpath   s    r   c                   @   s$   e Zd ZdZddd„Zddd„ZdS )	ÚMTECorpusViewz0
    Class for lazy viewing the MTE Corpus.
    Nc                 C   s   t  | |||¡ d S r   )r   Ú__init__)ÚselfZfileidÚtagspecÚelt_handlerr   r   r   r      s    zMTECorpusView.__init__c              	   C   s   t tdd„ t | |||¡ƒƒS )Nc                 S   s   | d uS r   r   ©Úxr   r   r   Ú<lambda>   ó    z*MTECorpusView.read_block.<locals>.<lambda>)ÚlistÚfilterr   Ú
read_block)r   Ústreamr   r   r   r   r   r      s    þÿzMTECorpusView.read_block)N)NN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r   r   r   r   r      s   
r   c                   @   sê   e Zd ZdZdddœZdZdZdZdZd	Z	d
d„ Z
edd„ ƒZedd„ ƒZedd„ ƒZedd„ ƒZedd„ ƒZedd„ ƒZedd„ ƒZedd„ ƒZedd„ ƒZdd„ Zd d!„ Zd"d#„ Zd$d%„ Zd&d'„ Zd(d)„ Zd*d+„ Zd,d-„ Zd.d/„ Zd0S )1ÚMTEFileReaderz©
    Class for loading the content of the multext-east corpus. It
    parses the xml files and does some tag-filtering depending on the
    given method parameters.
    zhttps://www.tei-c.org/ns/1.0z%https://www.w3.org/XML/1998/namespace)ZteiÚxmlz{https://www.tei-c.org/ns/1.0}z'{https://www.w3.org/XML/1998/namespace}zTEI/text/body/div/div/p/s/(w|c)zTEI/text/body/div/div/p/szTEI/text/body/div/div/pc                 C   s
   || _ d S r   )Ú_MTEFileReader__file_path)r   Ú	file_pathr   r   r   r   3   s    zMTEFileReader.__init__c                 C   s   |j S r   )Útext©ÚclsÚeltÚcontextr   r   r   Ú	_word_elt6   s    zMTEFileReader._word_eltc                    s   ‡ fdd„t |dˆ jƒD ƒS )Nc                    s   g | ]}ˆ   |d ¡‘qS r   )r(   ©Ú.0Úw©r%   r   r   Ú
<listcomp><   r   z+MTEFileReader._sent_elt.<locals>.<listcomp>Ú*©r   r
   r$   r   r,   r   Ú	_sent_elt:   s    zMTEFileReader._sent_eltc                    s   ‡ fdd„t |dˆ jƒD ƒS )Nc                    s   g | ]}ˆ   |d ¡‘qS r   )r0   ©r*   Úsr,   r   r   r-   @   r   z+MTEFileReader._para_elt.<locals>.<listcomp>r.   r/   r$   r   r,   r   Ú	_para_elt>   s    zMTEFileReader._para_eltc                 C   sÊ   d|j vr|jdfS | jdkr8| jdkr8|j|j d fS | jdkrb| jdkrb|jt |j d ¡fS t dt dd| j¡ d ¡}| 	|j d ¡rÂ| jdkrª|j|j d fS |jt |j d ¡fS nd S d S )	NZanaÚ ÚmsdÚ	universalú^ú-Ú.z.*$)
Úattribr#   Ú_MTEFileReader__tagsÚ_MTEFileReader__tagsetÚMTETagConverterÚmsd_to_universalÚreÚcompileÚsubÚmatch)r%   r&   r'   Útagsr   r   r   Ú_tagged_word_eltB   s    


þzMTEFileReader._tagged_word_eltc                    s*   t tdd„ ‡ fdd„t|dˆ jƒD ƒƒƒS )Nc                 S   s   | d uS r   r   r   r   r   r   r   \   r   z0MTEFileReader._tagged_sent_elt.<locals>.<lambda>c                    s   g | ]}ˆ   |d ¡‘qS r   )rD   r)   r,   r   r   r-   ]   r   z2MTEFileReader._tagged_sent_elt.<locals>.<listcomp>r.   ©r   r   r   r
   r$   r   r,   r   Ú_tagged_sent_eltX   s    þÿzMTEFileReader._tagged_sent_eltc                    s*   t tdd„ ‡ fdd„t|dˆ jƒD ƒƒƒS )Nc                 S   s   | d uS r   r   r   r   r   r   r   e   r   z0MTEFileReader._tagged_para_elt.<locals>.<lambda>c                    s   g | ]}ˆ   |d ¡‘qS r   )rF   r1   r,   r   r   r-   f   r   z2MTEFileReader._tagged_para_elt.<locals>.<listcomp>r.   rE   r$   r   r,   r   Ú_tagged_para_elta   s    þÿzMTEFileReader._tagged_para_eltc                 C   s(   d|j vr|jdfS |j|j d fS d S )NZlemmar4   )r:   r#   r$   r   r   r   Ú_lemma_word_eltj   s    

zMTEFileReader._lemma_word_eltc                    s   ‡ fdd„t |dˆ jƒD ƒS )Nc                    s   g | ]}ˆ   |d ¡‘qS r   )rH   r)   r,   r   r   r-   s   r   z1MTEFileReader._lemma_sent_elt.<locals>.<listcomp>r.   r/   r$   r   r,   r   Ú_lemma_sent_eltq   s    zMTEFileReader._lemma_sent_eltc                    s   ‡ fdd„t |dˆ jƒD ƒS )Nc                    s   g | ]}ˆ   |d ¡‘qS r   )rI   r1   r,   r   r   r-   w   r   z1MTEFileReader._lemma_para_elt.<locals>.<listcomp>r.   r/   r$   r   r,   r   Ú_lemma_para_eltu   s    zMTEFileReader._lemma_para_eltc                 C   s   t | jtjtjƒS r   )r   r!   r   Ú	word_pathr(   ©r   r   r   r   Úwordsy   s    ÿzMTEFileReader.wordsc                 C   s   t | jtjtjƒS r   )r   r!   r   Ú	sent_pathr0   rL   r   r   r   Úsents~   s    ÿzMTEFileReader.sentsc                 C   s   t | jtjtjƒS r   )r   r!   r   Ú	para_pathr3   rL   r   r   r   Úparasƒ   s    ÿzMTEFileReader.parasc                 C   s   t | jtjtjƒS r   )r   r!   r   rK   rH   rL   r   r   r   Úlemma_wordsˆ   s    ÿzMTEFileReader.lemma_wordsc                 C   s   |t _|t _t| jt jt jƒS r   )r   r<   r;   r   r!   rK   rD   ©r   ÚtagsetrC   r   r   r   Útagged_words�   s
    ÿzMTEFileReader.tagged_wordsc                 C   s   t | jtjtjƒS r   )r   r!   r   rN   rI   rL   r   r   r   Úlemma_sents”   s    ÿzMTEFileReader.lemma_sentsc                 C   s   |t _|t _t| jt jt jƒS r   )r   r<   r;   r   r!   rN   rF   rS   r   r   r   Útagged_sents™   s
    ÿzMTEFileReader.tagged_sentsc                 C   s   t | jtjtjƒS r   )r   r!   r   rP   rJ   rL   r   r   r   Úlemma_paras    s    ÿzMTEFileReader.lemma_parasc                 C   s   |t _|t _t| jt jt jƒS r   )r   r<   r;   r   r!   rP   rG   rS   r   r   r   Útagged_paras¥   s
    ÿzMTEFileReader.tagged_parasN)r   r   r   r   r
   Ztag_nsZxml_nsrK   rN   rP   r   Úclassmethodr(   r0   r3   rD   rF   rG   rH   rI   rJ   rM   rO   rQ   rR   rU   rV   rW   rX   rY   r   r   r   r   r   "   sJ   þ








r   c                   @   s:   e Zd ZdZdddddddd	d
ddddœZedd„ ƒZdS )r=   zu
    Class for converting msd tags to universal tags, more conversion
    options are currently not implemented.
    ZADJZADPZADVZCONJZDETZNOUNZNUMZPRTZPRONZVERBr9   ÚX)ÚAÚSÚRÚCÚDÚNÚMÚQÚPÚVr9   r8   c                 C   s4   | d dks| d n| d }|t jvr*d}t j| S )zö
        This function converts the annotation from the Multex-East to the universal tagset
        as described in Chapter 5 of the NLTK-Book

        Unknown Tags will be mapped to X. Punctuation marks are not supported in MSD tags, so
        r   ú#é   r8   )r=   Úmapping_msd_universal)ÚtagZ	indicatorr   r   r   r>   Â   s    
z MTETagConverter.msd_to_universalN)r   r   r   r   rh   Ústaticmethodr>   r   r   r   r   r=   ­   s    ôr=   c                   @   s|   e Zd ZdZddd„Zdd„ Zddd	„Zdd
d„Zddd„Zd dd„Z	d!dd„Z
d"dd„Zd#dd„Zd$dd„Zd%dd„ZdS )&ÚMTECorpusReaderzæ
    Reader for corpora following the TEI-p5 xml scheme, such as MULTEXT-East.
    MULTEXT-East contains part-of-speech-tagged words with a quite precise tagging
    scheme. These tags can be converted to the Universal tagset
    NÚutf8c                 C   s   t  | |||¡ d| _dS )a.  
        Construct a new MTECorpusreader for a set of documents
        located at the given root directory.  Example usage:

            >>> root = '/...path to corpus.../'
            >>> reader = MTECorpusReader(root, 'oana-*.xml', 'utf8') # doctest: +SKIP

        :param root: The root directory for this corpus. (default points to location in multext config file)
        :param fileids: A list or regexp specifying the fileids in this corpus. (default is oana-en.xml)
        :param encoding: The encoding of the given files (default is utf8)
        z00README.txtN)r   r   Z_readme)r   r   ÚfileidsÚencodingr   r   r   r   Ù   s    zMTECorpusReader.__init__c                    sP   |d u rˆ j }nt|tƒr |g}t‡ fdd„|ƒ}tdd„ |ƒ}|sLtdƒ |S )Nc                    s
   | ˆ j v S r   )Ú_fileidsr   rL   r   r   r   î   r   z+MTECorpusReader.__fileids.<locals>.<lambda>c                 S   s   | dvS )N)zoana-bg.xmlzoana-mk.xmlr   r   r   r   r   r   ð   r   z$No valid multext-east file specified)ro   Ú
isinstanceÚstrr   Úprint©r   rm   r   rL   r   Z	__fileidsè   s    
zMTECorpusReader.__fileidsc                    s   t ‡ fdd„ˆ  |¡D ƒƒS )z½
        :param fileids: A list specifying the fileids that should be used.
        :return: the given file(s) as a list of words and punctuation symbols.
        :rtype: list(str)
        c                    s$   g | ]}t tj ˆ j|¡ƒ ¡ ‘qS r   )r   Úosr	   ÚjoinÚ_rootrM   ©r*   ÚfrL   r   r   r-   ü   s   ÿz)MTECorpusReader.words.<locals>.<listcomp>©r   Ú_MTECorpusReader__fileidsrs   r   rL   r   rM   õ   s
    
þÿzMTECorpusReader.wordsc                    s   t ‡ fdd„ˆ  |¡D ƒƒS )zõ
        :param fileids: A list specifying the fileids that should be used.
        :return: the given file(s) as a list of sentences or utterances,
                 each encoded as a list of word strings
        :rtype: list(list(str))
        c                    s$   g | ]}t tj ˆ j|¡ƒ ¡ ‘qS r   )r   rt   r	   ru   rv   rO   rw   rL   r   r   r-   
  s   ÿz)MTECorpusReader.sents.<locals>.<listcomp>ry   rs   r   rL   r   rO     s
    
þÿzMTECorpusReader.sentsc                    s   t ‡ fdd„ˆ  |¡D ƒƒS )a  
        :param fileids: A list specifying the fileids that should be used.
        :return: the given file(s) as a list of paragraphs, each encoded as a list
                 of sentences, which are in turn encoded as lists of word string
        :rtype: list(list(list(str)))
        c                    s$   g | ]}t tj ˆ j|¡ƒ ¡ ‘qS r   )r   rt   r	   ru   rv   rQ   rw   rL   r   r   r-     s   ÿz)MTECorpusReader.paras.<locals>.<listcomp>ry   rs   r   rL   r   rQ     s
    
þÿzMTECorpusReader.parasc                    s   t ‡ fdd„ˆ  |¡D ƒƒS )a  
        :param fileids: A list specifying the fileids that should be used.
        :return: the given file(s) as a list of words, the corresponding lemmas
                 and punctuation symbols, encoded as tuples (word, lemma)
        :rtype: list(tuple(str,str))
        c                    s$   g | ]}t tj ˆ j|¡ƒ ¡ ‘qS r   )r   rt   r	   ru   rv   rR   rw   rL   r   r   r-   &  s   ÿz/MTECorpusReader.lemma_words.<locals>.<listcomp>ry   rs   r   rL   r   rR     s
    
þÿzMTECorpusReader.lemma_wordsr5   r4   c                    s<   ˆdksˆdkr0t ‡ ‡‡fdd„ˆ  |¡D ƒƒS tdƒ dS )a;  
        :param fileids: A list specifying the fileids that should be used.
        :param tagset: The tagset that should be used in the returned object,
                       either "universal" or "msd", "msd" is the default
        :param tags: An MSD Tag that is used to filter all parts of the used corpus
                     that are not more precise or at least equal to the given tag
        :return: the given file(s) as a list of tagged words and punctuation symbols
                 encoded as tuples (word, tag)
        :rtype: list(tuple(str, str))
        r6   r5   c                    s(   g | ] }t tj ˆ j|¡ƒ ˆˆ¡‘qS r   )r   rt   r	   ru   rv   rU   rw   ©r   rC   rT   r   r   r-   9  s   ýÿz0MTECorpusReader.tagged_words.<locals>.<listcomp>úUnknown tagset specified.N©r   rz   rr   ©r   rm   rT   rC   r   r{   r   rU   ,  s    üÿ	zMTECorpusReader.tagged_wordsc                    s   t ‡ fdd„ˆ  |¡D ƒƒS )aB  
        :param fileids: A list specifying the fileids that should be used.
        :return: the given file(s) as a list of sentences or utterances, each
                 encoded as a list of tuples of the word and the corresponding
                 lemma (word, lemma)
        :rtype: list(list(tuple(str, str)))
        c                    s$   g | ]}t tj ˆ j|¡ƒ ¡ ‘qS r   )r   rt   r	   ru   rv   rV   rw   rL   r   r   r-   L  s   ÿz/MTECorpusReader.lemma_sents.<locals>.<listcomp>ry   rs   r   rL   r   rV   C  s
    
þÿzMTECorpusReader.lemma_sentsc                    s<   ˆdksˆdkr0t ‡ ‡‡fdd„ˆ  |¡D ƒƒS tdƒ dS )aH  
        :param fileids: A list specifying the fileids that should be used.
        :param tagset: The tagset that should be used in the returned object,
                       either "universal" or "msd", "msd" is the default
        :param tags: An MSD Tag that is used to filter all parts of the used corpus
                     that are not more precise or at least equal to the given tag
        :return: the given file(s) as a list of sentences or utterances, each
                 each encoded as a list of (word,tag) tuples
        :rtype: list(list(tuple(str, str)))
        r6   r5   c                    s(   g | ] }t tj ˆ j|¡ƒ ˆˆ¡‘qS r   )r   rt   r	   ru   rv   rW   rw   r{   r   r   r-   _  s   ýÿz0MTECorpusReader.tagged_sents.<locals>.<listcomp>r|   Nr}   r~   r   r{   r   rW   R  s    üÿ	zMTECorpusReader.tagged_sentsc                    s   t ‡ fdd„ˆ  |¡D ƒƒS )am  
        :param fileids: A list specifying the fileids that should be used.
        :return: the given file(s) as a list of paragraphs, each encoded as a
                 list of sentences, which are in turn encoded as a list of
                 tuples of the word and the corresponding lemma (word, lemma)
        :rtype: list(List(List(tuple(str, str))))
        c                    s$   g | ]}t tj ˆ j|¡ƒ ¡ ‘qS r   )r   rt   r	   ru   rv   rX   rw   rL   r   r   r-   r  s   ÿz/MTECorpusReader.lemma_paras.<locals>.<listcomp>ry   rs   r   rL   r   rX   i  s
    
þÿzMTECorpusReader.lemma_parasc                    s<   ˆdksˆdkr0t ‡ ‡‡fdd„ˆ  |¡D ƒƒS tdƒ dS )a  
        :param fileids: A list specifying the fileids that should be used.
        :param tagset: The tagset that should be used in the returned object,
                       either "universal" or "msd", "msd" is the default
        :param tags: An MSD Tag that is used to filter all parts of the used corpus
                     that are not more precise or at least equal to the given tag
        :return: the given file(s) as a list of paragraphs, each encoded as a
                 list of sentences, which are in turn encoded as a list
                 of (word,tag) tuples
        :rtype: list(list(list(tuple(str, str))))
        r6   r5   c                    s(   g | ] }t tj ˆ j|¡ƒ ˆˆ¡‘qS r   )r   rt   r	   ru   rv   rY   rw   r{   r   r   r-   †  s   ýÿz0MTECorpusReader.tagged_paras.<locals>.<listcomp>r|   Nr}   r~   r   r{   r   rY   x  s    üÿ	zMTECorpusReader.tagged_paras)NNrl   )N)N)N)N)Nr5   r4   )N)Nr5   r4   )N)Nr5   r4   )r   r   r   r   r   rz   rM   rO   rQ   rR   rU   rV   rW   rX   rY   r   r   r   r   rk   Ò   s   








rk   )r   rt   r?   Ú	functoolsr   Znltk.corpus.readerr   r   Znltk.corpus.reader.xmldocsr   r   r   r   r=   rk   r   r   r   r   Ú<module>   s    %