a
     ˜ëjà-  ã                   @   sÚ   d dl mZ d dlmZmZ d dlmZ d dlmZ d dl	m
Z
mZ d dlmZmZmZ dd„ Zd	d
„ ZG dd„ dƒZG dd„ deƒZG dd„ deƒZeddƒZeddƒZeddƒZG dd„ deƒZG dd„ deeƒZdS )é    )Ú
namedtuple)ÚpartialÚwraps)ÚCategorizedCorpusReader)ÚPlaintextCorpusReader)ÚconcatÚread_blankline_block)Úblankline_tokenizeÚsent_tokenizeÚword_tokenizec                    s   t ˆ ƒ‡ fdd„ƒ}|S )zœ
    A decorator that allows a function to be called with
    a single string of comma-separated values which become
    individual function arguments.
    c                     sœ   t ƒ }| D ]N}t|tƒr4| dd„ | d¡D ƒ¡ q
t|t ƒrN| t|ƒ¡ q
| |¡ q
| ¡ D ]*\}}t|tƒrbdd„ | d¡D ƒ||< qbˆ |i |¤ŽS )Nc                 S   s   h | ]}|  ¡ ’qS © ©Ústrip©Ú.0Úpartr   r   úZ/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/nltk/corpus/reader/markdown.pyÚ	<setcomp>   ó    z?comma_separated_string_args.<locals>.wrapper.<locals>.<setcomp>ú,c                 S   s   h | ]}|  ¡ ’qS r   r   r   r   r   r   r      r   )ÚlistÚ
isinstanceÚstrÚappendÚsplitÚsetÚitems)ÚargsÚkwargsÚ_argsÚargÚnameÚvalue©Úfuncr   r   Úwrapper   s    


z,comma_separated_string_args.<locals>.wrapper)r   )r$   r%   r   r#   r   Úcomma_separated_string_args
   s    r&   c                 C   s    t | ƒ}|r| |d ¡gS |S ©Nr   )r   Úrender)ÚstreamÚparserÚblockr   r   r   Úread_parse_blankline_block#   s    r,   c                   @   sT   e Zd Zdd„ Zdd„ Zdd„ Zedd„ ƒZed	d
„ ƒZedd„ ƒZ	edd„ ƒZ
dS )ÚMarkdownBlockc                 C   s   || _ d| _d S )Né   )ÚcontentÚtruncate_at)Úselfr/   r   r   r   Ú__init__+   s    zMarkdownBlock.__init__c                 C   s   | j j› dtt| ƒƒ› d�S )Nz	(content=ú))Ú	__class__Ú__name__Úreprr   ©r1   r   r   r   Ú__repr__/   s    zMarkdownBlock.__repr__c                 C   s,   | j d | j… › t| j ƒ| jkr$dnd› �S )Nz...Ú )r/   r0   Úlenr7   r   r   r   Ú__str__2   s    ÿÿzMarkdownBlock.__str__c                 C   s   | j S ©N©r/   r7   r   r   r   Úraw8   s    zMarkdownBlock.rawc                 C   s
   t | jƒS r<   )r   r/   r7   r   r   r   Úwords<   s    zMarkdownBlock.wordsc                 C   s   dd„ t | jƒD ƒS )Nc                 S   s   g | ]}t |ƒ‘qS r   ©r   ©r   Úsentr   r   r   Ú
<listcomp>B   r   z'MarkdownBlock.sents.<locals>.<listcomp>)r
   r/   r7   r   r   r   Úsents@   s    zMarkdownBlock.sentsc                 C   s   dd„ t | jƒD ƒS )Nc                 S   s   g | ]}d d„ t |ƒD ƒ‘qS )c                 S   s   g | ]}t |ƒ‘qS r   r@   rA   r   r   r   rC   G   r   z2MarkdownBlock.paras.<locals>.<listcomp>.<listcomp>)r
   ©r   Úparar   r   r   rC   F   s   ÿz'MarkdownBlock.paras.<locals>.<listcomp>©r	   r/   r7   r   r   r   ÚparasD   s    þzMarkdownBlock.parasN)r5   Ú
__module__Ú__qualname__r2   r8   r;   Úpropertyr>   r?   rD   rH   r   r   r   r   r-   *   s   


r-   c                       s@   e Zd Z‡ fdd„Zedd„ ƒZedd„ ƒZedd„ ƒZ‡  ZS )	Ú	CodeBlockc                    s   || _ tƒ j|Ž  d S r<   )ÚlanguageÚsuperr2   )r1   rM   r   ©r4   r   r   r2   M   s    zCodeBlock.__init__c                 C   s   dd„ | j  ¡ D ƒS )Nc                 S   s   g | ]}t |ƒ‘qS r   r@   ©r   Úliner   r   r   rC   S   r   z#CodeBlock.sents.<locals>.<listcomp>©r/   Ú
splitlinesr7   r   r   r   rD   Q   s    zCodeBlock.sentsc                 C   s
   | j  ¡ S r<   rR   r7   r   r   r   ÚlinesU   s    zCodeBlock.linesc                 C   s   dd„ t | jƒD ƒS )Nc                 S   s   g | ]}d d„ |  ¡ D ƒ‘qS )c                 S   s   g | ]}t |ƒ‘qS r   r@   rP   r   r   r   rC   \   r   z.CodeBlock.paras.<locals>.<listcomp>.<listcomp>)rS   rE   r   r   r   rC   [   s   ÿz#CodeBlock.paras.<locals>.<listcomp>rG   r7   r   r   r   rH   Y   s    þzCodeBlock.paras)	r5   rI   rJ   r2   rK   rD   rT   rH   Ú__classcell__r   r   rO   r   rL   L   s   

rL   c                       s   e Zd Z‡ fdd„Z‡  ZS )ÚMarkdownSectionc                    s   || _ || _tƒ j|Ž  d S r<   )ÚheadingÚlevelrN   r2   )r1   rW   rX   r   rO   r   r   r2   b   s    zMarkdownSection.__init__)r5   rI   rJ   r2   rU   r   r   rO   r   rV   a   s   rV   ÚImagezlabel, src, titleÚLinkzlabel, href, titleÚListzis_ordered, itemsc                       s*   e Zd Zddœ‡ fdd„
Zdd„ Z‡  ZS )ÚMarkdownCorpusReaderN©r*   c                   sz   ddl m} ddlm} ddlm} || _| jd u rN|d|d�| _| j |¡ | dt	t
| jd�¡ tƒ j|i |¤Ž d S )	Nr   )Ú
MarkdownIt)ÚRendererPlain)Úfront_matter_pluginZ
commonmark)Zrenderer_clsZpara_block_readerr]   )Zmarkdown_itr^   Zmdit_plain.rendererr_   Zmdit_py_plugins.front_matterr`   r*   ZuseÚ
setdefaultr   r,   rN   r2   )r1   r*   r   r   r^   r_   r`   rO   r   r   r2   n   s    
ÿzMarkdownCorpusReader.__init__c                 C   s,   t ƒ }|  |¡D ]}| | j |¡¡ q|S r<   )r   Z_para_block_readerÚextendZ_word_tokenizerÚtokenize)r1   r)   r?   rF   r   r   r   Ú_read_word_block~   s    z%MarkdownCorpusReader._read_word_block)r5   rI   rJ   r2   rd   rU   r   r   rO   r   r\   m   s   r\   c                       s0  e Zd ZdZddœdd„Zed1‡ fdd„	ƒZed2‡ fd	d
„	ƒZed3‡ fdd„	ƒZed4‡ fdd„	ƒZ	ed5‡ fdd„	ƒZ
ed6‡ fdd„	ƒZdd„ Zdd„ Zed7dd„ƒZdd„ Zed8dd„ƒZdd„ Zed9dd „ƒZd!d"„ Zed:d#d$„ƒZd%d&„ Zed;d'd(„ƒZd)d*„ Zed<d+d,„ƒZd-d.„ Zed=d/d0„ƒZ‡  ZS )>ÚCategorizedMarkdownCorpusReadera  
    A reader for markdown corpora whose documents are divided into
    categories based on their file identifiers.

    Based on nltk.corpus.reader.plaintext.CategorizedPlaintextCorpusReader:
    https://www.nltk.org/_modules/nltk/corpus/reader/api.html#CategorizedCorpusReader
    Útags)Ú	cat_fieldc                   s�   g d¢}t ‡ fdd„|D ƒƒs(tƒ ˆ d< t | ˆ ¡ tj| g|¢R i ˆ ¤Ž | jdurŒ| jsŒ| jD ](}|  |¡}|rb|d  |g ¡| j|< qbdS )a  
        Initialize the corpus reader. Categorization arguments
        (``cat_pattern``, ``cat_map``, and ``cat_file``) are passed to
        the ``CategorizedCorpusReader`` constructor.  The remaining arguments
        are passed to the ``MarkdownCorpusReader`` constructor.
        )Zcat_patternÚcat_mapZcat_filec                 3   s   | ]}|ˆ v V  qd S r<   r   )r   r    ©r   r   r   Ú	<genexpr>–   r   z;CategorizedMarkdownCorpusReader.__init__.<locals>.<genexpr>rh   Nr   )	ÚanyÚdictr   r2   r\   Ú_mapÚ_fileidsÚmetadataÚget)r1   rg   r   r   Zcat_argsZfile_idro   r   ri   r   r2   Ž   s    


z(CategorizedMarkdownCorpusReader.__init__Nc                    s   t ƒ  |¡S r<   )rN   Ú
categories)r1   ÚfileidsrO   r   r   rq   ¥   s    z*CategorizedMarkdownCorpusReader.categoriesc                    s   |d u r| j S tƒ  |¡S r<   )rn   rN   rr   )r1   rq   rO   r   r   rr   ©   s    z'CategorizedMarkdownCorpusReader.fileidsc                    s   t ƒ  |  ||¡¡S r<   )rN   r>   Ú_resolve©r1   rr   rq   rO   r   r   r>   ²   s    z#CategorizedMarkdownCorpusReader.rawc                    s   t ƒ  |  ||¡¡S r<   )rN   r?   rs   rt   rO   r   r   r?   ¶   s    z%CategorizedMarkdownCorpusReader.wordsc                    s   t ƒ  |  ||¡¡S r<   )rN   rD   rs   rt   rO   r   r   rD   º   s    z%CategorizedMarkdownCorpusReader.sentsc                    s   t ƒ  |  ||¡¡S r<   )rN   rH   rs   rt   rO   r   r   rH   ¾   s    z%CategorizedMarkdownCorpusReader.parasc                    s*   t ‡ ‡fdd„ˆjˆ ||¡dd�D ƒƒS )Nc                    s    g | ]\}}ˆj |ˆ |d �‘qS ))Úencoding)Z
CorpusView)r   ÚpathÚenc©Úreaderr1   r   r   rC   Æ   s   ÿzECategorizedMarkdownCorpusReader.concatenated_view.<locals>.<listcomp>T)Zinclude_encoding)r   Zabspathsrs   )r1   ry   rr   rq   r   rx   r   Úconcatenated_viewÄ   s    ÿþÿz1CategorizedMarkdownCorpusReader.concatenated_viewc                    s*   ddl m‰  ‡ fdd„| j | ¡ ¡D ƒS )Nr   ©Ú	safe_loadc                    s    g | ]}|j d krˆ |jƒ‘qS )Zfront_matter)Útyper/   ©r   Útr{   r   r   rC   Ñ   s   
þzCCategorizedMarkdownCorpusReader.metadata_reader.<locals>.<listcomp>)Zyamlr|   r*   ÚparseÚread©r1   r)   r   r{   r   Úmetadata_readerÎ   s    
þz/CategorizedMarkdownCorpusReader.metadata_readerc                 C   s   |   | j||¡S r<   )rz   rƒ   rt   r   r   r   ro   ×   s    z(CategorizedMarkdownCorpusReader.metadatac           
         s„   ˆ j  | ¡ ¡}tdd„ |ƒ}tdd„ |ƒ}tƒ }t||ƒD ]4\}}| |¡}| ||¡}	| |||	d … ¡ q<‡ fdd„|D ƒS )Nc                 S   s   | j dko| jdkS )Nr   Zblockquote_open©rX   r}   ©r   r   r   r   Ú<lambda>Þ   r   zCCategorizedMarkdownCorpusReader.blockquote_reader.<locals>.<lambda>c                 S   s   | j dko| jdkS )Nr   Zblockquote_closer„   r…   r   r   r   r†   á   r   é   c                    s(   g | ] }t ˆ jjj|ˆ jjd d�ƒ‘qS )N©Úenv)r-   r*   Úrendererr(   Úoptions©r   r+   r7   r   r   rC   è   s   ýÿzECategorizedMarkdownCorpusReader.blockquote_reader.<locals>.<listcomp>©r*   r€   r�   Úfilterr   ÚzipÚindexr   )
r1   r)   ÚtokensÚopening_tokensÚclosing_tokensÚblockquotesÚoÚcÚopening_indexÚclosing_indexr   r7   r   Úblockquote_readerÛ   s    ÿÿ

üz1CategorizedMarkdownCorpusReader.blockquote_readerc                 C   s   |   | j||¡S r<   )rz   r™   rt   r   r   r   r”   ï   s    z+CategorizedMarkdownCorpusReader.blockquotesc                 C   s   dd„ | j  | ¡ ¡D ƒS )Nc                 S   s.   g | ]&}|j d kr|jdv rt|j|jƒ‘qS )r   )ZfenceZ
code_block)rX   r}   rL   Úinfor/   r~   r   r   r   rC   ô   s   ûþzECategorizedMarkdownCorpusReader.code_block_reader.<locals>.<listcomp>)r*   r€   r�   r‚   r   r   r   Úcode_block_readeró   s    ûz1CategorizedMarkdownCorpusReader.code_block_readerc                 C   s   |   | j||¡S r<   )rz   r›   rt   r   r   r   Úcode_blocksý   s    z+CategorizedMarkdownCorpusReader.code_blocksc                 C   s$   dd„ t dd„ | j | ¡ ¡ƒD ƒS )Nc              	   S   s<   g | ]4}|j D ](}|jd krt|j| d¡| d¡ƒ‘qqS )ÚimageÚsrcÚtitle)Úchildrenr}   rY   r/   ÚattrGet)r   Úinline_tokenÚchild_tokenr   r   r   rC     s   

÷ýz@CategorizedMarkdownCorpusReader.image_reader.<locals>.<listcomp>c                 S   s
   | j dkS ©NÚinline©r}   r…   r   r   r   r†   	  r   z>CategorizedMarkdownCorpusReader.image_reader.<locals>.<lambda>©rŽ   r*   r€   r�   r‚   r   r   r   Úimage_reader  s
    ÿúz,CategorizedMarkdownCorpusReader.image_readerc                 C   s   |   | j||¡S r<   )rz   r¨   rt   r   r   r   Úimages  s    z&CategorizedMarkdownCorpusReader.imagesc                 C   s$   dd„ t dd„ | j | ¡ ¡ƒD ƒS )Nc              	   S   sN   g | ]F}t |jƒD ]6\}}|jd krt|j|d  j| d¡| d¡ƒ‘qqS )Z	link_openr‡   ÚhrefrŸ   )Ú	enumerater    r}   rZ   r/   r¡   )r   r¢   Úir£   r   r   r   rC     s   
÷ýz?CategorizedMarkdownCorpusReader.link_reader.<locals>.<listcomp>c                 S   s
   | j dkS r¤   r¦   r…   r   r   r   r†     r   z=CategorizedMarkdownCorpusReader.link_reader.<locals>.<lambda>r§   r‚   r   r   r   Úlink_reader  s
    ÿúz+CategorizedMarkdownCorpusReader.link_readerc                 C   s   |   | j||¡S r<   )rz   r­   rt   r   r   r   Úlinks!  s    z%CategorizedMarkdownCorpusReader.linksc           
         s�   | j  | ¡ ¡}d‰t‡fdd„|ƒ}d‰ t‡ fdd„|ƒ}tƒ }t||ƒD ]4\}}| |¡}| ||¡}	| |||	d … ¡ qLdd„ |D ƒS )	N)Zbullet_list_openÚordered_list_openc                    s   | j dko| jˆ v S r'   r„   r…   )Úopening_typesr   r   r†   )  r   z=CategorizedMarkdownCorpusReader.list_reader.<locals>.<lambda>)Zbullet_list_closeZordered_list_closec                    s   | j dko| jˆ v S r'   r„   r…   )Úclosing_typesr   r   r†   -  r   r‡   c                 S   s*   g | ]"}t |d  jdkdd„ |D ƒƒ‘qS )r   r¯   c                 S   s   g | ]}|j r|j ‘qS r   r=   r~   r   r   r   rC   7  r   zJCategorizedMarkdownCorpusReader.list_reader.<locals>.<listcomp>.<listcomp>)r[   r}   )r   r‘   r   r   r   rC   4  s
   üþz?CategorizedMarkdownCorpusReader.list_reader.<locals>.<listcomp>r�   )
r1   r)   r‘   r’   r“   Zlist_blocksr•   r–   r—   r˜   r   )r±   r°   r   Úlist_reader%  s"    ÿÿ
ûz+CategorizedMarkdownCorpusReader.list_readerc                 C   s   |   | j||¡S r<   )rz   r²   rt   r   r   r   Úlists<  s    z%CategorizedMarkdownCorpusReader.listsc                    sˆ   t ƒ t ƒ  }}ˆ j | ¡ ¡D ]H}|jdkrX|jdkrX|sF| |¡ qf| |¡ |g}q|r| |¡ q|rv| |¡ ‡ fdd„|D ƒS )Nr   Zheading_openc              
      s>   g | ]6}t |d  j|d j d¡ˆ jjj|ˆ jjdd�ƒ‘qS )r‡   r   ú#Nrˆ   )rV   r/   ZmarkupÚcountr*   rŠ   r(   r‹   rŒ   r7   r   r   rC   M  s   ûýzBCategorizedMarkdownCorpusReader.section_reader.<locals>.<listcomp>)r   r*   r€   r�   rX   r}   r   )r1   r)   Zsection_blocksr+   r   r   r7   r   Úsection_reader@  s    


úz.CategorizedMarkdownCorpusReader.section_readerc                 C   s   |   | j||¡S r<   )rz   r¶   rt   r   r   r   ÚsectionsV  s    z(CategorizedMarkdownCorpusReader.sections)N)N)NN)NN)NN)NN)NN)NN)NN)NN)NN)NN)NN)r5   rI   rJ   Ú__doc__r2   r&   rq   rr   r>   r?   rD   rH   rz   rƒ   ro   r™   r”   r›   rœ   r¨   r©   r­   r®   r²   r³   r¶   r·   rU   r   r   rO   r   re   …   sH   
	
re   N)Úcollectionsr   Ú	functoolsr   r   Znltk.corpus.reader.apir   Znltk.corpus.reader.plaintextr   Znltk.corpus.reader.utilr   r   Znltk.tokenizer	   r
   r   r&   r,   r-   rL   rV   rY   rZ   r[   r\   re   r   r   r   r   Ú<module>   s   "


