a
    ¡˜ëj�%  ã                   @  sš  d dl mZ d dlmZ d dlmZ d dlZd dlZd dlm	Z	 d dl
mZmZ d dlmZ d dlmZmZ d d	lmZ d d
lmZ dd„ Zdd„ Zdd„ ZG dd„ dƒZdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd „ Z d!d"„ Z!d#d$„ Z"d%d&„ Z#d'd(œd)d*„Z$d+d,„ Z%d-d.„ Z&d'd(œd/d0„Z'd1d2„ Z(d3d4„ Z)d5d6„ Z*d7d8„ Z+d9d:„ Z,d;d<„ Z-d=d>„ Z.ej/ 0d?d@dAg¡dBdCœdDdE„ƒZ1dFdG„ Z2dHdI„ Z3dJdK„ Z4dS )Lé    )Úannotations)ÚAny)ÚpatchN)ÚMockFixture)Úassert_round_trips_through_JSONÚexample_doc_path)Úchunk_by_title)ÚElementTypeÚTitle©Úpartition_md)Ú#UNSTRUCTURED_INCLUDE_DEBUG_METADATAc                  C  sj   t dƒ} t| d�}t|ƒdks"J ‚ddd„ |D ƒvs8J ‚t|d tƒsJJ ‚trfdd„ |D ƒd	hksfJ ‚d S )
Nú	README.md©Úfilenamer   Z	PageBreakc                 S  s   g | ]
}|j ‘qS © )Úcategory)Ú.0Úelemr   r   úb/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/test_unstructured/partition/test_md.pyÚ
<listcomp>   ó    z3test_partition_md_from_filename.<locals>.<listcomp>c                 S  s   h | ]}|j j’qS r   )ÚmetadataZdetection_origin©r   Úelementr   r   r   Ú	<setcomp>   r   z2test_partition_md_from_filename.<locals>.<setcomp>Zmd)r   r   ÚlenÚ
isinstancer
   r   ©r   Úelementsr   r   r   Útest_partition_md_from_filename   s    
r    c                  C  sP   t dƒ} t| dƒ�}t|d�}W d   ƒ n1 s20    Y  t|ƒdksLJ ‚d S )Nr   Úrb©Úfiler   )r   Úopenr   r   )r   Úfr   r   r   r   Útest_partition_md_from_file   s    (r&   c                  C  sh   t tdƒƒ�} |  ¡ }W d   ƒ n1 s*0    Y  t|d�}t|ƒdksNJ ‚tdd„ |D ƒƒsdJ ‚d S )Nr   ©Útextr   c                 s  s   | ]}|j jd u V  qd S ©N©r   r   ©r   Úer   r   r   Ú	<genexpr>*   r   z.test_partition_md_from_text.<locals>.<genexpr>)r$   r   Úreadr   r   Úall©r%   r(   r   r   r   r   Útest_partition_md_from_text#   s
    &
r1   c                   @  s"   e Zd Zi fddddœdd„ZdS )ÚMockResponseÚstrÚintzdict[str, Any]©r(   Ústatus_codeÚheadersc                 C  s    || _ || _|dk | _|| _d S )Ni,  )r(   r6   Úokr7   )Úselfr(   r6   r7   r   r   r   Ú__init__.   s    
zMockResponse.__init__N)Ú__name__Ú
__module__Ú__qualname__r:   r   r   r   r   r2   -   s   r2   c                  C  s®   t dƒ} t| ƒ�}| ¡ }W d   ƒ n1 s.0    Y  t|dddid�}tjtd|d��}tdd	�}W d   ƒ n1 sz0    Y  t|ƒd
ks”J ‚t	dd„ |D ƒƒsªJ ‚d S )Nr   éÈ   úContent-Typeútext/markdownr5   Úget©Zreturn_valueúhttps://fake.url©Úurlr   c                 s  s   | ]}|j jd u V  qd S r)   r*   r+   r   r   r   r-   C   r   z-test_partition_md_from_url.<locals>.<genexpr>)
r   r$   r.   r2   r   ÚobjectÚrequestsr   r   r/   )r   r%   r(   ÚresponseÚ_r   r   r   r   Útest_partition_md_from_url5   s    
&ý(rJ   c               	   C  s²   t dƒ} t| ƒ�}| ¡ }W d   ƒ n1 s.0    Y  t|dddid�}tjtd|d��D}t t	¡� t
dd	� W d   ƒ n1 s†0    Y  W d   ƒ n1 s¤0    Y  d S )
Nr   iô  r?   z	text/htmlr5   rA   rB   rC   rD   ©r   r$   r.   r2   r   rF   rG   ÚpytestÚraisesÚ
ValueErrorr   ©r   r%   r(   rH   rI   r   r   r   Ú6test_partition_md_from_url_raises_with_bad_status_codeF   s    
&ýrP   c               	   C  s²   t dƒ} t| ƒ�}| ¡ }W d   ƒ n1 s.0    Y  t|dddid�}tjtd|d��D}t t	¡� t
dd	� W d   ƒ n1 s†0    Y  W d   ƒ n1 s¤0    Y  d S )
Nr   r>   r?   zapplication/jsonr5   rA   rB   rC   rD   rK   rO   r   r   r   Ú7test_partition_md_from_url_raises_with_bad_content_typeT   s    
&ýrQ   c                   C  s4   t  t¡� tƒ  W d   ƒ n1 s&0    Y  d S r)   )rL   rM   rN   r   r   r   r   r   Ú,test_partition_md_raises_with_none_specifiedb   s    rR   c                  C  sr   t dƒ} t| ƒ�}| ¡ }W d   ƒ n1 s.0    Y  t t¡� t| |d� W d   ƒ n1 sd0    Y  d S )Nr   )r   r(   )r   r$   r.   rL   rM   rN   r   )r   r%   r(   r   r   r   Ú0test_partition_md_raises_with_too_many_specifiedg   s
    
&rS   c                  C  s6   t tdƒƒ} t| ƒdksJ ‚tdd„ | D ƒƒs2J ‚d S )Nr   r   c                 s  s   | ]}|j jd kV  qdS )r   Nr*   r+   r   r   r   r-   w   r   zRtest_partition_md_from_filename_gets_filename_from_filename_arg.<locals>.<genexpr>©r   r   r   r/   ©r   r   r   r   Ú?test_partition_md_from_filename_gets_filename_from_filename_args   s    rV   c                  C  sb   t tdƒdƒ�} t| d�}W d   ƒ n1 s.0    Y  t|ƒdksHJ ‚tdd„ |D ƒƒs^J ‚d S )Nr   r!   r"   r   c                 s  s   | ]}|j jd u V  qd S r)   r*   r+   r   r   r   r-      r   zAtest_partition_md_from_file_gets_filename_None.<locals>.<genexpr>)r$   r   r   r   r/   ©r%   r   r   r   r   Ú.test_partition_md_from_file_gets_filename_Nonez   s    (rX   c                  C  s:   t tdƒdd�} t| ƒdks J ‚tdd„ | D ƒƒs6J ‚d S )Nr   úorig-name.md)Úmetadata_filenamer   c                 s  s   | ]}|j jd kV  qdS ©rY   Nr*   r   r   r   r   r-   †   r   zLtest_partition_md_from_filename_prefers_metadata_filename.<locals>.<genexpr>rT   rU   r   r   r   Ú9test_partition_md_from_filename_prefers_metadata_filename‚   s    r\   c                  C  sT   t tdƒdƒ�} t| dd�}W d   ƒ n1 s00    Y  tdd„ |D ƒƒsPJ ‚d S )Nr   r!   rY   )r#   rZ   c                 s  s   | ]}|j jd kV  qdS r[   r*   r+   r   r   r   r-   �   r   zHtest_partition_md_from_file_prefers_metadata_filename.<locals>.<genexpr>©r$   r   r   r/   rW   r   r   r   Ú5test_partition_md_from_file_prefers_metadata_filename‰   s    *r^   c                    sJ   d‰ t tdƒƒ} t‡ fdd„| D ƒƒsFJ dˆ › dt| d jjƒ› �ƒ‚d S )Nr@   r   c                 3  s   | ]}|j jˆ kV  qd S r)   )r   Úfiletyper+   ©ZMD_MIME_TYPEr   r   r-   –   r   zOtest_partition_md_gets_the_MD_MIME_type_in_metadata_filetype.<locals>.<genexpr>zExpected all elements to have 'z' as their filetype, but got: r   )r   r   r/   Úreprr   r_   rU   r   r`   r   Ú<test_partition_md_gets_the_MD_MIME_type_in_metadata_filetype“   s    ÿÿrb   r   )Úmockerc                   s<   d‰ | j dˆ d� ttdƒƒ}t‡ fdd„|D ƒƒs8J ‚d S )Nú2029-07-05T09:24:28ú0unstructured.partition.md.get_last_modified_daterB   r   c                 3  s   | ]}|j jˆ kV  qd S r)   ©r   Zlast_modifiedr+   ©Úfilesystem_last_modifiedr   r   r-   §   r   zVtest_partition_md_from_file_path_gets_last_modified_from_filesystem.<locals>.<genexpr>©r   r   r   r/   )rc   r   r   rg   r   ÚCtest_partition_md_from_file_path_gets_last_modified_from_filesystemŸ   s    ÿrj   c                  C  sR   t tdƒdƒ�} t| d�}W d   ƒ n1 s.0    Y  tdd„ |D ƒƒsNJ ‚d S )Nr   r!   r"   c                 s  s   | ]}|j jd u V  qd S r)   rf   r+   r   r   r   r-   ®   r   zFtest_partition_md_from_file_gets_last_modified_None.<locals>.<genexpr>r]   rW   r   r   r   Ú3test_partition_md_from_file_gets_last_modified_Noneª   s    (rk   c                  C  sX   t tdƒƒ�} |  ¡ }W d   ƒ n1 s*0    Y  t|d�}tdd„ |D ƒƒsTJ ‚d S )Nr   r'   c                 s  s   | ]}|j jd u V  qd S r)   rf   r+   r   r   r   r-   ·   r   zFtest_partition_md_from_text_gets_last_modified_None.<locals>.<genexpr>©r$   r   r.   r   r/   r0   r   r   r   Ú3test_partition_md_from_text_gets_last_modified_None±   s    &
rm   c                   sD   d}d‰ | j d|d� ttdƒˆ d�}t‡ fdd„|D ƒƒs@J ‚d S )	Nrd   ú2020-07-05T09:24:28re   rB   r   ©Úmetadata_last_modifiedc                 3  s   | ]}|j jˆ kV  qd S r)   rf   r+   ro   r   r   r-   Å   r   zRtest_partition_md_from_file_path_prefers_metadata_last_modified.<locals>.<genexpr>ri   )rc   rh   r   r   ro   r   Ú?test_partition_md_from_file_path_prefers_metadata_last_modifiedº   s    ÿÿrq   c                    s\   d‰ t tdƒdƒ�} t| ˆ d�}W d   ƒ n1 s40    Y  t‡ fdd„|D ƒƒsXJ ‚d S )Nrn   r   r!   )r#   rp   c                 3  s   | ]}|j jˆ kV  qd S r)   rf   r+   ro   r   r   r-   Í   r   zMtest_partition_md_from_file_prefers_metadata_last_modified.<locals>.<genexpr>r]   rW   r   ro   r   Ú:test_partition_md_from_file_prefers_metadata_last_modifiedÈ   s    *rr   c                    sb   d‰ t tdƒƒ�} |  ¡ }W d   ƒ n1 s.0    Y  t|ˆ d�}t‡ fdd„|D ƒƒs^J ‚d S )Nrn   r   )r(   rp   c                 3  s   | ]}|j jˆ kV  qd S r)   rf   r+   ro   r   r   r-   ×   r   zMtest_partition_md_from_text_prefers_metadata_last_modified.<locals>.<genexpr>rl   r0   r   ro   r   Ú:test_partition_md_from_text_prefers_metadata_last_modifiedÐ   s
    &rs   c                  C  sJ   t tdƒƒ�} |  ¡ }W d   ƒ n1 s*0    Y  t|d�}t|ƒ d S )Nr   r'   )r$   r   r.   r   r   r0   r   r   r   Útest_partition_md_with_jsonÝ   s    &
rt   c                  C  s@   t dƒ} t| ƒ}t| dd�}t|ƒ}||ks0J ‚||ks<J ‚d S )Nr   Zby_title)Zchunking_strategy)r   r   r   )r   r   Zchunk_elementsÚchunksr   r   r   Ú3test_add_chunking_strategy_by_title_on_partition_mdä   s    rv   c                  C  s(   d} t | d�}|d jjdgks$J ‚d S )Nzexample-docs/README.mdr   r   Úeng)r   r   Ú	languagesr   r   r   r   Ú0test_partition_md_element_metadata_has_languagesî   s    
ry   c                  C  sD   d} t | dd�}dd„ |D ƒ}|dgddgdgdgdggks@J ‚d S )Nz*example-docs/language-docs/eng_spa_mult.mdT)r   Zdetect_language_per_elementc                 S  s   g | ]}|j j‘qS r   )r   rx   r   r   r   r   r   ÷   r   zJtest_partition_md_respects_detect_language_per_element.<locals>.<listcomp>rw   Zspar   )r   r   Zlangsr   r   r   Ú6test_partition_md_respects_detect_language_per_elementô   s    rz   c                  C  s:   t dƒ} t| d�}t|ƒdks"J ‚|d jtjks6J ‚d S )Nzsimple-table.mdr   r   )r   r   r   r   r	   ZTABLEr   r   r   r   Útest_partition_md_parse_tableû   s    
r{   r   zumlauts-utf8.mdzumlauts-non-utf8.mdr3   r   c                 C  s:   t | ƒ} t| d�}t|ƒdks"J ‚|d j d¡s6J ‚d S )Nr   r   éÿÿÿÿu   Ã¤Ã¶Ã¼ÃŸ)r   r   r   r(   Úendswithr   r   r   r   Útest_partition_md_with_umlauts  s    
r~   c                  C  s"   d} t | d�}t|ƒdksJ ‚d S )Nz‰```
<?xml version="1.0"?>
<sparql xmlns="http://www.w3.org/2005/sparql-results#">
  <head></head>
  <boolean>true</boolean>
</sparql>
```r'   é   ©r   r   ©Zxml_contentr   r   r   r   Ú,test_partition_md_xml_processing_instruction
  s    
r‚   c                  C  s"   d} t | d�}t|ƒdksJ ‚d S )Nz‹```
  <?xml version="1.0"?>
<sparql xmlns="http://www.w3.org/2005/sparql-results#">
  <head></head>
  <boolean>true</boolean>
</sparql>
```r'   r   r€   r�   r   r   r   Ú9test_partition_md_xml_processing_instruction_with_indents  s    
rƒ   c                  C  s"   d} t | d�}t|ƒdksJ ‚d S )Nz&```
    <?php echo "hello"; ?>
    ```r'   r   r€   )Zphp_contentr   r   r   r   Ú0test_partition_md_non_xml_processing_instruction$  s    
r„   )5Ú
__future__r   Útypingr   Zunittest.mockr   rL   rG   Zpytest_mockr   Ztest_unstructured.unit_utilsr   r   Zunstructured.chunking.titler   Zunstructured.documents.elementsr	   r
   Zunstructured.partition.mdr   Z&unstructured.partition.utils.constantsr   r    r&   r1   r2   rJ   rP   rQ   rR   rS   rV   rX   r\   r^   rb   rj   rk   rm   rq   rr   rs   rt   rv   ry   rz   r{   ÚmarkZparametrizer~   r‚   rƒ   r„   r   r   r   r   Ú<module>   sP   

	
