a
    ¡˜ëj)"  ã                   @  sÜ  d Z ddlmZ ddlZddlmZ ddlmZ ddlm	Z	 ddl
mZmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ej dddg¡ddœdd„ƒZdd„ Zej dddg¡ddœdd„ƒZdd„ Zej dddg¡ddœdd„ƒZej dddg¡ddœdd„ƒZdd„ Zdd „ Zej dddg¡ddœd!d"„ƒZej dddg¡ddœd#d$„ƒZ d%d&„ Z!d'd(„ Z"d)d*œd+d,„Z#d-d.„ Z$d)d*œd/d0„Z%d1d2„ Z&ej dddg¡ddœd3d4„ƒZ'd5d6„ Z(d7d8„ Z)d9d:„ Z*d;d<„ Z+dS )=z3Test-suite for `unstructured.partition.xml` module.é    )ÚannotationsN)ÚMockerFixture)Úexample_doc_path)Úchunk_by_title)ÚNarrativeTextÚTitle)Úpartition_json)Ú#UNSTRUCTURED_INCLUDE_DEBUG_METADATA)Úpartition_xml)Úelements_to_jsonÚfilenameúfactbook.xmlúfactbook-utf-16.xmlÚstr©r   c                 C  sZ   t | ƒ}t|dd�}|d jdks&J ‚|d jj| ks:J ‚trVdd„ |D ƒdhksVJ ‚d S )NF©r   Úxml_keep_tagsr   úUnited Statesc                 S  s   h | ]}|j j’qS © )ÚmetadataZdetection_origin©Ú.0Úelementr   r   úc/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/test_unstructured/partition/test_xml.pyÚ	<setcomp>   ó    z3test_partition_xml_from_filename.<locals>.<setcomp>Úxml)r   r
   Útextr   r   r	   ©r   Ú	file_pathÚelementsr   r   r   Ú test_partition_xml_from_filename   s    r!   c                  C  s<   t tdƒddd�} | d jdks$J ‚| d jjdks8J ‚d S )Nr   FÚtest)r   Úmetadata_filenamer   r   )r
   r   r   r   r   ©r    r   r   r   Ú7test_partition_xml_from_filename_with_metadata_filename   s
    
ÿr%   c                 C  sj   t | ƒ}t|dƒ�}t|d|d�}W d   ƒ n1 s60    Y  |d jdksRJ ‚|d jj| ksfJ ‚d S ©NÚrbF©Úfiler   r#   r   r   ©r   Úopenr
   r   r   r   ©r   r   Úfr    r   r   r   Útest_partition_xml_from_file%   s
    ,r.   c                  C  sf   t tdƒdƒ�} t| ddd�}W d   ƒ n1 s20    Y  |d jdksNJ ‚|d jjdksbJ ‚d S )Nr   r'   Fr"   r(   r   r   )r+   r   r
   r   r   r   ©r-   r    r   r   r   Ú3test_partition_xml_from_file_with_metadata_filename/   s    ,r0   c                 C  sj   t | ƒ}t|dƒ�}t|d|d�}W d   ƒ n1 s60    Y  |d jdksRJ ‚|d jj| ksfJ ‚d S r&   r*   r,   r   r   r   Útest_partition_xml_from_file_rb7   s
    ,r1   c                 C  s>   t | ƒ}t|dd�}d|d jv s&J ‚|d jj| ks:J ‚d S )NTr   ú<leader>Joe Biden</leader>r   )r   r
   r   r   r   r   r   r   r   Ú;test_partition_xml_from_filename_with_tags_default_encodingA   s    r3   c                  C  sV   t tdƒƒ�} |  ¡ }W d   ƒ n1 s*0    Y  t|dd�}d|d jv sRJ ‚d S )Nr   T)r   r   r2   r   )r+   r   Úreadr
   r   )r-   r   r    r   r   r   Ú&test_partition_xml_from_text_with_tagsJ   s    &r5   c                   C  s@   t  t¡�" ttdƒddd� W d   ƒ n1 s20    Y  d S )Nr   Túutf-8)r   Úencoding)ÚpytestÚraisesÚUnicodeDecodeErrorr
   r   r   r   r   r   Ú@test_partition_xml_from_filename_with_tags_raises_encoding_errorR   s    r;   c                 C  sj   t | ƒ}t|dƒ�}t|d|d�}W d   ƒ n1 s60    Y  d|d jv sRJ ‚|d jj| ksfJ ‚d S ©Nr'   Tr(   r2   r   r*   r,   r   r   r   Ú7test_partition_xml_from_file_with_tags_default_encodingW   s
    ,r=   c                 C  sj   t | ƒ}t|dƒ�}t|d|d�}W d   ƒ n1 s60    Y  d|d jv sRJ ‚|d jj| ksfJ ‚d S r<   r*   r,   r   r   r   Ú:test_partition_xml_from_file_rb_with_tags_default_encodinga   s
    ,r>   c               	   C  sj   t  t¡�L ttdƒdƒ�} t| ddd� W d   ƒ n1 s>0    Y  W d   ƒ n1 s\0    Y  d S )Nr   r'   Tr6   )r)   r   r7   )r8   r9   r:   r+   r   r
   )r-   r   r   r   Ú?test_partition_xml_from_file_rb_with_tags_raises_encoding_errork   s    ýr?   c                    sJ   d‰ t tdƒƒ} t‡ fdd„| D ƒƒsFJ dˆ › dt| d jjƒ› �ƒ‚d S )Nzapplication/xmlr   c                 3  s   | ]}|j jˆ kV  qd S )N)r   Úfiletype)r   Úe©ZXML_MIME_TYPEr   r   Ú	<genexpr>{   r   zQtest_partition_xml_gets_the_XML_mime_type_in_metadata_filetype.<locals>.<genexpr>zExpected all elements to have 'z' as their filetype, but got: r   )r
   r   ÚallÚreprr   r@   r$   r   rB   r   Ú>test_partition_xml_gets_the_XML_mime_type_in_metadata_filetypex   s    ÿÿrF   r   )Úmockerc                 C  s4   d}| j d|d� tdd�}|d jj|ks0J ‚d S )Nú2029-07-05T09:24:28ú1unstructured.partition.xml.get_last_modified_date©Zreturn_valueúexample-docs/factbook.xmlr   r   ©Úpatchr
   r   Úlast_modified)rG   Zmocked_last_modification_dater    r   r   r   ÚDtest_partition_xml_from_file_path_gets_last_modified_from_filesystem„   s    þ
rO   c                  C  sL   t ddƒ�} t| d�}W d   ƒ n1 s*0    Y  |d jjd u sHJ ‚d S )NrK   r'   )r)   r   ©r+   r
   r   rN   r/   r   r   r   Ú4test_partition_xml_from_file_gets_last_modified_None‘   s    (rQ   c                 C  s:   d}d}| j d|d� td|d�}|d jj|ks6J ‚d S )NrH   z2020-07-05T09:24:28rI   rJ   rK   )r   Úmetadata_last_modifiedr   rL   )rG   Zfilesystem_last_modifiedrR   r    r   r   r   Ú@test_partition_xml_from_file_path_prefers_metadata_last_modified˜   s    ÿþrS   c                  C  sN   t ddƒ�} t| dd�}W d   ƒ n1 s,0    Y  |d jjdksJJ ‚d S )NrK   r'   rH   )r)   rR   r   rP   r/   r   r   r   Ú;test_partition_xml_from_file_prefers_metadata_last_modified¨   s    *rT   c                 C  s˜   t | ƒ}t|dd�}tt|ƒd�}t|ƒt|ƒks6J ‚|d jj|d jjksRJ ‚|d jj|d jjksnJ ‚tt|ƒƒD ]}|| || kszJ ‚qzd S )NFr   ©r   r   )	r   r
   r   r   Úlenr   Zpage_numberr   Úrange)r   r   r    Ztest_elementsÚir   r   r   Útest_partition_xml_with_json²   s    rY   c                  C  sh   d} t | d�}|d tdƒks"J ‚t|d tƒs4J ‚t|d ƒ d¡sJJ ‚t|d ƒ ¡  d¡sdJ ‚d S )NzÝ<xml>
        <parrot>
            <name>Conure</name>
            <description>A conure is a very friendly bird.
            Conures are feathery and like to dance.
            </description>
        </parrot>
    </xml>rU   r   ZConureé   z!A conure is a very friendly bird.z'Conures are feathery and like to dance.)r
   r   Ú
isinstancer   r   Ú
startswithÚstripÚendswith)Zxml_textr    r   r   r   Ú-test_partition_xml_with_narrative_line_breaksÀ   s    	
r_   c                  C  s@   t dƒ} t| ƒ}t| dd�}t|ƒ}||ks0J ‚||ks<J ‚d S )Nr   Zby_title)Zchunking_strategy)r   r
   r   )r   r    Zchunk_elementsÚchunksr   r   r   Ú+test_add_chunking_strategy_on_partition_xmlÑ   s    ra   c                  C  s*   t dƒ} t| ƒ}|d jjdgks&J ‚d S )Nr   r   Úeng)r   r
   r   Ú	languages)r   r    r   r   r   Ú1test_partition_xml_element_metadata_has_languagesÚ   s    rd   c                  C  sD   t tdƒdd�} dd„ | D ƒ}|dgddgdgdgdggks@J ‚d S )Nzlanguage-docs/eng_spa_mult.xmlT)Zdetect_language_per_elementc                 S  s   g | ]}|j j‘qS r   )r   rc   r   r   r   r   Ú
<listcomp>ä   r   zKtest_partition_xml_respects_detect_language_per_element.<locals>.<listcomp>rb   Zspa)r
   r   )r    Zlangsr   r   r   Ú7test_partition_xml_respects_detect_language_per_elementà   s
    ÿrf   ),Ú__doc__Ú
__future__r   r8   Zpytest_mockr   Ztest_unstructured.unit_utilsr   Zunstructured.chunking.titler   Zunstructured.documents.elementsr   r   Zunstructured.partition.jsonr   Z&unstructured.partition.utils.constantsr	   Zunstructured.partition.xmlr
   Zunstructured.staging.baser   ÚmarkZparametrizer!   r%   r.   r0   r1   r3   r5   r;   r=   r>   r?   rF   rO   rQ   rS   rT   rY   r_   ra   rd   rf   r   r   r   r   Ú<module>   sL   
					
	