a
    ¡˜ëjÎ&  ã                   @  sÐ  d Z ddlmZ ddlZddlmZmZ ddlZddlm	Z	 ddl
mZmZmZmZmZmZ ddlmZ ddlmZmZmZmZmZmZmZmZmZ dd	lmZ dd
lm Z  ddœdd„Z!dddœdd„Z"dddœdd„Z#dddœdd„Z$dd„ Z%dd„ Z&ddœdd„Z'd d!„ Z(d"d#„ Z)d$d%„ Z*d&d'„ Z+d(d)„ Z,d*d+œd,d-„Z-d*d+œd.d/„Z.d0d1„ Z/d2d3„ Z0ej1 2d4i d5fd6did5fd6d7id7fg¡dd8d9d:œd;d<„ƒZ3d=d>„ Z4d?d@„ Z5dAdB„ Z6dCdD„ Z7e 8¡ ddEœdFdG„ƒZ9dS )Hz3Test suite for `unstructured.partition.doc` module.é    )ÚannotationsN)ÚAnyÚIterator)ÚMockFixture)ÚANYÚCaptureFixtureÚFixtureRequestÚassert_round_trips_through_JSONÚexample_doc_pathÚmethod_mock)Úchunk_elements)	ÚAddressÚCompositeElementÚElementÚListItemÚNarrativeTextÚTableÚ
TableChunkÚTextÚTitle)Úpartition_doc)Úpartition_docxr   )Úrequestc                 C  s(   t dƒ}t dƒ}t|ƒt|ƒks$J ‚d S )Nú
simple.doczsimple.docx)r
   r   r   )r   Údoc_file_pathZdocx_file_path© r   úc/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/test_unstructured/partition/test_doc.pyÚ)test_partition_doc_matches_partition_docx%   s    r   zlist[Element]zCaptureFixture[str])Úexpected_elementsÚcapsysc                 C  sV   t tdƒƒ}|| ksJ ‚tdd„ |D ƒƒs.J ‚| ¡ jdks@J ‚| ¡ jdksRJ ‚d S )Nr   c                 s  s   | ]}|j jtd ƒkV  qdS )Ú N)ÚmetadataZfile_directoryr
   ©Ú.0Úer   r   r   Ú	<genexpr>3   ó    z3test_partition_doc_from_filename.<locals>.<genexpr>r    )r   r
   ÚallÚ
readouterrÚoutÚerr)r   r   Úelementsr   r   r   Ú test_partition_doc_from_filename/   s
    r,   c                 C  sn   t tdƒdƒ�}t|dd�}W d   ƒ n1 s00    Y  || ksFJ ‚| ¡ jdksXJ ‚| ¡ jdksjJ ‚d S )Nr   ÚrbzMS Word 2007 XML©ÚfileZlibre_office_filterr    )Úopenr
   r   r(   r)   r*   ©r   r   Úfr+   r   r   r   Ú5test_partition_doc_from_file_with_libre_office_filter8   s
    *r3   c                 C  s„   t tdƒdƒ�}t|d d�}W d   ƒ n1 s00    Y  || ksFJ ‚| ¡ jdksXJ ‚| ¡ jdksjJ ‚tdd„ |D ƒƒs€J ‚d S )Nr   r-   r.   r    c                 s  s   | ]}|j jd u V  qd S ©N©r!   Úfilenamer"   r   r   r   r%   L   r&   zKtest_partition_doc_from_file_with_no_libre_office_filter.<locals>.<genexpr>)r0   r
   r   r(   r)   r*   r'   r1   r   r   r   Ú8test_partition_doc_from_file_with_no_libre_office_filterC   s    *r7   c               	   C  sp   t dƒ} t| dƒ�J}tjtdd�� t| |d� W d   ƒ n1 sD0    Y  W d   ƒ n1 sb0    Y  d S )Nr   r-   ú2Exactly one of filename and file must be specified©Úmatch)r6   r/   )r
   r0   ÚpytestÚraisesÚ
ValueErrorr   )r   r2   r   r   r   ÚEtest_partition_doc_raises_when_both_a_filename_and_file_are_specifiedO   s    r>   c                   C  s8   t jtdd�� tƒ  W d   ƒ n1 s*0    Y  d S )Nr8   r9   )r;   r<   r=   r   r   r   r   r   ÚVtest_partition_doc_raises_when_neither_a_file_path_nor_a_file_like_object_are_providedW   s    r?   zpathlib.Path)Útmp_pathc                 C  sH   t | d ƒ}tjtdd�� t|d� W d   ƒ n1 s:0    Y  d S )Nzasdf.doczasdf.doc does not existr9   )r6   )Ústrr;   r<   r=   r   )r@   Zdoc_filenamer   r   r   Ú&test_partition_raises_with_missing_doc\   s    rB   c                  C  s6   t tdƒƒ} t| ƒdksJ ‚tdd„ | D ƒƒs2J ‚d S )Nr   r   c                 s  s   | ]}|j jd kV  qdS )r   Nr5   r"   r   r   r   r%   j   r&   zStest_partition_doc_from_filename_gets_filename_from_filename_arg.<locals>.<genexpr>©r   r
   Úlenr'   ©r+   r   r   r   Ú@test_partition_doc_from_filename_gets_filename_from_filename_argf   s    rF   c                  C  sb   t tdƒdƒ�} t| d�}W d   ƒ n1 s.0    Y  t|ƒdksHJ ‚tdd„ |D ƒƒs^J ‚d S )Nr   r-   )r/   r   c                 s  s   | ]}|j jd u V  qd S r4   r5   r"   r   r   r   r%   r   r&   zBtest_partition_doc_from_file_gets_filename_None.<locals>.<genexpr>)r0   r
   r   rD   r'   ©r2   r+   r   r   r   Ú/test_partition_doc_from_file_gets_filename_Nonem   s    (rH   c                  C  s:   t tdƒdd�} t| ƒdks J ‚tdd„ | D ƒƒs6J ‚d S )Nr   Útest)Úmetadata_filenamer   c                 s  s   | ]}|j jd kV  qdS ©rI   Nr5   ©r#   Úelementr   r   r   r%   y   r&   zMtest_partition_doc_from_filename_prefers_metadata_filename.<locals>.<genexpr>rC   rE   r   r   r   Ú:test_partition_doc_from_filename_prefers_metadata_filenameu   s    rN   c                  C  sT   t tdƒdƒ�} t| dd�}W d   ƒ n1 s00    Y  tdd„ |D ƒƒsPJ ‚d S )Nr   r-   rI   )r/   rJ   c                 s  s   | ]}|j jd kV  qdS rK   r5   r"   r   r   r   r%   €   r&   zItest_partition_doc_from_file_prefers_metadata_filename.<locals>.<genexpr>)r0   r
   r   r'   rG   r   r   r   Ú6test_partition_doc_from_file_prefers_metadata_filename|   s    *rO   c                    sJ   d‰ t tdƒƒ} t‡ fdd„| D ƒƒsFJ dˆ › dt| d jjƒ› �ƒ‚d S )Nzapplication/mswordr   c                 3  s   | ]}|j jˆ kV  qd S r4   )r!   Úfiletyper"   ©ZDOC_MIME_TYPEr   r   r%   ‰   r&   zQtest_partition_doc_gets_the_DOC_MIME_type_in_metadata_filetype.<locals>.<genexpr>zExpected all elements to have 'z' as their filetype, but got: r   )r   r
   r'   Úreprr!   rP   rE   r   rQ   r   Ú>test_partition_doc_gets_the_DOC_MIME_type_in_metadata_filetype†   s    ÿÿrS   r   )Úmockerc                   s<   d‰ | j dˆ d� ttdƒƒ}t‡ fdd„|D ƒƒs8J ‚d S )Nú2029-07-05T09:24:28ú1unstructured.partition.doc.get_last_modified_date©Zreturn_valuezfake.docc                 3  s   | ]}|j jˆ kV  qd S r4   ©r!   Zlast_modifiedr"   ©Úfilesystem_last_modifiedr   r   r%   š   r&   zItest_partition_doc_pulls_last_modified_from_filesystem.<locals>.<genexpr>©Úpatchr   r
   r'   )rT   r+   r   rY   r   Ú6test_partition_doc_pulls_last_modified_from_filesystem’   s    ÿr]   c                   sD   d}d‰ | j d|d� ttdƒˆ d�}t‡ fdd„|D ƒƒs@J ‚d S )	NrU   z2020-07-05T09:24:28rV   rW   r   ©Zmetadata_last_modifiedc                 3  s   | ]}|j jˆ kV  qd S r4   rX   r"   r^   r   r   r%   ª   r&   zRtest_partition_doc_prefers_metadata_last_modified_when_provided.<locals>.<genexpr>r[   )rT   rZ   r+   r   r^   r   Ú?test_partition_doc_prefers_metadata_last_modified_when_provided�   s    ÿÿr_   c                  C  s&   t tdƒƒ} tdd„ | D ƒƒs"J ‚d S )Nr   c                 s  s   | ]}|j jd gkV  qdS )ÚengN©r!   Ú	languagesr"   r   r   r   r%   ²   r&   z=test_partition_doc_adds_languages_metadata.<locals>.<genexpr>)r   r
   r'   rE   r   r   r   Ú*test_partition_doc_adds_languages_metadata°   s    rc   c                  C  s@   t tdƒdd�} dd„ | D ƒdgddgdgdgdggks<J ‚d S )Nzlanguage-docs/eng_spa_mult.docT)Zdetect_language_per_elementc                 S  s   g | ]}|j j‘qS r   ra   r"   r   r   r   Ú
<listcomp>¹   r&   zOtest_partition_doc_respects_detect_language_per_element_arg.<locals>.<listcomp>r`   Zspa)r   r
   rE   r   r   r   Ú;test_partition_doc_respects_detect_language_per_element_argµ   s    ÿûre   )ÚkwargsÚexpected_valueZhi_resÚstrategyÚautozdict[str, Any]z
str | None)r   rf   rg   c                 C  sd   ddl m} dddœdd„}t| |d|d	�}ttd
ƒfi |¤Ž\}| t¡ |jd|› �ks`J ‚d S )Nr   )Ú_DocxPartitionerrj   zIterator[Element])ÚselfÚreturnc                 s  s   t d| jj› �ƒV  d S )Nústrategy == )r   Z_optsrh   )rk   r   r   r   Úfake_iter_document_elementsÎ   s    z_test_partition_odt_forwards_strategy_arg_to_partition_docx.<locals>.fake_iter_document_elementsZ_iter_document_elements)Zside_effectr   rm   )Úunstructured.partition.docxrj   r   r   r
   Zassert_called_once_withr   Útext)r   rf   rg   rj   rn   Z_iter_elements_rM   r   r   r   Ú:test_partition_odt_forwards_strategy_arg_to_partition_docxÅ   s    ü
rq   c                  C  sÒ   g d¢} g d¢}t tdƒƒ}t|d tƒs.J ‚|d jj| ksBJ ‚|d jj|ksVJ ‚|d tdƒksjJ ‚|d jj| ks~J ‚|d jj|ks’J ‚|d tdƒks¦J ‚|d jjd u sºJ ‚|d jjd u sÎJ ‚d S )	N)ZboldZitalicúbold-italicrr   )ÚbÚirs   rt   zfake-doc-emphasized-text.docr   é   z$I am a bold italic bold-italic text.é   zI am a normal text.)r   r
   Ú
isinstancer   r!   Zemphasized_text_contentsZemphasized_text_tagsr   )Z!expected_emphasized_text_contentsZexpected_emphasized_text_tagsr+   r   r   r   Ú)test_partition_doc_grabs_emphasized_textsÞ   s    rx   c                   C  s   t ttdƒƒƒ dS )zCElements produced can be serialized then deserialized without loss.r   N)r	   r   r
   r   r   r   r   Ú+test_partition_doc_round_trips_through_jsonñ   s    ry   c                  C  sF   t dƒ} t| ƒ}t| dd�}tdd„ |D ƒƒs2J ‚|t|ƒksBJ ‚d S )Nr   Úbasic)Zchunking_strategyc                 s  s   | ]}t |tttfƒV  qd S r4   )rw   r   r   r   )r#   Úcr   r   r   r%   ü   r&   zYtest_partition_doc_chunks_elements_when_chunking_strategy_is_specified.<locals>.<genexpr>)r
   r   r'   r   )Údocument_pathr+   Úchunksr   r   r   ÚFtest_partition_doc_chunks_elements_when_chunking_strategy_is_specifiedö   s
    r~   c                  C  sT   t dƒ} dd„ t| ƒD ƒ}dd„ t| ƒD ƒ}||ks8J ‚t|ƒtt|ƒƒksPJ ‚d S )Nzduplicate-paragraphs.docc                 S  s   g | ]
}|j ‘qS r   ©ÚidrL   r   r   r   rd     r&   zStest_partition_doc_assigns_deterministic_and_unique_element_ids.<locals>.<listcomp>c                 S  s   g | ]
}|j ‘qS r   r   rL   r   r   r   rd     r&   )r
   r   rD   Úset)r|   ÚidsZids_2r   r   r   Ú?test_partition_doc_assigns_deterministic_and_unique_element_ids  s
    rƒ   )rl   c                	   C  s4   t dƒtdƒtdƒtdƒtdƒtdƒtdƒtdƒgS )	Nz&These are a few of my favorite things:ZParrotsZHockeyZAnalysisz4This is my first thought. This is my second thought.zThis is my third thought.Z2023zDOYLESTOWN, PA 18901)r   r   r   r   r   r   r   r   r   r     s    ør   ):Ú__doc__Ú
__future__r   ÚpathlibÚtypingr   r   r;   Zpytest_mockr   Ztest_unstructured.unit_utilsr   r   r   r	   r
   r   Zunstructured.chunking.basicr   Zunstructured.documents.elementsr   r   r   r   r   r   r   r   r   Zunstructured.partition.docr   ro   r   r   r,   r3   r7   r>   r?   rB   rF   rH   rN   rO   rS   r]   r_   rc   re   ÚmarkZparametrizerq   rx   ry   r~   rƒ   Zfixturer   r   r   r   r   Ú<module>   sJ    ,
	

þ