a
    ¡˜ëjU5  ã                
   @   s  d dl Z d dlmZ d dlZd dlZd dlmZ d dlm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZ d d	lmZmZmZmZmZmZmZmZmZmZ d dlmZ d d
lmZ G dd„ de	j ƒZ!G dd„ de	j"ƒZ#dd„ Z$dd„ Z%ej& 'ddedfdedfdedfdedfg¡dd„ ƒZ(dd„ Z)dd„ Z*d d!„ Z+d"d#„ Z,ej& 'd$e -¡ ¡e.e/e d$œd%d&„ƒZ0ej& 'd'ej1d(fej2d)fej3d(fej4d)fej5d)fej6d(fg¡e.e7d'œd*d+„ƒZ8d,d-„ Z9d.d/„ Z:G d0d1„ d1ƒZ;d2d3„ Z<d4d5„ Z=d6d7„ Z>ej& 'd8d9d:g¡d;d<„ ƒZ?d=d>„ Z@ed?ƒfd@dA„ZAdS )Bé    N)ÚPool)ÚImage)Úlayout)Ú
TextRegion©ÚLayoutElement)Úexample_doc_path)Ú
PixelSpace)
ÚTYPE_TO_TEXT_ELEMENT_MAPÚCheckBoxÚCoordinatesMetadataÚElementTypeÚFigureCaptionÚHeaderÚListItemÚNarrativeTextÚTextÚTitle)Úcommonc                   @   s*   e Zd Zeejdœdd„Zedd„ ƒZdS )ÚMockPageLayout©ÚnumberÚimagec                 C   s   || _ || _d S ©Nr   )Úselfr   r   © r   úm/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/test_unstructured/partition/common/test_common.pyÚ__init__    s    zMockPageLayout.__init__c                 C   s4   t ddd d�t ddd d�t ddd d�t dd	d d�gS )
NÚHeadlinez#Charlie Brown and the Great Pumpkin)ÚtypeÚtextÚbboxÚSubheadlinezThe Beginningr   z/This time Charlie Brown had it really tricky...r   z#Another book title in the same pager   ©r   r   r   r   Úelements$   s*    ýýýýðzMockPageLayout.elementsN)Ú__name__Ú
__module__Ú__qualname__Úintr   r   Úpropertyr$   r   r   r   r   r      s   r   c                   @   s   e Zd Zedd„ ƒZdS )ÚMockDocumentLayoutc                 C   s   t dt dd¡d�gS )Né   Ú1)r+   r+   r   )r   r   Únewr#   r   r   r   Úpages?   s    ÿzMockDocumentLayout.pagesN)r%   r&   r'   r)   r.   r   r   r   r   r*   >   s   r*   c                  C   sr   dddgddgddgdd	ggd d
dœ} t ddd�}tj| |d�}|td
ddgddgddgdd	gg|d�ksnJ ‚d S )Nr   r+   é   é   é   é   é   é   é   úSome lovely text)r   ÚcoordinatesÚcoordinate_systemr    é
   é   ©ÚwidthÚheight©r8   ©r    r7   r8   )r	   r   Únormalize_layout_elementr   ©Úlayout_elementr8   Úelementr   r   r   Ú"test_normalize_layout_element_dictF   s    üþýrD   c                  C   s@   ddddœ} t ddd�}tj| |d�}|tdd|d	�ks<J ‚d S )
NZFigure)©r+   r/   ©r0   r1   )r2   r3   )r4   r5   r6   ©r   r7   r    r9   r:   r;   r>   r?   )r	   r   r@   ÚImageElementrA   r   r   r   Ú*test_normalize_layout_element_dict_captionY   s    ýþýrI   )Úelement_typeÚexpected_typeÚexpected_depthr   r   r+   r"   r/   r   c                 C   s`   | ddgddgddgddggd	d
œ}t ddd�}tj||d�}|jj|ksNJ ‚t||ƒs\J ‚d S )Nr+   r/   r0   r1   r2   r3   r4   r5   r6   rG   r9   r:   r;   r>   )r	   r   r@   ÚmetadataZcategory_depthÚ
isinstance)rJ   rK   rL   rB   r8   rC   r   r   r   Ú&test_normalize_layout_element_headlinek   s    ýrO   c                  C   sp   dddgddgddgdd	ggd
dœ} t ddd�}tj| |d�}|td
ddgddgddgdd	gg|d�kslJ ‚d S )Nr   r+   r/   r0   r1   r2   r3   r4   r5   r6   rG   r9   r:   r;   r>   r?   )r	   r   r@   r   rA   r   r   r   Ú1test_normalize_layout_element_dict_figure_caption€   s    ýþýrP   c                  C   sp   dddgddgddgdd	ggd
dœ} t ddd�}tj| |d�}|td
ddgddgddgdd	gg|d�kslJ ‚d S )NZMiscr+   r/   r0   r1   r2   r3   r4   r5   r6   rG   r9   r:   r;   r>   r?   )r	   r   r@   r   rA   r   r   r   Ú'test_normalize_layout_element_dict_misc’   s    ýþýrQ   c                  C   sJ   t jddddddd�} tdd	d
�}tj| |d�}|tdd|d�ksFJ ‚d S )Nr   r+   r/   r0   r1   r6   ©r   Úx1Úy1Zx2Úy2r    r9   r:   r;   r>   ©rE   )r+   r1   rF   )r0   r/   r?   ©r   Úfrom_coordsr	   r   r@   r   rA   r   r   r   Ú,test_normalize_layout_element_layout_element¤   s$    úþýrY   c                  C   sJ   t jddddddd�} tdd	d
�}tj| |d�}|tdd|d�ksFJ ‚d S )Nr   r+   r/   r0   r1   r6   rR   r9   r:   r;   r>   rV   r?   rW   rA   r   r   r   Ú;test_normalize_layout_element_layout_element_narrative_text¹   s$    úþýrZ   )rJ   Úexpected_element_classc                 C   sJ   t j| dddddd�}tddd	�}tj||d
�}||dd|d�ksFJ ‚d S )Nr+   r/   r0   r1   r6   rR   r9   r:   r;   r>   rV   r?   )r   rX   r	   r   r@   )rJ   r[   rB   r8   rC   r   r   r   ÚMtest_normalize_layout_element_layout_element_maps_to_appropriate_text_elementÎ   s$    úþýr\   )rJ   Úexpected_checkedFTc                 C   sX   t j| dddddd�}tddd	�}tj||d
�}t|tƒs>J ‚|t|d|d�ksTJ ‚d S )Nr+   r/   r0   r1   Ú rR   r9   r:   r;   r>   rV   )Úcheckedr7   r8   )r   rX   r	   r   r@   rN   r   )rJ   r]   rB   r8   rC   r   r   r   Ú'test_normalize_layout_element_checkableê   s&    úþýr`   c                  C   sd   t jddddddd�} tdd	d
�}tj| |d�}|tdd|d�tdd|d�tdd|d�gks`J ‚d S )NÚListr+   r/   r0   r1   z61. I'm so cool! 2. You're cool too. 3. We're all cool!rR   r9   r:   r;   r>   úI'm so cool!rV   r?   úYou're cool too.úWe're all cool!©r   rX   r	   r   r@   r   ©rB   r8   r$   r   r   r   Ú-test_normalize_layout_element_enumerated_list  s<    úþýýýõrg   c                  C   sd   t jddddddd�} tdd	d
�}tj| |d�}|tdd|d�tdd|d�tdd|d�gks`J ‚d S )Nra   r+   r/   r0   r1   z3* I'm so cool! * You're cool too. * We're all cool!rR   r9   r:   r;   r>   rb   rV   r?   rc   rd   re   rf   r   r   r   Ú+test_normalize_layout_element_bulleted_list,  s<    úþýýýõrh   c                   @   s   e Zd Zdd„ ZdS )ÚMockRunOutputc                 C   s   || _ || _|| _d S r   )Ú
returncodeÚstdoutÚstderr)r   rj   rk   rl   r   r   r   r   O  s    zMockRunOutput.__init__N)r%   r&   r'   r   r   r   r   r   ri   M  s   ri   c                 C   sD   ddl m} dd„ }|  |d|¡ tjdddd	� d
|jv s@J ‚d S )Nr   )Ú
subprocessc                  _   s   t dd ¡ d ¡ ƒS )Nr+   zan error occurredzerror details)ri   Úencode)ÚargsÚkwargsr   r   r   Úmock_runX  s    z9test_convert_office_doc_captures_errors.<locals>.mock_runÚrunzno-real.docxzfake-directoryÚdocx)Ztarget_formatz4soffice failed to convert to format docx with code 1)Z$unstructured.partition.common.commonrm   Úsetattrr   Úconvert_office_docr    )ZmonkeypatchZcaplogrm   rq   r   r   r   Ú'test_convert_office_doc_captures_errorsU  s
    rv   c                     s    dd„ dD ƒ} | D ] }|j dd� |d jdd� qtdƒ‰ td	ƒ�,}| tj‡ fd
d„| D ƒ¡ W d   ƒ n1 sv0    Y  t dd„ | D ƒ¡d	ksœJ ‚d S )Nc                 S   s   g | ]}t  |¡‘qS r   ©ÚpathlibÚPath©Ú.0Úpathr   r   r   Ú
<listcomp>a  ó    zNtest_convert_office_docs_avoids_concurrent_call_to_soffice.<locals>.<listcomp>)z
/tmp/proc1z
/tmp/proc2z
/tmp/proc3T)Úexist_okúsimple.docx©Ú
missing_okú
simple.docr0   c                    s   g | ]}ˆ |f‘qS r   r   rz   ©Zfile_to_convertr   r   r}   h  r~   c                 S   s   g | ]}|d    ¡ ‘qS ©r€   ©Úis_filerz   r   r   r   r}   j  r~   ©	ÚmkdirÚunlinkr   r   Ústarmapr   ru   ÚnpÚsum©Zpaths_to_saver|   Úpoolr   r„   r   Ú:test_convert_office_docs_avoids_concurrent_call_to_soffice`  s    
:r�   c                     s¢   dd„ dD ƒ} | D ]"}|j ddd� |d jdd� qtdƒ‰ td	ƒ�,}| tj‡ fd
d„| D ƒ¡ W d   ƒ n1 sx0    Y  t dd„ | D ƒ¡d	k sžJ ‚d S )Nc                 S   s   g | ]}t  |¡‘qS r   rw   rz   r   r   r   r}   n  s   zBtest_convert_office_docs_respects_wait_timeout.<locals>.<listcomp>)z/tmp/wait/proc1z/tmp/wait/proc2z/tmp/wait/proc3T)Úparentsr   r€   r�   rƒ   r0   c                    s   g | ]}ˆ |d ddf‘qS )rs   Nr   r   rz   r„   r   r   r}   {  r~   c                 S   s   g | ]}|d    ¡ ‘qS r…   r†   rz   r   r   r   r}   €  r~   rˆ   rŽ   r   r„   r   Ú.test_convert_office_docs_respects_wait_timeoutm  s    ÿ
ü"	r’   ©r    Úexpected)u@   <table><tbody><tr><td>ðŸ‘¨\U+1F3FBðŸ”§</td></tr></tbody></table>T)z6<table><tbody><tr><td>Hello!</td></tr></tbody></table>Fc                 C   s   t  | ¡|u sJ ‚d S r   )r   Zcontains_emojir“   r   r   r   Útest_contains_emojiƒ  s    r•   c                  C   s(   t ƒ } t | jd ¡}t|tƒs$J ‚d S )Nr   )r*   r   Zget_page_image_metadatar.   rN   Údict)ÚdocrM   r   r   r   Ú2test_get_page_image_metadata_and_coordinate_systemŽ  s    r˜   z img/layout-parser-paper-fast.jpgc           
      C   sÈ   t jdddddd�t jddd	d
dd�g}dd„ |D ƒ}t | ¡}tj||jd�}t|ƒt|ƒksdJ ‚dd„ |D ƒtj	hks~J ‚|j\}}t
||d�}t||ƒD ]$\}}	|jjt|	jj|d�ksžJ ‚qžd S )Ng     `d@g     À\@g     @|@g      `@z(LayoutParser: A Unified Toolkit for Deep)r    g     €c@g     €`@g     �|@g     `b@z&Learning Based Document Image Analysisc                 S   s$   g | ]}t |j|j|jtjd �‘qS ))r!   r    Úsourcer   )r   r!   r    r™   r   ÚUNCATEGORIZED_TEXT)r{   Úrr   r   r   r}   §  s   úüz-test_ocr_data_to_elements.<locals>.<listcomp>)Úocr_dataZ
image_sizec                 S   s   h | ]
}|j ’qS r   )Úcategory)r{   Úelr   r   r   Ú	<setcomp>¸  r~   z,test_ocr_data_to_elements.<locals>.<setcomp>r;   )ZpointsÚsystem)r   rX   r   Úopenr   Zocr_data_to_elementsÚsizeÚlenr   rš   r	   ÚziprM   r7   r   r!   )
ÚfilenameZtext_regionsrœ   r   r$   Zimage_widthZimage_heightr8   rž   Z	layout_elr   r   r   Útest_ocr_data_to_elements”  s@    ûûøù	
þ
þr¦   )Brx   Úmultiprocessingr   ÚnumpyrŒ   ZpytestZPILr   Z unstructured_inference.inferencer   Z)unstructured_inference.inference.elementsr   Z.unstructured_inference.inference.layoutelementr   Ztest_unstructured.unit_utilsr   Z"unstructured.documents.coordinatesr	   Zunstructured.documents.elementsr
   r   r   r   r   r   r   r   r   r   rH   Zunstructured.partition.commonr   Z
PageLayoutr   ZDocumentLayoutr*   rD   rI   ÚmarkZparametrizerO   rP   rQ   rY   rZ   ÚitemsÚstrr   r\   ZCHECK_BOX_UNCHECKEDZCHECK_BOX_CHECKEDZRADIO_BUTTON_UNCHECKEDZRADIO_BUTTON_CHECKEDZCHECKEDZ	UNCHECKEDÚboolr`   rg   rh   ri   rv   r�   r’   r•   r˜   r¦   r   r   r   r   Ú<module>   s~   0üþ	
þþúþ!!þþ
ÿ