a
    ¡˜ëj Ý  ã                   @  sü	  d dl mZ d dlZd dlZd dlZd dlZd dlZd dlmZ d dl	m
Z
 d dlmZ d dlmZ d dlZd dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZmZ d dlmZ d dlmZm Z  d dl!m"Z" d dl#m$Z$ d dl%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/ d dl0m1Z1 d dl2m3Z3m4Z4 d dl5m6Z6m7Z7 d dl8m9Z9 d dl:m;Z;m<Z<m=Z=m>Z>m?Z?m@Z@mAZA G dd„ dƒZBdd„ ZCdd„ ZDdd„ ZEdd „ ZFG d!d"„ d"ejƒZGG d#d$„ d$ejƒZHG d%d&„ d&ejƒZIG d'd(„ d(ejƒZJejK Ld)e d*ƒdfd+g¡d,d-„ ƒZMd.d/„ ZNejK Ld0g d1¢¡ejK Ld2eAjOd3d3d4hd5hfeAjOd6d6d7hd5hfeAjPd4h d8£h d9£feAjQd3h d:£d;hfg¡e d<ƒfd=d>„ƒƒZRejS TejUd?d@i¡e d*ƒfdAdB„ƒZVejK LdCd@dDg¡e d*ƒfdEdF„ƒZWe d*ƒfdGdH„ZXe d*ƒfdIdJ„ZYe d*ƒfdKdL„ZZe d*ƒfdMdN„Z[e d*ƒfdOdP„Z\e d*ƒfdQdR„Z]dSdT„ Z^e d*ƒfdUdV„Z_e d*ƒfdWdX„Z`e d*ƒfdYdZ„Zae d*ƒfd[d\„Zbe d*ƒfd]d^„Zce d*ƒfd_d`„Zde d*ƒfdadb„Zee d*ƒfdcdd„Zfdedf„ ZgejK Ldgdhdig¡djdk„ ƒZhejK LdleAjOeAjPeAjQg¡dmdn„ ƒZiejK Ldgdhdig¡dodp„ ƒZjdqdr„ Zkdsdt„ Zle duƒfdvdw„Zmdxdy„ Zne d*ƒfdzd{„Zod|d}„ Zpe d*ƒfd~d„Zqe d*ƒfd€d�„ZrejK Ld0g d1¢¡ejK LdleAjseAjPeAjOeAjQg¡ejK Ld‚ddƒg¡d„d…d…d†d‡œdˆd‰„ƒƒƒZtejK LdleAjOeAjPg¡d…dŠœd‹dŒ„ƒZue d*ƒfd�dŽ„Zvd�d�„ Zwd‘d’„ Zxd“d”„ ZyejK Ld•e d*ƒg¡d–d—„ ƒZzejK Ld˜e d*ƒd™fe d*ƒdšfg¡d›dœ„ ƒZ{ejK Ld˜e d�ƒd™fe d�ƒdšfg¡dždŸ„ ƒZ|d d¡„ Z}d¢d£„ Z~e d¤ƒfd¥d¦„Ze d§ƒfd¨d©„Z€dªd«„ Z�ejK Ld¬d­g d®¢fd¯g d°¢fg¡d±d²„ ƒZ‚ejK Ld³d´dµg¡d¶d·„ ƒZƒd¸d¹„ Z„dºd»„ Z…ejK Ld¼eAjPd½feAjQd½feAjQd¾fg¡d¿dÀ„ ƒZ†ejK LdÁdÂdÃgdÄe$dÄdÄƒd3d fdÅdÆdÇdÈdÉdÊœdËd3dÌœg dÍ¢g dÎ¢dÏg dÐ¢dÑœdÅdÆdÒdÓdÔœdËd3dÌœg dÍ¢g dÕ¢dÏdÖdÑœgd×e$dØd×ƒd3dÙfdÅdÆdÇdÈdÉdÊœdËd3dÌœg dÍ¢g dÎ¢dÏdÚdÑœdÅdÆdÒdÓdÔœdËd3dÌœg dÍ¢g dÕ¢dÏdÛdÑœgd×e$dØd×ƒd3d fg¡dÜdÝ„ ƒZ‡ejK Ld0g d1¢¡ejK LdÞe d*ƒdßfe dàƒdáfg¡dâdã„ ƒƒZˆdädå„ Z‰dædç„ ZŠe dèƒfdédê„Z‹ejK Ldëdìg¡dídî„ ƒZŒejK Ldëdïg¡dðdñ„ ƒZ�dòdódôd…dõœdöd÷„ZŽejK Ld0d•døg¡ejK Ldùdßdág¡e dúƒfdûdü„ƒƒZ�e dýƒfdþdÿ„Z�e ‘¡ �d �d„ ƒZ’e ‘¡ �d�d„ ƒZ“e ‘¡ �d�d„ ƒZ”e ‘¡ �d�d„ ƒZ•ejKjL�deAjO�dfeAjP�dfeAjQ�dfg�dg�d	��d
�d„ ƒZ–ejK L�ddádßg¡ejK L�d�d�dg¡dôd…�d�dœ�d�d„ƒƒZ—ejK L�dg �d¢¡�d�d„ ƒZ˜�d�d„ Z™eG �d�d„ �dƒƒZšejKj›�d�d��d�d„ ƒZœejK L�d e>d fe=d3fe?d3fg¡�d!�d"„ ƒZ��d#�d$„ ZžejK Ld0g d1¢¡ejK LdleAjOeAjPeAjQg¡e �d%ƒf�d&�d'„ƒƒZŸ�d(�d)„ Z �d*�d+„ Z¡dS (,  é    )ÚannotationsN)Ú	dataclass)ÚPath)ÚSpooledTemporaryFile)Úmock)ÚPDFPageCountError)ÚImage)ÚMockFixture)Úlayout)Ú	Rectangle)ÚDocumentLayoutÚ
PageLayout)ÚLayoutElement)Úassert_round_trips_through_JSONÚexample_doc_path)Úchunk_by_title)Ú
PixelSpace)
ÚCoordinatesMetadataÚElementÚElementMetadataÚElementTypeÚFooterÚHeaderÚListItemÚNarrativeTextÚTextÚTitle)ÚPageCountExceededError)ÚpdfÚ
strategies)ÚocrÚpdfminer_processing)Úget_uris_from_annots)ÚOCR_AGENT_PADDLEÚOCR_AGENT_TESSERACTÚSORT_MODE_BASICÚSORT_MODE_DONTÚSORT_MODE_XY_CUTÚ#UNSTRUCTURED_INCLUDE_DEBUG_METADATAÚPartitionStrategyc                   @  s   e Zd Zdd„ Zdd„ ZdS )ÚMockResponsec                 C  s   || _ || _d S ©N©Ústatus_codeÚresponse)Úselfr-   r.   © r0   úm/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/test_unstructured/partition/pdf_image/test_pdf.pyÚ__init__5   s    zMockResponse.__init__c                 C  s   | j S r+   )r.   ©r/   r0   r0   r1   Újson9   s    zMockResponse.jsonN)Ú__name__Ú
__module__Ú__qualname__r2   r4   r0   r0   r0   r1   r*   4   s   r*   c                 K  s   t di d�S )NéÈ   r,   ©r*   ©ÚurlÚkwargsr0   r0   r1   Úmock_healthy_get=   s    r=   c                 K  s   t di d�S ©Niô  r,   r9   r:   r0   r0   r1   Úmock_unhealthy_getA   s    r?   c                 K  s   t di d�S r>   r9   r:   r0   r0   r1   Úmock_unsuccessful_postE   s    r@   c                 K  s4   dddddœgdœddddœgdœgi}t d	|d
�S )NÚpagesr   r   ú#Charlie Brown and the Great Pumpkin)ÚtypeÚtext)ÚnumberÚelementsé   zA Charlie Brown Christmasr8   r,   r9   )r;   r<   r.   r0   r0   r1   Úmock_successful_postI   s    ÿþ
þùÿrH   c                   @  s   e Zd Zdddœdd„ZdS )ÚMockPageLayoutÚintr   ©rE   Úimagec                 C  sN   || _ || _dddœ| _d | _tjjddddddd�g| _tj 	| j¡| _
d S )Né
   ©ÚwidthÚheightr   g        g       @rB   )rC   Úx1Úy1Zx2Úy2rD   )rE   rL   Úimage_metadataZdetection_modelr
   r   Úfrom_coordsrF   ÚLayoutElementsÚ	from_listÚelements_array©r/   rE   rL   r0   r0   r1   r2   \   s    úÿ
zMockPageLayout.__init__N)r5   r6   r7   r2   r0   r0   r0   r1   rI   [   s   rI   c                   @  s4   e Zd Zdddœdd„Zedd„ ƒZedd	„ ƒZd
S )ÚMockSinglePageLayoutrJ   zImage.ImagerK   c                 C  s   || _ || _dddœ| _d S )NrM   rN   )rE   rL   rT   rY   r0   r0   r1   r2   o   s    zMockSinglePageLayout.__init__c                 C  s\   t ddtd d d d ƒd�t ddtd d d d ƒd�t ddtd d d d ƒd�t dd	td d d d ƒd�gS )
NZHeadlinerB   )rC   rD   ÚbboxZSubheadlinezThe Beginningr   z/This time Charlie Brown had it really tricky...r   z#Another book title in the same page)r   r   r3   r0   r0   r1   rF   t   s*    ýýýýðzMockSinglePageLayout.elementsc                 C  s   t j | j¡S r+   )r
   rV   rW   rF   r3   r0   r0   r1   rX   �   s    z#MockSinglePageLayout.elements_arrayN)r5   r6   r7   r2   ÚpropertyrF   rX   r0   r0   r0   r1   rZ   n   s
   
rZ   c                   @  s   e Zd Zedd„ ƒZdS )ÚMockDocumentLayoutc                 C  s(   t dt dd¡d�t dt dd¡d�gS )Nr   Ú1©rG   rG   rK   rG   )rI   r   Únewr3   r0   r0   r1   rA   “   s    þzMockDocumentLayout.pagesN©r5   r6   r7   r\   rA   r0   r0   r0   r1   r]   ’   s   r]   c                   @  s   e Zd Zedd„ ƒZdS )ÚMockSinglePageDocumentLayoutc                 C  s   t dt dd¡d�gS )NrG   r^   r_   rK   )rZ   r   r`   r3   r0   r0   r1   rA   œ   s    ÿz"MockSinglePageDocumentLayout.pagesNra   r0   r0   r0   r1   rb   ›   s   rb   )ÚfilenameÚfileú pdf/layout-parser-paper-fast.pdf)Ns   0000c                 C  sŽ   |   tddd„ ¡ |   tddd„ ¡ |   tddd„ ¡ |   tdd	d„ ¡ |   td
dd„ ¡ |   tddd„ ¡ t ||¡}|d jdksŠJ ‚d S )NÚprocess_data_with_modelc                  _  s   t ƒ S r+   ©r]   ©Úargsr<   r0   r0   r1   Ú<lambda>®   ó    z*test_partition_pdf_local.<locals>.<lambda>Úprocess_file_with_modelc                  _  s   t ƒ S r+   rg   rh   r0   r0   r1   rj   ³   rk   Zprocess_data_with_pdfminerc                  _  s   t ƒ S r+   rg   rh   r0   r0   r1   rj   ¸   rk   Zprocess_file_with_pdfminerc                  _  s   t ƒ S r+   rg   rh   r0   r0   r1   rj   ½   rk   Zprocess_data_with_ocrc                  _  s   t ƒ S r+   rg   rh   r0   r0   r1   rj   Â   rk   Úprocess_file_with_ocrc                  _  s   t ƒ S r+   rg   rh   r0   r0   r1   rj   Ç   rk   r   rB   )Úsetattrr
   r!   r    r   Ú_partition_pdf_or_image_localrD   )Úmonkeypatchrc   rd   Zpartition_pdf_responser0   r0   r1   Útest_partition_pdf_local£   s@    ýýýýýýrq   c                   C  sB   t  ttf¡�  tjdd dd� W d   ƒ n1 s40    Y  d S )NÚ F)rc   rd   Úis_image)ÚpytestÚraisesÚFileNotFoundErrorr   r   ro   r0   r0   r0   r1   Ú0test_partition_pdf_local_raises_with_no_filenameÎ   s    rw   Ú	file_mode)rc   ÚrbZspool)ÚstrategyÚstarting_page_numberÚexpected_page_numbersÚoriginrG   é   Úpdfmineré   é   >   r~   r�   é   >   r   Úocr_tesseractÚyolox>   rG   r€   r~   rƒ   ú,pdf/layout-parser-paper-with-empty-pages.pdfc              	     s   ‡ ‡fdd„}| dkr0t j|||d�}||ƒ nÌ| dkr|t|dƒ�(}t j|||d�}||ƒ W d   ƒ qü1 sp0    Y  n€t|dƒ�f}	tƒ �@}
|
 |	 ¡ ¡ |
 d¡ t j|
||d�}||ƒ W d   ƒ n1 sÔ0    Y  W d   ƒ n1 sò0    Y  d S )Nc                   sV   t | ƒdksJ ‚dd„ | D ƒˆ ks&J ‚trRtdd„ | D ƒƒ dd„ | D ƒˆksRJ ‚d S )NrM   c                 S  s   h | ]}|j j’qS r0   ©ÚmetadataÚpage_number©Ú.0Úelementr0   r0   r1   Ú	<setcomp>ì   rk   zitest_partition_pdf_outputs_valid_amount_of_elements_and_metadata_values.<locals>._test.<locals>.<setcomp>c                 S  s   g | ]}|j j|j|jf‘qS r0   )r‡   Údetection_originÚcategoryrD   r‰   r0   r0   r1   Ú
<listcomp>ï   s   ÿzjtest_partition_pdf_outputs_valid_amount_of_elements_and_metadata_values.<locals>._test.<locals>.<listcomp>c                 S  s   h | ]}|j j’qS r0   ©r‡   r�   r‰   r0   r0   r1   rŒ   ô   rk   )Úlenr(   Úprint©Úresult©r|   r}   r0   r1   Ú_testè   s    þÿzVtest_partition_pdf_outputs_valid_amount_of_elements_and_metadata_values.<locals>._testrc   )rc   rz   r{   ry   )rd   rz   r{   r   ©r   Úpartition_pdfÚopenr   ÚwriteÚreadÚseek)rx   rz   r{   r|   r}   rc   r–   r”   ÚfÚ	test_fileÚspooled_temp_filer0   r•   r1   ÚGtest_partition_pdf_outputs_valid_amount_of_elements_and_metadata_valuesÓ   s,    ÿ
ÿ(
ýr    ÚUNSTRUCTURED_HI_RES_MODEL_NAMEÚcheckboxc                 C  sp   |   tddd„ ¡ tjjtdtƒ d��6}tj|tj	d� |j
d d d	ksNJ ‚W d   ƒ n1 sb0    Y  d S )
NÚextractable_elementsc                  _  s   g S r+   r0   rh   r0   r0   r1   rj     rk   z<test_partition_pdf_with_model_name_env_var.<locals>.<lambda>rl   ©Zreturn_value©rc   rz   rG   Ú
model_namer¢   ©rn   r   r   ÚpatchÚobjectr
   r]   r˜   r)   ÚHI_RESÚ	call_args©rp   rc   Úmock_processr0   r0   r1   Ú*test_partition_pdf_with_model_name_env_var  s    ýr®   r¦   r„   c              	   C  sø   |   tddd„ ¡ tjjtdtƒ d��8}tj|tj	|d� |j
d d |ksPJ ‚W d   ƒ n1 sd0    Y  tjjtd	tƒ d��b}t|d
ƒ�8}tj|tj	|d� |j
d d |ks¸J ‚W d   ƒ n1 sÌ0    Y  W d   ƒ n1 sê0    Y  d S )Nr£   c                  _  s   g S r+   r0   rh   r0   r0   r1   rj   #  rk   z4test_partition_pdf_with_model_name.<locals>.<lambda>rl   r¤   )rc   rz   r¦   rG   r¦   rf   ry   )rd   rz   r¦   )rn   r   r   r¨   r©   r
   r]   r˜   r)   rª   r«   r™   )rp   r¦   rc   r­   r�   r0   r0   r1   Ú"test_partition_pdf_with_model_name  s4    ýý4ýýr¯   c                 C  sr   |   tddd„ ¡ tjjtdtƒ d��8}tj|tj	dd� |j
d d	 dksPJ ‚W d   ƒ n1 sd0    Y  d S )
Nr£   c                  _  s   g S r+   r0   rh   r0   r0   r1   rj   B  rk   z;test_partition_pdf_with_hi_res_model_name.<locals>.<lambda>rl   r¤   r¢   ©rc   rz   Úhi_res_model_namerG   r¦   r§   r¬   r0   r0   r1   Ú)test_partition_pdf_with_hi_res_model_name>  s    ýÿr²   c                 C  sr   |   tddd„ ¡ tjjtdtƒ d��8}tj|tj	dd� |j
d d	 dksPJ ‚W d   ƒ n1 sd0    Y  d S )
Nr£   c                  _  s   g S r+   r0   rh   r0   r0   r1   rj   S  rk   zDtest_partition_pdf_or_image_with_hi_res_model_name.<locals>.<lambda>rl   r¤   r¢   r°   rG   r¦   )rn   r   r   r¨   r©   r
   r]   Úpartition_pdf_or_imager)   rª   r«   r¬   r0   r0   r1   Ú2test_partition_pdf_or_image_with_hi_res_model_nameO  s    ýÿr´   c                 C  sZ   t j| tjd�}d}|d j|ks&J ‚|d jjdks:J ‚|d jjtj	 
| ¡ksVJ ‚d S )Nr¥   õP   LayoutParser: A Uniï¬�ed Toolkit for Deep Learning Based Document Image Analysisr�   úlayout-parser-paper-fast.pdf)r   r˜   r)   ÚAUTOrD   r‡   rc   Zfile_directoryÚosÚpathÚdirname)rc   rF   Útitler0   r0   r1   Ú%test_partition_pdf_with_auto_strategy`  s
    r¼   c                 C  s*   t j| d dd�}ddd„ |D ƒv s&J ‚d S )NT)rc   r;   Úinclude_page_breaksÚ	PageBreakc                 S  s   g | ]
}|j ‘qS r0   ©rŽ   ©rŠ   Úelemr0   r0   r1   r�   n  rk   z7test_partition_pdf_with_page_breaks.<locals>.<listcomp>©r   r˜   ©rc   rF   r0   r0   r1   Ú#test_partition_pdf_with_page_breaksj  s    rÄ   c                 C  s(   t j| d d�}ddd„ |D ƒvs$J ‚d S )N)rc   r;   r¾   c                 S  s   g | ]
}|j ‘qS r0   r¿   rÀ   r0   r0   r1   r�   u  rk   z:test_partition_pdf_with_no_page_breaks.<locals>.<listcomp>rÂ   rÃ   r0   r0   r1   Ú&test_partition_pdf_with_no_page_breaksq  s    rÅ   c                 C  s\   t j| d tjdd�}t|ƒdks$J ‚dd„ |D ƒddhks>J ‚|D ]}|jjdksBJ ‚qBd S )Nr€   )rc   r;   rz   r{   rM   c                 S  s   h | ]}|j j’qS r0   r†   r‰   r0   r0   r1   rŒ   €  rk   z8test_partition_pdf_with_fast_strategy.<locals>.<setcomp>r~   r¶   )r   r˜   r)   ÚFASTr‘   r‡   rc   ©rc   rF   r‹   r0   r0   r1   Ú%test_partition_pdf_with_fast_strategyx  s    
ÿrÈ   c                  C  sj   t dƒ} tj| d tjd�}t|ƒdks*J ‚|d jjjd d dk sHJ ‚|d jjjd d dk sfJ ‚d S )Nzpdf/negative-coords.pdf©rc   r;   rz   é   r   rG   )	r   r   r˜   r)   rÆ   r‘   r‡   ÚcoordinatesÚpointsrÃ   r0   r0   r1   Ú,test_partition_pdf_with_fast_neg_coordinates…  s
    rÍ   c                 C  sx   t j| d tjd�}d }|D ]}t|tƒr|} q2qt|jƒdksDJ ‚|j d¡sTJ ‚|j 	d¡sdJ ‚|j
jdkstJ ‚d S )NrÉ   iè  zAbstract. Recent advancesz https://layout-parser.github.io.r¶   )r   r˜   r)   rÆ   Ú
isinstancer   r‘   rD   Ú
startswithÚendswithr‡   rc   )rc   rF   Zfirst_narrative_elementr‹   r0   r0   r1   Ú(test_partition_pdf_with_fast_groups_text�  s    
rÑ   c                 C  sP   t | dƒ�"}tj|d tjd�}W d   ƒ n1 s20    Y  t|ƒdksLJ ‚d S )Nry   )rd   r;   rz   rM   )r™   r   r˜   r)   rÆ   r‘   )rc   r�   rF   r0   r0   r1   Ú/test_partition_pdf_with_fast_strategy_from_file�  s    0rÒ   c                 C  sf   t j|d tjdd�}t|ƒdks$J ‚ddd„ |D ƒv s:J ‚d| jvsHJ ‚|D ]}|jjdksLJ ‚qLd S )	NT)rc   r;   rz   r½   rM   r¾   c                 S  s   g | ]
}|j ‘qS r0   r¿   rÀ   r0   r0   r1   r�   °  rk   zItest_partition_pdf_with_fast_strategy_and_page_breaks.<locals>.<listcomp>ú'unstructured_inference is not installedr¶   )r   r˜   r)   rÆ   r‘   rD   r‡   rc   )Úcaplogrc   rF   r‹   r0   r0   r1   Ú5test_partition_pdf_with_fast_strategy_and_page_breaks¥  s    ürÕ   c                 C  s>   t  t¡�  tj| d dd� W d   ƒ n1 s00    Y  d S )NZmade_uprÉ   )rt   ru   Ú
ValueErrorr   r˜   ©rc   r0   r0   r1   Ú+test_partition_pdf_raises_with_bad_strategy·  s    rØ   c                 C  s‚   dd„ }|   td|¡ tdƒgg g}tjjtd|d��"}tj|d tj	d� W d   ƒ n1 s^0    Y  | 
¡  d|jv s~J ‚d S )	Nc                 S  s   | dvS ©N)Úunstructured_inferenceÚunstructured_pytesseractr0   ©Údepr0   r0   r1   Úmock_existsÃ  s    z:test_partition_pdf_falls_back_to_fast.<locals>.mock_existsÚdependency_existsúHello there!r£   r¤   rÉ   rÓ   ©rn   r   r   r   r¨   r©   r   r˜   r)   rª   Úassert_called_oncerD   ©rp   rÔ   rc   rÞ   Úmock_returnÚmock_partitionr0   r0   r1   Ú%test_partition_pdf_falls_back_to_fast¾  s    ý0ræ   c              	   C  s¸   dd„ }|   td|¡ tdƒgg g}tjjtd|d��P}tj td¡�"}tj|d tj	d� W d   ƒ n1 sn0    Y  W d   ƒ n1 sŒ0    Y  | 
¡  | ¡  d	|jv s´J ‚d S )
Nc                 S  s   | dvS ©N)rÛ   r0   rÜ   r0   r0   r1   rÞ   Ù  s    zHtest_partition_pdf_falls_back_to_fast_from_ocr_only.<locals>.mock_existsrß   rà   r£   r¤   Ú _partition_pdf_or_image_with_ocrrÉ   úpytesseract is not installed)rn   r   r   r   r¨   r©   r   r˜   r)   ÚOCR_ONLYrâ   Zassert_not_calledrD   )rp   rÔ   rc   rÞ   rä   rå   Zmock_partition_ocrr0   r0   r1   Ú3test_partition_pdf_falls_back_to_fast_from_ocr_onlyÔ  s$    ýþüNrë   c                 C  s�   dd„ }|   td|¡ |   tddd„ ¡ tdƒg}tjjtd|d	��"}tj|d tj	d
� W d   ƒ n1 sl0    Y  | 
¡  d|jv sŒJ ‚d S )Nc                 S  s   | dvS rç   r0   rÜ   r0   r0   r1   rÞ   ó  s    zJtest_partition_pdf_falls_back_to_hi_res_from_ocr_only.<locals>.mock_existsrß   r£   c                  _  s   g S r+   r0   rh   r0   r0   r1   rj   ÷  rk   zGtest_partition_pdf_falls_back_to_hi_res_from_ocr_only.<locals>.<lambda>rà   ro   r¤   rÉ   ré   )rn   r   r   r   r   r¨   r©   r˜   r)   rê   râ   rD   rã   r0   r0   r1   Ú5test_partition_pdf_falls_back_to_hi_res_from_ocr_onlyî  s    
ý0rì   c                 C  s~   dd„ }|   td|¡ tdƒg}tjjtd|d��"}tj|d tj	d� W d   ƒ n1 sZ0    Y  | 
¡  d|jv szJ ‚d S )	Nc                 S  s   | dvS )N)rÚ   r0   rÜ   r0   r0   r1   rÞ   
  s    z>test_partition_pdf_falls_back_to_ocr_only.<locals>.mock_existsrß   rà   rè   r¤   rÉ   rÓ   rá   rã   r0   r0   r1   Ú)test_partition_pdf_falls_back_to_ocr_only  s    
ý0rí   c                  C  sV   t dƒ} t d¡�0}tj| dd� |jd d s4J ‚W d   ƒ n1 sH0    Y  d S )Nre   z:unstructured.partition.pdf_image.ocr.process_file_with_ocrT)Úinfer_table_structurerG   rî   )r   r   r¨   r   r˜   r«   )rc   Zmock_process_file_with_modelr0   r0   r1   Ú(test_partition_pdf_uses_table_extraction  s    ÿrï   Úocr_modeZentire_pageZindividual_blocksc                 C  sˆ   t dƒ}tj|| dgtjdd�}dd„ |D ƒ}|d jjdgksDJ ‚t|ƒdksTJ ‚d	|d v sdJ ‚d
|d v stJ ‚d|d v s„J ‚d S )Núpdf/korean-text-with-tables.pdfÚkorT)rc   rð   Ú	languagesrz   rî   c                 S  s   g | ]}|j jr|j j‘qS r0   ©r‡   Ztext_as_html©rŠ   Úelr0   r0   r1   r�   4  rk   zItest_partition_pdf_hi_table_extraction_with_languages.<locals>.<listcomp>r   é   ú<table><thead><tr>ú</thead><tbody><tr>u   ì—…)r   r   r˜   r)   rª   r‡   ró   r‘   ©rð   rc   rF   Útabler0   r0   r1   Ú5test_partition_pdf_hi_table_extraction_with_languages$  s    ûrü   rz   c                 C  s4   t dƒ}tj|dg| d�}|d jjdgks0J ‚d S )Nrñ   rò   )rc   ró   rz   r   )r   r   r˜   r‡   ró   )rz   rc   rF   r0   r0   r1   Ú5test_partition_pdf_strategies_keep_languages_metadata>  s    	ýrý   c                 C  sŽ   t dƒ}tj|| tjdd�}dd„ |D ƒ}t|ƒdks:J ‚d|d v sJJ ‚d	|d v sZJ ‚d
|d v sjJ ‚d|d v szJ ‚d|d v sŠJ ‚d S )Núpdf/layout-parser-paper.pdfT)rc   rð   rz   rî   c                 S  s   g | ]}|j jr|j j‘qS r0   rô   rõ   r0   r0   r1   r�   _  rk   zLtest_partition_pdf_hi_res_ocr_mode_with_table_extraction.<locals>.<listcomp>r÷   rø   r   rù   z%Layouts of history Japanese documentsz9Layouts of scanned modern magazines and scientific reportz6Layouts of scanned US newspapers from the 20th century)r   r   r˜   r)   rª   r‘   rú   r0   r0   r1   Ú8test_partition_pdf_hi_res_ocr_mode_with_table_extractionP  s    ürÿ   c                  C  sz   t dƒ} tj| tjd�}d}d}|| j|ks2J ‚dd„ |D ƒddhksLJ ‚|| jjd us`J ‚t|| jjt	ƒsvJ ‚d S )	Núpdf/copy-protected.pdfr¥   rµ   é   c                 S  s   h | ]}|j j’qS r0   r†   r‰   r0   r0   r1   rŒ   n  rk   z:test_partition_pdf_with_copy_protection.<locals>.<setcomp>rG   r÷   )
r   r   r˜   r)   rª   rD   r‡   Zdetection_class_probrÎ   Úfloat)rc   rF   r»   Úidxr0   r0   r1   Ú'test_partition_pdf_with_copy_protectionh  s    r  c                  C  sh   t dƒ} tjjtdtƒ d��8}tj| tj	dd� |j
d d dksFJ ‚W d   ƒ n1 sZ0    Y  d S )Nr   rl   r¤   éd   )rc   rz   Úpdf_image_dpirG   r  )r   r   r¨   r©   r
   r]   r   r˜   r)   rª   r«   ©rc   r­   r0   r0   r1   Útest_partition_pdf_with_dpis  s    ÿr  úpdf/reliance.pdfc                 C  sL   t j| tjd�}t|ƒdks J ‚|d jjdks4J ‚|d jjdksHJ ‚d S )Nr¥   é2   r   rG   éÿÿÿÿr€   )r   r˜   r)   rÆ   r‘   r‡   rˆ   rÃ   r0   r0   r1   Ú0test_partition_pdf_requiring_recursive_text_grab|  s    r  c                  C  s,   t dƒ} tj| tjd�}t|ƒdks(J ‚d S )Nzpdf/loremipsum-flat.pdfr¥   r   )r   r   r˜   r)   rÆ   r‘   rÃ   r0   r0   r1   Ú'test_partition_pdf_text_not_extractableƒ  s    r  c                 C  sb   dd„ }|   td|¡ |   tddd„ ¡ t t¡� tj|d� W d   ƒ n1 sT0    Y  d S )Nc                 S  s   | dvS rÙ   r0   rÜ   r0   r0   r1   rÞ   �  s    zDtest_partition_pdf_fails_if_pdf_not_processable.<locals>.mock_existsrß   r£   c                  _  s   g S r+   r0   rh   r0   r0   r1   rj   ‘  rk   zAtest_partition_pdf_fails_if_pdf_not_processable.<locals>.<lambda>r×   )rn   r   r   rt   ru   rÖ   r˜   )rp   rc   rÞ   r0   r0   r1   Ú/test_partition_pdf_fails_if_pdf_not_processable‰  s
    r  c                  C  sÎ   t dƒ} tj| tjd�}d}tddd�}tt||d�d�}|d	 td
|d�ksRJ ‚t	|d t
ƒsdJ ‚t|d ƒ d¡szJ ‚t|d ƒ d¡s�J ‚d}tddd�}tt||d�d�}|d td|d�ksÊJ ‚d S )Núpdf/chevron-page.pdfr¥   ))çzÇ):’%h@ç`�Âõ(üQ@)r  ç`�Âõ(üV@)çF%u+z@r  )r  r  éd  é  rN   )rÌ   Úsystem)rË   r   zeastern mediterranean)r‡   rG   ZWezJordan and Egypt.))ç­ú\mÅêW@çØ4ï8E¥f@)r  çØ4ï8EEl@)ç¨5Í;NÙd@r  )r  r  r÷   z2.5)r   r   r˜   r)   rÆ   r   r   r   r   rÎ   r   ÚstrrÏ   rÐ   r   )rc   rF   Zexpected_coordinate_points_0Zexpected_coordinate_system_0Zexpected_elem_metadata_0Zexpected_coordinate_points_3Zexpected_coordinate_system_3Zexpected_elem_metadata_3r0   r0   r1   Ú/test_partition_pdf_fast_groups_text_in_text_box—  s4    þÿþþÿr  c                 C  s0   t j| d ddd�}|D ]}|jjdksJ ‚qd S )NTÚtest)rc   r;   r½   Úmetadata_filename)r   r˜   r‡   rc   rÇ   r0   r0   r1   Ú)test_partition_pdf_with_metadata_filename¿  s    ür  c                 C  s\   t | dƒ�$}tj|d tjdd�}W d   ƒ n1 s40    Y  |D ]}|jjdksBJ ‚qBd S )Nry   r  )rd   r;   rz   r  )r™   r   r˜   r)   rÆ   r‡   rc   )rc   r�   rF   r‹   r0   r0   r1   ÚFtest_partition_pdf_with_fast_strategy_from_file_with_metadata_filenameÌ  s    ü$r   Úmetadata_last_modifiedz2020-07-05T09:24:28r	   r  z
str | None)Úmockerrx   rz   r!  c              	   C  s,  t dƒ}d}|r|n|}|s(|dkr(d }| jd|d� |dkrPtj|||d�}n¾|dkr”t|dƒ� }tj|||d�}W d   ƒ n1 sˆ0    Y  nzt|dƒ�^}	tƒ �8}
|
 |	 ¡ ¡ |
 d	¡ tj|
||d�}W d   ƒ n1 sä0    Y  W d   ƒ n1 �s0    Y  d
d„ |D ƒ|hk�s(J ‚d S )Nr   z2029-07-05T09:24:28rc   z1unstructured.partition.pdf.get_last_modified_dater¤   )rc   rz   r!  ry   )rd   rz   r!  r   c                 S  s   h | ]}|j j’qS r0   )r‡   Zlast_modifiedrõ   r0   r0   r1   rŒ     rk   z3test_partition_pdf_metadata_date.<locals>.<setcomp>)	r   r¨   r   r˜   r™   r   rš   r›   rœ   )r"  rx   rz   r!  rc   Zfilesystem_last_modifiedZexpected_last_modifiedrF   r�   rž   rŸ   r0   r0   r1   Ú test_partition_pdf_metadata_dateÚ  sB    
ÿþýý&
ýDr#  ©rz   c                 C  s   t jtdƒ| d�}t|ƒ d S )Nre   r$  )r   r˜   r   r   )rz   rF   r0   r0   r1   Útest_partition_pdf_with_json  s
    þr%  c                 C  s>   t j| d�}t j| dd�}t|ƒ}||ks.J ‚||ks:J ‚d S )Nr×   Zby_title)Zchunking_strategy)r   r˜   r   )rc   rF   Zchunk_elementsÚchunksr0   r0   r1   Ú4test_add_chunking_strategy_by_title_on_partition_pdf  s
    r'  c                  C  sj   t dƒ} tj tdt ¡ ¡�:}tj| tj	dgd� |j
d d dksHJ ‚W d   ƒ n1 s\0    Y  d S )Núpdf/DA-1p.pdfrm   Úen©rc   rz   ró   rG   Úocr_languagesÚeng)r   r   r¨   r©   r    Z	MagicMockr   r˜   r)   rª   r«   r  r0   r0   r1   Ú2test_partition_pdf_formats_languages_for_tesseract&  s    r-  c                 C  s,   t dƒ}tj|tjdd� d| jv s(J ‚d S )Nr  r,  )rc   rz   r+  z*The ocr_languages kwarg will be deprecated)r   r   r˜   r)   rª   rD   )rÔ   rc   r0   r0   r1   Ú+test_partition_pdf_warns_with_ocr_languages-  s    r.  c                  C  sx   d} t jjtdt| ƒgd��H tjtdƒtjd�}t	|d tƒsDJ ‚|d j
| ksVJ ‚W d   ƒ n1 sj0    Y  d S )NzThis is Clearly a Titlero   r¤   re   r$  r   )r   r¨   r©   r   r   r³   r   r)   rª   rÎ   rD   )rD   rF   r0   r0   r1   Ú$test_partition_categorization_backup3  s    þr/  rc   c                 C  sn   t j| tjd�}d }|D ]}t|tƒr|} q0qt|ƒdk s@J ‚tdd„ |D ƒƒdksZJ ‚|j d¡sjJ ‚d S )Nr¥   é   c                 S  s   g | ]}t |tƒr|‘qS r0   )rÎ   r   r‰   r0   r0   r1   r�   K  rk   z.test_combine_numbered_list.<locals>.<listcomp>r~   z6character recognition, and other DIA tasks (Section 3))	r   r˜   r)   r·   rÎ   r   r‘   rD   rÐ   )rc   rF   Zfirst_list_elementr‹   r0   r0   r1   Útest_combine_numbered_list?  s    
ÿr1  r¥   ÚfastÚhi_resc                 C  sH   t j| |d�}ddddœddddœd	d
ddœg}|d jj|ksDJ ‚d S )Nr¥   Ú8zcite.gardner2018allennlpéŠ   )rD   r;   Ústart_indexZ34zcite.wolf2019huggingfaceé�   Z35zcite.wu2019detectron2é¨   r  )r   r˜   r‡   Úlinks)rc   rz   rF   r9  r0   r0   r1   Útest_partition_pdf_hyperlinksQ  s    ýýýõr:  zpdf/embedded-link.pdfc                 C  sF   t j| |d�}|d jjd d dks*J ‚t|d jjƒdksBJ ‚d S )Nr¥   r  rD   Z	capturingr÷   )r   r˜   r‡   r9  r‘   )rc   rz   rF   r0   r0   r1   Ú,test_partition_pdf_hyperlinks_multiple_linesn  s    r;  c                  C  sp   t j td¡�N} tjtdƒdtjd� |  ¡  d| j	j
v s>J ‚| j	j
d sNJ ‚W d   ƒ n1 sb0    Y  d S )Nro   re   r  )r¦   rz   r¦   ©r   r¨   r©   r   r˜   r   r)   rª   râ   r«   r<   ©Zmockpartitionr0   r0   r1   Ú"test_partition_pdf_uses_model_name{  s    þýr>  c                  C  sp   t j td¡�N} tjtdƒdtjd� |  ¡  d| j	j
v s>J ‚| j	j
d sNJ ‚W d   ƒ n1 sb0    Y  d S )Nro   re   r  )r±   rz   r±   r<  r=  r0   r0   r1   Ú)test_partition_pdf_uses_hi_res_model_name‹  s    þýr?  z"pdf/interface-config-guide-p93.pdfc              
   C  sT   zt j| dd�}W n, ty> } zd| ‚W Y d }~n
d }~0 0 t|ƒdksPJ ‚d S )Nr2  r¥   zPartitioning fail: %sé   )r   r˜   Ú	Exceptionr‘   )rc   rF   Úer0   r0   r1   Ú%test_partition_pdf_word_bbox_not_char›  s
    rC  zpdf/a1977-backus-p21.pdfc                 C  s   t j| dd� dS )z†Verify there is no regression for https://github.com/Unstructured-IO/unstructured/pull/2940,
    failing to map old parent_id's to newr2  r¥   NrÂ   r×   r0   r0   r1   Ú)test_partition_pdf_fast_no_mapping_errors¥  s    rD  c                  C  sH   t dƒ} t t¡�" tj| tjdd� W d   ƒ n1 s:0    Y  d S )Nr  r,  r*  )r   rt   ru   Ú	TypeErrorr   r˜   r)   rª   r×   r0   r0   r1   Ú9test_partition_pdf_raises_TypeError_for_invalid_languages­  s    rF  )Ú	thresholdÚexpectedgš™™™™™Ù?)TFFFFgš™™™™™¹?)TTFFFc                   sv   g d¢ddœg d¢ddœg d¢ddœg d¢ddœg d¢ddœg}d}t  ||d| ¡‰ ‡ fdd	„|D ƒ}||ksrJ ‚d S )
N)r   r   rG   rG   rG   )r[   rˆ   )r   r   r€   rG   r÷   )r   r   r   rG   )r€   r   r~   rG   c                   s   g | ]}|ˆ v ‘qS r0   r0   )rŠ   Ú
annotation©Úfilteredr0   r1   r�   Ä  rk   z9test_check_annotations_within_element.<locals>.<listcomp>)r   Z check_annotations_within_element)rG  rH  r   Zelement_bboxÚresultsr0   rJ  r1   Ú%test_check_annotations_within_element³  s    	ûrM  )ÚenvrH  )Nr„   )r  r  c                 C  s(   | d ur|  d| ¡ t ¡ |ks$J ‚d S )Nr¡   )Zsetenvr   Zdefault_hi_res_model)rN  rH  rp   r0   r0   r1   Útest_default_hi_res_modelÈ  s    rO  c                  C  sB   t dƒ} ztj| tjdd d� W n ty<   t d¡ Y n0 d S )Nr(  r,  )rc   rz   r+  r¦   ú3partition_pdf() raised AttributeError unexpectedly!©r   r   r˜   r)   rª   ÚAttributeErrorrt   Zfailr×   r0   r0   r1   Ú)test_partition_model_name_default_to_NoneÕ  s    ü
rS  c                  C  s@   t dƒ} ztj| tjd d� W n ty:   t d¡ Y n0 d S )Nr(  r°   rP  rQ  r×   r0   r0   r1   Ú0test_partition_hi_res_model_name_default_to_Noneâ  s    ý
rT  )rz   Úocr_funcz-unstructured_pytesseract.image_to_pdf_or_hocrz(unstructured_pytesseract.image_to_stringc              	   C  s²   G dd„ dt ƒ}tj|dƒd�}t ||¡�N t |¡�$ tjtdƒ| dd� W d   ƒ n1 sb0    Y  W d   ƒ n1 s€0    Y  |j	j
}d|v sžJ ‚|d dks®J ‚d S )	Nc                   @  s   e Zd ZdS )z7test_ocr_language_passes_through.<locals>.CallExceptionN)r5   r6   r7   r0   r0   r0   r1   ÚCallException  s   rV  zFunction called!)Zside_effectre   rò   )rz   r+  Úlang)rA  r   ZMockr¨   rt   ru   r   r˜   r   r«   r<   )rz   rU  rV  Zmock_ocr_funcr<   r0   r0   r1   Ú test_ocr_language_passes_throughî  s    ýBrX  )ÚannotsrP   Úcoordinate_systemrˆ   rH  ÚBSZBEi,  z/'Annot'z/'Link'z	/'Action'z/'URI'z"b'https://layout-parser.github.io')ÚTypeÚSÚURIz/'S')r]  ÚW)r   r   rG   )r   rG   rG   z/'I')g{®Gá s@g¢E¶óýl@gd;ßO�G|@g²�ï§ÆKm@)r\  ZSubtypeÚAr[  ZBorderÚCÚHZRectz/'GoTo'zb'cite.harley2015evaluation')r]  ÚD)r   rG   r   )ç{®GáD}@ç;ßO�—`@ç\�Âõ(~@g+‡ÙÎa@r  r  r÷   zI am not a tuple or list!)rd  re  rf  c                 C  s"   t | |||ƒ}t|ƒ|ksJ ‚d S r+   )r"   r‘   )rY  rP   rZ  rˆ   rH  Zannotation_listr0   r0   r1   Ú+test_get_uris_from_annots_string_annotation  s    Irg  )rc   rs   Fz img/layout-parser-paper-fast.jpgTc           	   	   C  sÄ  | dkr t j|tjdg|d�}nÈ| dkrjt|dƒ�&}t j|tjdg|d�}W d   ƒ qè1 s^0    Y  n~t|dƒ�d}tƒ �>}| | ¡ ¡ | d¡ t j|tjdg|d�}W d   ƒ n1 sÀ0    Y  W d   ƒ n1 sÞ0    Y  |d j	j
dgksþJ ‚|�r dd„ |D ƒd	hk�s<J ‚nd
d„ |D ƒd	dhk�s<J ‚|D ]^}|j	j�r@|j	jjD ]B}|d �rX|d	 �rX|d tju�s„J ‚|d	 tju�sXJ ‚�qX�q@t�rÀdd„ |D ƒdhk�sÀJ ‚d S )Nrc   r,  )rc   rz   ró   rs   ry   )rd   rz   ró   rs   r   c                 S  s   h | ]}|j j’qS r0   r†   rõ   r0   r0   r1   rŒ   Š  rk   z<test_partition_pdf_with_ocr_only_strategy.<locals>.<setcomp>rG   c                 S  s   h | ]}|j j’qS r0   r†   rõ   r0   r0   r1   rŒ   Œ  rk   r÷   c                 S  s   h | ]}|j j’qS r0   r�   r‰   r0   r0   r1   rŒ   ˜  rk   rƒ   )r   r˜   r)   rê   r™   r   rš   r›   rœ   r‡   ró   rË   rÌ   ÚmathÚnanr(   )	rx   rc   rs   rF   r�   rž   rŸ   r‹   Zpointr0   r0   r1   Ú)test_partition_pdf_with_ocr_only_strategy_  sJ    üü&
üB
rj  c                  C  s    t dƒ} tj| tjd�sJ ‚d S )Nzpdf/all-number-table.pdfr$  )r   r   r˜   r)   rê   r×   r0   r0   r1   Ú>test_partition_pdf_with_all_number_table_and_ocr_only_strategy›  s    rk  c                  C  s   t dƒ} tj| dd�sJ ‚d S )Nzpdf/pdf-bad-color-space.pdfr2  r$  )r   r   r˜   r×   r0   r0   r1   Ú)test_partition_pdf_with_bad_color_profile¤  s    rl  zpdf/header-test-doc.pdfc                 C  sP   t j| dd�}t|d tƒs J ‚t|d tƒs2J ‚dd„ |D ƒg d¢ksLJ ‚d S )Nr2  r$  r   r  c                 S  s   g | ]
}|j ‘qS r0   )rD   r‰   r0   r0   r1   r�   ¯  rk   zFtest_partition_pdf_with_fast_finds_headers_footers.<locals>.<listcomp>)zI Am A Headerr   zHere is a lovely sentences.zI Am A Footer)r   r˜   rÎ   r   r   rÃ   r0   r0   r1   Ú2test_partition_pdf_with_fast_finds_headers_footers©  s    rm  )rc   Úexpected_log)z-invalid-pdf-structure-pdfminer-entire-doc.pdfzRepairing the PDF document ...c                 C  s8   |  tj¡ tjtd| › �ƒd�s&J ‚||jv s4J ‚d S ©Nzpdf/r×   ©Z	set_levelÚloggingÚINFOr   r£   r   rD   ©rc   rn  rÔ   r0   r0   r1   Ú6test_extractable_elements_repair_invalid_pdf_structure·  s    rt  )z+invalid-pdf-structure-pdfminer-one-page.pdfzRepairing the PDF page 2 ...c                 C  s8   |  tj¡ tjtd| › �ƒd�s&J ‚||jvs4J ‚d S ro  rp  rs  r0   r0   r1   Útest_properly_patch_pdfminerÄ  s    	ru  zlist[Element]z	list[str]Úbool)rF   Úextract_image_block_typesÚextract_image_block_to_payloadÚtmpdirc                 C  s0  g }|D ]0}g }| D ]}|j |kr| |¡ q| |¡ q|D ]ì}t|ƒD ]Þ\}}|r¤|jjd usfJ ‚|jjdksvJ ‚t |jj¡}	t|	t	ƒs’J ‚|jj
d u s¢J ‚qJ|j tjkr´dnd}
tj t|ƒ|
› d|jj› d|d › d�¡}|jj
|ksôJ ‚tj |¡�sJ ‚|jjd u �sJ ‚|jjd u sJJ ‚qJq>d S )Nz
image/jpegrû   Úfigureú-rG   z.jpg)rŽ   ÚappendÚ	enumerater‡   Zimage_base64Zimage_mime_typeÚbase64Ú	b64decoderÎ   ÚbytesZ
image_pathr   ZTABLEr¸   r¹   Újoinr  rˆ   Úisfile)rF   rw  rx  ry  Zextracted_elementsZel_typeZextracted_elements_by_typerö   ÚiZ
image_dataÚbasenameZexpected_image_pathr0   r0   r1   Úassert_element_extractionÒ  s.    
"ÿr…  ry   rx  zpdf/embedded-images-tables.pdfc              	   C  s²   ddg}t  ¡ �Ž}| dkr0tj|d|||d�}n>t|dƒ�$}tj|d|||d�}W d   ƒ n1 sd0    Y  |d jjdks‚J ‚t||||ƒ W d   ƒ n1 s¤0    Y  d S )	Nr   ZTablerc   é   )rc   r{   rw  rx  Úextract_image_block_output_dirry   )rd   r{   rw  rx  r‡  r   )ÚtempfileÚTemporaryDirectoryr   r˜   r™   r‡   rˆ   r…  )rx   rx  rc   rw  ry  rF   r�   r0   r0   r1   Ú%test_partition_pdf_element_extractionó  s,    
ú	ú$ÿrŠ  zpdf/embedded-images.pdfc                 C  s0   t j| dd�}dd„ |D ƒ}t|ƒdks,J ‚d S )Nr3  r¥   c                 S  s   g | ]}|j tjkr|‘qS r0   )rŽ   r   ZIMAGErõ   r0   r0   r1   r�     rk   zEtest_partition_pdf_always_keep_all_image_elements.<locals>.<listcomp>r€   )r   r˜   r‘   )rc   rF   Zimage_elementsr0   r0   r1   Ú1test_partition_pdf_always_keep_all_image_elements  s    þr‹  c                   C  s   g d¢S ©N)
Z 27a6cb3e5a4ad399b2f865729bbd3840Z a90a54baba0093296a013d26b7acbc17Z 9be424e2d151dac4b5f36a85e9bbfe65Z 4631da875fb4996c63b2d80cea6b588eZ 6264f4eda97a049f4710f9bea0c01cbdZ abded7b2ff3a5542c88b4a831755ec24Z b781ea5123cb31e0571391b7b42cac75Z 033f27d2618ba4cda9068b267b5a731eZ 8982a12fcced30dd12ccbf61d14f30bfZ 41af2fd5df0cf47aa7e8ecca200d3ac6r0   r0   r0   r0   r1   Ú&expected_element_ids_for_fast_strategy!  s    r�  c                   C  s   g d¢S rŒ  r0   r0   r0   r0   r1   Ú(expected_element_ids_for_hi_res_strategy1  s    rŽ  c                   C  s   g d¢S )N)Z 272ab65cbe81795161128aea59599d83Z b38affd7bbbb3dddf5c85ba8b14d380dZ 65903214d456b8b3cba6faa6714bd9baZ 5b41ceae05dcfaeeac32ff8e82dc2ff1Z 6582fc6c6c595225feeddcc3263f0ae3Z 64b610c8f4274f1ce2175bf30814409dZ 8edde8bf2d3a68370dc4bd142c408ca4Z a052bc17696043efce2e4f4f28393a83r0   r0   r0   r0   r1   Ú%expected_element_ids_for_ocr_strategyA  s    r�  c                 C  s   |   | j¡S r+   )ZgetfixturevalueÚparam)Úrequestr0   r0   r1   Úexpected_idsO  s    r’  )rz   r’  )Zindirectc                 C  s6   t jtdƒ| dd�}dd„ |D ƒ}||ks2J dƒ‚d S )Nz%pdf/fake-memo-with-duplicate-page.pdfr÷   )rz   r{   c                 S  s   g | ]
}|j ‘qS r0   )Úidr‰   r0   r0   r1   r�   c  rk   z=test_unique_and_deterministic_element_ids.<locals>.<listcomp>z%Element IDs do not match expected IDs)r   r˜   r   )rz   r’  rF   Úidsr0   r0   r1   Ú)test_unique_and_deterministic_element_idsT  s    
ýr•  Úis_path)Úexample_docÚ	doc_pages)re   r÷   )zimg/DA-1p.pngrG   rJ   )r–  r—  r˜  c              	   C  sÊ  t  ¡ ��ª}d }t|ƒ}t|ƒj d¡ }| s<t|dƒ}d }tj|||t	j
d|d� t|ƒ}d }| rx|d t|ƒj }n:t|d  ¡ ƒd }	|r d|	jv s®J ‚nd|	jv s®J ‚|	}|d }
|
 ¡ sÆJ ‚t|
 ¡ ƒ}g d	¢}t|ƒt|ƒksîJ ‚|D ]}|
|› d
�  ¡ sòJ ‚qò|d }| ¡ �s$J ‚t| ¡ ƒ}g d¢}t|ƒ|t|ƒ k�sRJ ‚td|d ƒ}|D ]2}|D ]&}|d|› d|› d� |v �slJ ‚�ql�qd|�r¦| ¡  W d   ƒ n1 �s¼0    Y  d S )Nr   ry   T)rc   rd   rs   rz   ÚanalysisZanalyzed_image_output_dir_pathr™  r   rL   Zlayout_dump)Zobject_detectionr    r   Úfinalz.jsonZbboxes)Zod_modelr    r   rš  rG   ÚpageZ_layout_z.png)rˆ  r‰  r   r   ÚsuffixrÐ   r™   r   r˜   r)   rª   ÚstemÚlistÚiterdirÚnameÚexistsr‘   ÚrangeÚclose)r–  r—  r˜  Útemp_dirrd   rc   rs   Zanalysis_dirZfile_analysis_rootZgenerated_file_stem_pathZlayout_dump_dirZlayout_dump_filesZexpected_layout_dumpsZexpected_layout_dumpZ
bboxes_dirZbboxes_filesZexpected_rendersZexpected_pagesrö   r›  r0   r0   r1   Útest_analysis_artifacts_savedg  sT    	
ú	(r¥  )rc   Úpdf_hi_res_max_pagesÚexpected_error))r…   NF)r…   r€   T)r	  r€   F)r	  r÷   T)úimg/DA-1p.jpgNF)r¨  r÷   Fc                 C  sv   t | ƒj d¡ }|s0tjt| ƒtj||d� nBt 	t
¡�( tjt| ƒtj||d� W d   ƒ n1 sh0    Y  d S )Nr   )rc   rz   r¦  rs   )r   rœ  rÐ   r   r³   r   r)   rª   rt   ru   r   )rc   r¦  r§  rs   r0   r0   r1   Ú"test_pdf_hi_res_max_pages_argument¦  s    üür©  c                  C  sL   t ƒ } | jD ]}d |jjd d …d d …f< qt | ¡}|d jjd u sHJ ‚d S )Nr   )rb   rA   rX   Zelement_coordsr   Údocument_to_element_listr‡   rË   )Zlayout_elem_absent_coordinatesr›  rF   r0   r0   r1   ÚItest_document_to_element_list_omits_coord_system_when_coord_points_absentÅ  s
    

r«  c                   @  s   e Zd ZdZdZdZdS )Ú	MockImagei€  ià  ZJPGN)r5   r6   r7   rO   rP   Úformatr0   r0   r0   r1   r¬  Ï  s   r¬  z2no current layout model supports parent assignment)Úreasonc               	   C  s~   t jddddddd�} t jddddd| dd	�}tdtƒ d
�}tj | |g¡|_t 	|g¡}t
 |¡\}}|jj|jkszJ ‚d S )NrG   r÷   r€   r~   úblock 1r   ©rD   rC   úblock 2©rD   ÚparentrC   rK   )r   rU   r   r¬  r
   rV   rW   rX   r   Ú
from_pagesr   rª  r‡   Z	parent_idr“  )Úblock1Úblock2r›  ÚdocZel1Zel2r0   r0   r1   Ú,test_document_to_element_list_handles_parentÖ  s$    ù	þr¸  )Ú	sort_modeÚ
call_countc              	   C  sª   t jddddddd�}t jddddd|dd	�}tdtƒ d
�}tj ||g¡|_t 	|g¡}t
j td¡� }tj|d| d� W d   ƒ n1 sŽ0    Y  |j|ks¦J ‚d S )NrG   r÷   r€   r~   r¯  r   r°  r±  r²  rK   Zsort_page_elementsT)Zsortabler¹  )r   rU   r   r¬  r
   rV   rW   rX   r   r´  r   r¨   r©   r   rª  rº  )r¹  rº  rµ  r¶  r›  r·  Zmock_sort_page_elementsr0   r0   r1   Ú8test_document_to_element_list_doesnt_sort_on_sort_methodì  s&    ù	þ.r»  c                  C  sd   t ƒ } t | ¡}|d jjdks$J ‚|d jjdks8J ‚|d jjd u sLJ ‚|d jjdks`J ‚d S )Nr   rG   r÷   r€   )rb   r   rª  r‡   Zcategory_depth)Zlayout_with_hierarchiesrF   r0   r0   r1   Ú8test_document_to_element_list_sets_category_depth_titles  s    
r¼  zpdf/password.pdfc              	   C  sú   dd„ }| dkr*t j||dd�}||ƒ nÌ| dkrvt|dƒ�(}t j||dd�}||ƒ W d   ƒ qö1 sj0    Y  n€t|dƒ�f}tƒ �@}| | ¡ ¡ | d¡ t j||dd�}||ƒ W d   ƒ n1 sÎ0    Y  W d   ƒ n1 sì0    Y  d S )	Nc                 S  s&   t | ƒdksJ ‚| d jdks"J ‚d S )NrG   r   zFile with password)r‘   rD   r“   r0   r0   r1   r–      s    z/test_partition_pdf_with_password.<locals>._testrc   Úpassword)rc   rz   r½  ry   )rd   rz   r½  r   r—   )rx   rz   rc   r–   r”   r�   rž   rŸ   r0   r0   r1   Ú test_partition_pdf_with_password  s     
(
ÿr¾  c                 C  sV   t jtdƒtjdttd� | jd d dtdœks6J ‚| jd d dtdœksRJ ‚d S )	Nz&pdf/layout-parser-paper-with-table.pdfT)rc   rz   rî   Z	ocr_agentZtable_ocr_agentr   rG   r,  )ÚlanguageZocr_agent_moduler)  )r   r˜   r   r)   rª   r$   r#   Zcall_args_list)Zmock_ocr_get_instancer"  r0   r0   r1   Ú,test_partition_pdf_with_specified_ocr_agents7  s    û	þþrÀ  c                  C  sˆ   ddl m }  | tdƒƒD ]j}tj|tjdd�}tdƒD ]J}tj|tjdd�}t||ƒD ]$\}}|j|jksVJ d|›d�ƒ‚qV qq6qd S )	Nr   )Úglobrþ   F)rc   rz   rî   r~   zload two time f=z return differents results)	rÁ  r   r   r˜   r)   r·   r¢  ÚziprD   )rÁ  r�   Z
elements_1Ú_Z
elements_2Úe1Úe2r0   r0   r1   Útest_reproductible_pdf_loaderL  s     ýýrÆ  )¢Ú
__future__r   r~  rq  rh  r¸   rˆ  Zdataclassesr   Úpathlibr   r   Zunittestr   rt   Zpdf2image.exceptionsr   ZPILr   Zpytest_mockr	   Z unstructured_inference.inferencer
   Z)unstructured_inference.inference.elementsr   Z'unstructured_inference.inference.layoutr   r   Z.unstructured_inference.inference.layoutelementr   Ztest_unstructured.unit_utilsr   r   Zunstructured.chunking.titler   Z"unstructured.documents.coordinatesr   Zunstructured.documents.elementsr   r   r   r   r   r   r   r   r   r   Zunstructured.errorsr   Zunstructured.partitionr   r   Z unstructured.partition.pdf_imager    r!   Z4unstructured.partition.pdf_image.pdfminer_processingr"   Z&unstructured.partition.utils.constantsr#   r$   r%   r&   r'   r(   r)   r*   r=   r?   r@   rH   rI   rZ   r]   rb   ÚmarkZparametrizerq   rw   rÆ   rª   rê   r    r¨   ÚdictÚenvironr®   r¯   r²   r´   r¼   rÄ   rÅ   rÈ   rÍ   rÑ   rÒ   rÕ   rØ   ræ   rë   rì   rí   rï   rü   rý   rÿ   r  r  r  r  r  r  r  r   r·   r#  r%  r'  r-  r.  r/  r1  r:  r;  r>  r?  rC  rD  rF  rM  rO  rS  rT  rX  rg  rj  rk  rl  rm  rt  ru  r…  rŠ  r‹  Zfixturer�  rŽ  r�  r’  r•  r¥  r©  r«  r¬  Úskipr¸  r»  r¼  r¾  rÀ  rÆ  r0   r0   r0   r1   Ú<module>   sª  0$	$	
þþ
$üüú/þý"þ
þ
ÿ
ÿ
ÿ
ÿ
	ÿ
ÿ

þ
ÿ

ý
ý
ý
ý
	þþ
ýþ

þþ
	þ
)ÿ
ÿ
üþ	.	ÿ

þ


þþ


þþ
ÿ
ÿ


þþ
þþ
þþþ÷þ
ýôøñâ#ýôøñâÞþF


þþ4	ÿ
þþ
ýþ
!ý"ÿ



ýù	
þþ7þ
þ	ýüý