a
    ¡˜ëj‹§  ã                
   @  sø  d dl mZ d dlZd dlmZmZmZmZmZm	Z	m
Z
mZ d dlZd dlmZmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZmZ d dlmZm Z  d dl!m"Z" d dl#m$Z$m%Z%m&Z&m'Z'm(Z( d dl)m*Z* d dl+m,Z,m-Z- d dl.m/Z/ d dl0m1Z1 e�r:d dlm2Z2m3Z3 d dl4m5Z5 d dl6m7Z7 dZ8dZ9dšddddddœdd „Z:d!d"d#œd$d%„Z;d&d'd(œd)d*„Z<d+d,d-d.œd/d0„Z=d+d-d1œd2d3„Z>d+d+d4d5d6œd7d8„Z?d+d+d'd'd'd+d9œd:d;„Z@d+d+d'd4d<d=œd>d?„ZAe1d@ƒejBejCdAfd+d+dBd4d4dd+dCœdDdE„ƒZDe1d@ƒdFdGdd4dHdIœdJdK„ƒZEe1d@ƒd›dLddddMdNœdOdP„ƒZFdQdR„ ZGe9fdd'dSœdTdU„ZHe9fd'd'ddVœdWdX„ZIdYe9fd4dd'dZœd[d\„ZJdYe9fd4dd'dZœd]d^„ZKd_e9fd4dd'dZœd`da„ZLe1d@ƒd+dbdcœddde„ƒZMe1d@ƒdfdgddfdhœdidj„ƒZNdfdfdkœdldm„ZOe1d@ƒdœd&d4d&dnœdodp„ƒZPe*jQfd&d&d4ddqœdrds„ZRdFdtdFduœdvdw„ZSdxd4dyddzd{œd|d}„ZTd~dGdyddzd{œdd€„ZUd�d‚œdƒd„„ZVdzd…dd4dzd†œd‡dˆ„ZWd‰d4dŠd‹œdŒd�„ZXdzdŽd�œd�d‘„ZYd…d…d4d’œd“d”„ZZd…d4d#œd•d–„Z[d'dd—œd˜d™„Z\dS )�é    )ÚannotationsN)ÚTYPE_CHECKINGÚAnyÚBinaryIOÚIterableÚListÚOptionalÚUnionÚcast)ÚLTCharÚ	LTTextBox)Ú	PDFObjRef)Úopen_filename)Úinference_config)ÚFULL_PAGE_REGION_THRESHOLD)Ú	Rectangle)Ú
PixelSpaceÚ
PointSpace)ÚCoordinatesMetadataÚElementType)Úremove_control_characters)ÚPDFMinerConfigÚextract_image_objectsÚextract_text_objectsÚopen_pdfminer_pages_generatorÚrect_to_bbox)Ú
env_config)ÚSORT_MODE_BASICÚSource)Úsort_text_regions)Úrequires_dependencies)Ú
TextRegionÚTextRegions)ÚDocumentLayout©ÚLayoutElementsg{®Gáz„?é   Ú éÈ   ÚstrÚintzOptional[str]zOptional[PDFMinerConfig]z+tuple[List[List['TextRegion']], List[List]])ÚfilenameÚdpiÚpasswordÚpdfminer_configÚreturnc                 C  sT   t | dƒ�6}tt|ƒ}t||||d�\}}||fW  d   ƒ S 1 sF0    Y  d S )NÚrb)Úfiler,   r-   r.   )r   r
   r   Úprocess_data_with_pdfminer)r+   r,   r-   r.   ÚfpÚextracted_layoutÚlayouts_links© r6   ús/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/unstructured/partition/pdf_image/pdfminer_processing.pyÚprocess_file_with_pdfminer(   s    
ÿ
r8   zlist[int | float]Úbool)Úbboxr/   c                 C  s:   t dd„ | D ƒƒo8| d | d  dko8| d | d  dkS )Nc                 s  s   | ]}|d uV  qd S )Nr6   ©Ú.0Úxr6   r6   r7   Ú	<genexpr>7   ó    z!_validate_bbox.<locals>.<genexpr>é   r   é   é   )Úall©r:   r6   r6   r7   Ú_validate_bbox6   s    rE   r"   z
np.ndarray)Úregionsr/   c               	   G  sd   t  t jdd„ | D ƒdd�t jdd„ | D ƒdd�t jdd„ | D ƒdd�t jdd„ | D ƒdd�f¡jS )Nc                 S  s   g | ]
}|j ‘qS r6   )Úx1©r<   Úregionr6   r6   r7   Ú
<listcomp>>   r?   z._minimum_containing_coords.<locals>.<listcomp>r   ©Zaxisc                 S  s   g | ]
}|j ‘qS r6   )Úy1rH   r6   r6   r7   rJ   ?   r?   c                 S  s   g | ]
}|j ‘qS r6   )Úx2rH   r6   r6   r7   rJ   @   r?   c                 S  s   g | ]
}|j ‘qS r6   )Úy2rH   r6   r6   r7   rJ   A   r?   )ÚnpZvstackÚminÚmaxÚT)rF   r6   r6   r7   Ú_minimum_containing_coords:   s    üÿrS   r%   zIterable[ElementType]z	np.ndarry)Úinferred_layoutÚetypesr/   c                   sN   ‡ fdd„| j  ¡ D ƒ}t t| ƒf¡ t¡}|D ]}t || j|k¡}q2|S )Nc                   s   g | ]\}}|ˆ v r|‘qS r6   r6   )r<   ÚidxÚ
class_name©rU   r6   r7   rJ   I   s   þz,_inferred_is_elementtype.<locals>.<listcomp>)	Úelement_class_id_mapÚitemsrO   ÚzerosÚlenÚastyper9   Ú
logical_orÚelement_class_ids)rT   rU   Zinferred_text_idxZinferred_is_etypesrV   r6   rX   r7   Ú_inferred_is_elementtypeF   s    
þ
ÿr`   )rT   r/   c                 C  s   t | tjtjtjtjfd� S )z¨return a boolean array masking for each element if it is non-image type (True) or image like
    type (False); image types are ElementType.FIGURE/IMAGE/PAGE_BREAK/TABLErX   )r`   r   ÚFIGUREÚIMAGEZ
PAGE_BREAKZTABLE)rT   r6   r6   r7   Ú_inferred_is_textV   s    úþrc   Úfloatztuple[np.ndarray])r4   rT   Úsame_region_thresholdr/   c                 C  s¤   t |ƒdkr t dgt | ƒ ¡S t | ƒdkr6t g ¡S t| j|j|d�}|jdd� t¡}tj|dd�}|| }|  	|¡}|j
 ¡ |j
|< t| 	|¡|ƒ|j|< |S )aÌ  merge exstracted elements that have almost the same bounding box as an inferrred element into
    that inferred element: a) the inferred element bounding box is updated, if needed, to be able to
    bound the merged extracted element; b) the inferred element uses the extracted element's text as
    its text attribute. Return a boolean mask array indicating where (when True) an extracted
    element is merged therefore should be excluded from later analysisr   F©Ú	thresholdrB   rK   )r\   rO   ÚarrayÚ	boxes_iouÚelement_coordsÚsumr]   r9   ZargmaxÚsliceÚtextsÚcopyrS   )r4   rT   re   Úboxes_almost_sameZ%extracted_almost_the_same_as_inferredZfirst_matchZinferred_indices_to_updateÚextracted_to_remover6   r6   r7   Ú3_merge_extracted_into_inferred_when_almost_the_samef   s&    
ý
þ
rq   )r4   rT   Ú"extracted_is_subregion_of_inferredÚextracted_to_procÚinferred_to_procr/   c           
        sŒ   || }|| }t |jƒD ]\\}}t |¡d }	|	js8qd||	< d||< t| |g¡g‡ fdd„|	D ƒ¢R Ž |j|g< q|||< |||< |S )aC  merged extracted elements that are subregions of inferrred elements into those inferred
    elements: the inferred elements' bounding boxes expands, if needed, to include those subregion
    elements. Returns the modified inferred layout where some of its elements' bounding boxes may
    have expanded due to merging.
    r   Fc                   s   g | ]}ˆ   |g¡‘qS r6   )rl   )r<   Úmatch©r4   r6   r7   rJ   §   r?   zH_merge_extracted_that_are_subregion_of_inferred_text.<locals>.<listcomp>)Ú	enumeraterR   rO   ÚwhereÚsizerS   rl   rj   )
r4   rT   rr   rs   rt   Zinferred_to_iterZextracted_to_iterZinferred_indexZinferred_rowÚmatchesr6   rv   r7   Ú4_merge_extracted_that_are_subregion_of_inferred_text‹   s     
ÿþr{   z	np.ndaray)r4   rT   Úinferred_to_keepÚsubregion_thresholdr/   c                 C  sL   t |j| j|d�}t | j|j|d�}t ||j¡jdd� t¡}d||< |S )a  
    Marking elements in inferred layout to remove after merging when:
    - if the inferred element is subregion of an extracted element
    - and/or an extracted element is subregion of this inferred element
    Return updated mask on which inferred indices to keep (when True)
    rf   rB   rK   F)Ú&bboxes1_is_almost_subregion_of_bboxes2rj   rO   r^   rR   rk   r]   r9   )r4   rT   r|   r}   Z"inferred_is_subregion_of_extractedrr   Zinferred_to_remove_maskr6   r6   r7   ÚB_mark_non_table_inferred_for_removal_if_has_subregion_relationship®   s*    ýýþüûÿ
r   Zunstructured_inferenceé   Útuple)rT   r4   Úpage_image_sizere   r}   Ú
max_roundsr/   c                 C  sz  ddl m} t|ƒdkr| S t| ƒdkr,|S |\}}tdd||ƒ}	t |jdk¡d }
t|
ƒrŽt| |
¡j	|	gt
d�jdd� t¡}|
|  }
t| j	| |
¡j	|d�jdd� t¡}|  | ¡}t dgt|ƒ ¡}t |jdk¡d }t|ƒdk�r
| || |
¡f¡S t|ƒdk�r,| t |
|f¡¡S | |¡}t|||ƒ}t|ƒ}| }d}||k �røt|ƒ�røt|ƒ�rø|d7 }| ¡ }| ¡ }t|j	|j	|d�}t| |¡| |¡|| dd…|f ||ƒ}|j	|j	|< t ||¡�rTt ||¡�rT�qø�qTt |
|| f¡}t|ƒ�rTt |t|tjtjtjgƒ¡}t| |¡| |¡|| |ƒ||< | ¡  | | |¡| |¡f¡}|S )zumerge elements using array data structures; it also returns LayoutElements instead of
    collection of LayoutElementr   r$   rB   rf   rK   TN)Ú.unstructured_inference.inference.layoutelementr%   r\   r   rO   rx   r_   ri   rl   rj   r   rk   r]   r9   rh   Úconcatenaterq   rc   Úanyrn   r~   r{   Zarray_equalr^   r`   r   ra   rb   ZPICTUREr   Úsort)rT   r4   r‚   re   r}   rƒ   r%   ÚwÚhZfull_page_regionZimage_indices_to_keepZfull_page_image_maskro   Zinferred_layout_to_procr|   Ztext_element_indicesZextracted_text_layoutsrp   rt   rs   ÚroundsZinferred_to_proc_at_startZextracted_to_proc_startrr   Zupdated_inferredZextracted_to_keepZfinal_layoutr6   r6   r7   Ú1array_merge_inferred_layout_with_extracted_layoutÒ   sÈ    
ýûúÿ	

ýûúÿþÿ
ýýúÿÿ
ÿ
ýþþüÿþÿr‹   Úlistzint | floatztuple[LayoutElements, list])Úannotation_listÚpage_heightÚpage_numberÚ
coord_coefr/   c              
   C  sŠ  ddl m} g }g g g   }}}	tj}
|D �]}t|j|ƒ\}}}}||||f}t| ƒdkrœt|tƒrœt	| |||
ƒ}t
||ƒ\}}|D ]}| t||ƒ¡ q†t|dƒròt|ƒ}|D ]<}t|j|ƒ}t|ƒsÌq²| | ¡ ¡ | |¡ |	 d¡ q²q*t|ƒ}|D ]:}t|j|ƒ}t|ƒ�sqþ| d ¡ | |¡ |	 d¡ qþq*||t |¡ t |¡ t¡t |	¡tjtjdœt tjgt|	ƒ ¡d�|fS )Nr   r$   Úget_textrB   ©r   rB   )rj   rm   r_   rY   Úsources)r„   r%   r   ZPDF_ANNOTATION_THRESHOLDr   r:   r\   Ú
isinstancer   Ú check_annotations_within_elementÚget_words_from_objÚappendÚmap_bbox_and_indexÚhasattrr   rE   r‘   r   rO   rh   r]   Úobjectr   ZUNCATEGORIZED_TEXTrb   r   ÚPDFMINER)r�   Úpage_layoutrŽ   r�   r�   r%   Úurls_metadatarj   rm   Úelement_classÚannotation_thresholdÚobjrG   rL   rM   rN   r:   Úannotations_within_elementÚ_ÚwordsÚannotZinner_text_objectsZ	inner_objZ
inner_bboxZinner_image_objectsZimg_objr6   r6   r7   Ú!process_page_layout_from_pdfminerv  sV    
ü




ûør¥   z Optional[Union[bytes, BinaryIO]]z'tuple[List[LayoutElements], List[List]])r1   r,   r-   r.   r/   c                   s  ddl m} g }g }|d ‰ tt| ||d�ƒD ]â\}\}}	|	j|	j }
}g }t|
|d�}|jrnt|j|||ƒ}t	||	||ˆ ƒ\}}‡ fdd„|D ƒ}g }t
tjtjfdƒD ]4\}}| |j|k¡}| t|ƒrØt||ƒn|¡ qª| |¡}t|tƒ}t|ƒ}| |¡ | |¡ q.||fS )	zzLoads the image and word objects from a pdf using pdfplumber and the image renderings of the
    pdf pages using pdf2imager   r$   éH   )r-   r.   )ÚwidthÚheightc                   s8   g | ]0}‡ fd d„|d D ƒ|d |d |d dœ‘qS )c                   s   g | ]}|ˆ  ‘qS r6   r6   r;   ©Úcoefr6   r7   rJ   Õ  r?   z9process_data_with_pdfminer.<locals>.<listcomp>.<listcomp>r:   ÚtextÚuriÚstart_index)r:   r«   Úurlr­   r6   )r<   Úmetadatar©   r6   r7   rJ   Ó  s   ûüz.process_data_with_pdfminer.<locals>.<listcomp>r’   )r„   r%   rw   r   r§   r¨   r   ÚannotsÚget_urisr¥   Úzipr   Z#EMBEDDED_TEXT_SAME_REGION_THRESHOLDZ$EMBEDDED_IMAGE_SAME_REGION_THRESHOLDrl   r_   r—   r\   Úremove_duplicate_elementsr…   r   r   )r1   r,   r-   r.   r%   Zlayoutsr5   r�   Úpagerœ   r§   r¨   r�   Úcoordinate_systemZlayoutr�   ÚlinksZclean_layoutsrg   rž   Zelements_to_sortr6   r©   r7   r2   ²  sP    
ÿþ
ÿ
ù
þûÿý


r2   c                 C  s&   |j | | || || || ||d�S )zECreates a text region of the specified class with scaled coordinates.©r«   Úsource)Úfrom_coords)rG   rL   rM   rN   rª   r«   r¸   Úregion_classr6   r6   r7   Ú_create_text_regionù  s    úr»   )Úround_tor/   c                 C  sj   t | tjƒr|  |¡S tjt| ƒdftjd�}t| ƒD ](\}}|j|j	|j
|jg||dd…f< q6| |¡S )z.convert a list of boxes's coords into np arrayé   )ZdtypeN)r”   rO   ZndarrayÚroundr[   r\   Úfloat32rw   rG   rL   rM   rN   )Úbboxesr¼   ÚcoordsÚir:   r6   r6   r7   Úget_coords_from_bboxes  s    
"rÃ   )Úcoords1Úcoords2r¼   c              
   C  sÜ   t j| ddd�\}}}}t j|ddd�\}}}	}
t  t  |t  |	¡¡t  |t  |¡¡ d d¡t  t  |t  |
¡¡t  |t  |¡¡ d d¡ }|| d || d  }|	| d |
| d  }| |¡| |¡| |¡fS )zHcompute intersection area and own areas for two groups of bounding boxesr½   rB   rK   r   )rO   ÚsplitÚmaximumZminimumZ	transposer¾   )rÄ   rÅ   r¼   Zx11Zy11Zx12Zy12Zx21Zy21Zx22Zy22Ú
inter_areaÚ	boxa_areaÚ	boxb_arear6   r6   r7   Ú$areas_of_boxes_and_intersection_area  s    (ÿ.þrË   ç      à?)rg   r¼   r/   c           	      C  sJ   t | |d�}t ||d�}t|||d�\}}}|t |t¡ |k||jk@ S )zacompute if each element from bboxes1 is almost a subregion of one or more elements in
    bboxes2©r¼   ©rÃ   rË   rO   rÇ   ÚEPSILON_AREArR   ©	Zbboxes1Zbboxes2rg   r¼   rÄ   rÅ   rÈ   rÉ   rÊ   r6   r6   r7   r~   #  s    ÿÿr~   c                 C  s   t | |d�}t||||ƒS )z#compute iou for a group of elementsrÍ   )rÃ   ri   )rÀ   rg   r¼   rÁ   r6   r6   r7   Úboxes_self_iou4  s    rÑ   g      è?c           	      C  sJ   t | |d�}t ||d�}t|||d�\}}}|t t||j | ¡ |kS )z*compute iou between two groups of elementsrÍ   rÎ   rÐ   r6   r6   r7   ri   =  s    ÿri   zlist[TextRegions])Úlayout_elementsr/   c                 C  sb   ddl m}m} g }t| jƒD ]>\}}|dkr2|n|}| |j| j| | j| t	j
dœŽ¡ q|S )zÈa temporary solution to convert layout elements to a list of either EmbeddedTextRegion or
    ImageTextRegion; this should be made obsolete after we refactor the merging logic in inference
    libraryr   )ÚEmbeddedTextRegionÚImageTextRegionr·   )Ú)unstructured_inference.inference.elementsrÓ   rÔ   rw   r_   r—   r¹   rj   rm   r   r›   )rÒ   rÓ   rÔ   rF   rÂ   rž   rº   r6   r6   r7   Ú!pdfminer_elements_to_text_regionsJ  s    ýÿrÖ   z'DocumentLayout'zList[TextRegions])Úinferred_document_layoutr4   Úhi_res_model_namer/   c                 C  sî   ddl m} | j}tt||ƒƒD ]È\}\}}|j}| d¡}	| d¡}
|	|
f}i }t|j|ƒrtd|jj	vrtdddœ}t
|j|fd|i|¤Ž}t|tƒ}|j t¡|_t|jƒD ]2\}}|d	u rÒt| |g¡|d
�}t|ƒ|j|< q®||_q | S )z1Merge an inferred layout with an extracted layoutr   )ÚUnstructuredDetectronONNXModelr§   r¨   ZR_50rÌ   )re   r}   r‚   N)Útarget_regionÚsource_regions)Z,unstructured_inference.models.detectron2onnxrÙ   Úpagesrw   r²   Úimage_metadataÚgetr”   Zdetection_modelZ
model_pathr‹   Úelements_arrayr   r   rm   r]   rš   Ú aggregate_embedded_text_by_blockrl   r   )r×   r4   rØ   rÙ   Zinferred_pagesrÂ   Zinferred_pageZextracted_page_layoutrÝ   rˆ   r‰   Z
image_sizeZthreshold_kwargsZmerged_layoutr«   r6   r6   r7   Ú$merge_inferred_with_extracted_layouta  sD    ÿ


ÿ
þ
þýü

þrá   )Údocumentr/   c              
   C  s®   | j D ]¢}|jjtjk}|j | ¡j}|j |¡j}t|ƒdkst|ƒdkrPqt||t	j
ƒjdd�dk}t |¡d |  }|j t t t | ¡d |f¡¡¡|_q| S )z•Clean pdfminer elements from inside tables.

    This function removes elements sourced from PDFMiner that are subregions within table elements.
    r   rB   rK   )rÜ   rß   r“   r   r›   rl   rj   r\   r~   r   Ú-EMBEDDED_TEXT_AGGREGATION_SUBREGION_THRESHOLDrk   rO   rx   r‡   r…   )râ   r´   Zpdfminer_maskZnon_pdfminer_element_boxesZpdfminer_element_boxesZ&is_element_subregion_of_other_elementsZpdfminer_to_keepr6   r6   r7   Úclean_pdfminer_inner_elements˜  s*    
ýüûÿ	ÿrä   )Úelementsrg   r/   c                   s^   | j ‰ t dd¡}t ˆ jd d | ¡}tjˆ |dd�}‡ ‡fdd„|D ƒ}|  t |¡¡S )zMRemoves duplicate text elements extracted by PDFMiner from a document layout.ZUNST_MATMUL_MEMORY_CAP_IN_GBrB   r   g     @Ÿ@rK   c                   s,   g | ]$}t jt|ˆ ˆƒd d�jd d� ‘qS )rB   )ÚkrK   )rO   Ztriuri   r†   )r<   rÆ   ©rÁ   rg   r6   r7   rJ   Æ  r?   z-remove_duplicate_elements.<locals>.<listcomp>)	rj   ÚosÚgetenvrO   ÚceilÚshapeZarray_splitrl   r…   )rå   rg   Zmemory_cap_in_gbZn_splitÚsplitsZiousr6   rç   r7   r³   ·  s    r³   )rÚ   rÛ   rg   r/   c                 C  sZ   t |ƒdkst | ƒdkrdS t|j| j|ƒjdd� t¡}d dd„ | |¡jD ƒ¡}|S )zgExtracts the text aggregated from the elements of the given layout that lie within the given
    block.r   r'   rB   rK   ú c                 S  s   g | ]}|r|‘qS r6   r6   )r<   r«   r6   r6   r7   rJ   ß  r?   z4aggregate_embedded_text_by_block.<locals>.<listcomp>)	r\   r~   rj   rk   r]   r9   Újoinrl   rm   )rÚ   rÛ   rg   Úmaskr«   r6   r6   r7   rà   Ê  s    ýûúÿ
rà   r   )Ú
page_linksrI   r/   c                   s4   dd„ ˆ D ƒ}t ||gƒ}‡ fdd„t|ƒD ƒ}|S )Nc                 S  s   g | ]}t | d ¡Ž ‘qS rD   )r   rÞ   )r<   Úlinkr6   r6   r7   rJ   å  r?   z(get_links_in_element.<locals>.<listcomp>c                   sB   g | ]:\}}t |ƒrˆ |  d ¡ˆ |  d¡ˆ |  d¡dœ‘qS )r«   r®   r­   )r«   r®   r­   )r†   rÞ   )r<   rV   Úresult©rð   r6   r7   rJ   ç  s   ûý)r~   rw   )rð   rI   Zlinks_bboxesÚresultsr¶   r6   ró   r7   Úget_links_in_elementã  s    
ú
rõ   zPDFObjRef | list[PDFObjRef]zPixelSpace | PointSpacezlist[dict[str, Any]])r°   r¨   rµ   r�   r/   c                 C  s:   t | tƒrt| |||ƒS |  ¡ }|du r,g S t||||ƒS )a”  
    Extracts URI annotations from a single or a list of PDF object references on a specific page.
    The type of annots (list or not) depends on the pdf formatting. The function detectes the type
    of annots and then pass on to get_uris_from_annots function as a list.

    Args:
        annots (PDFObjRef | list[PDFObjRef]): A single or a list of PDF object references
            representing annotations on the page.
        height (float): The height of the page in the specified coordinate system.
        coordinate_system (PixelSpace | PointSpace): The coordinate system used to represent
            the annotations' coordinates.
        page_number (int): The page number from which to extract annotations.

    Returns:
        list[dict]: A list of dictionaries, each containing information about a URI annotation,
        including its coordinates, bounding box, type, URI link, and page number.
    N)r”   rŒ   Úget_uris_from_annotsÚresolve)r°   r¨   rµ   r�   Zresolved_annotsr6   r6   r7   r±   ô  s    
r±   zlist[PDFObjRef]c              	   C  sv  g }| D �]f}t |ƒ}t|tƒs"q| dd¡}|rt|tƒst|ƒdkrJq| dd¡}|rt|tƒst|ƒdkrrqt||ƒ\}	}
}}|	|
f|	|f||f||
ff}t||d�}d|vr¶qt |d ƒ}t|tƒsÎqd}d|v rôt|d tƒsôt|d ƒ}d}zD|d	k�rt t |d
 ƒƒ 	d¡}|dk�r:t t |d ƒƒ 	d¡}W n t
�yP   Y n0 | ||	|
||f|||dœ¡ q|S )a³  
    Extracts URI annotations from a list of PDF object references.

    Args:
        annots (list[PDFObjRef]): A list of PDF object references representing annotations on
            a page.
        height (int | float): The height of the page in the specified coordinate system.
        coordinate_system (PixelSpace | PointSpace): The coordinate system used to represent
            the annotations' coordinates.
        page_number (int): The page number from which to extract annotations.

    Returns:
        list[dict]: A list of dictionaries, each containing information about a URI annotation,
        including its coordinates, bounding box, type, URI link, and page number.
    ZSubtypeNz/'Link'ZRectr½   )ÚpointsÚsystemÚAÚSz/'URI'ÚURIzutf-8z/'GoTo'ÚD)Zcoordinatesr:   Útyper¬   r�   )Útry_resolver”   ÚdictrÞ   r   r)   r\   r   r   ÚdecodeÚ	Exceptionr—   )r°   r¨   rµ   r�   r�   Ú
annotationZannotation_dictÚsubtypeZrectrG   rL   rM   rN   rø   Zcoordinates_metadataZuri_dictZuri_typer¬   r6   r6   r7   rö     sT    

þ



ûÿ	rö   r   ©r¤   c                 C  s&   z
|   ¡ W S  ty    |  Y S 0 dS )z�
    Attempt to resolve a PDF object reference. If successful, returns the resolved object;
    otherwise, returns the original reference.
    N)r÷   r  r  r6   r6   r7   rÿ   Z  s    
rÿ   z!tuple[float, float, float, float])r�   Úelement_bboxr�   rŸ   r/   c                 C  sN   g }| D ]@}|d |krt |d ƒ}|rt||d ƒ| |kr| |¡ q|S )a½  
    Filter annotations that are within or highly overlap with a specified element on a page.

    Args:
        annotation_list (list[dict[str,Any]]): A list of dictionaries, each containing information
            about an annotation.
        element_bbox (tuple[float, float, float, float]): The bounding box coordinates of the
            specified element in the bbox format (x1, y1, x2, y2).
        page_number (int): The page number to which the annotations and element belong.
        annotation_threshold (float, optional): The threshold value (between 0.0 and 1.0)
            that determines the minimum overlap required for an annotation to be considered
            within the element. Default is 0.9.

    Returns:
        list[dict[str,Any]]: A list of dictionaries containing information about annotations
        that are within or highly overlap with the specified element on the given page, based on
        the specified threshold.
    r�   r:   )Úcalculate_bbox_areaÚcalculate_intersection_arear—   )r�   r  r�   rŸ   r¡   r  Zannotation_bbox_sizer6   r6   r7   r•   e  s    ÿÿr•   r   z)tuple[list[LTChar], list[dict[str, Any]]])r    r¨   r/   c              	   C  sN  g }g }d}| D �]2}d}d\}}}	}
d}t |ƒD �]\}}t|tƒ�r| |¡ | ¡ }|r†| ¡ s†| ||||	|
f|dœ¡ d}q2|s’| ¡ }|rÈ| ¡ |krÈ| ¡ }| ||||	|
f|dœ¡ d}t|ƒdkrþ|| }|j}||j	 }
|j
}	||j }n|j
}	||j	 }
||7 }q2| ||||	|
f|dœ¡ d}q2|t|ƒ7 }q||fS )a|  
    Extracts characters and word bounding boxes from a PDF text element.

    Args:
        obj (LTTextBox): The PDF text element from which to extract characters and words.
        height (float): The height of the page in the specified coordinate system.

    Returns:
        tuple[list[LTChar], list[dict[str,Any]]]: A tuple containing two lists:
            - list[LTChar]: A list of LTChar objects representing individual characters.
            - list[dict[str,Any]]]: A list of dictionaries, each containing information about
                a word, including its text, bounding box, and start index in the element's text.
    r   r'   )NNNN)r«   r:   r­   )rw   r”   r   r—   r‘   ÚstripÚisalnumr\   Zx0Zy0rG   rL   )r    r¨   Ú
charactersr£   Ztext_lenZ	text_lineÚwordrG   rL   rM   rN   r­   ÚindexÚ	characterÚcharr
  r6   r6   r7   r–   ‰  sP    

ÿÿ


ÿr–   zdict[str, Any])r£   r¤   c                 C  s4  t | ƒdkr d|d< d|d< |S t |d d t dd„ | D ƒ¡ d	 |d d
 t dd„ | D ƒ¡ d	  ¡}t |d d	 t dd„ | D ƒ¡ d	 |d d t dd„ | D ƒ¡ d	  ¡}t|ƒ}t|ƒ}d}||k�rt||d
 ƒD ]}|d7 }|| | d 7 }qèn| | d }| ¡ |d< | | d |d< |S )aq  
    Maps a bounding box annotation to the corresponding text and start index within a list of words.

    Args:
        words (list[dict[str,Any]]): A list of dictionaries, each containing information about
            a word, including its text, bounding box, and start index.
        annot (dict[str,Any]): The annotation dictionary to be mapped, which will be updated with
        "text" and "start_index" fields.

    Returns:
        dict: The updated annotation dictionary with "text" representing the mapped text and
            "start_index" representing the start index of the mapped text in the list of words.
    r   r'   r«   éÿÿÿÿr­   r:   c                 S  s   g | ]}|d  d ‘qS )r:   r   r6   ©r<   r  r6   r6   r7   rJ   à  r?   z&map_bbox_and_index.<locals>.<listcomp>r@   rB   c                 S  s   g | ]}|d  d ‘qS )r:   rB   r6   r  r6   r6   r7   rJ   á  r?   c                 S  s   g | ]}|d  d ‘qS )r:   r@   r6   r  r6   r6   r7   rJ   ä  r?   rA   c                 S  s   g | ]}|d  d ‘qS )r:   rA   r6   r  r6   r6   r7   rJ   å  r?   rí   )r\   rO   Úsqrtrh   Ú
try_argminÚranger	  )r£   r¤   Zdistance_from_bbox_startZdistance_from_bbox_endZclosest_startZclosest_endr«   r¢   r6   r6   r7   r˜   Í  s2    ""ÿÿ""ÿÿ
r˜   )Úbbox1Úbbox2r/   c                 C  sl   | \}}}}|\}}}}	t ||ƒ}
t ||ƒ}t||ƒ}t||	ƒ}|
|k rd||k rdt|
|||fƒ}|S dS dS )aû  
    Calculate the area of intersection between two bounding boxes.

    Args:
        bbox1 (tuple[float, float, float, float]): The coordinates of the first bounding box
            in the format (x1, y1, x2, y2).
        bbox2 (tuple[float, float, float, float]): The coordinates of the second bounding box
            in the format (x1, y1, x2, y2).

    Returns:
        float: The area of intersection between the two bounding boxes. If there is no
        intersection, the function returns 0.0.
    g        N)rQ   rP   r  )r  r  Zx1_1Zy1_1Zx2_1Zy2_1Zx1_2Zy1_2Zx2_2Zy2_2Zx_intersectionZy_intersectionZx2_intersectionZy2_intersectionZintersection_arear6   r6   r7   r  ø  s    




ÿr  c                 C  s    | \}}}}|| ||  }|S )a(  
    Calculate the area of a bounding box.

    Args:
        bbox (tuple[float, float, float, float]): The coordinates of the bounding box
            in the format (x1, y1, x2, y2).

    Returns:
        float: The area of the bounding box, computed as the product of its width and height.
    r6   )r:   rG   rL   rM   rN   Úarear6   r6   r7   r    s    r  )rh   r/   c                 C  s*   zt t | ¡ƒW S  ty$   Y dS 0 dS )a;  
    Attempt to find the index of the minimum value in a NumPy array.

    Args:
        array (np.ndarray): The NumPy array in which to find the minimum value's index.

    Returns:
        int: The index of the minimum value in the array. If the array is empty or an
        IndexError occurs, it returns -1.
    r  N)r*   rO   ZargminÚ
IndexError)rh   r6   r6   r7   r  *  s    r  )r'   r(   NN)Nr(   NN)rÌ   )]Ú
__future__r   rè   Útypingr   r   r   r   r   r   r	   r
   ÚnumpyrO   Zpdfminer.layoutr   r   Zpdfminer.pdftypesr   Zpdfminer.utilsr   Zunstructured_inference.configr   Z unstructured_inference.constantsr   rÕ   r   Z"unstructured.documents.coordinatesr   r   Zunstructured.documents.elementsr   r   Z0unstructured.partition.pdf_image.pdf_image_utilsr   Z/unstructured.partition.pdf_image.pdfminer_utilsr   r   r   r   r   Z#unstructured.partition.utils.configr   Z&unstructured.partition.utils.constantsr   r   Z$unstructured.partition.utils.sortingr   Zunstructured.utilsr    r!   r"   Z'unstructured_inference.inference.layoutr#   r„   r%   rÏ   ZDEFAULT_ROUNDr8   rE   rS   r`   rc   rq   r{   r   ZLAYOUT_SAME_REGION_THRESHOLDZLAYOUT_SUBREGION_THRESHOLDr‹   r¥   r2   r»   rÃ   rË   r~   rÑ   ri   rÖ   rá   rä   r³   rã   rà   rõ   r±   rö   rÿ   r•   r–   r˜   r  r  r  r6   r6   r6   r7   Ú<module>   sœ   (    ü%#$ú $;    üFÿÿ
ÿ6 þýG$D+"