a
    ¡˜ëj4Â  ã                    @  s  d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlm	Z	 d dl
mZmZmZmZmZ d dlZd dlZd dlmZmZmZmZ d dlmZ d dlmZ d dlmZ d d	lm Z  d d
l!m"Z" d dl#m$Z$ d dl%m&Z& d dl'm(Z(m)Z) d dl*m+Z+m,Z, d dl-m.Z.m/Z/m0Z0m1Z1mZm2Z2m3Z3m4Z4m5Z5m6Z6m7Z7 d dl8m9Z9 d dl:m;Z; d dl<m=Z= d dl>m?Z?m@Z@ d dlAmBZB d dlCmDZDmEZEmFZFmGZGmHZHmIZI d dlJmKZKmLZL d dlMmNZN d dlOmPZPmQZQmRZRmSZS d dlTmUZU d dlVmWZW d dlXmYZYmZZZm[Z[ d dl\m]Z]m^Z^m_Z_m`Z`maZambZbmcZc d dldmeZemfZfmgZg d dlhmiZimjZj d dlkmlZl d d lmmnZn d d!lompZpmqZqmrZrmsZsmtZtmuZu d d"lvmwZwmxZx d d#lymzZz d d$l{m|Z|m}Z} e�r¦ezƒ  ej~d%ejd&�Z€e}d'ƒd(d)œd*d+„ƒZ�e7ƒ e;e=j‚ƒe&ddd,eujƒd,ddddddd,ddd,d-d,d.ddddd/fd0d1d2d(d2d0d3d0d0d0d0d2d3d0d2d4d2d2d0d5d5d5d5d6d7d8œd9d:„ƒƒƒZ„d;dd,d,eujƒd,dddd,ddd,d-d,d.ddddd/epepfd(d<d2d2d(d2d3d0d0d2d3d0d2d4d2d2d0d5d5d5d5d(d(d6d7d=œd>d?„Z…d‡d(d<d3d0d4d0d@d6dAdBœ	dCdD„Z†dˆd(d1dEd0d4d0d@d6dAdBœ	dFdG„Z‡e}dHƒenjˆd-ddfdId(dEd0d5d4d0d@dAdJœ	dKdL„ƒZ‰d‰d(d<d4dMœdNdO„ZŠdŠd(d<d4dPdQœdRdS„Z‹e}d'ƒd;dd,d,d,ddetjŒj�ddddd,d,ddd,d,dd-d,d.dddepepfd(d<d2d2d2d3d0d(d0d0dTd0d2d2d3d0d2d2d0d4d2d2dTd0d@d(d(d6d7dUœdVdW„ƒZŽd,esfdAd2d(dXœdYdZ„Z�d‹d(d<d2d3d0d2d0d4d0d6d[œ
d\d]„Z�ddd-d,desfd^d3d0d4d2d0d(d6d7d_œ	d`da„Z‘d7dbœdcdd„Z’ded(dfœdgdh„Z“e ”didj¡dkdl„ ƒZ•d7dmd7dnœdodp„Z–dqdrdsdtœdudv„Z—dwdwdmdwdxœdydz„Z˜dŒd}d}d~d~d2dœd€d�„Z™d,d,dd.ddesdd-df
d‚d2d2d0d2d0d0d(d3d4dƒd6d7d„œd…d†„ZšdS )�é    )ÚannotationsN)ÚPath)ÚIOÚTYPE_CHECKINGÚAnyÚOptionalÚcast)ÚLTContainerÚLTImageÚLTItemÚ	LTTextBox)Úopen_filename)Úregister_heif_opener)ÚImage)Ú	PdfReader)ÚDocumentLayout)ÚLayoutElement)Úadd_chunking_strategy)Ú%clean_extra_whitespace_with_index_runÚ-index_adjustment_after_clean_extra_whitespace)Ú
PixelSpaceÚ
PointSpace)ÚCoordinatesMetadataÚElementÚElementMetadataÚElementTyper   ÚLinkÚListItemÚ	PageBreakÚTextÚTitleÚprocess_metadata)ÚPageCountExceededError)Úadd_metadata_with_filetype)ÚFileType)ÚloggerÚtrace_logger)ÚPARAGRAPH_PATTERN)Úadd_element_metadataÚexactly_oneÚget_page_image_metadataÚnormalize_layout_elementÚocr_data_to_elementsÚspooled_to_bytes_io_if_needed)Úcheck_language_argsÚprepare_languages_for_tesseract)Úget_last_modified_date)ÚExtractedLayoutDumperÚFinalLayoutDumperÚObjectDetectionLayoutDumperÚOCRLayoutDumper)Úsave_analysis_artifiacts)Úrun_form_extraction)Úcheck_element_types_to_extractÚconvert_pdf_to_imagesÚsave_elements)Ú check_annotations_within_elementÚclean_pdfminer_inner_elementsÚget_links_in_elementÚget_urisÚget_words_from_objÚmap_bbox_and_indexÚ$merge_inferred_with_extracted_layout)ÚPDFMinerConfigÚopen_pdfminer_pages_generatorÚrect_to_bbox)Údetermine_pdf_or_image_strategyÚvalidate_strategy)Úelement_from_text)Ú
env_config)ÚOCR_AGENT_TESSERACTÚSORT_MODE_BASICÚSORT_MODE_DONTÚSORT_MODE_XY_CUTÚOCRModeÚPartitionStrategy)Úcoord_has_valid_pointsÚsort_page_elements)Úpatch_psparser)ÚfirstÚrequires_dependenciesz\s+)ÚpatternÚflagsZunstructured_inferenceÚstr)Úreturnc                  C  s   ddl m}  tj d| ¡S )Nr   ©ÚDEFAULT_MODELZUNSTRUCTURED_HI_RES_MODEL_NAME)Z"unstructured_inference.models.baserX   ÚosÚenvironÚgetrW   © r\   úY/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/unstructured/partition/pdf.pyÚdefault_hi_res_modelr   s    r^   Fé   Tg®Gáz®Ç?zOptional[str]zOptional[IO[bytes]]ÚboolzOptional[list[str]]ÚintzOptional[float]r   zlist[Element])ÚfilenameÚfileÚinclude_page_breaksÚstrategyÚinfer_table_structureÚocr_languagesÚ	languagesÚmetadata_filenameÚmetadata_last_modifiedÚchunking_strategyÚhi_res_model_nameÚextract_images_in_pdfÚextract_image_block_typesÚextract_image_block_output_dirÚextract_image_block_to_payloadÚstarting_page_numberÚextract_formsÚform_extraction_skip_tablesÚpasswordÚpdfminer_line_marginÚpdfminer_char_marginÚpdfminer_line_overlapÚpdfminer_word_marginÚkwargsrV   c                 K  sR   t | |d� t|pg |ƒ}tf | |||||||
||||||||||||dœ|¤ŽS )ay  Parses a pdf document into a list of interpreted elements.
    Parameters
    ----------
    filename
        A string defining the target filename path.
    file
        A file-like object as bytes --> open(filename, "rb").
    strategy
        The strategy to use for partitioning the PDF. Valid strategies are "hi_res",
        "ocr_only", and "fast". When using the "hi_res" strategy, the function uses
        a layout detection model to identify document elements. When using the
        "ocr_only" strategy, partition_pdf simply extracts the text from the
        document using OCR and processes it. If the "fast" strategy is used, the text
        is extracted directly from the PDF. The default strategy `auto` will determine
        when a page can be extracted using `fast` mode, otherwise it will fall back to `hi_res`.
    infer_table_structure
        Only applicable if `strategy=hi_res`.
        If True, any Table elements that are extracted will also have a metadata field
        named "text_as_html" where the table's text content is rendered into an html string.
        I.e., rows and cells are preserved.
        Whether True or False, the "text" field is always present in any Table element
        and is the text content of the table (no structure).
    languages
        The languages present in the document, for use in partitioning and/or OCR. To use a language
        with Tesseract, you'll first need to install the appropriate Tesseract language pack.
    metadata_last_modified
        The last modified date for the document.
    hi_res_model_name
        The layout detection model used when partitioning strategy is set to `hi_res`.
    extract_images_in_pdf
        Only applicable if `strategy=hi_res`.
        If True, any detected images will be saved in the path specified by
        'extract_image_block_output_dir' or stored as base64 encoded data within metadata fields.
        Deprecation Note: This parameter is marked for deprecation. Future versions will use
        'extract_image_block_types' for broader extraction capabilities.
    extract_image_block_types
        Only applicable if `strategy=hi_res`.
        Images of the element type(s) specified in this list (e.g., ["Image", "Table"]) will be
        saved in the path specified by 'extract_image_block_output_dir' or stored as base64
        encoded data within metadata fields.
    extract_image_block_to_payload
        Only applicable if `strategy=hi_res`.
        If True, images of the element type(s) defined in 'extract_image_block_types' will be
        encoded as base64 data and stored in two metadata fields: 'image_base64' and
        'image_mime_type'.
        This parameter facilitates the inclusion of element data directly within the payload,
        especially for web-based applications or APIs.
    extract_image_block_output_dir
        Only applicable if `strategy=hi_res` and `extract_image_block_to_payload=False`.
        The filesystem path for saving images of the element type(s)
        specified in 'extract_image_block_types'.
    extract_forms
        Whether the form extraction logic should be run
        (results in adding FormKeysValues elements to output).
    form_extraction_skip_tables
        Whether the form extraction logic should ignore regions designated as Tables.
    pdfminer_line_margin
        If two lines are close together they are considered to be part of the same paragraph.
        The margin is specified relative to the height of a line.
    pdfminer_char_margin
        If two characters are closer together than this margin they are considered part of
        the same line. The margin is specified relative to the width of the character.
    pdfminer_line_overlap
        If two characters have more overlap than this they are considered to be on the same line.
        The overlap is specified relative to the minimum height of both characters.
    pdfminer_word_margin
        If two characters on the same line are further apart than this margin then they are
        considered to be two separate words, and an intermediate space will be added for
        readability. The margin is specified relative to the width of the character.
    ©rb   rc   )rb   rc   rd   re   rf   rh   rj   rl   rm   rn   ro   rp   rq   rr   rs   rt   ru   rv   rw   rx   )r)   r.   Úpartition_pdf_or_image)rb   rc   rd   re   rf   rg   rh   ri   rj   rk   rl   rm   rn   ro   rp   rq   rr   rs   rt   ru   rv   rw   rx   ry   r\   r\   r]   Úpartition_pdf}   s4    dìër|   Ú zOptional[bytes | IO[bytes]])rb   rc   Úis_imagerd   re   rf   rh   rj   rl   rm   rn   ro   rp   rq   rr   rs   rt   ru   rv   rw   rx   Ú	ocr_agentÚtable_ocr_agentry   rV   c                  K  s  |du rdg}t ƒ  t||ƒ | r*t| ƒnd}t||||d�}g }d}|sÀz<tf | t|ƒ||p`||||dœ|¤Ž}tdd„ |D ƒƒ}W n8 ty¾ } z t 	|¡ t 
d¡ W Y d}~n
d}~0 0 t|||||	|
d	�}|duræ| d
¡ t|ƒ}|tjk�rxt ¡ �b t d¡ tf | t|ƒ||||||�p,||||	|
|||||||||dœ|¤Ž}|W  d  ƒ S 1 �sl0    Y  nŠ|tjk�rœtf ||dœ|¤Ž}|S |tjk�rt ¡ �@ tf | ||||||�pÊ|||dœ	|¤Ž}t|ƒ}W d  ƒ n1 �sø0    Y  |S )zCParses a pdf or image document into a list of interpreted elements.NÚeng)Zline_marginZchar_marginZline_overlapZword_marginF©rb   rc   rh   rj   rq   rt   Úpdfminer_configc                 s  s,   | ]$}|D ]}t |tƒo |j ¡ V  q
qd S ©N)Ú
isinstancer   ÚtextÚstrip)Ú.0Úpage_elementsÚelr\   r\   r]   Ú	<genexpr>:  s   þz)partition_pdf_or_image.<locals>.<genexpr>z3PDF text extraction failed, skip text extraction...)r~   Úpdf_text_extractablerf   rm   rn   r   Úignore)rb   rc   r~   rf   rd   rh   rg   rj   rl   rŒ   rm   rn   ro   rp   rq   rr   rs   rt   rƒ   r   r€   )Úextracted_elementsrd   )	rb   rc   rd   rh   rg   r~   rj   rq   rt   )r   rE   r0   rA   Úextractable_elementsr-   ÚanyÚ	Exceptionr%   ÚdebugÚinforD   Úseekr/   rM   ZHI_RESÚwarningsÚcatch_warningsÚsimplefilterÚ_partition_pdf_or_image_localZFASTÚ_partition_pdf_with_pdfparserZOCR_ONLYÚ _partition_pdf_or_image_with_ocrÚ$_process_uncategorized_text_elements) rb   rc   r~   rd   re   rf   rh   rj   rl   rm   rn   ro   rp   rq   rr   rs   rt   ru   rv   rw   rx   r   r€   ry   Úlast_modifiedrƒ   rŽ   rŒ   Úerg   ÚelementsÚout_elementsr\   r\   r]   r{   ý   sÂ     
üùø
þ
 ú	


ëê&þý
÷
ö(r{   zOptional[PDFMinerConfig]zlist[list[Element]])	rb   rc   rh   rj   rq   rt   rƒ   ry   rV   c              
   K  s2   t |tƒrt |¡}tf | ||||||dœ|¤ŽS )Nr‚   )r…   ÚbytesÚioÚBytesIOÚ_partition_pdf_with_pdfminer)rb   rc   rh   rj   rq   rt   rƒ   ry   r\   r\   r]   r�   �  s    


ùør�   z	list[str]c           
      K  sœ   |du rdg}t | |d� | rvt| dƒ�<}ttt |ƒ}tf || |||||dœ|¤Ž}	W d  ƒ q˜1 sj0    Y  n"|r˜tf || |||||dœ|¤Ž}	|	S )a‚  Partitions a PDF using PDFMiner instead of using a layoutmodel. Used for faster
    processing or detectron2 is not available.

    Implementation is based on the `extract_text` implementation in pdfminer.six, but
    modified to support tracking page numbers and working with file-like objects.

    ref: https://github.com/pdfminer/pdfminer.six/blob/master/pdfminer/high_level.py
    Nr�   rz   Úrb)Úfprb   rh   rj   rq   rt   rƒ   )r)   r   r   r   r    Ú_process_pdfminer_pages)
rb   rc   rh   rj   rq   rt   rƒ   ry   r¥   rž   r\   r\   r]   r£   §  s<    ùø&ùør£   Úpdfminerz	IO[bytes])	r¥   rb   rh   rj   Úannotation_thresholdrq   rt   rƒ   rV   c           $      K  sª  g }	t t| ||d�|d�D �]ˆ\}
\}}|j|j }}g }g }t||d�}|jr`t|j|||
ƒ}|D �](}t|j|ƒ\}}}}||||f}g }t	|ƒdkrÚt
|tƒrÚt|||
|ƒ}t||ƒ\}}|D ]}| t||ƒ¡ qÄt|dƒrð| ¡ g}nt|ƒ}t t|¡}|D ]„}t|ƒ\}}| ¡ �r||f||f||f||ff} t|| |d�}!t| |d�}"t||ƒ}#t||
|"||#|d�|!_d	|!j_| |!¡ �qqdt||ƒ}|	 |¡ q|	S )
z>Uses PDFMiner to split a document into pages and process them.)rt   rƒ   ©Ústart©ÚwidthÚheightr   Úget_text)ÚcoordinatesÚcoordinate_system©ÚpointsÚsystem)rb   Úpage_numberr¯   rœ   Úlinksrh   r§   )Ú	enumeraterB   r¬   r­   r   Zannotsr=   rC   ÚbboxÚlenr…   r   r:   r>   Úappendr?   Úhasattrr®   Ú_extract_textÚreÚsplitr'   r   r‡   rF   r   Ú_get_links_from_urls_metadatar   ÚmetadataÚdetection_originÚ_combine_list_elements)$r¥   rb   rh   rj   r¨   rq   rt   rƒ   ry   rž   r´   ÚpageZpage_layoutr¬   r­   r‰   Zannotation_listr°   ÚobjÚx1Úy1Úx2Úy2r·   Úurls_metadataZannotations_within_elementÚ_ÚwordsZannotZ_text_snippetsZ_textÚmoved_indicesr²   ÚelementZcoordinates_metadatarµ   r\   r\   r]   r¦   Ú  sv    þþ
ü

ýþ
ú
r¦   )rb   rc   rV   c                 C  s:   |rt |ƒ ¡ }| d¡ n| r.t | ƒ ¡ }ntdƒ‚|S )Nr   z-Either 'file' or 'filename' must be provided.)r   Zget_num_pagesr”   Ú
ValueError)rb   rc   Znumber_of_pagesr\   r\   r]   Ú_get_pdf_page_number1  s    rÎ   ÚNone)rb   rc   Úpdf_hi_res_max_pagesrV   c                 C  s(   |r$t | |d�}||kr$t||d�‚dS )z6Checks whether PDF exceeds pdf_hi_res_max_pages limit.rz   )Údocument_pagesrÐ   N)rÎ   r"   )rb   rc   rÐ   rÑ   r\   r\   r]   Ú#check_pdf_hi_res_max_pages_exceeded?  s    ÿrÒ   zOptional[int])rb   rc   r~   rf   rd   rh   rg   Úocr_modeÚ
model_namerl   Úpdf_image_dpirj   rŒ   rm   rn   ro   rp   ÚanalysisÚanalyzed_image_output_dir_pathrq   rr   rs   rÐ   rt   rƒ   r   r€   ry   rV   c           2      K  s  ddl m}m} ddlm}m} ddlm} m}! |sBt	| ||d� |	pN|pNt
ƒ }	|
du r\d}
d}"d}#d}$d}%tj}&|du �rL|| ||	|
|d�}'|r¢|!| |
||d	�ng g f\}(})|�r|sâtjrÒtttjƒd
 ƒ}ntt ¡ d
 ƒ}tj|dd� |&�st|'|	d�}"tdd„ |(D ƒd�}#tƒ }$t|'|(|	d�}*|| |*|(||||||
|$||d�}+nò||||	|
|d�}'t|dƒ�rt| d¡ |�rŠ| ||
||d�ng g f\}(})|�rú|�sÎtj�r¾tttjƒd
 ƒ}ntt ¡ d
 ƒ}|&�sút|'|	d�}"tdd„ |(D ƒd�}#tƒ }$t|'|(|	d�}*t|dƒ�r| d¡ |||*|(||||||
|$||d�}+t|+ƒ}+t|+fd||d|||)dœ|¤Ž},t|ƒ}|�r�t|,|tj| |||
||d�	 |D ]6}-|�r®|-tjk�r®�q”t|,||-| |||
||d�	 �q”g }.|,D ]†}/t |/t!ƒ�rî|�sî�qÔt |/t"ƒ�r|. #t$t%|/ƒ¡ nLt |/t&ƒ�rÔt' (t)d|/j*�p*d¡ +¡ |/_*|/j*�sHt |/t!ƒ�rÔ|. #t$t%|/ƒ¡ �qÔ|�r~t,|| |	|.|d�}0|. -|0¡ |�r|&�s”t.|.d�}%g }1|"�r¨|1 #|"¡ |#�r¸|1 #|#¡ |$�rÈ|1 #|$¡ |%�rØ|1 #|%¡ t/|1| |||tj0tjtj1tj2tj3tj4dœ
Ž |.S )z)Partition using package installed locallyr   )Úprocess_data_with_modelÚprocess_file_with_model)Úprocess_data_with_ocrÚprocess_file_with_ocr)Úprocess_data_with_pdfminerÚprocess_file_with_pdfminer)rb   rc   rÐ   NéÈ   )r~   rÔ   rÕ   rt   )rb   Údpirt   rƒ   Z	annotatedT)Úexist_ok)ÚlayoutrÔ   c                 S  s   g | ]}|  ¡ ‘qS r\   ©Úas_list©rˆ   rá   r\   r\   r]   Ú
<listcomp>«  ó    z1_partition_pdf_or_image_local.<locals>.<listcomp>)rá   )Úinferred_document_layoutÚextracted_layoutrl   )
rè   r~   rf   r   rg   rÓ   rÕ   Úocr_layout_dumperrt   r€   r”   )rc   rß   rt   rƒ   c                 S  s   g | ]}|  ¡ ‘qS r\   râ   rä   r\   r\   r]   rå   å  ræ   F)Úsortablerd   Úlast_modification_dateÚinfer_list_itemsrh   rq   Úlayouts_links)	rž   rq   Zelement_category_to_saverb   rc   r~   rÕ   rp   Zoutput_dir_pathú r}   )rc   rb   rÔ   rž   Zskip_table_regions)
rb   rc   r~   r×   Zskip_bboxesZskip_dump_odZ	draw_gridZdraw_captionÚresizeÚformat)5Ú'unstructured_inference.inference.layoutrØ   rÙ   Z$unstructured.partition.pdf_image.ocrrÚ   rÛ   Ú4unstructured.partition.pdf_image.pdfminer_processingrÜ   rÝ   rÒ   r^   rG   ZANALYSIS_DUMP_OD_SKIPZGLOBAL_WORKING_DIR_ENABLEDrU   r   ZGLOBAL_WORKING_PROCESS_DIRÚcwdrY   Úmakedirsr3   r1   r4   r@   rº   r”   r;   Údocument_to_element_listr7   r9   r   ZIMAGEr…   r   r   r¹   r   r   r   r¼   ÚsubÚ RE_MULTISPACE_INCLUDING_NEWLINESr†   r‡   r6   Úextendr2   r5   ZANALYSIS_BBOX_SKIPZANALYSIS_BBOX_DRAW_GRIDZANALYSIS_BBOX_DRAW_CAPTIONZANALYSIS_BBOX_RESIZEZANALYSIS_BBOX_FORMAT)2rb   rc   r~   rf   rd   rh   rg   rÓ   rÔ   rl   rÕ   rj   rŒ   rm   rn   ro   rp   rÖ   r×   rq   rr   rs   rÐ   rt   rƒ   r   r€   ry   rØ   rÙ   rÚ   rÛ   rÜ   rÝ   Zod_model_layout_dumperZextracted_layout_dumperré   Zfinal_layout_dumperZskip_analysis_dumprç   rè   rí   Zmerged_document_layoutZfinal_document_layoutrž   Zel_typerŸ   rŠ   ZformsZlayout_dumpersr\   r\   r]   r˜   M  s¨    ÿ
ûúüøÿþÿýôû
ýÿûÿþÿý
ôÿõô÷÷

ý
û
ÿ



õr˜   )rŽ   rd   Ú	sort_modec                 K  sJ   g }| D ]<}t |tƒ}|tkr(t ||ƒ}||7 }|r| tdd�¡ q|S )z!Partitions a PDF using pdfparser.r}   ©r†   )rO   rI   r¹   r   )rŽ   rd   rù   ry   rž   r‰   Úsorted_page_elementsr\   r\   r]   r™   m  s    

r™   )
rb   rc   rd   rh   rg   r~   rj   rq   rt   ry   c	              
   K  sº   g }
|rpg }|durt  |¡nt  | ¡}| |¡ t||d�D ].\}}tf ||||||dœ|	¤Ž}|
 |¡ q>nFtt| ||d�|d�D ].\}}tf ||||||dœ|	¤Ž}|
 |¡ q†|
S )zkPartitions an image or PDF using OCR. For PDFs, each page is converted
    to an image prior to processing.Nr©   )Úimagerh   rg   r´   rd   rj   )rt   )ÚPILImageÚopenr¹   r¶   Ú+_partition_pdf_or_image_with_ocr_from_imagerø   r8   )rb   rc   rd   rh   rg   r~   rj   rq   rt   ry   rž   Zimagesrü   r´   r‰   r\   r\   r]   rš   …  s@    
úù	ÿúù	rš   zPILImage.Image)	rü   rh   rg   r´   rd   rj   rù   ry   rV   c                 K  s„   ddl m} |j|d�}	|	 ¡ r$t}|	j| d�}
t|| j||d�}t|
 	¡ | j
|d�}|}|tkrlt||ƒ}|r€| tdd�¡ |S )	zQExtract `unstructured` elements from an image using OCR and perform partitioning.r   )ÚOCRAgent)Úlanguage)rü   )rœ   Úfiletyper´   rh   )Z
image_sizeZcommon_metadatar}   rú   )Z5unstructured.partition.utils.ocr_models.ocr_interfacer   Z	get_agentZis_text_sortedrJ   Zget_layout_elements_from_imager   rð   r,   rã   ÚsizerO   r¹   r   )rü   rh   rg   r´   rd   rj   rù   ry   r   r   Zocr_datar¿   r‰   rû   r\   r\   r]   rÿ   ·  s,    ü	ý
rÿ   )rž   c                 C  sP   g }| D ]B}t |dƒr<|jtjkr<ttt|ƒjƒ}|j|_n|}| 	|¡ q|S )z¸Processes a list of elements, creating a new list where elements with the
    category `UncategorizedText` are replaced with corresponding
    elements created from their text content.Úcategory)
rº   r  r   ZUNCATEGORIZED_TEXTrF   r   r   r†   r¿   r¹   )rž   rŸ   rŠ   Znew_elr\   r\   r]   r›   æ  s    
r›   r   )ÚitemrV   c                 C  sT   t | dƒr|  ¡ S t| tƒr>d}| D ]}|t|ƒp4d7 }q$|S t| ttfƒrPdS dS )zrRecursively extracts text from PDFMiner objects to account
    for scenarios where the text is in a sub-container.r®   r}   Ú
)rº   r®   r…   r	   r»   r   r
   )r  r†   Úchildr\   r\   r]   r»   ÷  s    

r»   zpdfminer.pdfinterpz!PDFPageInterpreter.init_resourcesc                 C  s   |d }d|v r|d= | |ƒS )Nr   Z
ColorSpacer\   )ÚwrappedÚinstanceÚargsry   Ú	resourcesr\   r\   r]   Ú#pdfminer_interpreter_init_resources  s    r  zPixelSpace | PointSpace)rž   r°   rV   c                 C  sz   d}g }| D ]h}t |tƒr.|}|j}|jj}n<|rjt|jj|d�rj|› d|j› �|_t|||d�}| ¡  | |¡ q|S )zECombine elements that should be considered a single ListItem element.N)r¯   Úboundaryrî   )Úelement1Úelement2r°   )	r…   r   r†   r¿   r¯   Úcheck_coords_within_boundaryÚ"_combine_coordinates_into_element1Úpopr¹   )rž   r°   Ztmp_elementZupdated_elementsrÌ   Ztmp_textZ
tmp_coordsr\   r\   r]   rÁ     s(    

þýrÁ   zlist[dict[str, Any]]z
np.ndarrayz
list[Link])rÈ   rË   rV   c              	   C  s`   g }| D ]R}t  t¡�4 | |d |d t|d |ƒdœ¡ W d  ƒ q1 sP0    Y  q|S )z+Extracts links from a list of URL metadata.r†   ÚuriÚstart_index)r†   Úurlr  N)Ú
contextlibÚsuppressÚ
IndexErrorr¹   r   )rÈ   rË   rµ   r  r\   r\   r]   r¾   3  s    þýÿ$
r¾   r   )r  r  r°   rV   c                 C  sÎ   t | jjjd d |jjjd d ƒ}t| jjjd d |jjjd d ƒ}t | jjjd d |jjjd d ƒ}t| jjjd d |jjjd d ƒ}||f||f||f||ff}t||d�| j_t | ¡S )zXCombine the coordiantes of two elements and apply the updated coordiantes to `elements1`r   é   r_   r±   )Úminr¿   r¯   r²   Úmaxr   ÚcopyÚdeepcopy)r  r  r°   rÄ   rÆ   rÅ   rÇ   r²   r\   r\   r]   r  G  s,    þþþþþ
r  çš™™™™™É?ç333333Ó?r   Úfloat)r¯   r  Úhorizontal_thresholdÚvertical_thresholdrV   c                 C  sö   t | ƒs,t |ƒs,t d| › d|› d�¡ dS |jd d }|jd d }|jd d }|jd d }|| }|| }	| jd d |||  ko¸| jd d |||  k o¸| jd d |k}
| jd d |||	  k oì| jd d |||	  k}|
oô|S )a”  Checks if the coordinates are within boundary thresholds.
    Parameters
    ----------
    coordinates
        a CoordinatesMetadata input
    boundary
        a CoordinatesMetadata to compare against
    vertical_threshold
        a float ranges from [0,1] to scale the vertical (y-axis) boundary
    horizontal_threshold
        a float ranges from [0,1] to scale the horizontal (x-axis) boundary
    zcoordinates z and boundary z did not pass validationFr   r  r_   )rN   r&   Zdetailr²   )r¯   r  r!  r"  Zboundary_x_minZboundary_x_maxZboundary_y_minZboundary_y_maxÚ
line_widthZline_heightZx_within_boundaryZy_within_boundaryr\   r\   r]   r  c  s*    ÿÿýÿþr  r   zOptional[list[list]])Údocumentrê   rd   rë   rì   Úsource_formatrÀ   rù   rh   rq   rí   ry   rV   c                   sÊ  g }t | jƒ}t| j|	d�D �]¦\}‰g }tˆƒ}| d¡}| d¡}| d¡}g }|
rl|
d rl|
||	  nd}dd„ ˆjj ¡ D ƒ}|r¢t‡fd	d
„|D ƒƒ}nd}ˆj 	¡ D �]|‰ |râ|rât
 tˆ jdt
jƒ¡sât||d�}nd}tˆ |||rö|ndd�}t|tƒ�rR|D ]}|�r ||j_||j_�q| |¡ | ‡ fdd„|D ƒ¡ q°nˆ|�rdt|ˆ jƒng |j_|�rz||j_tˆ ddƒ|j_tˆ ddƒ|j_t|tƒ�rÂ|jjdu �rÂ|�rÂd|j_| |¡ | ˆ |f¡ |jj�rî|jjjnd}t ˆ dƒ�rˆ j!nd}t"|f|||||jj|||dœ|¤Ž q°|D ]B\‰ }t ˆ dƒ�r4ˆ j#du�r4t$‡ fdd
„|D ƒƒ}|j%|j_&�q4|}|�r–|t'k�r–t(||ƒ}|�rº|||	 k �rº| t)dd�¡ | |¡ q|S )zDConverts a DocumentLayout object to a list of unstructured elements.r©   rð   r¬   r­   r   Nc                 S  s   g | ]\}}|d v r|‘qS ))ZHeadlineZSubheadliner\   )rˆ   ÚidxZ
class_typer\   r\   r]   rå   ±  s   þz,document_to_element_list.<locals>.<listcomp>c                 3  s    | ]}t  ˆ jj|k¡V  qd S r„   )Únpr�   Úelements_arrayZelement_class_ids)rˆ   r&  )rÂ   r\   r]   r‹   ·  s   ÿz+document_to_element_list.<locals>.<genexpr>FrÄ   r«   Úhtml)r°   rì   r%  c                   s   g | ]}ˆ |f‘qS r\   r\   )rˆ   rŠ   ©Úlayout_elementr\   r]   rå   Ô  ræ   Útext_as_htmlÚtable_as_cellsÚ
image_path)r´   r  r¯   r°   Úcategory_depthr.  rÀ   rh   Úparentc                 3  s    | ]\}}|ˆ j u r|V  qd S r„   )r0  )rˆ   Zl_elrŠ   r*  r\   r]   r‹      ræ   r}   rú   )*r¸   Zpagesr¶   r*   r[   r(  Zelement_class_id_mapÚitemsr�   Ziter_elementsr'  ÚisnanÚgetattrr·   Únanr   r+   r…   Úlistr¿   rœ   r´   rø   r<   rµ   r,  r-  r    r/  r¹   r¯   r²   rº   r.  r(   r0  rQ   ÚidZ	parent_idrJ   rO   r   )r$  rê   rd   rë   rì   r%  rÀ   rù   rh   rq   rí   ry   rž   Z	num_pagesr´   r‰   Zpage_image_metadataZimage_formatZimage_widthZimage_heightZtranslation_mappingrµ   Zhead_line_type_class_idsZhas_headliner°   rÌ   rŠ   r¯   Zel_image_pathZelement_parentrû   r\   )r+  rÂ   r]   rõ   �  s²    



ÿÿý
þþ
ÿþý
ü
ÿÿ
ÿÿÿ÷
öÿ
rõ   )r}   NNNr_   NN)r_   NN)r}   N)r}   NN)	r}   NFNNFNr_   N)r  r  )›Ú
__future__r   r  r  r¡   rY   r¼   r•   Úpathlibr   Útypingr   r   r   r   r   Únumpyr'  ZwraptZpdfminer.layoutr	   r
   r   r   Zpdfminer.utilsr   Zpi_heifr   ZPILr   rý   Zpypdfr   rñ   r   Z.unstructured_inference.inference.layoutelementr   Zunstructured.chunkingr   Zunstructured.cleaners.corer   r   Z"unstructured.documents.coordinatesr   r   Zunstructured.documents.elementsr   r   r   r   r   r   r   r   r    r!   Zunstructured.errorsr"   Z unstructured.file_utils.filetyper#   Zunstructured.file_utils.modelr$   Zunstructured.loggerr%   r&   Zunstructured.nlp.patternsr'   Z$unstructured.partition.common.commonr(   r)   r*   r+   r,   r-   Z"unstructured.partition.common.langr.   r/   Z&unstructured.partition.common.metadatar0   Z5unstructured.partition.pdf_image.analysis.layout_dumpr1   r2   r3   r4   Z/unstructured.partition.pdf_image.analysis.toolsr5   Z0unstructured.partition.pdf_image.form_extractionr6   Z0unstructured.partition.pdf_image.pdf_image_utilsr7   r8   r9   rò   r:   r;   r<   r=   r>   r?   r@   Z/unstructured.partition.pdf_image.pdfminer_utilsrA   rB   rC   Z!unstructured.partition.strategiesrD   rE   Zunstructured.partition.textrF   Z#unstructured.partition.utils.configrG   Z&unstructured.partition.utils.constantsrH   rI   rJ   rK   rL   rM   Z$unstructured.partition.utils.sortingrN   rO   Zunstructured.patches.pdfminerrP   Zunstructured.utilsrQ   rR   ÚcompileÚDOTALLr÷   r^   ZPDFZAUTOr|   r{   r�   r£   ZPDF_ANNOTATION_THRESHOLDr¦   rÎ   rÒ   Z	FULL_PAGEÚvaluer˜   r™   rš   rÿ   r›   r»   Zpatch_function_wrapperr  rÁ   r¾   r  r  rõ   r\   r\   r\   r]   Ú<module>   sˆ  4 $	 
éF~é@        ù    ù 3ø"W  þ   ýåJ  #ý         ÷"4ù /

  ü.õ