a
    ¡˜ëj{<  ã                   @  sê  d dl mZ d dlZd dlZd dlmZmZmZ d dlm	Z	 d dl
mZ d dlmZmZmZmZmZmZ d dlZd dlZd dlmZmZ d dlmZmZmZmZmZmZmZm Z m!Z! d d	l"m#Z# d d
l$m%Z%m&Z& erìd dl'm(Z( d dl)m*Z* dKddddddœdd„Z+dddddddœdd„Z,dLdddddddddddd dd!œd"d#„Z-ddd$œd%d&„Z.d'd(„ Z/dMd+d+d+dd,d-œd.d/„Z0d d0d1œd2d3„Z1ed4ƒZ2d5d6d7œd8d9„Z3d:d;d7œd<d=„Z4d+dd>œd?d@„Z5dAdBdCœdDdE„Z6dNdFdGdddddHœdIdJ„Z7dS )Oé    )ÚannotationsN)ÚBufferedReaderÚBytesIOÚTextIOWrapper)ÚSpooledTemporaryFile)Úsleep)ÚIOÚTYPE_CHECKINGÚAnyÚOptionalÚTypeVarÚcast)ÚCoordinateSystemÚ
PixelSpace)	ÚTYPE_TO_TEXT_ELEMENT_MAPÚCheckBoxÚCoordinatesMetadataÚElementÚElementMetadataÚElementTypeÚListItemÚ	PageBreakÚText)Úlogger)ÚENUMERATED_BULLETS_REÚUNICODE_BULLETS_RE)Ú
PageLayout)ÚLayoutElementTÚhtmlz(LayoutElement | Element | dict[str, Any]zOptional[CoordinateSystem]ÚboolzOptional[str]zElement | list[Element])Úlayout_elementÚcoordinate_systemÚinfer_list_itemsÚsource_formatÚreturnc                 C  s¼  t | tƒr|dkr| S t | tƒr*tdd�S t | tƒs>|  ¡ }n| }| dd¡}|r\| d¡nd}| d¡}| d¡}| d	d¡}	d}
|	rŽ|	j}
|r¶t |ttt	t
jfƒr¶tt	|ƒd
�}ntƒ }||||
dœ}|tjkrü|rèt|fi |¤ŽS tf d|i|¤ŽS n¼|tv �r^t |tƒ�sJ ‚t| }|f d|i|¤Ž}|tjk�rFd|j_n|tjk�rZd|j_|S |tjtjtjtjtjtjfv �r¦|tjtjtjfv }tf d|i|¤ŽS tf d|i|¤ŽS dS )zSConverts an unstructured_inference LayoutElement object to an unstructured Element.r   Ú ©Útextr'   ÚcoordinatesNÚtypeÚprobÚsource)Zdetection_class_prob)r(   r!   ÚmetadataÚdetection_originé   é   Úchecked)Ú
isinstancer   r   ÚdictÚto_dictÚgetÚvalueÚintÚstrÚfloatÚnumbersÚNumberr   r   ÚLISTÚlayout_list_to_list_itemsr   r   ZHEADLINEr,   Úcategory_depthZSUB_HEADLINEZCHECK_BOX_CHECKEDZCHECK_BOX_UNCHECKEDZRADIO_BUTTON_CHECKEDZRADIO_BUTTON_UNCHECKEDZCHECKEDZ	UNCHECKEDr   r   )r    r!   r"   r#   Zlayout_dictr'   r(   Zelement_typer*   Z
aux_originÚoriginZclass_prob_metadataZcommon_kwargsZ_element_classr0   © r?   úc/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/unstructured/partition/common/common.pyÚnormalize_layout_element!   sŽ    





ü
ÿþÿþ
ÿþ
úýÿþÿþrA   z)Optional[tuple[tuple[float, float], ...]]zOptional[ElementMetadata]zlist[Element])r'   r(   r!   r,   r-   r$   c           	      C  sr   | rt  | ¡ng }t|ƒdkr0| r,t | ¡ng }g }|D ]4}t| ¡ ƒdkr8t| ¡ ||||d�}| |¡ q8|S )z=Converts a list LayoutElement to a list of ListItem elements.r.   r   )r'   r(   r!   r,   r-   )r   ÚsplitÚlenr   Ústripr   Úappend)	r'   r(   r!   r,   r-   Zsplit_itemsZ
list_itemsZtext_segmentÚitemr?   r?   r@   r<   y   s    ûr<   r   zOptional[int]zOptional[list[str]]r
   )ÚelementÚfilenameÚfiletypeÚpage_numberÚurlÚtext_as_htmlr(   r!   Ú
image_pathr-   Ú	languagesÚkwargsr$   c                 K  s,  |dur|durt ||d�nd}t| dƒr>t| jƒdkr>| jnd}|rTdd„ |D ƒnd}|rjdd„ |D ƒnd}|r€dd„ |D ƒnd}t| d	ƒr¢t| jƒdkr¢| jnd}|r¸d
d„ |D ƒnd}|rÎdd„ |D ƒnd}| jjrâ| jjnd}t||||||||||||||
d�}| j |¡ |	du�r(|	| j_	| S )z�Adds document metadata to the document element.

    Document metadata includes information like the filename, source url, and page number.
    N)ZpointsÚsystemÚlinksr   c                 S  s   g | ]}|  d ¡‘qS )rK   ©r4   ©Ú.0Úlinkr?   r?   r@   Ú
<listcomp>³   ó    z(add_element_metadata.<locals>.<listcomp>c                 S  s   g | ]}|  d ¡‘qS r&   rR   rS   r?   r?   r@   rV   ´   rW   c                 S  s   g | ]}|  d ¡‘qS )Ústart_indexrR   rS   r?   r?   r@   rV   µ   rW   Úemphasized_textsc                 S  s   g | ]}|  d ¡‘qS r&   rR   ©rT   Zemphasized_textr?   r?   r@   rV   ¼   rW   c                 S  s   g | ]}|  d ¡‘qS )ÚtagrR   rZ   r?   r?   r@   rV   Á   rW   )r(   rH   rI   rJ   rK   rL   Ú	link_urlsÚ
link_textsÚlink_start_indexesÚemphasized_text_contentsÚemphasized_text_tagsr=   rM   rN   )
r   ÚhasattrrC   rQ   rY   r,   r=   r   Úupdater-   )rG   rH   rI   rJ   rK   rL   r(   r!   rM   r-   rN   rO   Zcoordinates_metadatarQ   r\   r]   r^   rY   r_   r`   Údepthr,   r?   r?   r@   Úadd_element_metadata—   s`    üüþú"ÿÿýÿýÿýò
rd   )Úlayout_elementsr$   c                 C  sV   g }t ƒ }| D ]B}t|ƒ}t|tƒr@|D ]
}||_q(| |¡ q||_| |¡ q|S )z•Removes document metadata from the document element.

    Document metadata includes information like the filename, source url, and page number.
    )r   rA   r1   Úlistr,   ÚextendrE   )re   Úelementsr,   r    rG   Ú_elementr?   r?   r@   Úremove_element_metadataÝ   s    
rj   c                  C  sN   t  ¡ D ]@} zd|  ¡  ¡ v r&W  dS W q t jt jt jfyF   Y q0 qdS )NÚsofficeTF)ÚpsutilZprocess_iterÚnameÚlowerZNoSuchProcessZAccessDeniedZZombieProcess)Úprocr?   r?   r@   Ú_is_soffice_runningð   s    rp   Údocxé
   r7   r6   )Úinput_filenameÚoutput_directoryÚtarget_formatÚtarget_filterÚwait_for_soffice_ready_time_outc           
      C  sö   |dur|› d|› �}ddd|d|| g}znd}d}t j|d	d
�}|j ¡  ¡ }	||k r”|	dkr”||7 }tƒ rvt|ƒ qNt j|d	d
�}|j ¡  ¡ }	qNW n ty°   tdƒ‚Y n0 t 	|	¡ |j
dksÎ|	dkròt d||j
¡ t |j ¡  ¡ ¡ dS )a  Converts a .doc/.ppt file to a .docx/.pptx file using the libreoffice CLI.

    Parameters
    ----------
    input_filename: str
        The name of the .doc file to convert to .docx
    output_directory: str
        The output directory for the convert .docx file
    target_format: str
        The desired output format
    target_filter: str
        The output filter name to use when converting. See references below
        for details.
    wait_for_soffice_ready_time_out: int
        The max wait time in seconds for soffice to become available to run

    References
    ----------
    https://stackoverflow.com/questions/52277264/convert-doc-to-docx-using-soffice-not-working
    https://git.libreoffice.org/core/+/refs/heads/master/filter/source/config/fragments/filters

    Nú:rk   z
--headlessz--convert-toz--outdirr   gš™™™™™¹?T)Úcapture_outputr%   a  soffice command was not found. Please install libreoffice
on your system and try again.

- Install instructions: https://www.libreoffice.org/get-help/install-howto/
- Mac: https://formulae.brew.sh/cask/libreoffice
- Debian: https://wiki.debian.org/LibreOfficez3soffice failed to convert to format %s with code %i)Ú
subprocessÚrunÚstdoutÚdecoderD   rp   r   ÚFileNotFoundErrorr   ÚinfoÚ
returncodeÚerrorÚstderr)
rs   rt   ru   rv   rw   ÚcommandZ	wait_timeZ
sleep_timeÚoutputÚmessager?   r?   r@   Úconvert_office_docú   s>    ù	
ÿ
	
ÿr†   ÚNone)rO   r$   c                  K  sr   t dd„ |  ¡ D ƒƒdkrnt|  ¡ ƒ}t|ƒdkrXdd |dd… ¡› d|d › d	�}n|d
 › d	�}t|ƒ‚dS )zª
    Verify arguments; exactly one of all keyword arguments must not be None.

    Example:
        >>> exactly_one(filename=filename, file=file, text=text, url=url)
    c                 S  s   g | ]}|d uo|dk‘qS )Nr%   r?   )rT   Úargr?   r?   r@   rV   O  rW   zexactly_one.<locals>.<listcomp>r.   zExactly one of z, Néÿÿÿÿz and z must be specified.r   )ÚsumÚvaluesrf   ÚkeysrC   ÚjoinÚ
ValueError)rO   Únamesr…   r?   r?   r@   Úexactly_oneH  s    &r�   Ú_Tz _T | SpooledTemporaryFile[bytes]z_T | BytesIO)Úfiler$   c                 C  s*   t | tƒr&|  d¡ ttt|  ¡ ƒƒS | S )aã  Convert `file` to `BytesIO` when it is a `SpooledTemporaryFile`.

    Note that `file` does not need to be IO[bytes]. It can be `None` or `bytes` and this function
    will not complain.

    In Python <3.11, `SpooledTemporaryFile` does not implement `.readable()` or `.seekable()` which
    triggers an exception when the file is loaded by certain packages. In particular, the stdlib
    `zipfile.Zipfile` raises on opening a `SpooledTemporaryFile` as does `Pandas.read_csv()`.
    r   )r1   r   Úseekr   r   ÚbytesÚread)r’   r?   r?   r@   Úspooled_to_bytes_io_if_needed[  s    


r–   zbytes | IO[bytes]r”   c                 C  s˜   t | tƒr| S t | tƒr8|  d¡ |  ¡ }|  d¡ |S t | tƒrJ|  ¡ S t | ttfƒrŒt	| j
dƒ�}| ¡ W  d  ƒ S 1 s‚0    Y  tdƒ‚dS )zºExtract the bytes from `file` without preventing it from being read again later.

    As a convenience to simplify client code, also returns `file` unchanged if it is already bytes.
    r   ÚrbNzInvalid file-like object type)r1   r”   r   r“   r•   r   Úgetvaluer   r   Úopenrm   rŽ   )r’   Zf_bytesÚfr?   r?   r@   Úconvert_to_bytesm  s    




&r›   )Úsr$   c                 C  s   t t | ¡ƒS )zÏ
    Check if the input string contains any emoji characters.

    Parameters:
    - s (str): The input string to check.

    Returns:
    - bool: True if the string contains any emoji, False otherwise.
    )r   ÚemojiZemoji_count)rœ   r?   r?   r@   Úcontains_emoji…  s    rž   r   zdict[str, Any])Úpager$   c                 C  sl   t | ddƒ}t | ddƒ}|r0|j}|j}|j}n0|rT| d¡}| d¡}| d¡}nd}d}d}|||dœS )z:Retrieve image metadata and coordinate system from a page.ÚimageNÚimage_metadataÚformatÚwidthÚheight)r¢   r£   r¤   )Úgetattrr¢   r£   r¤   r4   )rŸ   r    r¡   Zimage_formatÚimage_widthÚimage_heightr?   r?   r@   Úget_page_image_metadata“  s"    

ýr¨   zlist['LayoutElement']ztuple[int | float, int | float])Úocr_dataÚ
image_sizeÚcommon_metadatar"   r#   r$   c                 C  sX   |\}}t ||d�}g }| D ]6}	t|	|||r0|ndd�}
|rH|
j |¡ | |
¡ q|S )zNConvert OCR layout data into `unstructured` elements with associated metadata.)r£   r¤   r   )r!   r"   r#   )r   rA   r,   rb   rE   )r©   rª   r«   r"   r#   r¦   r§   r!   rh   r    rG   r?   r?   r@   Úocr_data_to_elements­  s    	
ür¬   )NTr   )
NNNNNNNNNN)rq   Nrr   )NTN)8Ú
__future__r   r9   rz   Úior   r   r   Útempfiler   Útimer   Útypingr   r	   r
   r   r   r   r�   rl   Z"unstructured.documents.coordinatesr   r   Zunstructured.documents.elementsr   r   r   r   r   r   r   r   r   Zunstructured.loggerr   Zunstructured.nlp.patternsr   r   Z'unstructured_inference.inference.layoutr   Z.unstructured_inference.inference.layoutelementr   rA   r<   rd   rj   rp   r†   r�   r‘   r–   r›   rž   r¨   r¬   r?   r?   r?   r@   Ú<module>   s^    ,   üX           õ(F   ûN   û