a
    ¡˜ëjÓ  ã                   @  s¢  d dl mZ d dlZd dlZd dlmZmZmZmZ d dl	m
Z
 d dlmZmZ d dlmZ d dlmZmZmZmZmZmZmZmZmZmZ d dlmZ d d	lmZ d d
lm Z m!Z! d dl"m#Z# d dl$m%Z%m&Z& d dl'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z- e%ej.ƒe
d4ddddddœdddddddddœdd„ƒƒZ/d5dddddœdd„Z0dd d!d"œd#d$„Z1dd%d&œd'd(„Z2d6ddd!d%d*œd+d,„Z3d7ddd!d%d*œd.d/„Z4dd0d1œd2d3„Z5dS )8é    )ÚannotationsN)ÚIOÚAnyÚCallableÚLiteral)Úadd_chunking_strategy)Úauto_paragraph_grouperÚclean_bullets)ÚCoordinateSystem)
ÚAddressÚElementÚElementMetadataÚEmailAddressÚFooterÚHeaderÚListItemÚNarrativeTextÚTextÚTitle)Úread_txt_file)ÚFileType)ÚPARAGRAPH_PATTERNÚUNICODE_BULLETS_RE)Úexactly_one)Úapply_metadataÚget_last_modified_date)Úis_bulleted_textÚis_email_addressÚis_possible_narrative_textÚis_possible_numbered_listÚis_possible_titleÚis_us_city_state_zipÚtext)ÚfileÚencodingr"   Úparagraph_grouperÚdetection_originz
str | NonezIO[bytes] | Nonez,Callable[[str], str] | Literal[False] | Noner   zlist[Element])Úfilenamer#   r$   r"   r%   r&   ÚkwargsÚreturnc                K  s  |dur |  ¡ dkr |s | s g S t| ||d� d}| durLt| |d�\}}n*|durft||d�\}}n|durvt|ƒ}|du r€n|dur’||ƒ}nt|ƒ}t|ƒ}g }	t| r´t| ƒndd�}
||
_|D ]6}|  ¡ }|rÆt	|ƒsÆt
|ƒ}t |
¡|_|	 |¡ qÆ|	S )aÄ  Partition a .txt documents into its constituent paragraph elements.

    If paragraphs are below "min_partition" or above "max_partition" boundaries,
    they are combined or split.
    Parameters
    ----------
    filename
        A string defining the target filename path.
    file
        A file-like object using "rb" mode --> open(filename, "rb").
    encoding
        The encoding method used to decode the input bytes when drawn from `filename` or `file`.
        Defaults to "utf-8".
    text
        The string representation of the .txt document.
    paragrapher_grouper
        A str -> str function for fixing paragraphs that are interrupted by line breaks
        for formatting purposes.
    NÚ )r'   r#   r"   )r'   r$   )r#   r$   F)Zlast_modified)Ústripr   r   Ústrr   Ú_split_by_paragraphr   r   r&   Ú_is_empty_bulletÚelement_from_textÚcopyÚdeepcopyÚmetadataÚappend)r'   r#   r$   r"   r%   r&   r(   Ú	file_textZfile_contentÚelementsr2   ZctextÚelement© r7   úZ/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/unstructured/partition/text.pyÚpartition_text(   s8    
ÿr9   r,   z&tuple[tuple[float, float], ...] | NonezCoordinateSystem | Noner   )r"   ÚcoordinatesÚcoordinate_systemr)   c                 C  sÊ   t ||ƒrt| ||d�S t||ƒr0t| ||d�S t| ƒrNt| ƒ}t|||d�S t| ƒr`t| d�S t	| ƒrvt
| ||d�S t| ƒrŒt| ||d�S t| ƒr¢t| ||d�S t| ƒr¸t| ||d�S t| ||d�S d S )N)r"   r:   r;   ©r"   )Ú_is_in_header_positionr   Ú_is_in_footer_positionr   r   r	   r   r   r   r!   r   r   r   r   r    r   r   )r"   r:   r;   Z
clean_textr7   r7   r8   r/   o   sd    
ý
ýý
ýýýýýr/   ztuple[tuple[float, float], ...]r
   Úfloat)r:   r;   r)   c                 C  s$   t dd„ | D ƒƒt| ƒ }||j S )Nc                 s  s   | ]}|d  V  qdS )é   Nr7   )Ú.0Z
coordinater7   r7   r8   Ú	<genexpr>²   ó    z)_get_height_percentage.<locals>.<genexpr>)ÚsumÚlenÚheight)r:   r;   Zavg_yr7   r7   r8   Ú_get_height_percentage®   s    rG   Úbool)r"   r)   c                 C  s   t t | ¡ot| ƒdkƒS )z(Checks if input text is an empty bullet.r@   )rH   r   ÚmatchrE   r<   r7   r7   r8   r.   ¶   s    r.   çÃõ(\�Âí?)r:   r;   Ú	thresholdr)   c                 C  s&   | du s|du rdS t | |ƒ}||kS )zZChecks to see if the position of the text indicates that the text belongs
    to a footer.NF©rG   ©r:   r;   rK   Zheight_percentager7   r7   r8   r>   »   s    
r>   çìQ¸…ë±?c                 C  s&   | du s|du rdS t | |ƒ}||k S )zVChecks to see if the position of the text indicates that the text belongs to a header.NFrL   rM   r7   r7   r8   r=   É   s    
r=   z	list[str])r4   r)   c                 C  s   t  t|  ¡ ¡S )zSplit text into paragraphs.)ÚreÚsplitr   r+   )r4   r7   r7   r8   r-   Ö   s    r-   )N)NN)rJ   )rN   )6Ú
__future__r   r0   rO   Útypingr   r   r   r   Zunstructured.chunkingr   Zunstructured.cleaners.corer   r	   Z"unstructured.documents.coordinatesr
   Zunstructured.documents.elementsr   r   r   r   r   r   r   r   r   r   Z unstructured.file_utils.encodingr   Zunstructured.file_utils.modelr   Zunstructured.nlp.patternsr   r   Z$unstructured.partition.common.commonr   Z&unstructured.partition.common.metadatar   r   Z unstructured.partition.text_typer   r   r   r   r    r!   ZTXTr9   r/   rG   r.   r>   r=   r-   r7   r7   r7   r8   Ú<module>   sB   0 
 ÿù$G  ý? ý ý