a
    ¡˜ëj©—  ã                J   @  sö  U d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlmZ d dlmZ d dlmZmZmZmZmZmZ d dlmZmZmZ d dlmZmZmZ d dlm Z  d d	l!m"Z"m#Z# d
Z$de%d< dZ&de%d< ej'G dd„ dƒƒZ(ej'G dd„ dƒƒZ)G dd„ deƒZ*G dd„ deƒZ+G dd„ deƒZ,G dd„ dƒZ-G dd„ dej.ƒZ/edƒZ0dddœd d!„Z1d"d#œd$d%„Z2G d&d'„ d'ƒZ3G d(d)„ d)ej4ƒZ5G d*d+„ d+e5ƒZ6G d,d-„ d-e5ƒZ7G d.d/„ d/e7ƒZ8G d0d1„ d1e7ƒZ9G d2d3„ d3e7ƒZ:G d4d5„ d5e7ƒZ;G d6d7„ d7e7ƒZ<G d8d9„ d9e7ƒZ=G d:d;„ d;e7ƒZ>G d<d=„ d=e7ƒZ?G d>d?„ d?e7ƒZ@G d@dA„ dAe7ƒZAG dBdC„ dCe7ƒZBG dDdE„ dEeBƒZCG dFdG„ dGe7ƒZDG dHdI„ dIe7ƒZEG dJdK„ dKe7ƒZFG dLdM„ dMe7ƒZGG dNdO„ dOe7ƒZHG dPdQ„ dQe7ƒZIe3jJe=e3jKe=e3jLe=e3jMe=e3jNe=e3jOe7e3jPe9e3jQe;e3jRe;e3jSe;e3jTe;e3jUe;e3jVe;e3jWe;e3jXe;e3jYe<e3jZe<e3j[e<e3j\eDe3j]eDe3j^eEe3j_eEe3j`eEe3jae:e3jbe:e3jce@e3jde@e3jee@e3jfeBe3jge>e3jhe?e3jie8e3jjeAe3jkeFe3jleGe3jmeHe3jneIi%ZodRe%dS< dTdUdVœdWdX„ZpdUdTdYœdZd[„ZqdS )\é    )ÚannotationsN)Úgroupby)ÚMappingProxyType)ÚAnyÚCallableÚ	FrozenSetÚOptionalÚSequenceÚcast)Ú	ParamSpecÚ	TypeAliasÚ	TypedDict)ÚTYPE_TO_COORDINATE_SYSTEM_MAPÚCoordinateSystemÚRelativeCoordinateSystem)Ú#UNSTRUCTURED_INCLUDE_DEBUG_METADATA)Úget_call_args_applying_defaultsÚlazypropertyztuple[float, float]r   ÚPointztuple[Point, ...]ÚPointsc                   @  s€   e Zd ZU dZdZded< dZded< dZded< dZded< dZ	ded	< dZ
ded
< dZded< dd„ Zeddœdd„ƒZdS )ÚDataSourceMetadataz@Metadata fields that pertain to the data source of the document.NúOptional[str]ÚurlÚversionzOptional[dict[str, Any]]Úrecord_locatorÚdate_createdÚdate_modifiedÚdate_processedzOptional[list[dict[str, Any]]]Úpermissions_datac                 C  s   dd„ | j  ¡ D ƒS )Nc                 S  s   i | ]\}}|d ur||“qS ©N© )Ú.0ÚkeyÚvaluer    r    ú^/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/unstructured/documents/elements.pyÚ
<dictcomp>+   ó    z.DataSourceMetadata.to_dict.<locals>.<dictcomp>)Ú__dict__Úitems©Úselfr    r    r$   Úto_dict*   s    zDataSourceMetadata.to_dictúdict[str, Any]©Ú
input_dictc                   s8   dd„ t  | ¡D ƒ‰ ‡ fdd„| ¡ D ƒ}| f i |¤ŽS )Nc                 S  s   g | ]
}|j ‘qS r    )Úname)r!   Úfr    r    r$   Ú
<listcomp>0   r&   z0DataSourceMetadata.from_dict.<locals>.<listcomp>c                   s   i | ]\}}|ˆ v r||“qS r    r    )r!   ÚkÚv©Zsupported_fieldsr    r$   r%   1   r&   z0DataSourceMetadata.from_dict.<locals>.<dictcomp>)ÚdcÚfieldsr(   )Úclsr.   Úargsr    r4   r$   Ú	from_dict-   s    zDataSourceMetadata.from_dict)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   Ú__annotations__r   r   r   r   r   r   r+   Úclassmethodr9   r    r    r    r$   r      s   
r   c                   @  s\   e Zd ZU dZded< ded< dddœdd„Zd	d
dœdd„Zdd„ Zeddœdd„ƒZ	dS )ÚCoordinatesMetadataz?Metadata fields that pertain to the coordinates of the element.úOptional[Points]ÚpointsúOptional[CoordinateSystem]Úsystem©rB   rD   c                 C  s8   |d u r|d us |d ur(|d u r(t dƒ‚|| _|| _d S )NzNCoordinates points should not exist without coordinates system and vice versa.)Ú
ValueErrorrB   rD   )r*   rB   rD   r    r    r$   Ú__init__=   s     ÿzCoordinatesMetadata.__init__r   Úbool©ÚotherÚreturnc                 C  s*   t |tƒsdS t| j|jk| j|jkgƒS ©NF)Ú
isinstancer@   ÚallrB   rD   ©r*   rJ   r    r    r$   Ú__eq__F   s    


þÿzCoordinatesMetadata.__eq__c                 C  sL   | j | jd u rd nt| jjjƒ| jd u r,d n| jj| jd u r@d n| jjdœS )N)rB   rD   Úlayout_widthÚlayout_height)rB   rD   ÚstrÚ	__class__r:   ÚwidthÚheightr)   r    r    r$   r+   P   s
    üzCoordinatesMetadata.to_dictr,   r-   c           	      C  sœ   dddœdd„}|  d¡}|d ur*||ƒnd }|  d¡}|  d¡}|  d	¡}|d u rXd n6|d
krftƒ n(|d urŒ|d urŒ|tv rŒt| ||ƒnd }| ||d�S )NzSequence[Sequence[float]]r   )Úsequence_of_sequencesrK   c                 S  sP   g }| D ]>}t |tƒr,| ttt|ƒƒ¡ qt |tƒr| tt|ƒ¡ qt|ƒS r   )rM   ÚlistÚappendr
   r   Útuple)rW   rB   Úseqr    r    r$   Úconvert_to_points[   s    

z8CoordinatesMetadata.from_dict.<locals>.convert_to_pointsrB   rD   rQ   rR   r   rE   )Úgetr   r   )	r7   r.   r\   Zinput_pointsrB   Zsystem_namerU   rV   rD   r    r    r$   r9   X   s&    




ÿÿþýüózCoordinatesMetadata.from_dictN)
r:   r;   r<   r=   r>   rG   rP   r+   r?   r9   r    r    r    r$   r@   6   s   
	
r@   c                   @  s*   e Zd ZU dZded< ded< ded< dS )	ÚLinkz#Metadata related to extracted linksr   ÚtextrS   r   ÚintÚstart_indexN)r:   r;   r<   r=   r>   r    r    r    r$   r^   �   s   
r^   c                   @  s&   e Zd ZU ded< ded< ded< dS )ÚFormKeyOrValuerS   r_   r   Zlayout_element_idzOptional[Text]Úcustom_elementN©r:   r;   r<   r>   r    r    r    r$   rb   ‰   s   
rb   c                   @  s&   e Zd ZU ded< ded< ded< dS )ÚFormKeyValuePairrb   r"   zOptional[FormKeyOrValue]r#   ÚfloatÚ
confidenceNrd   r    r    r    r$   re   �   s   
re   c                '      sF  e Zd ZU dZded< ded< ded< ded	< d
ed< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded< d ed!< d"ed#< d$ed%< ded&< ded'< ded(< ded)< ded*< ded+< ded,< ded-< ded.< ded/< ded0< d1ed2< ded3< edgƒZdXddddddd
dddd5dddddddddd ddd"dd$dddddddd1ddd6d7œ%d8d9„Zd:d;d<œd=d>„Zd?d6d@œdAdB„Z	d?dCd6dDœ‡ fdEdF„Z
edGd dHœdIdJ„ƒZedKdLœdMdN„ƒZedKdLœdOdP„ƒZdGdLœdQdR„Zd d6d<œdSdT„ZedUdLœdVdW„ƒZ‡  ZS )YÚElementMetadataz>Fully-dynamic replacement for dataclass-based ElementMetadata.r   Úattached_to_filenamezOptional[int]Úcategory_depthzOptional[CoordinatesMetadata]ÚcoordinateszOptional[DataSourceMetadata]Údata_sourcezOptional[float]Údetection_class_probÚdetection_originzOptional[list[str]]Úemphasized_text_contentsÚemphasized_text_tagsÚfile_directoryÚfilenameÚfiletypeÚ	image_urlÚ
image_pathÚimage_base64Úimage_mime_typeÚheader_footer_typezOptional[bool]Úis_continuationz Optional[list[FormKeyValuePair]]Úkey_value_pairsÚ	languagesÚlast_modifiedÚ
link_textsÚ	link_urlszOptional[list[int]]Úlink_start_indexeszOptional[list[Link]]ÚlinkszOptional[list[Element]]Úorig_elementsÚ	page_nameÚpage_numberÚ	parent_idÚbcc_recipientÚcc_recipientÚemail_message_idÚ	sent_fromÚsent_toÚsubjectÚ	signatureÚtext_as_htmlzOptional[dict[str, str | int]]Útable_as_cellsr   NzOptional[str | pathlib.Path]ÚNone)%ri   r…   rj   r†   rk   rl   rm   ro   rp   rq   rr   rs   rx   rv   rw   rt   ru   ry   r{   r|   r   r}   r~   r€   r‡   r�   r‚   rƒ   r„   rˆ   r‰   r‹   rŠ   r�   rŒ   r   rK   c%           '      C  s  || _ || _|| _|| _|| _|| _|| _|| _|	| _t	|t
jƒrJt|ƒn|}tj |pZd¡\}%}&|
pl|%pld | _|&pvd | _|| _|| _|| _|| _|| _|| _|| _|| _|| _|| _|| _|| _|| _|| _|| _ || _!|| _"|| _#|| _$|| _%| | _&|!| _'|#| _(|"| _)|$| _*d S )NÚ )+ri   r…   rj   r†   rk   rl   rm   ro   rp   rM   ÚpathlibÚPathrS   ÚosÚpathÚsplitrq   rr   rs   rx   rv   rw   rt   ru   ry   r{   r|   r}   r~   r   r€   r‡   r�   r‚   rƒ   r„   rˆ   r‰   r‹   rŠ   rŒ   r�   r   )'r*   ri   r…   rj   r†   rk   rl   rm   ro   rp   rq   rr   rs   rx   rv   rw   rt   ru   ry   r{   r|   r   r}   r~   r€   r‡   r�   r‚   rƒ   r„   rˆ   r‰   r‹   rŠ   r�   rŒ   r   Zdirectory_pathÚ	file_namer    r    r$   rG   Ù   sL    '
zElementMetadata.__init__ÚobjectrH   rI   c                 C  s   t |tƒsdS | j|jkS )zÄImplments equivalence, like meta == other_meta.

        All fields at all levels must match. Unpopulated fields are not considered except when
        populated in one and not the other.
        F)rM   rh   r6   rO   r    r    r$   rP   -  s    
zElementMetadata.__eq__rS   )Ú	attr_namerK   c                 C  s"   || j v rdS td|› d�ƒ‚dS )z)Only called when attribute doesn't exist.Nz+'ElementMetadata' object has no attribute 'ú')Ú_known_field_namesÚAttributeError)r*   r—   r    r    r$   Ú__getattr__7  s    
zElementMetadata.__getattr__r   )Ú_ElementMetadata__nameÚ_ElementMetadata__valuerK   c                   sD   |d u r || j v rt| |ƒ d S ts2|| jv r2d S tƒ  ||¡ d S r   )r'   Údelattrr   ÚDEBUG_FIELD_NAMESÚsuperÚ__setattr__)r*   rœ   r�   ©rT   r    r$   r¡   =  s    

zElementMetadata.__setattr__r,   )Ú	meta_dictrK   c                 C  s’   ddl m} t |¡}tƒ }| ¡ D ]h\}}|dkrBt |¡|_q$|dkrXt	 |¡|_
q$|dkrl||ƒ|_q$|dkr€t|ƒ|_q$t|||ƒ q$|S )z»Construct from a metadata-dict.

        This would generally be a dict formed using the `.to_dict()` method and stored as JSON
        before "rehydrating" it using this method.
        r   )Ú!elements_from_base64_gzipped_jsonrk   rl   r�   rz   )Úunstructured.staging.baser¤   ÚcopyÚdeepcopyrh   r(   r@   r9   rk   r   rl   r�   Ú#_kvform_rehydrate_internal_elementsrz   Úsetattr)r7   r£   r¤   r*   Ú
field_nameÚfield_valuer    r    r$   r9   G  s    
zElementMetadata.from_dictzMappingProxyType[str, Any]©rK   c                   s   t ‡ fdd„ˆ j ¡ D ƒƒS )züPopulated metadata fields in this object as a read-only dict.

        Basically `self.__dict__` but it needs a little filtering to remove entries like
        "_known_field_names". Note this is a *snapshot* and will not reflect later changes.
        c                   s*   i | ]"\}}|  d ¡s|ˆ jvr||“qS )Ú_)Ú
startswithrŸ   ©r!   rª   r«   r)   r    r$   r%   i  s   þz*ElementMetadata.fields.<locals>.<dictcomp>)r   r'   r(   r)   r    r)   r$   r6   a  s
    
þÿzElementMetadata.fieldsc                   s$   ˆj ‰ t‡ ‡fdd„ˆj ¡ D ƒƒS )a,  Populated non-ad-hoc fields in this object as a read-only dict.

        Only fields declared at the top of this class are included. Ad-hoc fields added to this
        instance by assignment are not. Note this is a *snapshot* and will not reflect changes that
        occur after this call.
        c                   s(   i | ] \}}|ˆ v r|ˆj vr||“qS r    )rŸ   r¯   ©Zknown_field_namesr*   r    r$   r%   z  s   þz0ElementMetadata.known_fields.<locals>.<dictcomp>)r™   r   r'   r(   r)   r    r°   r$   Úknown_fieldsp  s    þÿzElementMetadata.known_fieldsc                 C  sª   ddl m} t t| jƒ¡}| jD ]}| |d¡ q"dd„ | ¡ D ƒ}| j	dur^| j	 
¡ |d< | jdurv| j 
¡ |d< | jdurŽ|| jƒ|d< | jdur¦t| jƒ|d	< |S )
z½Convert this metadata to dict form, suitable for JSON serialization.

        The returned dict is "sparse" in that no key-value pair appears for a field with value
        `None`.
        r   )Úelements_to_base64_gzipped_jsonNc                 S  s&   i | ]\}}|g kr|i kr||“qS r    r    )r!   rª   r#   r    r    r$   r%   �  s   þz+ElementMetadata.to_dict.<locals>.<dictcomp>rk   rl   r�   rz   )r¥   r²   r¦   r§   Údictr6   rŸ   Úpopr(   rk   r+   rl   r�   rz   Ú_kvform_pairs_to_dict)r*   r²   r£   rª   r    r    r$   r+   �  s     
þ



zElementMetadata.to_dictc                 C  s6   t |tƒstdƒ‚|j ¡ D ]\}}t| ||ƒ qdS )ak  Update self with all fields present in `other`.

        Semantics are like those of `dict.update()`.

        - fields present in both `self` and `other` will be updated to the value in `other`.
        - fields present in `other` but not `self` will be added to `self`.
        - fields present in `self` but not `other` are unchanged.
        - `other` is unchanged.
        - both ad-hoc and known fields participate in update with the same semantics.

        Note that fields listed in DEBUG_FIELD_NAMES are skipped in this process. Those can only be
        updated by direct assignment to the instance.
        z@argument to '.update()' must be an instance of 'ElementMetadata'N)rM   rh   rF   r6   r(   r©   )r*   rJ   rª   r«   r    r    r$   Úupdate¢  s    
zElementMetadata.updatezFrozenSet[str]c                 C  s
   t | jƒS )aK  field-names for non-user-defined fields, available on all ElementMetadata instances.

        Note that the first call to this lazyproperty adds a `"_known_field_names"` item to the
        `__dict__` of this instance, so this be called *before* iterating through `self.__dict__`
        to avoid a mid-iteration mutation.
        )Ú	frozensetr>   r)   r    r    r$   r™   ¶  s    
z"ElementMetadata._known_field_names)$NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN)r:   r;   r<   r=   r>   r·   rŸ   rG   rP   r›   r¡   r?   r9   Úpropertyr6   r±   r+   r¶   r   r™   Ú__classcell__r    r    r¢   r$   rh   •   s´   

                                    ÛXT

!rh   c                   @  s6   e Zd ZdZdZdZdZdZdZe	ddœd	d
„ƒZ
dS )ÚConsolidationStrategya  Methods by which a metadata field can be consolidated across a collection of elements.

    These are assigned to `ElementMetadata` field-names immediately below. Metadata consolidation is
    part of the chunking process and may arise elsewhere as well.
    ZdropÚfirstZstring_concatenateÚLIST_CONCATENATEZlist_uniquez dict[str, ConsolidationStrategy]r¬   c              (   C  s¢   | j | j | j | j| j| j | j| j| j| j| j | j | j | j| j| j| j| j| j| j| j | j| j| j| j| j | j| j| j | j | j| j | j | j | j | j| j | j | jdœ'S )a7  Mapping from ElementMetadata field-name to its consolidation strategy.

        Note that only _TextSection objects ("pre-chunks" containing only `Text` elements that are
        not `Table`) have their metadata consolidated, so these strategies are only applicable for
        non-Table Text elements.
        )'ri   r†   r…   rj   rk   rl   rm   rn   ro   rp   rq   rr   rs   rx   rt   ru   rv   rw   ry   r{   r|   r}   r~   r   r€   r‡   Zmax_charactersr�   r‚   rƒ   r„   rˆ   r‰   r‹   rŠ   rŒ   r�   r   rz   )ÚFIRSTÚDROPr¼   ÚLIST_UNIQUEÚSTRING_CONCATENATE©r7   r    r    r$   Úfield_consolidation_strategiesÙ  sP    	Ùz4ConsolidationStrategy.field_consolidation_strategiesN)r:   r;   r<   r=   r¾   r½   rÀ   r¼   r¿   r?   rÂ   r    r    r    r$   rº   Ã  s   rº   Ú_Púlist[Element])ÚelementsrK   c                 C  sd   dd„ | D ƒ}dd„ t |ƒD ƒ}dd„ t| |ƒD ƒ}| D ]&}|jj}|r8||vrRq8|| |j_q8| S )a“  Converts `id` and `parent_id` of elements from UUIDs to hashes.

    This function ensures deterministic IDs by:
    1. Converting each element's UUID into a hash.
    2. Updating the `parent_id` to match the new hash ID of parent elements.

    Args:
        elements: A list of Element objects to update.

    Returns:
        List of updated Element objects with hashes for `id` and `parent_id`.
    c                 S  s   g | ]}|j j‘qS r    )Úmetadatarƒ   )r!   Úer    r    r$   r1     r&   z+assign_and_map_hash_ids.<locals>.<listcomp>c                 S  s&   g | ]\}}t |ƒD ]\}}|‘qqS r    )Ú	enumerate)r!   r­   ÚgroupZseq_on_pager    r    r$   r1     s   c                 S  s   i | ]\}}|j | |¡“qS r    )ÚidÚ
id_to_hash)r!   ÚelementZseq_on_page_counterr    r    r$   r%   #  s   ÿz+assign_and_map_hash_ids.<locals>.<dictcomp>)r   ÚziprÆ   r„   )rÅ   Zpage_numbersZpage_seq_pairsZold_to_new_mappingrÇ   r„   r    r    r$   Úassign_and_map_hash_ids  s    ÿþrÎ   zDCallable[[Callable[_P, list[Element]]], Callable[_P, list[Element]]]r¬   c                  C  s   dddœdd„} | S )a$  Post-process element-metadata for this document.

    This decorator adds a post-processing step to a document partitioner.

    - Adds `metadata_filename` parameter to docstring if not present.
    - Updates element.id to a UUID when `unique_element_ids` argument is provided and True.

    zCallable[_P, list[Element]])ÚfuncrK   c                   sN   ˆ j r*dˆ jjv r*dˆ j vr*ˆ  j d7  _ t ˆ ¡ddddœ‡ fdd„ƒ}|S )	NZmetadata_filenamezT
Metadata Parameters:
	metadata_filename:
		The filename to use in element metadata.z_P.argsz	_P.kwargsrÄ   )r8   ÚkwargsrK   c                    sD   ˆ | i |¤Ž}t ˆ g| ¢R i |¤Ž}| dd¡}|du r@t|ƒ}|S )NÚunique_element_idsF)r   r]   rÎ   )r8   rÐ   rÅ   Z	call_argsrÑ   ©rÏ   r    r$   ÚwrapperG  s    z4process_metadata.<locals>.decorator.<locals>.wrapper)r=   Ú__code__Úco_varnamesÚ	functoolsÚwraps)rÏ   rÓ   r    rÒ   r$   Ú	decorator<  s    
ÿþÿ
z#process_metadata.<locals>.decoratorr    )rØ   r    r    r$   Úprocess_metadata2  s    
rÙ   c                   @  sÈ   e Zd ZdZdZdZdZdZdZdZ	dZ
d	Zd
ZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZ dZ!d Z"d!Z#d"Z$d#Z%d$Z&d%Z'd&Z(d'Z)d(Z*d)Z+d*Z,d+Z-d,Z.e/d-d.„ ƒZ0d/S )0ÚElementTypeÚTitleÚTextÚUncategorizedTextÚNarrativeTextZBulletedTextZ	ParagraphZAbstractZ	ThreadingZFormz
Field-NameÚValuer^   ÚCompositeElementÚImageZPictureÚFigureCaptionZFigureZCaptionÚListÚListItemz	List-itemZCheckedZ	UncheckedZCheckBoxCheckedZCheckBoxUncheckedZRadioButtonCheckedZRadioButtonUncheckedÚAddressÚEmailAddressÚ	PageBreakÚFormulaÚTableÚHeaderZHeadlineZSubheadlinezPage-headerzSection-headerÚFooterZFootnotezPage-footerÚ
PageNumberÚCodeSnippetÚFormKeysValuesÚDocumentDatac                   s   ‡ fdd„t ˆ ƒD ƒS )z¬
        Convert class attributes to a dictionary.

        Returns:
            dict: A dictionary where keys are attribute names and values are attribute values.
        c                   s0   i | ](}t tˆ |ƒƒs| d ¡s|tˆ |ƒ“qS )Ú__)ÚcallableÚgetattrr®   )r!   ÚattrrÁ   r    r$   r%   �  s   þz'ElementType.to_dict.<locals>.<dictcomp>)ÚdirrÁ   r    rÁ   r$   r+   …  s    
þzElementType.to_dictN)1r:   r;   r<   ÚTITLEÚTEXTÚUNCATEGORIZED_TEXTÚNARRATIVE_TEXTÚBULLETED_TEXTÚ	PARAGRAPHÚABSTRACTÚ	THREADINGÚFORMÚ
FIELD_NAMEÚVALUEÚLINKÚCOMPOSITE_ELEMENTÚIMAGEÚPICTUREÚFIGURE_CAPTIONÚFIGUREÚCAPTIONÚLISTÚ	LIST_ITEMÚLIST_ITEM_OTHERZCHECKEDZ	UNCHECKEDZCHECK_BOX_CHECKEDZCHECK_BOX_UNCHECKEDZRADIO_BUTTON_CHECKEDZRADIO_BUTTON_UNCHECKEDÚADDRESSÚEMAIL_ADDRESSÚ
PAGE_BREAKÚFORMULAÚTABLEÚHEADERÚHEADLINEÚSUB_HEADLINEÚPAGE_HEADERÚSECTION_HEADERÚFOOTERÚFOOTNOTEÚPAGE_FOOTERÚPAGE_NUMBERÚCODE_SNIPPETÚFORM_KEYS_VALUESÚDOCUMENT_DATAr?   r+   r    r    r    r$   rÚ   W  s\   rÚ   c                   @  s|   e Zd ZU dZded< dZd dddd	dd
œdd„Zdd„ Zd!ddddœdd„Zdddœdd„Z	e
dd„ ƒZddœdd„ZdS )"ÚElementaï  An element is a semantically-coherent component of a document, often a paragraph.

    There are a few design principles that are followed when creating an element:
    1. It will always have an ID, which by default is a random UUID.
    2. Asking for an ID should always return a string, it can never be None.
    3. ID is lazy, meaning it will be generated when asked for the first time.
    4. When deterministic behavior is needed, the ID can be converted.
        to a hash based on its text `element.id_to_hash(position)`
    4. Even if the `text` attribute is not defined in a subclass, it will default to a blank string.
    6. Assigning a string ID manually is possible, but is meant to be used
        only for deserialization purposes.
    rS   r_   rÝ   Nr   ú)Optional[tuple[tuple[float, float], ...]]rC   úOptional[ElementMetadata]©Ú
element_idrk   Úcoordinate_systemrÆ   rn   c                 C  sv   |d urt |tƒstdƒ‚|| _|d u r.tƒ n|| _|d usD|d urTt||d�| j_|| j_t	| dƒrl| j
nd| _
d S )Nz'element_id must be of type str or None.rE   r_   r�   )rM   rS   rF   Ú_element_idrh   rÆ   r@   rk   rn   Úhasattrr_   )r*   r  rk   r   rÆ   rn   r    r    r$   rG   ¥  s    ÿ
zElement.__init__c                 C  s   | j S r   ©r_   r)   r    r    r$   Ú__str__»  s    zElement.__str__Tr   rH   rA   )Ú
new_systemÚin_placerK   c                   sf   ˆj jdu s(ˆj jjdu s(ˆj jjdu r,dS t‡ ‡fdd„ˆj jjD ƒƒ}|rb|ˆj j_ˆ ˆj j_|S )z¯Converts the element location coordinates to a new coordinate system.

        If inplace is true, changes the coordinates in place and updates the coordinate system.
        Nc                 3  s(   | ] \}}ˆj jjjˆ ||d �V  qdS ))r%  ÚxÚyN)rÆ   rk   rD   Ú!convert_coordinates_to_new_system)r!   r'  r(  ©r%  r*   r    r$   Ú	<genexpr>Ì  s   û
ýz<Element.convert_coordinates_to_new_system.<locals>.<genexpr>)rÆ   rk   rD   rB   rZ   )r*   r%  r&  Znew_coordinatesr    r*  r$   r)  ¾  s    
ÿþýú	

z)Element.convert_coordinates_to_new_systemr`   )Úsequence_numberrK   c                 C  s@   | j j› | j› | j j› |› �}t | ¡ ¡ ¡ dd… | _| j	S )a  Calculates and assigns a deterministic hash as an ID.

        The hash ID is based on element's text, sequence number on page,
        page number and its filename.

        Args:
            sequence_number: index on page

        Returns: new ID value
        Né    )
rÆ   rr   r_   rƒ   ÚhashlibÚsha256ÚencodeÚ	hexdigestr!  rÊ   )r*   r,  Údatar    r    r$   rË   Û  s    zElement.id_to_hashc                 C  s   | j d u rtt ¡ ƒ| _ | j S r   )r!  rS   ÚuuidÚuuid4r)   r    r    r$   rÊ   ê  s    
z
Element.idr,   r¬   c                 C  s   d | j | j| j ¡ dœS )N)Útyper  r_   rÆ   )rÊ   r_   rÆ   r+   r)   r    r    r$   r+   ð  s
    üzElement.to_dict)NNNNN)T)r:   r;   r<   r=   r>   ÚcategoryrG   r$  r)  rË   r¸   rÊ   r+   r    r    r    r$   r  ”  s    
     ú ÿ
r  c                      sT   e Zd ZdZdddddddd	œ‡ fd
d„Zdddœdd„Zddœ‡ fdd„Z‡  ZS )ÚCheckBoxzyA checkbox with an attribute indicating whether its checked or not.

    Primarily used in documents that are forms.
    NFr   r  rC   rH   r  )r  rk   r   ÚcheckedrÆ   rn   c                   s.   |r|nt ƒ }tƒ j|||||d� || _d S )Nr  )rh   r    rG   r8  )r*   r  rk   r   r8  rÆ   rn   r¢   r    r$   rG   ÿ  s    	ûzCheckBox.__init__r–   rI   c                 C  s.   t |tƒsdS t| j|jk| jj|jjkfƒS rL   )rM   r7  rN   r8  rÆ   rk   rO   r    r    r$   rP     s    

þÿzCheckBox.__eq__r,   r¬   c                   s*   t ƒ  ¡ }d|d< | j|d< | j|d< |S )ú-Serialize to JSON-compatible (str keys) dict.r7  r5  r8  r  )r    r+   r8  rÊ   ©r*   Úoutr¢   r    r$   r+     s
    


zCheckBox.to_dict)NNNFNN)r:   r;   r<   r=   rG   rP   r+   r¹   r    r    r¢   r$   r7  ù  s         ù
r7  c                	      sj   e Zd ZdZddddddddd	œ‡ fd
d„Zddœdd„Zdd„ Zddœdd„Zddœ‡ fdd„Z‡  Z	S )rÜ   z:Base element for capturing free text from within document.NrS   r   r  rC   r  zOptional[list[float]])r_   r  rk   r   rÆ   rn   Ú
embeddingsc                   s4   |r|nt ƒ }|| _|| _tƒ j|||||d� d S )N)r  rÆ   rk   r   rn   )rh   r_   r<  r    rG   )r*   r_   r  rk   r   rÆ   rn   r<  r¢   r    r$   rG   (  s    
ûzText.__init__r–   )rJ   c                 C  sB   t |tƒsdS t| j|jk| jj|jjk| j|jk| j|jkfƒS rL   )rM   rÜ   rN   r_   rÆ   rk   r6  r<  rO   r    r    r$   rP   >  s    



üÿzText.__eq__c                 C  s   | j S r   r#  r)   r    r    r$   r$  J  s    zText.__str__zCallable[[str], str])Úcleanersc                 G  s4   | j }|D ]}||ƒ}q
t|tƒs*tdƒ‚|| _ dS )z£Applies a cleaning brick to the text element.

        The function that's passed in should take a string as input and produce a string as
        output.
        z%Cleaner produced a non-string output.N)r_   rM   rS   rF   )r*   r=  Zcleaned_textZcleanerr    r    r$   ÚapplyM  s    

z
Text.applyr,   r¬   c                   s<   t ƒ  ¡ }| j|d< | j|d< | j|d< | jr8| j|d< |S )r9  r  r5  r_   r<  )r    r+   rÊ   r6  r_   r<  r:  r¢   r    r$   r+   \  s    




zText.to_dict)NNNNNN)
r:   r;   r<   r=   rG   rP   r$  r>  r+   r¹   r    r    r¢   r$   rÜ   %  s         ø rÜ   c                   @  s   e Zd ZdZd ZdS )rè   z,An element containing formulas in a documentN©r:   r;   r<   r=   r6  r    r    r    r$   rè   g  s   rè   c                   @  s   e Zd ZdZd ZdS )rà   aA  A chunk formed from text (non-Table) elements.

    Only produced by chunking. An instance may be formed by combining one or more sequential
    elements produced by partitioning. It it also used when text-splitting an "oversized" element,
    a single element that by itself is larger than the requested chunk size.
    Nr?  r    r    r    r$   rà   m  s   rà   c                   @  s   e Zd ZdZd ZdS )râ   z>An element for capturing text associated with figure captions.Nr?  r    r    r    r$   râ   x  s   râ   c                   @  s   e Zd ZdZd ZdS )rÞ   z–NarrativeText is an element consisting of multiple, well-formulated sentences. This
    excludes elements such titles, headers, footers, and captions.Nr?  r    r    r    r$   rÞ   ~  s   rÞ   c                   @  s   e Zd ZdZd ZdS )rä   z;ListItem is a NarrativeText element that is part of a list.Nr?  r    r    r    r$   rä   …  s   rä   c                   @  s   e Zd ZdZd ZdS )rÛ   z$A text element for capturing titles.Nr?  r    r    r    r$   rÛ   ‹  s   rÛ   c                   @  s   e Zd ZdZd ZdS )rå   z'A text element for capturing addresses.Nr?  r    r    r    r$   rå   ‘  s   rå   c                   @  s   e Zd ZdZd ZdS )ræ   z&A text element for capturing addressesNr?  r    r    r    r$   ræ   —  s   ræ   c                   @  s   e Zd ZdZejZdS )rá   z,A text element for capturing image metadata.N)r:   r;   r<   r=   rÚ   r  r6  r    r    r    r$   rá   �  s   rá   c                   @  s   e Zd ZdZd ZdS )rç   z%An element for capturing page breaks.Nr?  r    r    r    r$   rç   £  s   rç   c                   @  s   e Zd ZdZd ZdS )ré   z An element for capturing tables.Nr?  r    r    r    r$   ré   ©  s   ré   c                   @  s   e Zd ZdZdZdS )Ú
TableChunkz*An element for capturing chunks of tables.ré   Nr?  r    r    r    r$   r@  ¯  s   r@  c                   @  s   e Zd ZdZd ZdS )rê   z*An element for capturing document headers.Nr?  r    r    r    r$   rê   µ  s   rê   c                   @  s   e Zd ZdZd ZdS )rë   z*An element for capturing document footers.Nr?  r    r    r    r$   rë   »  s   rë   c                   @  s   e Zd ZdZd ZdS )rí   z'An element for capturing code snippets.Nr?  r    r    r    r$   rí   Á  s   rí   c                   @  s   e Zd ZdZd ZdS )rì   z&An element for capturing page numbers.Nr?  r    r    r    r$   rì   Ç  s   rì   c                   @  s   e Zd ZdZd ZdS )rî   z1An element for capturing Key-Value dicts (forms).Nr?  r    r    r    r$   rî   Í  s   rî   c                   @  s   e Zd ZdZd ZdS )rï   zœAn element for capturing document-level data,
    particularly for large data that does not make sense to
    represent across each element in the document.Nr?  r    r    r    r$   rï   Ó  s   rï   zdict[str, type[Text]]ÚTYPE_TO_TEXT_ELEMENT_MAPzlist[dict[str, Any]]zlist[FormKeyValuePair])Úkv_pairsrK   c                 C  sˆ   ddl m} | D ]h}|d d dur@||d d gƒ\|d d< |d dur|d d dur||d d gƒ\|d d< qttt | ƒS )zÒ
    The key_value_pairs metadata field contains (in the vast majority of cases)
    nested Text elements. Those need to be turned from dicts into Elements explicitly,
    e.g. when partition_json is used.
    r   )Úelements_from_dictsr"   rc   Nr#   )r¥   rC  r
   rX   re   )rB  rC  Úkv_pairr    r    r$   r¨     s    ÿÿr¨   )Úorig_kv_pairsrK   c                 C  st   t  | ¡}|D ]`}|d d dur:|d d  ¡ |d d< |d dur|d d dur|d d  ¡ |d d< q|S )zÜ
    The key_value_pairs metadata field contains (in the vast majority of cases)
    nested Text elements. Those need to be turned from Elements to dicts recursively,
    e.g. when FormKeysValues.to_dict() is used.

    r"   rc   Nr#   )r¦   r§   r+   )rE  rB  rD  r    r    r$   rµ     s    
rµ   )rÚ
__future__r   Úabcr¦   Zdataclassesr5   ÚenumrÖ   r.  r’   r�   r3  Ú	itertoolsr   Útypesr   Útypingr   r   r   r   r	   r
   Ztyping_extensionsr   r   r   Z"unstructured.documents.coordinatesr   r   r   Z&unstructured.partition.utils.constantsr   Zunstructured.utilsr   r   r   r>   r   Z	dataclassr   r@   r^   rb   re   rh   ÚEnumrº   rÃ   rÎ   rÙ   rÚ   ÚABCr  r7  rÜ   rè   rà   râ   rÞ   rä   rÛ   rå   ræ   rá   rç   ré   r@  rê   rë   rí   rì   rî   rï   rõ   r  r  r  rþ   r÷   r  rö   rø   rú   rû   rü   rý   rÿ   r   r  rù   r	  r  r  r  r  r  r  r  r  r  r  r  r
  r  r  r  r  r  r  r  rA  r¨   rµ   r    r    r    r$   Ú<module>   s¼    J  0I#%=e,B	Ú*