a
    ¡˜ëjé,  ã                   @  s  d Z ddlmZ ddlZddlZddlZddlZddlZddl	m
Z
mZmZmZ ddlmZ ddlmZmZ ddlmZ ddlmZ dd	lmZ ed
ƒZdddœdd„Zg d¢g d¢dœZefddddœdd„Zd#dddœdd„Zdddœdd „Zdddœd!d"„ZdS )$z>Helpers used across multiple partitioners to compute metadata.é    )ÚannotationsN)ÚAnyÚCallableÚIteratorÚSequence)Ú	ParamSpec)ÚElementÚElementMetadata)ÚFileType)Úapply_lang_metadata)Úget_call_args_applying_defaultsÚ_PÚstrz
str | None)ÚfilenameÚreturnc                 C  s.   t j | ¡sdS tj t j | ¡¡}| d¡S )a
  Modification time of file at path `filename`, if it exists.

    Returns `None` when `filename` is not a path to a file on the local filesystem.

    Otherwise returns date and time in ISO 8601 string format (YYYY-MM-DDTHH:MM:SS) like
    "2024-03-05T17:02:53".
    Nz%Y-%m-%dT%H:%M:%S%z)ÚosÚpathÚisfileÚdtÚdatetimeÚfromtimestampÚgetmtimeÚstrftime)r   Zmodify_date© r   úe/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/unstructured/partition/common/metadata.pyÚget_last_modified_date   s    r   )	ÚTextÚUncategorizedTextÚNarrativeTextÚListItemÚBulletedTextÚTableÚFigureCaptionÚCheckBoxr!   )
ÚTitler   r   r   r   r    r!   r"   r#   r!   )r$   ÚHeaderzSequence[Element]zdict[str, list[str]]úlist[Element])ÚelementsÚrulesetr   c           
      C  sÀ   g }| D ]®}|j jdurqd}t|ddƒ}t|j ddƒp:d}|sBq|r¤|d }t|dƒ}t|j ddƒphd}	||krz|	|k s’||krš|| |g ¡v rš|j}q¤| ¡  qB||j _| |¡ qt| ƒS )aK  Sets `.metadata.parent_id` for each element it applies to.

    `parent_id` assignment is based on the element's category and depth. The importance of an
    element's category is determined by a rule set. The rule set trumps category_depth. That is,
    category_depth is only relevant when elements are of the same category.
    NÚcategoryZcategory_depthr   éÿÿÿÿ)ÚmetadataÚ	parent_idÚgetattrÚgetÚidÚpopÚappendÚlist)
r'   r(   ÚstackÚelementr,   Zelement_categoryZelement_category_depthZtop_elementZtop_element_categoryZtop_element_category_depthr   r   r   Úset_element_hierarchy@   sB    	
ýú
ÿþüû
r5   zFileType | NonezDCallable[[Callable[_P, list[Element]]], Callable[_P, list[Element]]])Ú	file_typer   c                   s   dddœ‡ fdd„}|S )aÑ  Post-process element-metadata for this document.

    This decorator adds a post-processing step to a partitioner, primarily to apply metadata that
    is common to all partitioners. It assumes the following responsibilities:

      - Hash element-ids. Computes and applies SHA1 hash element.id when `unique_element_ids`
        argument is False.

      - Element Hierarchy. Computes and applies `parent_id` metadata based on `category_depth`
        etc. added by partitioner.

      - Language metadata. Computes and applies `language` metadata based on a language detection
        model.

      - Apply `filetype` (MIME-type) metadata. There are three cases; first one in this order that
        applies is used:

          - `metadata_file_type` argument is present in call, use that.
          - `file_type` decorator argument is populated, use that.
          - `file_type` decorator argument is omitted or None, don't apply `.metadata.filetype`
            (assume the partitioner will do that for itself, like `partition_image()`.

      - Replace `filename` with `metadata_filename` when present.

      - Replace `last_modified` with `metadata_last_modified` when present.

      - Apply `url` metadata when present.
    zCallable[_P, list[Element]])Úfuncr   c                   s&   t  ˆ ¡ddddœ‡‡ fdd„ƒ}|S )a  The decorator function itself.

        This function is returned by the `apply_metadata()` function and is the actual decorator.
        Think of `apply_metadata()` as a factory function that configures this decorator, in
        particular by setting its `file_type` value.
        z_P.argsz	_P.kwargsr&   )ÚargsÚkwargsr   c                    s  ˆ| i |¤Ž}t ˆg| ¢R i |¤Ž}t|ƒ}| d¡}| dd¡}tt|||d�ƒ}i }| d¡pdˆ }|d urx|j|d< | d¡pŠ| d¡}|r˜||d< | d	¡}	|	r®|	|d
< | d¡}
|
rÄ|
|d< |D ]$}|jjrÖqÈ|j t	f i |¤Ž¡ qÈ| dd¡}|du �rt
|ƒ}t|ƒ}|S )NÚ	languagesÚdetect_language_per_elementF)r'   r:   r;   Úmetadata_file_typeZfiletypeZmetadata_filenamer   Úmetadata_last_modifiedZlast_modifiedÚurlÚunique_element_ids)r   Ú _uniqueify_elements_and_metadatar.   r2   r   Z	mime_typer+   Zattached_to_filenameÚupdater	   Ú_assign_hash_idsr5   )r8   r9   r'   Z	call_argsr:   r;   Zmetadata_kwargsr<   r   r=   r>   r4   r?   )r6   r7   r   r   Úwrapper    sD    
ýÿ	



z2apply_metadata.<locals>.decorator.<locals>.wrapper)Ú	functoolsÚwraps)r7   rC   ©r6   )r7   r   Ú	decorator˜   s    Qz!apply_metadata.<locals>.decoratorr   )r6   rG   r   rF   r   Úapply_metadatax   s     \rH   ©r'   r   c                 C  sD   dd„ | D ƒ}dd„ t  |¡D ƒ}t| |ƒD ]\}}| |¡ q,| S )a.  Converts `.id` of each element from UUID to hash.

    The hash is based on the `.text` of the element, but also on its page-number and sequence number
    on that page. This provides for deterministic results even when the document is split into one
    or more fragments for parallel processing.
    c                 S  s   g | ]}|j j‘qS r   )r+   Zpage_number)Ú.0Úer   r   r   Ú
<listcomp>ÿ   ó    z$_assign_hash_ids.<locals>.<listcomp>c                 S  s&   g | ]\}}t |ƒD ]\}}|‘qqS r   )Ú	enumerate)rJ   Ú_ÚgroupZseq_on_pager   r   r   rL      s   þ)Ú	itertoolsÚgroupbyÚzipZ
id_to_hash)r'   Zpage_numbersZpage_seq_numbersr4   Zseq_on_page_counterr   r   r   rB   ÷   s    þrB   c                 C  s   dddœdd„}t || ƒƒS )zÞEnsure each of `elements` and their metadata are unique instances.

    This prevents hard-to-diagnose bugs downstream when mutating one element unexpectedly also
    mutates others because they are the same instance.
    r&   zIterator[Element]rI   c                 s  sp   t ƒ }t ƒ }| D ]Z}t|ƒ|v r*t |¡}t|jƒ|v rFt |j¡|_| t|ƒ¡ | t|jƒ¡ |V  qdS )zLSubstitute deep-copies of any non-unique elements or metadata in `elements`.N)Úsetr/   ÚcopyÚdeepcopyr+   Úadd)r'   Zseen_elementsZseen_metadatar4   r   r   r   Úiter_unique_elements  s    
z>_uniqueify_elements_and_metadata.<locals>.iter_unique_elements)r2   )r'   rX   r   r   r   r@     s    r@   )N) Ú__doc__Ú
__future__r   rU   r   r   rD   rQ   r   Útypingr   r   r   r   Ztyping_extensionsr   Zunstructured.documents.elementsr   r	   Zunstructured.file_utils.modelr
   Z"unstructured.partition.common.langr   Zunstructured.utilsr   r   r   ZHIERARCHY_RULE_SETr5   rH   rB   r@   r   r   r   r   Ú<module>   s.   ôÿ9 ÿ