a
    ¡˜ëjª  ã                   @  s¶   d Z ddlmZ ddlmZmZmZ ddlmZm	Z	m
Z
mZmZmZmZ ddlmZ ddlmZ ddddddddœd	d
dd
dd
d
dddœ	dd„Zd	dddœdd„ZG dd„ de
ƒZdS )zdImplementation of chunking by title.

Main entry point is the `@add_chunking_strategy()` decorator.
é    )Úannotations)ÚIterableÚIteratorÚOptional)ÚCHUNK_MULTI_PAGE_DEFAULTÚBoundaryPredicateÚChunkingOptionsÚPreChunkCombinerÚ
PreChunkerÚis_on_next_pageÚis_title)ÚElement)ÚlazypropertyN©Úcombine_text_under_n_charsÚinclude_orig_elementsÚmax_charactersÚmultipage_sectionsÚnew_after_n_charsÚoverlapÚoverlap_allzIterable[Element]zOptional[int]zOptional[bool]zlist[Element])	Úelementsr   r   r   r   r   r   r   Úreturnc          	   	   C  s"   t j|||||||d�}t| |ƒS )a�	  Uses title elements to identify sections within the document for chunking.

    Splits off into a new CompositeElement when a title is detected or if metadata changes, which
    happens when page numbers or sections change. Cuts off sections once they have exceeded a
    character length of max_characters.

    Parameters
    ----------
    elements
        A list of unstructured elements. Usually the output of a partition function.
    combine_text_under_n_chars
        Combines elements (for example a series of titles) until a section reaches a length of
        n characters. Defaults to `max_characters` which combines chunks whenever space allows.
        Specifying 0 for this argument suppresses combining of small chunks. Note this value is
        "capped" at the `new_after_n_chars` value since a value higher than that would not change
        this parameter's effect.
    include_orig_elements
        When `True` (default), add elements from pre-chunk to the `.metadata.orig_elements` field
        of the chunk(s) formed from that pre-chunk. Among other things, this allows access to
        original-element metadata that cannot be consolidated and is dropped in the course of
        chunking.
    max_characters
        Chunks elements text and text_as_html (if present) into chunks of length
        n characters (hard max)
    multipage_sections
        If True, sections can span multiple pages. Defaults to True.
    new_after_n_chars
        Cuts off new sections once they reach a length of n characters (soft max). Defaults to
        `max_characters` when not specified, which effectively disables any soft window.
        Specifying 0 for this argument causes each element to appear in a chunk by itself (although
        an element with text longer than `max_characters` will be still be split into two or more
        chunks).
    overlap
        Specifies the length of a string ("tail") to be drawn from each chunk and prefixed to the
        next chunk as a context-preserving mechanism. By default, this only applies to split-chunks
        where an oversized element is divided into multiple chunks by text-splitting.
    overlap_all
        Default: `False`. When `True`, apply overlap between "normal" chunks formed from whole
        elements and not subject to text-splitting. Use this with caution as it entails a certain
        level of "pollution" of otherwise clean semantic chunk boundaries.
    r   )Ú_ByTitleChunkingOptionsÚnewÚ_chunk_by_title)	r   r   r   r   r   r   r   r   Úopts© r   úZ/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/unstructured/chunking/title.pyÚchunk_by_title   s    4ù	r   r   )r   r   r   c                 C  s&   t t | |¡|d� ¡ }dd„ |D ƒS )z-Implementation of actual "by-title" chunking.)r   c                 S  s   g | ]}|  ¡ D ]}|‘qqS r   )Ziter_chunks)Ú.0Z	pre_chunkÚchunkr   r   r   Ú
<listcomp>_   ó    z#_chunk_by_title.<locals>.<listcomp>)r	   r
   Ziter_pre_chunksZiter_combined_pre_chunks)r   r   Z
pre_chunksr   r   r   r   W   s    ÿ
r   c                      s\   e Zd ZdZeddœdd„ƒZeddœdd„ƒZed	dœd
d„ƒZddœ‡ fdd„Z‡  Z	S )r   a9  Adds the by-title-specific chunking options to the base case.

    `by_title`-specific options:

    combine_text_under_n_chars
        A remedy to over-chunking caused by elements mis-identified as Title elements.
        Every Title element would start a new chunk and this setting mitigates that, at the
        expense of sometimes violating legitimate semantic boundaries.
    multipage_sections
        Indicates that page-boundaries should not be respected while chunking, i.e. elements
        appearing on two different pages can appear in the same chunk.
    ztuple[BoundaryPredicate, ...]©r   c                   s   ddœ‡ fdd„}t |ƒ ƒS )a(  The semantic-boundary detectors to be applied to break pre-chunks.

        For the `by_title` strategy these are sections indicated by a title (section-heading), an
        explicit section metadata item (only present for certain document types), and optionally
        page boundaries.
        zIterator[BoundaryPredicate]r$   c                   3  s   t V  ˆ jstƒ V  d S )N)r   r   r   r   ©Úselfr   r   Úiter_boundary_predicatesy   s    zM_ByTitleChunkingOptions.boundary_predicates.<locals>.iter_boundary_predicates)Útuple)r&   r'   r   r%   r   Úboundary_predicatesp   s    	z+_ByTitleChunkingOptions.boundary_predicatesÚintc                 C  s   | j  d¡}|du r| jS |S )a<  Combine consecutive text pre-chunks if former is smaller than this and both will fit.

        - Does not combine text chunks if together they would exceed the chunking window.
        - Defaults to `max_characters` when not specified.
        - Is reduced to `new_after_n_chars` when it exceeds that value.
        r   N)Ú_kwargsÚgetÚhard_max©r&   Ú	arg_valuer   r   r   r   €   s    	z2_ByTitleChunkingOptions.combine_text_under_n_charsÚboolc                 C  s    | j  d¡}|du rtS t|ƒS )z0When False, break pre-chunks on page-boundaries.r   N)r+   r,   r   r0   r.   r   r   r   r   Œ   s    z*_ByTitleChunkingOptions.multipage_sectionsÚNonec                   sL   t ƒ  ¡  | jdk r$td| j› �ƒ‚| j| jkrHtd| j› d| j› �ƒ‚dS )z2Raise ValueError if request option-set is invalid.r   z8'combine_text_under_n_chars' argument must be >= 0, got zR'combine_text_under_n_chars' argument must not exceed `max_characters` value, got z > N)ÚsuperÚ	_validater   Ú
ValueErrorr-   r%   ©Ú	__class__r   r   r3   ’   s    

ÿÿ	ÿÿÿz!_ByTitleChunkingOptions._validate)
Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r)   r   r   r3   Ú__classcell__r   r   r5   r   r   b   s   )r:   Ú
__future__r   Útypingr   r   r   Zunstructured.chunking.baser   r   r   r	   r
   r   r   Zunstructured.documents.elementsr   Zunstructured.utilsr   r   r   r   r   r   r   r   Ú<module>   s   $	÷"@