a
    ¡˜ëj™  ã                	   @  sŠ   d Z ddlmZ ddlmZmZ ddlmZmZ ddl	m
Z
 ddddddœdd	d
d
d
d	ddœdd„Zddddœdd„ZG dd„ deƒZdS )a“  Implementation of baseline chunking.

This is the "plain-vanilla" chunking strategy. All the fundamental chunking behaviors are present in
this strategy and also in all other strategies. Those are:

- Maximally fill each chunk with sequential elements.
- Isolate oversized elements and divide (only) those chunks by text-splitting.
- Overlap when requested.

"Fancier" strategies add higher-level semantic-unit boundaries to be respected. For example, in the
by-title strategy, section boundaries are respected, meaning a chunk never contains text from two
different sections. When a new section is detected the current chunk is closed and a new one
started.
é    )Úannotations)ÚIterableÚOptional)ÚChunkingOptionsÚ
PreChunker)ÚElementN©Úinclude_orig_elementsÚmax_charactersÚnew_after_n_charsÚoverlapÚoverlap_allzIterable[Element]zOptional[bool]zOptional[int]zlist[Element])Úelementsr	   r
   r   r   r   Úreturnc                C  s   t j|||||d�}t| |ƒS )av  Combine sequential `elements` into chunks, respecting specified text-length limits.

    Produces a sequence of `CompositeElement`, `Table`, and `TableChunk` elements (chunks).

    Parameters
    ----------
    elements
        A list of unstructured elements. Usually the output of a partition function.
    include_orig_elements
        When `True` (default), add elements from pre-chunk to the `.metadata.orig_elements` field
        of the chunk(s) formed from that pre-chunk. Among other things, this allows access to
        original-element metadata that cannot be consolidated and is dropped in the course of
        chunking.
    max_characters
        Hard maximum chunk length. No chunk will exceed this length. A single element that exceeds
        this length will be divided into two or more chunks using text-splitting.
    new_after_n_chars
        A chunk that of this length or greater is not extended to include the next element, even if
        that element would fit without exceeding `max_characters`. A "soft max" length that can be
        used in conjunction with `max_characters` to limit most chunks to a preferred length while
        still allowing larger elements to be included in a single chunk without resorting to
        text-splitting. Defaults to `max_characters` when not specified, which effectively disables
        any soft window. Specifying 0 for this argument causes each element to appear in a chunk by
        itself (although an element with text longer than `max_characters` will be still be split
        into two or more chunks).
    overlap
        Specifies the length of a string ("tail") to be drawn from each chunk and prefixed to the
        next chunk as a context-preserving mechanism. By default, this only applies to split-chunks
        where an oversized element is divided into multiple chunks by text-splitting.
    overlap_all
        Default: `False`. When `True`, apply overlap between "normal" chunks formed from whole
        elements and not subject to text-splitting. Use this with caution as it produces a certain
        level of "pollution" of otherwise clean semantic chunk boundaries.
    r   )Ú_BasicChunkingOptionsÚnewÚ_chunk_elements)r   r	   r
   r   r   r   Úopts© r   úZ/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/unstructured/chunking/basic.pyÚchunk_elements   s    ,ûr   r   )r   r   r   c                 C  s   dd„ t  | |¡D ƒS )z(Implementation of actual basic chunking.c                 S  s   g | ]}|  ¡ D ]}|‘qqS r   )Ziter_chunks)Ú.0Z	pre_chunkÚchunkr   r   r   Ú
<listcomp>S   s   þz#_chunk_elements.<locals>.<listcomp>)r   Ziter_pre_chunks)r   r   r   r   r   r   O   s    
þr   c                   @  s   e Zd ZdZdS )r   zOptions for `basic` chunking.N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r   r   r   Z   s   )r   Ú
__future__r   Útypingr   r   Zunstructured.chunking.baser   r   Zunstructured.documents.elementsr   r   r   r   r   r   r   r   Ú<module>   s   ù7