a
    ¡˜ëj?! ã                   @  s€  d Z ddlmZ ddlmZmZ ddlZddlmZ ddl	m
Z
mZmZmZmZmZmZmZmZmZmZmZmZmZ ddlmZmZmZ ddlmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z& G d	d
„ d
ƒZ'G dd„ dƒZ(G dd„ dƒZ)G dd„ dƒZ*G dd„ dƒZ+G dd„ dƒZ,G dd„ dƒZ-G dd„ dƒZ.G dd„ dƒZ/G dd„ dƒZ0G dd„ dƒZ1G dd „ d ƒZ2G d!d"„ d"ƒZ3G d#d$„ d$ƒZ4dS )%z<Unit-test suite for the `unstructured.chunking.base` module.é    )Úannotations)ÚAnyÚSequenceN)Úfragment_fromstring)ÚChunkingOptionsÚPreChunkÚPreChunkBuilderÚPreChunkCombinerÚ
PreChunkerÚ_CellAccumulatorÚ_ChunkerÚ_HtmlTableSplitterÚ_PreChunkAccumulatorÚ_RowAccumulatorÚ_TableChunkerÚ_TextSplitterÚis_on_next_pageÚis_title)ÚHtmlCellÚHtmlRowÚ	HtmlTable)
ÚCheckBoxÚCompositeElementÚElementÚElementMetadataÚImageÚ	PageBreakÚTableÚ
TableChunkÚTextÚTitlec                   @  sü   e Zd ZdZej dg d¢¡ddœdd„ƒZdd	„ Zej d
ddg¡ddd
œdd„ƒZ	ej dddidfddidfddidfi dfg¡dddœdd„ƒZ
ej dddg¡ddœdd„ƒZdd „ Zd!d"„ Zd#d$„ Zd%d&„ Zd'd(„ Zd)d*„ Zd+d,„ Zd-d.„ ZdS )/ÚDescribeChunkingOptionszIUnit-test suite for `unstructured.chunking.base.ChunkingOptions` objects.Úmax_characters)r   éÿÿÿÿéÖÿÿÿÚint©r"   c                 C  sF   t jtd|› �d�� t|d� ¡  W d   ƒ n1 s80    Y  d S )Nz+'max_characters' argument must be > 0, got ©Úmatchr&   ©ÚpytestÚraisesÚ
ValueErrorr   Ú	_validate)Úselfr"   © r/   úc/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/test_unstructured/chunking/test_base.pyÚ/it_rejects_max_characters_not_greater_than_zero2   s
    þzGDescribeChunkingOptions.it_rejects_max_characters_not_greater_than_zeroc                 C  s4   zt dd� ¡  W n ty.   t d¡ Y n0 dS )a:  Caller can specify `max_characters` arg without specifying any others.

        In particular, When `combine_text_under_n_chars` is not specified it defaults to the value
        of `max_characters`; it has no fixed default value that can be greater than `max_characters`
        and trigger an exception.
        é2   r&   z3did not accept `max_characters` as option by itselfN)r   r-   r,   r*   Úfail©r.   r/   r/   r0   Ú=it_does_not_complain_when_specifying_max_characters_by_itself:   s    zUDescribeChunkingOptions.it_does_not_complain_when_specifying_max_characters_by_itself)Úcombine_text_under_n_charsÚexpected_value)Nr   )é*   r8   z
int | Nonec                 C  s   t |d�}|j|ksJ ‚dS )a’  Subclasses can store `combine_text_under_n_chars` but must validate and enable it.

        The `combine_text_under_n_chars` option is not used by all chunkers and its behavior can
        differ between subtypes. It is present in and stored by the contructur but it defaults to
        `0` (no pre-chunk combining) and must be overridden by subclasses to give it the desired
        behavior.
        ©r6   N)r   r6   )r.   r6   r7   Úoptsr/   r/   r0   ÚQit_accepts_combine_text_under_n_chars_in_constructor_but_defaults_to_no_combiningF   s    
ziDescribeChunkingOptions.it_accepts_combine_text_under_n_chars_in_constructor_but_defaults_to_no_combining)Úkwargsr7   Úinclude_orig_elementsTFNzdict[str, Any]Úboolc                 C  s   t f i |¤Žj|u sJ ‚d S ©N)r   r=   )r.   r<   r7   r/   r/   r0   Ú?it_knows_whether_to_include_orig_elements_in_the_chunk_metadataV   s    zWDescribeChunkingOptions.it_knows_whether_to_include_orig_elements_in_the_chunk_metadataÚn_charsr#   r$   )rA   c                 C  sF   t jtd|› �d�� t|d� ¡  W d   ƒ n1 s80    Y  d S )Nz/'new_after_n_chars' argument must be >= 0, got r'   ©Únew_after_n_charsr)   )r.   rA   r/   r/   r0   Ú1it_rejects_new_after_n_chars_for_n_less_than_zerod   s
    þzIDescribeChunkingOptions.it_rejects_new_after_n_chars_for_n_less_than_zeroc                 C  sB   t jtdd��  tddd� ¡  W d   ƒ n1 s40    Y  d S )NzE'overlap' argument must be less than `max_characters`, got 300 >= 200r'   éÈ   i,  ©r"   Úoverlapr)   r4   r/   r/   r0   Ú/it_rejects_overlap_not_less_than_max_charactersl   s
    þzGDescribeChunkingOptions.it_rejects_overlap_not_less_than_max_charactersc                 C  sF   t dd�}z| ¡  W n ty2   t d¡ Y n0 |jdksBJ ‚dS )zPCaller can specify `new_after_n_chars` arg without specifying any other options.rE   rB   z6did not accept `new_after_n_chars` as option by itselfN©r   r-   r,   r*   r3   Úsoft_max©r.   r:   r/   r/   r0   Ú@it_does_not_complain_when_specifying_new_after_n_chars_by_itselfs   s    
zXDescribeChunkingOptions.it_does_not_complain_when_specifying_new_after_n_chars_by_itselfc                 C  s$   t dd�}| ¡  |jdks J ‚dS )z¯Specifying `new_after_n_chars=0` places each element into its own pre-chunk.

        This puts each element into its own chunk, although long chunks are still split.
        r   rB   N)r   r-   rJ   rK   r/   r/   r0   ÚIit_accepts_0_for_new_after_n_chars_to_put_each_element_into_its_own_chunk}   s    
zaDescribeChunkingOptions.it_accepts_0_for_new_after_n_chars_to_put_each_element_into_its_own_chunkc                 C  sH   t ddd�}z| ¡  W n ty4   t d¡ Y n0 |jdksDJ ‚dS )zá`new_after_n_chars` > `max_characters` doesn't affect chunking behavior.

        So rather than raising an exception or warning, we just cap that value at `max_characters`
        which is the behavioral equivalent.
        i¼  i+  ©r"   rC   z@did not accept `new_after_n_chars` greater than `max_characters`NrI   rK   r/   r/   r0   Ú;it_silently_accepts_new_after_n_chars_greater_than_maxchars‡   s    zSDescribeChunkingOptions.it_silently_accepts_new_after_n_chars_greater_than_maxcharsc                 C  s   t dd�jdksJ ‚d S )Né
   ©rG   )r   rG   r4   r/   r/   r0   Ú2it_knows_how_much_overlap_to_apply_to_split_chunks•   s    zJDescribeChunkingOptions.it_knows_how_much_overlap_to_apply_to_split_chunksc                 C  s   t ddd�jdksJ ‚d S )NrP   T©rG   Úoverlap_all©r   Zinter_chunk_overlapr4   r/   r/   r0   ÚSand_it_uses_the_same_value_for_inter_chunk_overlap_when_asked_to_overlap_all_chunks˜   s    zkDescribeChunkingOptions.and_it_uses_the_same_value_for_inter_chunk_overlap_when_asked_to_overlap_all_chunksc                 C  s   t dd�jdksJ ‚d S )NrP   rQ   r   rU   r4   r/   r/   r0   Ú-but_it_does_not_overlap_pre_chunks_by_default›   s    zEDescribeChunkingOptions.but_it_does_not_overlap_pre_chunks_by_defaultc                 C  s   t ƒ jdksJ ‚d S )Nz

)r   Ztext_separatorr4   r/   r/   r0   Ú"it_knows_the_text_separator_stringž   s    z:DescribeChunkingOptions.it_knows_the_text_separator_string)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r*   ÚmarkÚparametrizer1   r5   r;   r@   rD   rH   rL   rM   rO   rR   rV   rW   rX   r/   r/   r/   r0   r!   /   s6   ÿ


üþ	

r!   c                   @  s   e Zd ZdZdd„ ZdS )ÚDescribePreChunkerzDUnit-test suite for `unstructured.chunking.base.PreChunker` objects.c                 C  s6  t dƒtdƒtdƒt dƒtdƒtdƒtƒ g}tddd	�}tj||d
�}t|ƒ}t|tƒs\J ‚|j	t dƒtdƒgksvJ ‚t|ƒ}t|tƒsŒJ ‚|j	tdƒgks J ‚t|ƒ}t|tƒs¶J ‚|j	t dƒtdƒgksÐJ ‚t|ƒ}t|tƒsæJ ‚|j	tdƒtƒ gksþJ ‚t
 t¡� t|ƒ W d   ƒ n1 �s(0    Y  d S )NúLorem Ipsumz8Lorem ipsum dolor sit amet, consectetur adipiscing elit.zBSed do eiusmod tempor incididunt ut labore et dolore magna aliqua.zUt EnimzHUt enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi.z#Ut aliquip ex ea commodo consequat.é–   éA   rN   ©r:   )r    r   r   r   r
   Ziter_pre_chunksÚnextÚ
isinstancer   Ú	_elementsr*   r+   ÚStopIteration)r.   Úelementsr:   Úpre_chunk_iterÚ	pre_chunkr/   r/   r0   ÚGit_gathers_elements_into_pre_chunks_respecting_the_specified_chunk_sizeª   s@    ù
þ
ÿ
þ
zZDescribePreChunker.it_gathers_elements_into_pre_chunks_respecting_the_specified_chunk_sizeN)rY   rZ   r[   r\   rk   r/   r/   r/   r0   r_   §   s   r_   c                   @  sÊ   e Zd ZdZdd„ Zdd„ Zdd„ Zdd	„ Zej	 
d
edƒedƒfedƒedƒfg¡ddd
œdd„ƒZej	 
dedƒedƒg¡ddœdd„ƒZdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zd d!„ Zd"d#„ Zd$S )%ÚDescribePreChunkBuilderzAUnit-test suite for `unstructured.chunking.base.PreChunkBuilder`.c                 C  s0   t tdd�d�}|jdksJ ‚|jdks,J ‚d S )Nr2   r&   rc   r   )r   r   Ú_text_lengthÚ_remaining_space©r.   Úbuilderr/   r/   r0   Úit_is_empty_on_constructionØ   s    z3DescribePreChunkBuilder.it_is_empty_on_constructionc                 C  sh   t tdd�d�}| tdƒ¡ |jdks,J ‚|jdks:J ‚| tdƒ¡ |jdksVJ ‚|jd	ksdJ ‚d S )
Nra   r&   rc   ÚIntroductioné   éˆ   úbLorem ipsum dolor sit amet consectetur adipiscing elit. In rhoncus ipsum sedlectus porta volutpat.ép   é$   )r   r   Úadd_elementr    rm   rn   r   ro   r/   r/   r0   Ú#it_accumulates_elements_added_to_itÞ   s    ÿÿz;DescribePreChunkBuilder.it_accumulates_elements_added_to_itc                 C  s"   t tƒ d�}| td ƒ¡sJ ‚d S ©Nrc   )r   r   Úwill_fitr   ro   r/   r/   r0   Ú)it_will_fit_when_element_has_none_as_textî   s    zADescribePreChunkBuilder.it_will_fit_when_element_has_none_as_textc                 C  s"   t tƒ d�}| tdƒ¡sJ ‚d S )Nrc   áè  abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd abcd )r   r   r{   r   ro   r/   r/   r0   Ú+it_will_fit_an_oversized_element_when_emptyò   s    zCDescribePreChunkBuilder.it_will_fit_an_oversized_element_when_empty)Úexisting_elementÚnext_elementÚabcdr}   úHeading
Cell textr   c                 C  s(   t tƒ d�}| |¡ | |¡r$J ‚d S rz   )r   r   rx   r{   )r.   r   r€   rp   r/   r/   r0   Ú+but_not_when_it_already_contains_an_elementö   s    

zCDescribePreChunkBuilder.but_not_when_it_already_contains_an_elementÚelementzFruits
Mango©r„   c                 C  s,   t tƒ d�}| tdƒ¡ | |¡s(J ‚d S )Nrc   r‚   )r   r   rx   r   r{   )r.   r„   rp   r/   r/   r0   ÚIit_will_accept_another_element_that_fits_when_it_already_contains_a_table  s    zaDescribePreChunkBuilder.it_will_accept_another_element_that_fits_when_it_already_contains_a_tablec                 C  s6   t tddd�d�}| tdƒ¡ | tdƒ¡r2J ‚d S )Néd   r2   rN   rc   ú7Lorem ipsum dolor sit amet consectetur adipiscing elit.zIn rhoncus ipsum.©r   r   rx   r   r{   ro   r/   r/   r0   ÚBit_will_not_fit_an_element_when_it_already_exceeds_the_soft_maxlen  s
    ÿzZDescribePreChunkBuilder.it_will_not_fit_an_element_when_it_already_exceeds_the_soft_maxlenc                 C  s4   t tdd�d�}| tdƒ¡ | tdƒ¡r0J ‚d S )Nr‡   r&   rc   rˆ   z,In rhoncus ipsum sed lectus portos volutpat.r‰   ro   r/   r/   r0   ÚQand_it_will_not_fit_an_element_when_that_would_cause_it_to_exceed_the_hard_maxlen  s    ÿÿziDescribePreChunkBuilder.and_it_will_not_fit_an_element_when_that_would_cause_it_to_exceed_the_hard_maxlenc                 C  s4   t tdd�d�}| tdƒ¡ | tdƒ¡s0J ‚d S )Nr‡   r&   rc   rˆ   z+In rhoncus ipsum sed lectus porto volutpat.r‰   ro   r/   r/   r0   Ú$but_it_will_fit_an_element_that_fits!  s
    ÿz<DescribePreChunkBuilder.but_it_will_fit_an_element_that_fitsc                 C  s€   t tdd�d�}| tdƒ¡ | tdƒ¡ t| ¡ ƒ}|jdksFJ ‚|jdksTJ ‚t	|t
ƒsbJ ‚|jtdƒtdƒgks|J ‚d S )Nra   r&   rc   rr   ru   r   )r   r   rx   r    r   rd   Úflushrm   rn   re   r   rf   )r.   rp   rj   r/   r/   r0   Ú?it_generates_a_PreChunk_when_flushed_and_resets_itself_to_empty*  s"    ÿÿÿþzWDescribePreChunkBuilder.it_generates_a_PreChunk_when_flushed_and_resets_itself_to_emptyc                 C  sH   t tdd�d�}t| ¡ ƒ}|g ks(J ‚|jdks6J ‚|jdksDJ ‚d S )Nra   r&   rc   r   )r   r   Úlistr�   rm   rn   )r.   rp   Ú
pre_chunksr/   r/   r0   Ú8but_it_does_not_generate_a_pre_chunk_on_flush_when_emptyE  s
    zPDescribePreChunkBuilder.but_it_does_not_generate_a_pre_chunk_on_flush_when_emptyc                 C  sÈ   t ddd�}t|d�}| tdƒ¡ t| ¡ ƒd }t|tƒsBJ ‚|jdksPJ ‚| t	dƒ¡ t| ¡ ƒd }t|tƒs|J ‚|jdksŠJ ‚| td	ƒ¡ t| ¡ ƒd }t|tƒs¶J ‚|jd
ksÄJ ‚d S )Né   TrS   rc   rˆ   r   ú+In rhoncus ipsum sed lectus porta volutpat.z<dipiscing elit.

In rhoncus ipsum sed lectus porta volutpat.z%Donec semper facilisis metus finibus.z6porta volutpat.

Donec semper facilisis metus finibus.)
r   r   rx   r   r�   r�   re   r   Ú_textr   )r.   r:   rp   rj   r/   r/   r0   ÚBit_computes_overlap_from_each_pre_chunk_and_applies_it_to_the_nextN  s    
zZDescribePreChunkBuilder.it_computes_overlap_from_each_pre_chunk_and_applies_it_to_the_nextc                 C  sL   t tdd�d�}| tdƒ¡ | tdƒ¡ |jdks:J ‚|jdksHJ ‚d S )Nr2   r&   rc   ZabcdeZfghijrs   rw   )r   r   rx   r   rm   rn   ro   r/   r/   r0   ÚLit_considers_separator_length_when_computing_text_length_and_remaining_spaced  s
    zdDescribePreChunkBuilder.it_considers_separator_length_when_computing_text_length_and_remaining_spaceN)rY   rZ   r[   r\   rq   ry   r|   r~   r*   r]   r^   r   r   rƒ   r†   rŠ   r‹   rŒ   rŽ   r‘   r•   r–   r/   r/   r/   r0   rl   Õ   s*   þþ		rl   c                   @  sz  e Zd ZdZej ddddgdddgdfdddgdddgdfdddgdddgd	fdddgdd
dgd	fdddgdd
dgd	fg¡ddddddœdd„ƒZdd„ Zdd„ Z	dd„ Z
ej dg d¢¡ddddœdd„ƒZdd„ Zej ddd g¡dddœd!d"„ƒZej d#edƒedƒgd$d%fedƒedƒedƒgd&d'fedƒedƒgd(d)fedƒedƒedƒgdd*fg¡d+ddd#œd,d-„ƒZd.S )/ÚDescribePreChunkzBUnit-test suite for `unstructured.chunking.base.PreChunk` objects.)Úoverlap_pfxÚtextsÚother_overlap_pfxÚother_textsr7   ZfooÚbarZbazTÚ ZfobFZbahZdahÚstrz	list[str]r>   c           	      C  sJ   t ƒ }tdd„ |D ƒ||d�}tdd„ |D ƒ||d�}||k|u sFJ ‚d S )Nc                 S  s   g | ]}t |ƒ‘qS r/   ©r   ©Ú.0Útr/   r/   r0   Ú
<listcomp>’  ó    z[DescribePreChunk.it_knows_when_it_is_equal_to_another_PreChunk_instance.<locals>.<listcomp>©Úoverlap_prefixr:   c                 S  s   g | ]}t |ƒ‘qS r/   rŸ   r    r/   r/   r0   r£   ”  r¤   )r   r   )	r.   r˜   r™   rš   r›   r7   r:   rj   Úother_pre_chunkr/   r/   r0   Ú6it_knows_when_it_is_equal_to_another_PreChunk_instancez  s    ÿzGDescribePreChunk.it_knows_when_it_is_equal_to_another_PreChunk_instancec                 C  s    t g dtƒ d�}|dksJ ‚d S )Nr�   r¥   r8   )r   r   ©r.   rj   r/   r/   r0   Ú@and_it_knows_it_is_NOT_equal_to_an_object_that_is_not_a_PreChunk™  s    zQDescribePreChunk.and_it_knows_it_is_NOT_equal_to_an_object_that_is_not_a_PreChunkc                 C  s.   t td ƒtdƒgdtƒ d�}|jdks*J ‚d S )NZhellor�   r¥   )r   r   r   r   r”   r©   r/   r/   r0   Ú'it_can_handle_element_with_none_as_text�  s    ÿz8DescribePreChunk.it_can_handle_element_with_none_as_textc                 C  sT   t tdƒtdƒtdƒgdtƒ d�}t| ¡ ƒ}t|ƒdks>J ‚|d jdksPJ ‚dS )zFRegression test for AttributeError when Image elements have None text.Nzhello worldr�   r¥   é   r   )r   r   r   r   r�   Úiter_chunksÚlenÚtext)r.   rj   Úchunksr/   r/   r0   Ú2it_can_chunk_elements_with_none_text_without_error£  s    ýzCDescribePreChunk.it_can_chunk_elements_with_none_text_without_error)r"   r6   r7   ))r‡   éI   T)r‡   éH   F)éc   r²   Fr%   c                 C  sN   t ||ddd�}ttdƒgd|d�}ttdƒgd|d�}| |¡|u sJJ ‚d	S )
zOThis allows `PreChunkCombiner` to operate without knowing `PreChunk` internals.é   T)r"   r6   rG   rT   z2Lorem ipsum dolor sit amet consectetur adipiscing.ze feugiat efficitur.r¥   zIn rhoncus sum sed lectus.zsectetur adipiscing.N)r   r   r   Zcan_combine)r.   r"   r6   r7   r:   rj   Znext_pre_chunkr/   r/   r0   ÚBit_knows_when_it_can_combine_itself_with_another_PreChunk_instance±  s"    üýýzSDescribePreChunk.it_knows_when_it_can_combine_itself_with_another_PreChunk_instancec                 C  sº   t ƒ }ttdƒtdƒgd|d�}ttdƒtdƒgd|d�}| |¡}|ttdƒtdƒtdƒtdƒgd|d�ksrJ ‚|ttdƒtdƒgd|d�ks”J ‚|ttdƒtdƒgd|d�ks¶J ‚dS )	z¤.combine() produces a new pre-chunk by appending the elements of `other_pre-chunk`.

        Note that neither the original or other pre_chunk are mutated.
        rˆ   r“   zfeugiat efficitur.r¥   z/Donec semper facilisis metus finibus malesuada.zCVivamus magna nibh, blandit eu dui congue, feugiat efficitur velit.zporta volupat.N)r   r   r   Úcombine)r.   r:   rj   r§   Znew_pre_chunkr/   r/   r0   Ú4it_can_combine_itself_with_another_PreChunk_instanceÕ  sN    þúþú	
üøþúþúzEDescribePreChunk.it_can_combine_itself_with_another_PreChunk_instance)r¯   r7   )r“   úctus porta volutpat.)z1In rhoncus ipsum sed lect us   portas volutpat.  zus portas volutpat.c                 C  s.   t t|ƒgdtddd�d�}|j|ks*J ‚d S )Nr�   rµ   TrS   r¥   )r   r   r   Zoverlap_tail)r.   r¯   r7   rj   r/   r/   r0   Ú?it_computes_its_overlap_tail_for_use_in_inter_pre_chunk_overlap  s    ÿzPDescribePreChunk.it_computes_its_overlap_tail_for_use_in_inter_pre_chunk_overlap)rh   r¦   r7   zbah da bing.zbah da bing.

foo

barzda bang.zda bang.

foo

barzbah da boom.zbah da boom.

foozfoo

barz
list[Text]c                 C  s"   t ||tƒ d�}|j|ksJ ‚dS )z÷._text is the "joined" text of the pre-chunk elements.

        The text-segment contributed by each element is separated from the next by a blank line
        ("

"). An element that contributes no text does not give rise to a separator.
        r¥   N)r   r   r”   )r.   rh   r¦   r7   rj   r/   r/   r0   Ú7it_knows_the_concatenated_text_of_the_pre_chunk_to_help   s    zHDescribePreChunk.it_knows_the_concatenated_text_of_the_pre_chunk_to_helpN)rY   rZ   r[   r\   r*   r]   r^   r¨   rª   r«   r±   r¶   r¸   rº   r   r   r»   r/   r/   r/   r0   r—   w  sH   öþþ:üþ	üþ	r—   c                   @  sX   e Zd ZdZdd„ Zdd„ Zdd„ Zdd	„ Zd
d„ Zdd„ Z	dd„ Z
dd„ Zdd„ ZdS )ÚDescribe_ChunkerzBUnit-test suite for `unstructured.chunking.base._Chunker` objects.c                 C  s    t dƒtdƒg}tddd�}t|d|d�}| ¡ }t|ƒ}|tdƒksJJ ‚|j|ju sZJ ‚|jj	|ksjJ ‚t
 t¡� t|ƒ W d   ƒ n1 s’0    Y  d S )Nrr   zcLorem ipsum dolor sit amet consectetur adipiscing elit. In rhoncus ipsum sed lectus porta volutpat.rE   T©r"   r=   z‡e feugiat efficitur.

Introduction

Lorem ipsum dolor sit amet consectetur adipiscing elit. In rhoncus ipsum sed lectus porta volutpat.©r¯   r:   )r    r   r   r   Ú_iter_chunksrd   r   ÚmetadataÚ_consolidated_metadataÚorig_elementsr*   r+   rg   )r.   rh   r:   ÚchunkerÚ
chunk_iterÚchunkr/   r/   r0   ÚLit_generates_a_single_chunk_from_its_elements_if_they_together_fit_in_window=  s(    ÿþú	ÿ
z]Describe_Chunker.it_generates_a_single_chunk_from_its_elements_if_they_together_fit_in_windowc                 C  sÖ   d}t |ƒg}tddd�}t|||d�}| ¡ }t|ƒ}|tdƒksHJ ‚|j|ju sXJ ‚|jj|kshJ ‚t|ƒ}|tdƒks€J ‚|j|j	u s�J ‚|jj|ks J ‚t
 t¡� t|ƒ W d   ƒ n1 sÈ0    Y  d S )NzçLorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.rE   Tr½   r¾   zÆLorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi utz aliquip ex ea commodo consequat.)r   r   r   r¿   rd   r   rÀ   rÁ   rÂ   Z_continuation_metadatar*   r+   rg   )r.   r¯   rh   r:   rÃ   rÄ   rÅ   r/   r/   r0   ÚIbut_it_generates_split_chunks_when_its_single_element_exceeds_window_size\  s$    ÿ
ÿ
zZDescribe_Chunker.but_it_generates_split_chunks_when_its_single_element_exceeds_window_sizec                 C  sX   d}t dddgdd�}t||d�g}tj||tdd	�d
�}dd„ |D ƒg d¢ksTJ ‚d S )Nú0'Lorem ipsum dolor' means 'Thank you very much'.r   úfoo.docxÚlatÚf87731e0©Úcategory_depthÚfilenameÚ	languagesÚ	parent_id©rÀ   rµ   r&   rc   c                 S  s   g | ]}|j j‘qS r/   )rÀ   Úis_continuation)r¡   Úcr/   r/   r0   r£   Œ  r¤   zkDescribe_Chunker.and_it_adds_the_is_continuation_flag_for_second_and_later_split_chunks.<locals>.<listcomp>)NTT)r   r   r   r­   r   )r.   r¯   rÀ   rh   rÄ   r/   r/   r0   ÚFand_it_adds_the_is_continuation_flag_for_second_and_later_split_chunks  s    üzWDescribe_Chunker.and_it_adds_the_is_continuation_flag_for_second_and_later_split_chunksc                 C  sX   t ƒ }tjtd|d�gdtƒ d�}t t¡� t|ƒ W d   ƒ n1 sJ0    Y  d S )Nz  rÑ   r�   r¾   )	r   r   r­   r   r   r*   r+   rg   rd   )r.   rÀ   rÄ   r/   r/   r0   Ú>but_it_generates_no_chunks_when_the_pre_chunk_contains_no_textŽ  s    ýzODescribe_Chunker.but_it_generates_no_chunks_when_the_pre_chunk_contains_no_textc              	   C  s~   t dtdddgdd�d�tdtd	dd
ddgd�d�g}d}t||tƒ d�}|jdd	gddgdgddggd
gdgdœkszJ ‚d S )Nr`   r   rÉ   rÊ   rË   rÌ   rÑ   ú9'Lorem ipsum dolor' means 'Thank you very much' in Latin.r¬   ú
sprite.pngÚeng©rÍ   rÎ   Ú
image_pathrÏ   úFLorem Ipsum

'Lorem ipsum dolor' means 'Thank you very much' in Latin.r¾   )rÍ   rÎ   rÏ   rÚ   rÐ   )r    r   r   r   r   Ú_all_metadata_values)r.   rh   r¯   rÃ   r/   r/   r0   ÚCit_extracts_all_populated_metadata_values_from_the_elements_to_helpš  s8    üþ	üþö÷zTDescribe_Chunker.it_extracts_all_populated_metadata_values_from_the_elements_to_helpc                 C  s’   t dddgdd�}d|_t ddddd	gd
�}d|_td|d�td|d�g}d}t||tƒ d�}|jddgddgdgdgdd	ggdgdœksŽJ ‚d S )Nr   rÉ   rÊ   rË   rÌ   g×£p=
×ã?r¬   r×   rØ   rÙ   g×£p=
×û?r`   rÑ   rÈ   rÛ   r¾   )rÍ   rÎ   rÚ   rÏ   rÐ   )r   ZcoefficientZquotientr    r   r   r   rÜ   )r.   rÀ   Z
metadata_2rh   r¯   rÃ   r/   r/   r0   Ú;but_it_discards_ad_hoc_metadata_fields_during_consolidationÀ  s4    üü

þûzLDescribe_Chunker.but_it_discards_ad_hoc_metadata_fields_during_consolidationc           
      C  s’   t dd�}tdd�}td|d�}td|d�}||g}d}t|||d	�}|j}|j}	|	d us^J ‚|	||gksnJ ‚|	d
 |u s~J ‚|	d |u sŽJ ‚d S )NT©r=   zfoo.pdf)rÎ   r`   rÑ   rÈ   rÛ   r¾   r   r¬   )r   r   r    r   r   rÁ   rÂ   )
r.   r:   rÀ   r„   Ú	element_2rh   r¯   rÃ   Zconsolidated_metadatarÂ   r/   r/   r0   ÚAand_it_adds_the_pre_chunk_elements_to_metadata_when_so_instructedà  s    

zRDescribe_Chunker.and_it_adds_the_pre_chunk_elements_to_metadata_when_so_instructedc              
   C  sŽ   t dƒtdtddddgddgd	gd
�d�tdtdddgddgdd	gd�d�g}d}t||tƒ d�}|j}|dg d¢g d¢d	dgdœksŠJ ‚dS )zÿ._meta_kwargs is used like `ElementMetadata(**self._meta_kwargs)` to construct metadata.

        Only non-None fields should appear in the dict and each field value should be the
        consolidation of the values across the pre_chunk elements.
        r�   r`   rÉ   r   ÚLoremÚIpsumÚbÚirÊ   )rÎ   rÍ   Úemphasized_text_contentsÚemphasized_text_tagsrÏ   rÑ   rÖ   zbar.docxÚipsumrØ   )rÎ   ræ   rç   rÏ   rÛ   r¾   )râ   rã   râ   rè   )rä   rå   rå   rä   N)r   r    r   r   r   r   Z_meta_kwargs)r.   rh   r¯   rÃ   Zmeta_kwargsr/   r/   r0   ÚPit_forms_ElementMetadata_constructor_kwargs_by_applying_consolidation_strategiesó  s<    úþ÷þóüzaDescribe_Chunker.it_forms_ElementMetadata_constructor_kwargs_by_applying_consolidation_strategiesc           	      C  s¬   t dd�}tdƒ}tdƒ}tdttdƒgd�d�}|||g}d	}t|||d
�}|j}||||gksfJ ‚|d |u svJ ‚|d |us†J ‚|d jjd u sšJ ‚|j|u s¨J ‚d S )NTrß   rr   rˆ   r“   zPorta volupat.)rÂ   rÑ   zrIntroduction

Lorem ipsum dolor sit amet consectetur adipiscing elit.

In rhoncus ipsum sed lectus porta volutpat.r¾   r   é   )	r   r    r   r   r   r   Ú_orig_elementsrÀ   rÂ   )	r.   r:   r„   rà   Z	element_3rh   r¯   rÃ   rÂ   r/   r/   r0   Ú.it_computes_the_original_elements_list_to_help!  s"    
þ
ÿz?Describe_Chunker.it_computes_the_original_elements_list_to_helpN)rY   rZ   r[   r\   rÆ   rÇ   rÔ   rÕ   rÝ   rÞ   rá   ré   rì   r/   r/   r/   r0   r¼   :  s   #& .r¼   c                   @  s„   e Zd ZdZdd„ Zdd„ Zdd„ Zdd	„ Zd
d„ Zdd„ Z	e
j dddg¡ddddœdd„ƒZdd„ Zdd„ Zdd„ Zdd„ ZdS )ÚDescribe_TableChunkerzGUnit-test suite for `unstructured.chunking.base._TableChunker` objects.c                 C  s–   d}d}t jt|t|d�d�dtdd�d�}t|ƒ}t|tƒsBJ ‚|jd	ksPJ ‚|jj	d
ks`J ‚t
 t¡� t|ƒ W d   ƒ n1 sˆ0    Y  d S )Nzž<table>
<thead>
<tr><th>Header Col 1 </th><th>Header Col 2 </th></tr>
</thead>
<tbody>
<tr><td>Lorem ipsum  </td><td>adipiscing   </td></tr>
</tbody>
</table>ú3Header Col 1  Header Col 2
Lorem ipsum   adipiscing©Útext_as_htmlrÑ   r¹   é¯   r&   r¥   zHctus porta volutpat.
Header Col 1  Header Col 2
Lorem ipsum   adipiscingzr<table><tr><td>Header Col 1</td><td>Header Col 2</td></tr><tr><td>Lorem ipsum</td><td>adipiscing</td></tr></table>)r   r­   r   r   r   rd   re   r¯   rÀ   rð   r*   r+   rg   ©r.   Ú
html_tableZ
text_tablerÄ   rÅ   r/   r/   r0   Ú>it_uses_its_table_as_the_sole_chunk_when_it_fits_in_the_windowA  s$    ÿ
ýÿÿzTDescribe_TableChunker.it_uses_its_table_as_the_sole_chunk_when_it_fits_in_the_windowc                 C  s^   d}t jtdt|d�d�dtdd�d�}t t¡� t|ƒ W d   ƒ n1 sP0    Y  d S )	Nz/<table><tr><td/><td>  	  
   </td></tr></table>z  	  
  rï   rÑ   z	volutpat.rñ   r&   r¥   )	r   r­   r   r   r   r*   r+   rg   rd   )r.   ró   rÄ   r/   r/   r0   Ú>but_not_when_the_table_is_is_empty_or_contains_only_whitespaceb  s    ýzTDescribe_TableChunker.but_not_when_the_table_is_is_empty_or_contains_only_whitespacec                 C  s˜   t dtdd�d�}tdd�}t |d|¡}t|ƒ}t|t ƒs@J ‚|jj|gksRJ ‚|jj	dksbJ ‚t
 t¡� t|ƒ W d   ƒ n1 sŠ0    Y  d S )Nzfoo barz<table>foo bar</table>rï   rÑ   Trß   r�   )r   r   r   r   r­   rd   re   rÀ   rÂ   rð   r*   r+   rg   ©r.   Útabler:   rÄ   rÅ   r/   r/   r0   ÚIand_it_includes_the_original_table_element_in_metadata_when_so_instructedn  s    
z_Describe_TableChunker.and_it_includes_the_original_table_element_in_metadata_when_so_instructedc                 C  sB   t  tdƒdtdd�¡}t|ƒ}t|tƒs.J ‚|jjd u s>J ‚d S )NZfoobarr�   Frß   )r   r­   r   r   rd   re   rÀ   rÂ   )r.   rÄ   rÅ   r/   r/   r0   Úbut_not_when_instructed_not_to|  s    ÿz4Describe_TableChunker.but_not_when_instructed_not_toc                 C  sè   d}d}t jt|t|d�d�dtddd�d	�}t|ƒ}t|tƒsDJ ‚|jd
ksRJ ‚|j	j
dksbJ ‚|j	jd u srJ ‚t|ƒ}t|tƒsˆJ ‚|jdks–J ‚|j	j
dks¦J ‚|j	js²J ‚t t¡� t|ƒ W d   ƒ n1 sÚ0    Y  d S )Naž              <table>
            <thead>
            <tr><th>Header Col 1   </th><th>Header Col 2  </th></tr>
            </thead>
            <tbody>
            <tr><td>Lorem ipsum    </td><td>A Link example</td></tr>
            <tr><td>Consectetur    </td><td>adipiscing elit</td></tr>
            <tr><td>Nunc aliquam   </td><td>id enim nec molestie</td></tr>
            </tbody>
            </table>
        z©Header Col 1   Header Col 2
Lorem ipsum    dolor sit amet
Consectetur    adipiscing elit
Nunc aliquam   id enim nec molestie
Vivamus quis   nunc ipsum donec ac fermentumrï   rÑ   r�   r‡   ©Ú
ú )r"   Útext_splitting_separatorsr¥   zPHeader Col 1 Header Col 2 Lorem ipsum A Link example Consectetur adipiscing elitz«<table><tr><td>Header Col 1</td><td>Header Col 2</td></tr><tr><td>Lorem ipsum</td><td>A Link example</td></tr><tr><td>Consectetur</td><td>adipiscing elit</td></tr></table>z!Nunc aliquam id enim nec molestiezJ<table><tr><td>Nunc aliquam</td><td>id enim nec molestie</td></tr></table>)r   r­   r   r   r   rd   re   r   r¯   rÀ   rð   rÒ   r*   r+   rg   rò   r/   r/   r0   ÚKit_splits_its_table_into_TableChunks_when_the_table_text_exceeds_the_window†  s4    ÿ
ýÿÿÿzaDescribe_TableChunker.it_splits_its_table_into_TableChunks_when_the_table_text_exceeds_the_windowc                 C  s¶   t dtdd�d�}tddd�}tj|d|d	�}t|ƒ}t|tƒsDJ ‚|jd
ksRJ ‚|j	j
|gksdJ ‚|j	jrpJ ‚t|ƒ}t|tƒs†J ‚|jdks”J ‚|j	j
|gks¦J ‚|j	js²J ‚dS )zGEven though text and html are split, the orig_elements metadata is not.z8Header Col 1   Header Col 2
Lorem ipsum   dolor sit ametú<table/>rï   rÑ   é   Tr½   r�   r¥   zHeader Col 1   Header Col 2zLorem ipsum   dolor sit ametN)r   r   r   r   r­   rd   re   r   r¯   rÀ   rÂ   rÒ   rö   r/   r/   r0   ÚLand_it_includes_the_whole_original_Table_in_each_metadata_when_so_instructedº  s     þzbDescribe_TableChunker.and_it_includes_the_whole_original_Table_in_each_metadata_when_so_instructed)r¯   r¦   r7   )r“   r�   r“   )r“   r¹   z@ctus porta volutpat.
In rhoncus ipsum sed lectus porta volutpat.rž   c                 C  s&   t t|ƒ|tƒ d�}|j|ks"J ‚d S )Nr¥   )r   r   r   Z_text_with_overlap)r.   r¯   r¦   r7   Útable_chunkerr/   r/   r0   Ú7it_includes_its_overlap_prefix_in_its_text_when_presentÐ  s    ÿzMDescribe_TableChunker.it_includes_its_overlap_prefix_in_its_text_when_presentc                 C  sX   t dtdd�d�}t|dtƒ d�}|j}|jdks6J ‚|j|gksFJ ‚|j|usTJ ‚d S )NúLorem ipsumrÿ   rï   rÑ   r�   r¥   )r   r   r   r   Ú	_metadatarð   rÂ   )r.   r÷   r  rÀ   r/   r/   r0   Ú+it_computes_metadata_for_each_chunk_to_helpç  s    zADescribe_TableChunker.it_computes_metadata_for_each_chunk_to_helpc                 C  s6   t tdtdd�d�dtdd�d�}|jjd u s2J ‚d S )	Nr  rÿ   rï   rÑ   r�   Frß   r¥   )r   r   r   r   r  rÂ   )r.   r  r/   r/   r0   Ú;but_it_omits_orig_elements_from_metadata_when_so_instructedô  s    ýzQDescribe_TableChunker.but_it_omits_orig_elements_from_metadata_when_so_instructedc                 C  s‚   t dtdt dƒgd�d�}t|dtƒ d�}|j}t|ƒdks@J ‚|d	 }||ksTJ ‚||us`J ‚|jjd u spJ ‚|j|u s~J ‚d S )
Nr  rÿ   r`   )rð   rÂ   rÑ   r�   r¥   r¬   r   )r   r   r   r   rë   r®   rÀ   rÂ   )r.   r÷   r  rÂ   Zorig_elementr/   r/   r0   rì   ý  s    þzDDescribe_TableChunker.it_computes_the_original_elements_list_to_helpc                 C  s0   t dƒ}tt |dtƒ ¡ƒ}t|ƒdks,J ‚dS )zFRegression test for AttributeError when Table elements have None text.Nr�   r   )r   r�   r   r­   r   r®   )r.   r÷   r°   r/   r/   r0   Ú-it_handles_table_with_none_text_without_error  s    zCDescribe_TableChunker.it_handles_table_with_none_text_without_errorN)rY   rZ   r[   r\   rô   rõ   rø   rù   rþ   r  r*   r]   r^   r  r  r  rì   r  r/   r/   r/   r0   rí   >  s$   !
4úþ	rí   c                   @  s(   e Zd ZdZdd„ Zdd„ Zdd„ ZdS )	ÚDescribe_HtmlTableSplitterzDUnit-test suite for `unstructured.chunking.base._HtmlTableSplitter`.c                 C  s4   t dd�}t d¡}tt ||¡ƒg d¢ks0J ‚d S )Né(   r&   aP  
            <table border="1" class="dataframe">
              <tbody>
                <tr>
                  <td>Stanley
              Cups</td>
                  <td></td>
                  <td></td>
                </tr>
                <tr>
                  <td>Team</td>
                  <td>Location</td>
                  <td>Stanley Cups</td>
                </tr>
                <tr>
                  <td>Blues</td>
                  <td>STL</td>
                  <td>1</td>
                </tr>
                <tr>
                  <td>Flyers</td>
                  <td>PHI</td>
                  <td>2</td>
                </tr>
                <tr>
                  <td>Maple Leafs</td>
                  <td>TOR</td>
                  <td>13</td>
                </tr>
              </tbody>
            </table>
            ))z'Stanley Cups Team Location Stanley Cupszs<table><tr><td>Stanley Cups</td><td/><td/></tr><tr><td>Team</td><td>Location</td><td>Stanley Cups</td></tr></table>)zBlues STL 1 Flyers PHI 2zj<table><tr><td>Blues</td><td>STL</td><td>1</td></tr><tr><td>Flyers</td><td>PHI</td><td>2</td></tr></table>)zMaple Leafs TOR 13zC<table><tr><td>Maple Leafs</td><td>TOR</td><td>13</td></tr></table>©r   r   Zfrom_html_textr�   r   Ziter_subtables©r.   r:   ró   r/   r/   r0   Ú=it_splits_an_HTML_table_on_whole_row_boundaries_when_possible%  s
    
ÿ#zXDescribe_HtmlTableSplitter.it_splits_an_HTML_table_on_whole_row_boundaries_when_possiblec                 C  s4   t dd�}t d¡}tt ||¡ƒg d¢ks0J ‚d S )Né]   r&   a.  
            <html><body><table>
              <tr>
                <td>Lorem ipsum dolor sit amet.</td>
                <td>   Consectetur    adipiscing     elit.   </td>
                <td>
                  Laboris nisi ut
                  aliquip ex ea commodo.
                </td>
              </tr>
              <tr>
                <td>Duis</td>
                <td>Dolor</td>
              </tr>
              <tr>
                <td>Duis</td>
                <td>Cillum</td>
              </tr>
            </table></body></html>
            ))z8Lorem ipsum dolor sit amet. Consectetur adipiscing elit.za<table><tr><td>Lorem ipsum dolor sit amet.</td><td>Consectetur adipiscing elit.</td></tr></table>)z&Laboris nisi ut aliquip ex ea commodo.zG<table><tr><td>Laboris nisi ut aliquip ex ea commodo.</td></tr></table>)zDuis Dolor Duis CillumzX<table><tr><td>Duis</td><td>Dolor</td></tr><tr><td>Duis</td><td>Cillum</td></tr></table>r  r  r/   r/   r0   ÚEand_it_splits_an_oversized_row_on_an_even_cell_boundary_when_possible_  s
    
ÿz`Describe_HtmlTableSplitter.and_it_splits_an_oversized_row_on_an_even_cell_boundary_when_possiblec                 C  s4   t dd�}t d¡}tt ||¡ƒg d¢ks0J ‚d S )Nr‡   r&   a«  
            <table>
              <thead>
                <tr>
                  <td>
                    Lorem ipsum dolor sit amet,
                    consectetur adipiscing elit.
                    Sed do eiusmod tempor
                    incididunt ut labore et dolore magna aliqua.
                  </td>
                  <td> Ut enim ad minim veniam.           </td>
                  <td> Quis nostrud exercitation ullamco. </td>
                </tr>
              </thead>
              <tbody>
                <tr><td>Duis aute irure dolor</td></tr>
                <tr><td>In reprehenderit voluptate.</td></tr>
              </tbody>
            </table
            ))z?Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed doz`<table><tr><td>Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do</td></tr></table>)z;eiusmod tempor incididunt ut labore et dolore magna aliqua.z\<table><tr><td>eiusmod tempor incididunt ut labore et dolore magna aliqua.</td></tr></table>)z;Ut enim ad minim veniam. Quis nostrud exercitation ullamco.zd<table><tr><td>Ut enim ad minim veniam.</td><td>Quis nostrud exercitation ullamco.</td></tr></table>)z1Duis aute irure dolor In reprehenderit voluptate.zc<table><tr><td>Duis aute irure dolor</td></tr><tr><td>In reprehenderit voluptate.</td></tr></table>r  r  r/   r/   r0   Ú8and_it_splits_an_oversized_cell_on_an_even_word_boundary�  s
    
ÿzSDescribe_HtmlTableSplitter.and_it_splits_an_oversized_cell_on_an_even_word_boundaryN)rY   rZ   r[   r\   r  r  r  r/   r/   r/   r0   r	  "  s   :.r	  c                   @  sp   e Zd ZdZdd„ Zdd„ Zdd„ Zej 	dd	d
g¡ddœdd„ƒZ
dd„ Zej 	dddg¡ddœdd„ƒZdS )ÚDescribe_TextSplitterzGUnit-test suite for `unstructured.chunking.base._TextSplitter` objects.c                 C  sf   t dddd�}t|ƒ}d}||ƒ\}}|dks2J ‚|dks>J ‚||ƒ\}}|dksVJ ‚|dksbJ ‚d S )	Nr2   rú   rP   ©r"   rý   rG   zULorem ipsum dolor amet consectetur adipiscing.  
  In rhoncus ipsum sed lectus porta.ú.Lorem ipsum dolor amet consectetur adipiscing.z,ipiscing. In rhoncus ipsum sed lectus porta.r�   ©r   r   ©r.   r:   Úsplitr¯   ÚsÚ	remainderr/   r/   r0   Ú.it_splits_on_a_preferred_separator_when_it_canÇ  s    ÿzDDescribe_TextSplitter.it_splits_on_a_preferred_separator_when_it_canc                 C  sŠ   t dddd�}t|ƒ}d}||ƒ\}}|dks2J ‚|dks>J ‚||ƒ\}}|dksVJ ‚|d	ksbJ ‚||ƒ\}}|d	kszJ ‚|d
ks†J ‚d S )Nr
  rú   rP   r  zZLorem ipsum dolor amet consectetur adipiscing. In rhoncus ipsum sed lectus porta volutpat.z"Lorem ipsum dolor amet consecteturzAnsectetur adipiscing. In rhoncus ipsum sed lectus porta volutpat.z&nsectetur adipiscing. In rhoncus ipsumz$cus ipsum sed lectus porta volutpat.r�   r  r  r/   r/   r0   ÚMand_it_splits_on_the_next_available_separator_when_the_first_is_not_availableÝ  s    ÿzcDescribe_TextSplitter.and_it_splits_on_the_next_available_separator_when_the_first_is_not_availablec                 C  sŠ   t dddd�}t|ƒ}d}||ƒ\}}|dks2J ‚|dks>J ‚||ƒ\}}|dksVJ ‚|d	ksbJ ‚||ƒ\}}|d
kszJ ‚|dks†J ‚d S )Nr   rú   rP   r  úPLoremipsumdolorametconsecteturadipiscingelit. In rhoncus ipsum sed lectus porta.ZLoremipsumdolorametconsecteturz<onsecteturadipiscingelit. In rhoncus ipsum sed lectus porta.zonsecteturadipiscingelit. Inz)gelit. In rhoncus ipsum sed lectus porta.zgelit. In rhoncus ipsum sedzipsum sed lectus porta.r  r  r/   r/   r0   Ú8and_it_splits_on_an_arbitrary_character_as_a_last_resortñ  s    zNDescribe_TextSplitter.and_it_splits_on_an_arbitrary_character_as_a_last_resortr¯   r  zLorem ipsum dolor.rž   )r¯   c                 C  s<   t ddd�}t|ƒ}||ƒ\}}||ks,J ‚|dks8J ‚d S )Né.   rP   rF   r�   r  )r.   r¯   r:   r  r  r  r/   r/   r0   Ú9it_does_not_split_a_string_that_is_not_longer_than_maxlen  s
    zODescribe_TextSplitter.it_does_not_split_a_string_that_is_not_longer_than_maxlenc                 C  sD   t ddd�}t|ƒ}d}||ƒ\}}|dks0J ‚t|ƒdks@J ‚d S )Né&   rP   rF   r  Z&Loremipsumdolorametconsecteturadipisci)r   r   r®   )r.   r:   r  r¯   r  Ú_r/   r/   r0   ÚEit_fills_the_window_when_falling_back_to_an_arbitrary_character_split  s    z[Describe_TextSplitter.it_fills_the_window_when_falling_back_to_an_arbitrary_character_splitÚ
separatorsrú   )rü   zSequence[str])r"  c                 C  sB   t d|dd�}t|ƒ}d}||ƒ\}}|dks2J ‚|dks>J ‚d S )Nr2   rP   r  zPLorem ipsum dolor amet consectetur adipiscing.   

 In rhoncus ipsum sed lectus.r  z&ipiscing. In rhoncus ipsum sed lectus.r  )r.   r"  r:   r  r¯   r  r  r/   r/   r0   Ú%it_strips_whitespace_around_the_split  s    z;Describe_TextSplitter.it_strips_whitespace_around_the_splitN)rY   rZ   r[   r\   r  r  r  r*   r]   r^   r  r!  r#  r/   r/   r/   r0   r  Ä  s   þþ	
r  c                   @  s|   e Zd ZdZdd„ Zdd„ Zej dg d¢¡dd	dœd
d„ƒZ	ej dg d¢¡dd	dœdd„ƒZ
dd„ Zdd„ Zdd„ ZdS )ÚDescribe_CellAccumulatorzBUnit-test suite for `unstructured.chunking.base._CellAccumulator`.c                 C  s   t dd�}|jg ksJ ‚d S ©Nr‡   ©Úmaxlen)r   Ú_cells©r.   Úaccumr/   r/   r0   rq   ,  s    
z4Describe_CellAccumulator.it_is_empty_on_constructionc                 C  s8   t dƒ}t|ƒ}tdd�}| |¡ |j|gks4J ‚d S )Nz<td>foobar</td>r‡   r&  )r   r   r   Úadd_cellr(  )r.   ÚtdÚcellr*  r/   r/   r0   ry   1  s
    

z<Describe_CellAccumulator.it_accumulates_elements_added_to_it)Ú	cell_htmlr7   )©z<td/>T©z<td>Lorem Ipsum.</td>T)z<td>Lorem Ipsum dolor sit.</td>T©z$<td>Lorem Ipsum dolor sit amet.</td>Frž   r>   c                 C  s<   t dd�}tt|ƒƒ}td|j›�ƒ | |¡|u s8J ‚d S )Né   r&  z
cell.text=)r   r   r   Úprintr¯   r{   ©r.   r.  r7   r*  r-  r/   r/   r0   Ú;it_will_fit_a_cell_with_text_shorter_than_maxlen_when_empty:  s    
zTDescribe_CellAccumulator.it_will_fit_a_cell_with_text_shorter_than_maxlen_when_empty)r/  r0  )z<td>Lorem Ipsum amet.</td>T)z<td>Lorem Ipsum dolor.</td>Fr1  c                 C  s>   t dd�}| ttdƒƒ¡ tt|ƒƒ}| |¡|u s:J ‚d S )Né,   r&  z#<td>abcdefghijklmnopqrstuvwxyz</td>)r   r+  r   r   r{   r4  r/   r/   r0   ÚLand_it_will_fit_a_cell_with_text_shorter_than_remaining_space_when_not_emptyM  s    
zeDescribe_CellAccumulator.and_it_will_fit_a_cell_with_text_shorter_than_remaining_space_when_not_emptyc                 C  sV   t dd�}| ttdƒƒ¡ t| ¡ ƒ\}}|dks8J ‚|dksDJ ‚|jg ksRJ ‚d S )Nr‡   r&  ú<td>abcde fghij klmno</td>úabcde fghij klmnoú2<table><tr><td>abcde fghij klmno</td></tr></table>©r   r+  r   r   rd   r�   r(  ©r.   r*  r¯   Úhtmlr/   r/   r0   ÚGit_generates_a_TextAndHtml_pair_and_resets_itself_to_empty_when_flusheda  s    
z`Describe_CellAccumulator.it_generates_a_TextAndHtml_pair_and_resets_itself_to_empty_when_flushedc                 C  sh   t dd�}| ttdƒƒ¡ | ttdƒƒ¡ t| ¡ ƒ\}}|dksJJ ‚|dksVJ ‚|jg ksdJ ‚d S )Nr‡   r&  r8  z<td>pqrst uvwxy z</td>úabcde fghij klmno pqrst uvwxy zzH<table><tr><td>abcde fghij klmno</td><td>pqrst uvwxy z</td></tr></table>r;  r<  r/   r/   r0   Ú7and_the_HTML_contains_as_many_cells_as_were_accumulatedk  s    
zPDescribe_CellAccumulator.and_the_HTML_contains_as_many_cells_as_were_accumulatedc                 C  sD   t dd�}t t¡� t| ¡ ƒ W d   ƒ n1 s60    Y  d S r%  )r   r*   r+   rg   rd   r�   r)  r/   r/   r0   Ú6but_it_does_not_generate_a_TextAndHtml_pair_when_emptyv  s    
zODescribe_CellAccumulator.but_it_does_not_generate_a_TextAndHtml_pair_when_emptyN)rY   rZ   r[   r\   rq   ry   r*   r]   r^   r5  r7  r>  r@  rA  r/   r/   r/   r0   r$  )  s    	þ	
þ


r$  c                   @  s|   e Zd ZdZdd„ Zdd„ Zej dg d¢¡dd	dœd
d„ƒZ	ej dg d¢¡dd	dœdd„ƒZ
dd„ Zdd„ Zdd„ ZdS )ÚDescribe_RowAccumulatorzAUnit-test suite for `unstructured.chunking.base._RowAccumulator`.c                 C  s   t dd�}|jg ksJ ‚d S r%  )r   Ú_rowsr)  r/   r/   r0   rq   €  s    
z3Describe_RowAccumulator.it_is_empty_on_constructionc                 C  s4   t dd�}ttdƒƒ}| |¡ |j|gks0J ‚d S )Nr‡   r&  z!<tr><td>foo</td><td>bar</td></tr>)r   r   r   Úadd_rowrC  )r.   r*  Úrowr/   r/   r0   Úit_accumulates_rows_added_to_it…  s    

z7Describe_RowAccumulator.it_accumulates_rows_added_to_it)Úrow_htmlr7   ))z<tr/>T)z<tr><td/></tr>T)z<tr><td>Lorem Ipsum.</td></tr>T)z'<tr><td>Lorem Ipsum dolor sit</td></tr>T)z(<tr><td>Lorem</td><td>Sit amet</td></tr>T)z-<tr><td>Lorem Ipsum dolor sit amet.</td></tr>F)z0<tr><td>Lorem Ipsum</td><td>Dolor sit.</td></tr>Frž   r>   c                 C  s,   t dd�}tt|ƒƒ}| |¡|u s(J ‚d S )Né   r&  )r   r   r   r{   ©r.   rG  r7   r*  rE  r/   r/   r0   Ú:it_will_fit_a_row_with_text_shorter_than_maxlen_when_empty�  s    
zRDescribe_RowAccumulator.it_will_fit_a_row_with_text_shorter_than_maxlen_when_emptyc                 C  s>   t dd�}| ttdƒƒ¡ tt|ƒƒ}| |¡|u s:J ‚dS )z9There is no overhead beyond row HTML for additional rows.é0   r&  z,<tr><td>abcdefghijklmnopqrstuvwxyz</td></tr>N)r   rD  r   r   r{   rI  r/   r/   r0   ÚKand_it_will_fit_a_row_with_text_shorter_than_remaining_space_when_not_empty¡  s    
zcDescribe_RowAccumulator.and_it_will_fit_a_row_with_text_shorter_than_remaining_space_when_not_emptyc                 C  sV   t dd�}| ttdƒƒ¡ t| ¡ ƒ\}}|dks8J ‚|dksDJ ‚|jg ksRJ ‚d S )Nr‡   r&  ú#<tr><td>abcde fghij klmno</td></tr>r9  r:  ©r   rD  r   r   rd   r�   rC  r<  r/   r/   r0   r>  ¸  s    
z_Describe_RowAccumulator.it_generates_a_TextAndHtml_pair_and_resets_itself_to_empty_when_flushedc                 C  sh   t dd�}| ttdƒƒ¡ | ttdƒƒ¡ t| ¡ ƒ\}}|dksJJ ‚|dksVJ ‚|jg ksdJ ‚d S )Nr‡   r&  rM  z<tr><td>pqrst uvwxy z</td></tr>r?  zQ<table><tr><td>abcde fghij klmno</td></tr><tr><td>pqrst uvwxy z</td></tr></table>rN  r<  r/   r/   r0   Ú6and_the_HTML_contains_as_many_rows_as_were_accumulatedÂ  s    
ÿzNDescribe_RowAccumulator.and_the_HTML_contains_as_many_rows_as_were_accumulatedc                 C  sD   t dd�}t t¡� t| ¡ ƒ W d   ƒ n1 s60    Y  d S r%  )r   r*   r+   rg   rd   r�   r)  r/   r/   r0   rA  Ò  s    
zNDescribe_RowAccumulator.but_it_does_not_generate_a_TextAndHtml_pair_when_emptyN)rY   rZ   r[   r\   rq   rF  r*   r]   r^   rJ  rL  r>  rO  rA  r/   r/   r/   r0   rB  }  s    þþ
rB  c                   @  s0   e Zd ZdZdd„ Zdd„ Zdd„ Zdd	„ Zd
S )ÚDescribePreChunkCombinerzBUnit-test suite for `unstructured.chunking.base.PreChunkCombiner`.c              	   C  sþ   t ddd�}ttdƒtdƒgd|d�ttdƒgd|d�ttdƒtd	ƒgd|d�ttd
ƒtdƒgd|d�g}t||d� ¡ }t|ƒ}t|tƒs�J ‚|j	tdƒtdƒtdƒtdƒtd	ƒtd
ƒtdƒgksÈJ ‚t
 t¡� t|ƒ W d   ƒ n1 sð0    Y  d S )Néú   ©r"   r6   r`   rˆ   r�   r¥   r‚   ú
Mauris Necú;Mauris nec urna non augue vulputate consequat eget et nisi.úSed Orciú?Sed orci quam, eleifend sit amet vehicula, elementum ultricies.rc   )r   r   r    r   r   r	   Úiter_combined_pre_chunksrd   re   rf   r*   r+   rg   ©r.   r:   r�   ri   rj   r/   r/   r0   Ú'it_combines_sequential_small_pre_chunksá  sL    þúþúþúîù
	z@DescribePreChunkCombiner.it_combines_sequential_small_pre_chunksc                 C  s
  t ddd�}ttdƒtdƒgd|d�ttdƒtd	ƒgd|d�ttd
ƒtdƒgd|d�g}t||d� ¡ }t|ƒ}t|tƒs~J ‚|jtdƒtdƒtdƒtd	ƒgks¤J ‚t|ƒ}t|tƒsºJ ‚|jtd
ƒtdƒgksÔJ ‚t	 
t¡� t|ƒ W d   ƒ n1 sü0    Y  d S )NrQ  éP   rR  r`   rˆ   r�   r¥   rS  rT  rU  rV  rc   ©r   r   r    r   r	   rW  rd   re   rf   r*   r+   rg   rX  r/   r/   r0   Ú/it_respects_the_specified_combination_threshold  sP    þúþú	þúîü
þ
zHDescribePreChunkCombiner.it_respects_the_specified_combination_thresholdc                 C  s
  t ddd�}ttdƒtdƒgd|d�ttdƒtdƒgd|d�ttd	ƒtd
ƒgd|d�g}t||d� ¡ }t|ƒ}t|tƒs~J ‚|jtdƒtdƒtdƒtdƒgks¤J ‚t|ƒ}t|tƒsºJ ‚|jtd	ƒtd
ƒgksÔJ ‚t	 
t¡� t|ƒ W d   ƒ n1 sü0    Y  d S )NrE   rR  r`   rˆ   r�   r¥   rS  rT  rU  rV  rc   r[  rX  r/   r/   r0   Ú*it_respects_the_hard_maximum_window_lengthB  sP    þúþú	þúîü
þ
zCDescribePreChunkCombiner.it_respects_the_hard_maximum_window_lengthc                 C  s  t ddd�}ttdƒgd|d�ttdƒgd|d�ttdƒgd|d�g}t|t ddd�ƒ ¡ }t|ƒ}t|tƒsrJ ‚|jtdƒgks†J ‚t|ƒ}t|tƒsœJ ‚|jtdƒgks°J ‚t|ƒ}t|tƒsÆJ ‚|jtdƒgksÚJ ‚t	 
t¡� t|ƒ W d  ƒ n1 �s0    Y  dS )	z=Such as occurs when a single element exceeds the window size.ra   rR  r`   r�   r¥   z³Lorem ipsum dolor sit amet consectetur adipiscing elit. Mauris nec urna non augue vulputate consequat eget et nisi. Sed orci quam, eleifend sit amet vehicula, elementum ultricies.zVulputate ConsequatNr[  rX  r/   r/   r0   Ú3it_accommodates_and_isolates_an_oversized_pre_chunkv  s<    ÿÿ÷óÿÿÿ
zLDescribePreChunkCombiner.it_accommodates_and_isolates_an_oversized_pre_chunkN)rY   rZ   r[   r\   rY  r\  r]  r^  r/   r/   r/   r0   rP  Þ  s
   .34rP  c                   @  s    e Zd ZdZdd„ Zdd„ ZdS )ÚDescribe_PreChunkAccumulatorzFUnit-test suite for `unstructured.chunking.base._PreChunkAccumulator`.c                 C  sz  t dd�}t|d�}ttdƒtdƒgd|d�}| |¡s<J ‚| |¡ ttdƒtd	ƒgd
|d�}| |¡snJ ‚| |¡ ttdƒtdƒgd|d�}| |¡s J ‚| |¡ | ¡ }t|ƒ}t	 
t¡� t|ƒ W d   ƒ n1 sâ0    Y  t|tƒsúJ ‚|jtdƒtdƒtdƒtd	ƒtdƒtdƒgk�s.J ‚|jdk�s>J ‚t	 
t¡� t| ¡ ƒ W d   ƒ n1 �sl0    Y  d S )Niô  r9   rc   r`   rˆ   z
elementum.r¥   rS  rT  z	sit amet.rU  zDSed orci quam, eleifend sit amet vehicula, elementum ultricies quam.z
consequat.)r   r   r   r    r   r{   Zadd_pre_chunkr�   rd   r*   r+   rg   re   rf   Z_overlap_prefix)r.   r:   r*  rj   ri   r/   r/   r0   ÚHit_generates_a_combined_PreChunk_when_flushed_and_resets_itself_to_empty¦  sZ    

þú
þú
þú
&ú	zeDescribe_PreChunkAccumulator.it_generates_a_combined_PreChunk_when_flushed_and_resets_itself_to_emptyc                 C  s(   t tdd�d�}t| ¡ ƒg ks$J ‚d S )Nra   r&   rc   )r   r   r�   r�   r)  r/   r/   r0   Ú7but_it_does_not_generate_a_PreChunk_on_flush_when_emptyá  s    zTDescribe_PreChunkAccumulator.but_it_does_not_generate_a_PreChunk_on_flush_when_emptyN)rY   rZ   r[   r\   r`  ra  r/   r/   r/   r0   r_  £  s   ;r_  c                	   @  sj   e Zd ZdZej dedƒededd�d�g¡dd	œd
d„ƒZ	dd„ Z
dd„ Zdd„ Zdd„ Zdd„ ZdS )ÚDescribe_is_on_next_pagea"  Unit-test suite for `unstructured.chunking.base.is_on_next_page()` function.

    `is_on_next_page()` is not itself a predicate, rather it returns a predicate on Element
    (`Callable[[Element], bool]`) that can be called repeatedly to detect section changes in an
    element stream.
    r„   r�   Úefghé   ©Zpage_numberrÑ   r   r…   c                 C  s   t ƒ }||ƒrJ ‚dS )z-The first page never represents a page-break.N)r   )r.   r„   Úpredr/   r/   r0   Ú1it_is_unconditionally_false_for_the_first_elementó  s    zJDescribe_is_on_next_page.it_is_unconditionally_false_for_the_first_elementc                 C  sD   t ƒ }|tdtdd�d�ƒr J ‚|tdƒƒr0J ‚|tdƒƒr@J ‚dS )zMAn element with a `None` page-number is assumed to continue the current page.r�   r¬   re  rÑ   rc  ÚijklN©r   r   r   ©r.   rf  r/   r/   r0   Ú2it_is_false_for_an_element_that_has_no_page_numberû  s    zKDescribe_is_on_next_page.it_is_false_for_an_element_that_has_no_page_numberc                 C  s^   t ƒ }|tdtdd�d�ƒr J ‚|tdƒƒr0J ‚|tdtdd�d�ƒrJJ ‚|tdƒƒrZJ ‚d S )Nr�   r¬   re  rÑ   rc  rh  Úmnopri  rj  r/   r/   r0   Ú7it_is_false_for_an_element_with_the_current_page_number  s
    zPDescribe_is_on_next_page.it_is_false_for_an_element_with_the_current_page_numberc                 C  s4   t ƒ }|tdƒƒrJ ‚|tdtdd�d�ƒr0J ‚d S )Nr�   rc  r¬   re  rÑ   ri  rj  r/   r/   r0   ÚCit_assigns_page_number_1_to_a_first_element_that_has_no_page_number	  s    z\Describe_is_on_next_page.it_assigns_page_number_1_to_a_first_element_that_has_no_page_numberc                 C  s>   t ƒ }|tdtdd�d�ƒr J ‚|tdtdd�d�ƒs:J ‚d S )Nr�   r¬   re  rÑ   rc  rê   ri  rj  r/   r/   r0   Ú@it_is_true_for_an_element_with_an_explicit_different_page_number  s    zYDescribe_is_on_next_page.it_is_true_for_an_element_with_an_explicit_different_page_numberc                 C  s‚   t ƒ }|tdtdd�d�ƒr J ‚|tdtdd�d�ƒs:J ‚|tdtdd�d�ƒrTJ ‚|tdƒƒrdJ ‚|td	td
d�d�ƒs~J ‚d S )Nr�   rd  re  rÑ   rc  rê   rh  rl  Zqrsté   ri  rj  r/   r/   r0   Ú2and_it_is_true_even_when_that_page_number_is_lower  s    zKDescribe_is_on_next_page.and_it_is_true_even_when_that_page_number_is_lowerN)rY   rZ   r[   r\   r*   r]   r^   r   r   rg  rk  rm  rn  ro  rq  r/   r/   r/   r0   rb  ë  s   ÿrb  c                   @  sF   e Zd ZdZdd„ Zej dedƒe	dƒe
dƒg¡dd	œd
d„ƒZdS )ÚDescribe_is_titlezFUnit-test suite for `unstructured.chunking.base.is_title()` predicate.c                 C  s   t tdƒƒsJ ‚d S )Nr�   )r   r    r4   r/   r/   r0   Úit_is_true_for_a_Title_element  s    z0Describe_is_title.it_is_true_for_a_Title_elementr„   r�   rî   r�   r   r…   c                 C  s   t |ƒrJ ‚d S r?   )r   )r.   r„   r/   r/   r0   Ú-and_it_is_false_for_any_other_element_subtype"  s    	z?Describe_is_title.and_it_is_false_for_any_other_element_subtypeN)rY   rZ   r[   r\   rs  r*   r]   r^   r   r   r   rt  r/   r/   r/   r0   rr    s   ýþrr  )5r\   Ú
__future__r   Útypingr   r   r*   Z	lxml.htmlr   Zunstructured.chunking.baser   r   r   r	   r
   r   r   r   r   r   r   r   r   r   Zunstructured.common.html_tabler   r   r   Zunstructured.documents.elementsr   r   r   r   r   r   r   r   r   r    r!   r_   rl   r—   r¼   rí   r	  r  r$  rB  rP  r_  rb  rr  r/   r/   r/   r0   Ú<module>   s8   @0x. # D   e #eTa FH1