a
    ¡˜ëjöK  ã                   @  sP  d Z ddlmZ ddlmZmZ ddlZddlmZm	Z	m
Z
mZ ddlmZ ddlmZmZ ddlmZ dd	lmZmZmZmZmZmZmZmZmZmZ dd
lmZ ddl m!Z! dd„ Z"dd„ Z#dd„ Z$dd„ Z%dd„ Z&dd„ Z'dd„ Z(dd„ Z)dd„ Z*dd„ Z+d d!„ Z,d"d#„ Z-d$d%„ Z.d&d'„ Z/d(d)„ Z0d*d+„ Z1G d,d-„ d-ƒZ2G d.d/„ d/ƒZ3dS )0z8Test suite for the `unstructured.chunking.title` module.é    )Úannotations)ÚAnyÚOptionalN)ÚFixtureRequestÚMockÚfunction_mockÚ
input_path)ÚCHUNK_MULTI_PAGE_DEFAULT)Ú_ByTitleChunkingOptionsÚchunk_by_title)ÚCoordinateSystem)
ÚCheckBoxÚCompositeElementÚCoordinatesMetadataÚElementÚElementMetadataÚListItemÚTableÚ
TableChunkÚTextÚTitle)Úpartition_html)Úelements_from_jsonc                  C  s>   t tdƒƒ} t| dd�}t|ƒdks(J ‚t|d tƒs:J ‚d S )Nzchunking/title_table_200.jsonr   ©Úcombine_text_under_n_charsé   ©r   r   r   ÚlenÚ
isinstancer   ©ÚelementsÚchunks© r"   úd/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/test_unstructured/chunking/test_title.pyÚ<test_it_chunks_text_followed_by_table_together_when_both_fit&   s    r$   c                  C  s>   t tdƒƒ} t| dd�}t|ƒdks(J ‚t|d tƒs:J ‚d S )Nzchunking/table_text_200.jsonr   r   r   r   r   r"   r"   r#   Ú<test_it_chunks_table_followed_by_text_together_when_both_fit/   s    r%   c                  C  s>   t tdƒƒ} t| ƒ}t|ƒdks$J ‚tdd„ |D ƒƒs:J ‚d S )Nzchunking/table_2000.jsoné   c                 s  s   | ]}t |tƒV  qd S ©N)r   r   ©Ú.0Úchunkr"   r"   r#   Ú	<genexpr>?   ó    z1test_it_splits_oversized_table.<locals>.<genexpr>)r   r   r   r   Úallr   r"   r"   r#   Útest_it_splits_oversized_table9   s    r.   c                  C  sF   t tdƒƒ} t| dd�}t|ƒdks(J ‚dd„ |D ƒttgksBJ ‚d S )Nz!chunking/long_text_table_200.jsonéú   ©Úmax_charactersé   c                 S  s   g | ]}t |ƒ‘qS r"   ©Útyper(   r"   r"   r#   Ú
<listcomp>H   r,   zLtest_it_starts_new_chunk_for_table_after_full_text_chunk.<locals>.<listcomp>)r   r   r   r   r   r   r   r"   r"   r#   Ú8test_it_starts_new_chunk_for_table_after_full_text_chunkB   s    r6   c                  C  sF   t tdƒƒ} t| dd�}t|ƒdks(J ‚dd„ |D ƒttgksBJ ‚d S )Nz&chunking/full_table_long_text_250.jsonr/   r0   r2   c                 S  s   g | ]}t |ƒ‘qS r"   r3   r(   r"   r"   r#   r5   Q   r,   zLtest_it_starts_new_chunk_for_text_after_full_table_chunk.<locals>.<listcomp>)r   r   r   r   r   r   r   r"   r"   r#   Ú8test_it_starts_new_chunk_for_text_after_full_table_chunkK   s    r7   c                  C  s>   t dƒtdƒg} t| dd�}|tdƒtdƒtdƒgks:J ‚d S )NZIntroductionzcLorem ipsum dolor sit amet consectetur adipiscing elit. In rhoncus ipsum sed lectus porta volutpat.é2   r0   z1Lorem ipsum dolor sit amet consectetur adipiscingz1elit. In rhoncus ipsum sed lectus porta volutpat.)r   r   r   r   r   r"   r"   r#   Ú8test_it_splits_a_large_text_element_into_multiple_chunksT   s    ÿþýr9   c                  C  s  t dƒtdƒtdƒtdƒt dƒtdƒtdƒt dƒtd	ƒtd
ƒtƒ g} t| ddd�}t|ƒdksbJ ‚|d }t|tƒsxJ ‚|jj	t dƒtdƒtdƒtdƒgks J ‚|d }t|tƒs¶J ‚|jj	t dƒtdƒtdƒgksØJ ‚|d }t|tƒsîJ ‚|jj	t dƒtd	ƒtd
ƒtƒ gk�sJ ‚d S )NúA Great DayúToday is a great day.úIt is sunny outside.úHeading
Cell textúAn Okay DayúToday is an okay day.úIt is rainy outside.ú	A Bad DayúToday is a bad day.úIt is storming outside.r   T©r   Úinclude_orig_elementsé   r   r2   )
r   r   r   r   r   r   r   r   ÚmetadataZorig_elements)r    r!   r*   r"   r"   r#   Ú*test_it_splits_elements_by_title_and_tablef   sJ    õü
ý
ürH   c                  C  s¨   t dtdgd�d�tdtdgd�d�tdƒtdƒt d	ƒtd
ƒtdƒt dƒtdƒtdƒtƒ g} t| ddd�}|tdƒtdƒtdƒgksˆJ ‚|d jtddgd�ks¤J ‚d S )Nr:   ZDay)Zemphasized_text_contents©rG   r;   Údayr<   r=   r>   r?   r@   rA   rB   rC   r   FrD   úKA Great Day

Today is a great day.

It is sunny outside.

Heading Cell textú8An Okay Day

Today is an okay day.

It is rainy outside.ú7A Bad Day

Today is a bad day.

It is storming outside.)r   r   r   r   r   r   r   rG   r   r"   r"   r#   Útest_chunk_by_title”   s.    õÿÿû
	rN   c                  C  s˜   t dtdd�d�tdtdd�d�tdtdd�d�tdƒt d	ƒtd
ƒtdƒt dƒtdƒtdƒtƒ g} t| ddd�}|tdƒtdƒtdƒtdƒgks”J ‚d S )Nr:   r   ©Zpage_numberrI   r;   r2   r<   r=   r>   r?   r@   rA   rB   rC   Fr   ©Úmultipage_sectionsr   z>Today is a great day.

It is sunny outside.

Heading Cell textrL   rM   ©r   r   r   r   r   r   r   r   r"   r"   r#   Ú,test_chunk_by_title_separates_by_page_number±   s.    õÿÿúrS   c                  C  s’   t dtdd�d�tdtdd�d�tdtdd�d�tdƒt d	ƒtd
ƒtdƒt dƒtdƒtdƒtƒ g} t| ddd�}|tdƒtdƒtdƒgksŽJ ‚d S ©Nr:   r   rO   rI   r;   r2   r<   r=   r>   r?   r@   rA   rB   rC   Tr   rP   rK   rL   rM   rR   r   r"   r"   r#   Ú&test_chuck_by_title_respects_multipageÍ   s,    õÿÿûrU   c                  C  s’   t dtdd�d�tdtdd�d�tdtdd�d�tdƒt d	ƒtd
ƒtdƒt dƒtdƒtdƒtƒ g} t| ddd�}|tdƒtdƒtdƒgksŽJ ‚d S rT   rR   r   r"   r"   r#   Ú'test_chunk_by_title_groups_across_pagesç   s,    õÿÿûrV   c                  C  s<   d} t | dd�}t | ƒ}t|ƒ}||ks,J ‚||ks8J ‚d S )Nú example-docs/example-10k-1p.htmlÚby_title)Úchunking_strategy)r   r   )ÚfilenameÚchunk_elementsr    r!   r"   r"   r#   Ú,test_add_chunking_strategy_on_partition_html  s    r\   c                  C  sž   d} t | ddddd�}t | ƒ}t|dddd�}|D ]$}t|tƒsDJ ‚t|jƒdks2J ‚q2|D ]$}t|tƒsnJ ‚t|jƒdks\J ‚q\||ksŽJ ‚||ksšJ ‚d S )NrW   rX   r   r8   éd   )rY   r   Únew_after_n_charsr1   )r   r^   r1   )r   r   r   r   r   Útext)rZ   r[   r    r!   r*   Zchunk_elementr"   r"   r#   Ú2test_add_chunking_strategy_respects_max_characters  s.    ûür`   c               	   C  sœ   t dtdd�d�tdtdd�d�tdtdd�d�t d	td
d�d�tdtdd�d�g} t| dd�}t|d ƒttdƒƒks|J ‚t|d ƒttdƒƒks˜J ‚d S )Nr:   ç      à?)Zdetection_class_probrI   r;   g×£p=
×ã?r<   g\�Âõ(\ç?r>   gáz®Gáê?r?   gffffffî?r   r   ú8A Great Day

Today is a great day.

It is sunny outside.r   ú"An Okay Day

Today is an okay day.)r   r   r   r   Ústrr   r   r"   r"   r#   Ú.test_chunk_by_title_drops_detection_class_prob&  sH    ÿþÿþÿþÿþÿþç ÿ
re   c                  C  sì   t dttdtddd�d�d�d�tdttd	td
d
d�d�d�d�tdttdtddd�d�d�d�t dttdtddd�d�d�d�tdttdtddd�d�d�d�g} t| dd�}t|d ƒttdƒƒksÌJ ‚t|d ƒttdƒƒksèJ ‚d S )Nr:   ))çš™™™™™¹?rf   )çš™™™™™É?rf   )rf   rg   ©rg   rg   rf   )ÚwidthÚheight)ZpointsÚsystem)ZcoordinatesrI   r;   )rh   )ç333333Ó?rg   )rg   rl   ©rl   rl   rg   r<   )rm   )çš™™™™™Ù?rl   )rl   rn   ©rn   rn   rl   r>   r?   )ro   )ra   rn   )rn   ra   )ra   ra   rn   r   r   rb   r   rc   )r   r   r   r   r   r   rd   r   r   r"   r"   r#   Ú(test_chunk_by_title_drops_extra_metadataN  sf    
ùÿþ
ùÿþ
ùÿþ
ùÿþ
ùÿþÇIÿ
rp   c                  C  sD   t dƒtdƒtdƒtdƒg} t| dd�}|tdƒtdƒgks@J ‚dS )	zHPreChunker includes length of separators when computing remaining space.zChunking Prioritiesz"Divide text into manageable chunkszPreserve semantic boundariesz!Minimize mid-text chunk-splittingés   r0   zUChunking Priorities

Divide text into manageable chunks

Preserve semantic boundariesN)r   r   r   r   r   r"   r"   r#   Ú4test_it_considers_separator_length_when_pre_chunking¡  s    üÿúrr   c                   @  sj   e Zd ZdZej dddidfddidfddidfi dfg¡ddd	d
œdd„ƒZe ¡ ddœdd„ƒZ	dS )ÚDescribe_chunk_by_titlezLUnit-test suite for `unstructured.chunking.title.chunk_by_title()` function.)ÚkwargsÚexpected_valuerE   TFNzdict[str, Any]Úboolr   )rt   ru   Ú_chunk_by_title_c                 C  s.   t g fi |¤Ž |jj\}}|j|u s*J ‚d S r'   )r   Z	call_argsÚargsrE   )Úselfrt   ru   rw   Ú_Úoptsr"   r"   r#   Ú,it_supports_the_include_orig_elements_optionÁ  s    zDDescribe_chunk_by_title.it_supports_the_include_orig_elements_optionr   )Úrequestc                 C  s
   t |dƒS )Nz+unstructured.chunking.title._chunk_by_title)r   )ry   r}   r"   r"   r#   rw   Ö  s    z(Describe_chunk_by_title._chunk_by_title_)
Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚpytestÚmarkÚparametrizer|   Zfixturerw   r"   r"   r"   r#   rs   ¾  s   


üþ	rs   c                   @  s”   e Zd ZdZej dddg¡ddœdd„ƒZd	d
„ Zdd„ Z	ej dddg¡ddddœdd„ƒZ
dd„ Zej ddddefg¡dddœdd„ƒZdS )ÚDescribe_ByTitleChunkingOptionszRUnit-test suite for `unstructured.chunking.title._ByTitleChunkingOptions` objects.Ún_charséÿÿÿÿiÖÿÿÿÚint)r†   c                 C  sD   t jtd|› �d�� tj|d� W d   ƒ n1 s60    Y  d S )Nz8'combine_text_under_n_chars' argument must be >= 0, got ©Úmatchr   ©r‚   ZraisesÚ
ValueErrorr
   Únew)ry   r†   r"   r"   r#   Ú:it_rejects_combine_text_under_n_chars_for_n_less_than_zeroÞ  s
    þzZDescribe_ByTitleChunkingOptions.it_rejects_combine_text_under_n_chars_for_n_less_than_zeroc                 C  s   t dd�}|jdksJ ‚dS )zSSpecifying `combine_text_under_n_chars=0` is how a caller disables chunk-combining.r   r   N)r
   r   ©ry   r{   r"   r"   r#   ÚFit_accepts_0_for_combine_text_under_n_chars_to_disable_chunk_combiningæ  s    
zfDescribe_ByTitleChunkingOptions.it_accepts_0_for_combine_text_under_n_chars_to_disable_chunk_combiningc                 C  s>   zt dd�}W n ty*   t d¡ Y n0 |jdks:J ‚dS )zUCaller can specify `combine_text_under_n_chars` arg without specifying other options.r8   r   z?did not accept `combine_text_under_n_chars` as option by itselfN)r
   rŒ   r‚   Úfailr   r�   r"   r"   r#   ÚIit_does_not_complain_when_specifying_combine_text_under_n_chars_by_itselfë  s
    ziDescribe_ByTitleChunkingOptions.it_does_not_complain_when_specifying_combine_text_under_n_chars_by_itself)r   r1   Úexpected_hard_max)éX  Niô  )r”   éÂ  r•   zOptional[int]c                 C  sL   t jtd|› d|› �d�� tj||d� W d  ƒ n1 s>0    Y  dS )aw  `combine_text_under_n_chars` > `max_characters` can produce behavior confusing to users.

        The behavior is no different from `combine_text_under_n_chars == max_characters`, but if
        `max_characters` is left to default (500) and `combine_text_under_n_chars` is set to a
        larger number like 1500 then it can look like chunk-combining isn't working.
        zR'combine_text_under_n_chars' argument must not exceed `max_characters` value, got z > r‰   )r1   r   Nr‹   )ry   r   r1   r“   r"   r"   r#   Ú;it_rejects_combine_text_under_n_chars_greater_than_maxcharsô  s    ÿÿýÿz[Describe_ByTitleChunkingOptions.it_rejects_combine_text_under_n_chars_greater_than_maxcharsc                 C  s@   zt jdd�}W n ty,   t d¡ Y n0 |jdks<J ‚dS )zPCaller can specify `new_after_n_chars` arg without specifying any other options.éÈ   )r^   z6did not accept `new_after_n_chars` as option by itselfN)r
   r�   rŒ   r‚   r‘   Zsoft_maxr�   r"   r"   r#   Ú@it_does_not_complain_when_specifying_new_after_n_chars_by_itself  s
    z`Describe_ByTitleChunkingOptions.it_does_not_complain_when_specifying_new_after_n_chars_by_itself)rQ   ru   )TT)FFNrv   c                 C  s   t |d�}|j|u sJ ‚d S )N)rQ   )r
   rQ   )ry   rQ   ru   r{   r"   r"   r#   Ú3it_knows_whether_to_break_chunks_on_page_boundaries  s    
zSDescribe_ByTitleChunkingOptions.it_knows_whether_to_break_chunks_on_page_boundaries)r~   r   r€   r�   r‚   rƒ   r„   rŽ   r�   r’   r–   r˜   r	   r™   r"   r"   r"   r#   r…   Û  s    	þ	þr…   )4r�   Ú
__future__r   Útypingr   r   r‚   Ztest_unstructured.unit_utilsr   r   r   r   Zunstructured.chunking.baser	   Zunstructured.chunking.titler
   r   Z"unstructured.documents.coordinatesr   Zunstructured.documents.elementsr   r   r   r   r   r   r   r   r   r   Zunstructured.partition.htmlr   Zunstructured.staging.baser   r$   r%   r.   r6   r7   r9   rH   rN   rS   rU   rV   r\   r`   re   rp   rr   rs   r…   r"   r"   r"   r#   Ú<module>   s8   0
	
			.	(S