a
    ¡˜ëjò9  ã                   @  sÔ  d dl mZ d dlZd dlZd dlmZmZ d dlZd dlm	Z	 d dl
mZmZ d dlmZ d dlmZ d dlmZmZmZmZ d d	lmZ d d
lmZ d dlmZ edd�edd�edd�edd�edd�edd�gZd dd¡ZdZej  !dg d¢¡dddœdd„ƒZ"dd„ Z#ej  !dg d ¢¡dd!œd"d#„ƒZ$ej  !d$d%d&e%fd'd&e&fg¡ddd(d$œd)d*„ƒZ'd+d,„ Z(d-d.„ Z)ej  !dg d ¢¡dd!œd/d0„ƒZ*d1d2„ Z+ej  !dg d ¢¡dd!œd3d4„ƒZ,d5d6„ Z-d7d8„ Z.d9d:„ Z/d;d<„ Z0d=d>„ Z1d?d@„ Z2dAdB„ Z3dCdD„ Z4dEdF„ Z5dGdH„ Z6dIdJ„ Z7dKdL„ Z8dMdN„ Z9dOdP„ Z:dQdR„ Z;dSdTœdUdV„Z<dWdX„ Z=dYdZ„ Z>dSdTœd[d\„Z?d]d^„ Z@d_d`„ ZAdadb„ ZBdcdd„ ZCej  !deg d¢¡ddfdeœdgdh„ƒZDdidj„ ZEdkdl„ ZFdmdn„ ZGdodp„ ZHdqdr„ ZIdsdt„ ZJdS )ué    )ÚannotationsN)ÚOptionalÚType)ÚMockerFixture)Úassert_round_trips_through_JSONÚexample_doc_path)Úchunk_by_title)Úgroup_broken_paragraphs)ÚAddressÚListItemÚNarrativeTextÚTitle)ÚFileType©Úpartition_text)Ú#UNSTRUCTURED_INCLUDE_DEBUG_METADATAz.This is a test document to use for unit tests.©ÚtextzDoylestown, PA 18901zImportant points:zHamburgers are deliciouszDogs are the bestzI love fuzzy blanketszÝThis is a story. This is a story that doesn't matter
 because it is just being used as an example. Hi. Hello. Howdy. Hola.
 The example is simple and repetitive and long and somewhat boring,
 but it serves a purpose. End.Ú
Ú zâThis is a story.

This is a story that doesn't matter because it is just being used as an example.

Hi.

Hello.

Howdy.

Hola.

The example is simple and repetitive and long and somewhat boring, but it serves a purpose.

End.
©ÚfilenameÚencoding))úfake-text.txtúutf-8©r   N)úfake-text-utf-16-be.txtz	utf-16-beÚstrzOptional[str]c                 C  sf   t t| ƒ|d�}t|ƒdks J ‚|tks,J ‚|D ]}|jj| ks0J ‚q0trbdd„ |D ƒdhksbJ ‚d S )N©r   r   c                 S  s   h | ]}|j j’qS © )ÚmetadataZdetection_origin©Ú.0Úelementr   r   úd/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/test_unstructured/partition/test_text.pyÚ	<setcomp>G   ó    z4test_partition_text_from_filename.<locals>.<setcomp>r   )r   r   ÚlenÚEXPECTED_OUTPUTr    r   r   )r   r   Úelementsr#   r   r   r$   Ú!test_partition_text_from_filename7   s    	r*   c                  C  s<   t tdƒddd�} | tksJ ‚| D ]}|jjdks"J ‚q"d S )Nr   r   Útest)r   Úmetadata_filename)r   r   r(   r    r   ©r)   r#   r   r   r$   Ú8test_partition_text_from_filename_with_metadata_filenameJ   s    
ÿr.   r   )zfake-text-utf-16.txtzfake-text-utf-16-le.txtzfake-text-utf-32.txt©r   c                 C  sF   t t| ƒƒ}t|ƒdksJ ‚|tks(J ‚|D ]}|jj| ks,J ‚q,d S )Nr   )r   r   r'   r(   r    r   )r   r)   r#   r   r   r$   Ú2test_partition_text_from_filename_default_encodingT   s
    r0   ©r   r   Úerrorr   zutf-16r   zType[BaseException]c                 C  sB   t  |¡�$ t| ƒ} t| |d� W d   ƒ n1 s40    Y  d S )Nr   )ÚpytestÚraisesr   r   r1   r   r   r$   Ú7test_partition_text_from_filename_raises_econding_errora   s    r5   c                  C  sr   t tdƒdƒ�} t| d�}W d   ƒ n1 s.0    Y  t|ƒdksHJ ‚|tksTJ ‚|D ]}|jjd u sXJ ‚qXd S ©Nr   Úrb©Úfiler   ©Úopenr   r   r'   r(   r    r   ©Úfr)   r#   r   r   r$   Útest_partition_text_from_filer   s    (r>   c                  C  sx   t dƒ} t| dƒ�}t|dd�}W d   ƒ n1 s40    Y  t|ƒdksNJ ‚|tksZJ ‚|D ]}|jjdks^J ‚q^d S )Nr   r7   r+   ©r9   r,   r   )r   r;   r   r'   r(   r    r   ©r   r=   r)   r#   r   r   r$   Ú4test_partition_text_from_file_with_metadata_filename|   s    *rA   c                 C  sr   t t| ƒdƒ�}t|d�}W d   ƒ n1 s.0    Y  t|ƒdksHJ ‚|tksTJ ‚|D ]}|jjd u sXJ ‚qXd S ©Nr7   r8   r   r:   r@   r   r   r$   Ú.test_partition_text_from_file_default_encoding†   s    (rC   c                  C  sr   t tdƒdƒ�} t| d�}W d   ƒ n1 s.0    Y  t|ƒdksHJ ‚|tksTJ ‚|D ]}|jjd u sXJ ‚qXd S r6   r:   r<   r   r   r$   Ú#test_partition_text_from_bytes_file“   s    (rD   c                 C  sr   t t| ƒdƒ�}t|d�}W d   ƒ n1 s.0    Y  t|ƒdksHJ ‚|tksTJ ‚|D ]}|jjd u sXJ ‚qXd S rB   r:   r@   r   r   r$   Ú4test_partition_text_from_bytes_file_default_encoding�   s    (rE   c                  C  s.   t tdƒddgd�} | d jjdgks*J ‚d S )Núbook-war-and-peace-1p.txtÚfastÚen)ZstrategyÚ	languagesr   Úeng©r   r   r    rI   ©r)   r   r   r$   Ú<test_text_partition_element_metadata_user_provided_languages«   s    ÿrM   c                  C  sx   t tdƒƒ�} |  ¡ }W d   ƒ n1 s*0    Y  t|d�}t|ƒdksNJ ‚|tksZJ ‚|D ]}|jjd u s^J ‚q^d S )Nr   r   r   )r;   r   Úreadr   r'   r(   r    r   )r=   r   r)   r#   r   r   r$   Útest_partition_text_from_text²   s    &
rO   c                   C  s   t dd�g ksJ ‚d S )Nr   r   r   r   r   r   r$   Ú5test_partition_text_from_text_works_with_empty_string¾   s    rP   c                   C  s4   t  t¡� tƒ  W d   ƒ n1 s&0    Y  d S ©N)r3   r4   Ú
ValueErrorr   r   r   r   r$   Ú.test_partition_text_raises_with_none_specifiedÂ   s    rS   c                  C  sr   t dƒ} t| ƒ�}| ¡ }W d   ƒ n1 s.0    Y  t t¡� t| |d� W d   ƒ n1 sd0    Y  d S )Nr   )r   r   )r   r;   rN   r3   r4   rR   r   )r   r=   r   r   r   r$   Ú2test_partition_text_raises_with_too_many_specifiedÇ   s
    
&rT   c                  C  sH   d} t | d�}|tdd�tdd�gks*J ‚|D ]}|jjd u s.J ‚q.d S )Nz6
    VERY IMPORTANT MEMO
    DOYLESTOWN, PA 18901
    r   zVERY IMPORTANT MEMOzDOYLESTOWN, PA 18901)r   r   r
   r    r   ©r   r)   r#   r   r   r$   Ú<test_partition_text_captures_everything_even_with_linebreaksÐ   s    
þ
rV   c                  C  sJ   d} t | td�}|tdd�tdd�gks,J ‚|D ]}|jjd u s0J ‚q0d S )NzYThe big brown fox
was walking down the lane.

At the end of the lane,
the fox met a bear.)r   Zparagraph_grouperz,The big brown fox was walking down the lane.r   z+At the end of the lane, the fox met a bear.)r   r	   r   r    r   rU   r   r   r$   Ú,test_partition_text_groups_broken_paragraphsÞ   s    ÿþ
rW   c                  C  sH   t tdƒƒ} t| ƒdksJ ‚| d j d¡s0J ‚| d j d¡sDJ ‚d S )Núnorwich-city.txtr   zIwan RobertséÿÿÿÿzExternal links)r   r   r'   r   Ú
startswithÚendswithrL   r   r   r$   Ú$test_partition_text_splits_long_textñ   s    r\   c                  C  sF   d} t | d�}t|ƒdksJ ‚|d j| ks0J ‚t|d tƒrBJ ‚d S )Nz--------------------r   é   r   )r   r'   r   Ú
isinstancer   )r   r)   r   r   r$   Ú*test_partition_text_doesnt_get_page_breaksø   s
    
r_   c                  C  s<   t tdƒƒ} tdd„ | D ƒƒs"J ‚tdd„ | D ƒƒs8J ‚d S )Nr   c                 s  s   | ]}|j jd kV  qdS r   ©r    r   ©r"   Úer   r   r$   Ú	<genexpr>  r&   zZtest_partition_text_from_filename_gets_filename_metadata_from_file_path.<locals>.<genexpr>c                 s  s   | ]}|j jtd ƒkV  qdS )r   N)r    Úfile_directoryr   ra   r   r   r$   rc     r&   ©r   r   ÚallrL   r   r   r$   ÚGtest_partition_text_from_filename_gets_filename_metadata_from_file_path  s    rg   c                  C  sh   t tdƒdƒ�} t| d�}W d   ƒ n1 s.0    Y  tdd„ |D ƒƒsNJ ‚tdd„ |D ƒƒsdJ ‚d S )Nr   r7   r8   c                 s  s   | ]}|j jd u V  qd S rQ   r`   ra   r   r   r$   rc     r&   zLtest_partition_text_from_file_gets_filename_metadata_None.<locals>.<genexpr>c                 s  s   | ]}|j jd u V  qd S rQ   ©r    rd   ra   r   r   r$   rc     r&   ©r;   r   r   rf   ©r=   r)   r   r   r$   Ú9test_partition_text_from_file_gets_filename_metadata_None
  s    (rk   c                  C  s@   t tdƒdd�} tdd„ | D ƒƒs&J ‚tdd„ | D ƒƒs<J ‚d S )Nr   z	a/b/c.txt)r,   c                 s  s   | ]}|j jd kV  qdS )zc.txtNr`   ra   r   r   r$   rc     r&   zNtest_partition_text_from_filename_prefers_metadata_filename.<locals>.<genexpr>c                 s  s   | ]}|j jd kV  qdS )za/bNrh   ra   r   r   r$   rc     r&   re   rL   r   r   r$   Ú;test_partition_text_from_filename_prefers_metadata_filename  s    rl   c                  C  sj   t tdƒdƒ�} t| dd�}W d   ƒ n1 s00    Y  tdd„ |D ƒƒsPJ ‚tdd„ |D ƒƒsfJ ‚d S )Nr   r7   z	d/e/f.txtr?   c                 s  s   | ]}|j jd kV  qdS )zf.txtNr`   ra   r   r   r$   rc     r&   zJtest_partition_text_from_file_prefers_metadata_filename.<locals>.<genexpr>c                 s  s   | ]}|j jd kV  qdS )zd/eNrh   ra   r   r   r$   rc     r&   ri   rj   r   r   r$   Ú7test_partition_text_from_file_prefers_metadata_filename  s    *rm   c                    sJ   d‰ t tdƒƒ} t‡ fdd„| D ƒƒsFJ dˆ › dt| d jjƒ› �ƒ‚d S )Nz
text/plainr   c                 3  s   | ]}|j jˆ kV  qd S rQ   ©r    Úfiletypera   ©ZTXT_MIME_TYPEr   r$   rc   '  r&   zRtest_partition_text_gets_the_TXT_MIME_type_in_metadata_filetype.<locals>.<genexpr>zExpected all elements to have 'z' as their filetype, but got: r   )r   r   rf   Úreprr    ro   rL   r   rp   r$   Ú?test_partition_text_gets_the_TXT_MIME_type_in_metadata_filetype$  s    ÿÿrr   c                  C  sB   t tdƒtjd�} tdd„ | D ƒƒs>J dt| d jjƒ› �ƒ‚d S )Nz	README.md)Zmetadata_file_typec                 s  s   | ]}|j jd kV  qdS )ztext/markdownNrn   ra   r   r   r$   rc   /  r&   zAtest_partition_text_prefers_metadata_file_type.<locals>.<genexpr>zJExpected all elements to have 'text/markdown' as their filetype, but got: r   )r   r   r   ZMDrf   rq   r    ro   rL   r   r   r$   Ú.test_partition_text_prefers_metadata_file_type-  s    ÿÿrs   r   )Úmockerc                   s<   d‰ | j dˆ d� ttdƒƒ}t‡ fdd„|D ƒƒs8J ‚d S )Nú2029-07-05T09:24:28ú2unstructured.partition.text.get_last_modified_date©Zreturn_valuer   c                 3  s   | ]}|j jˆ kV  qd S rQ   ©r    Zlast_modifiedra   ©Úfilesystem_last_modifiedr   r$   rc   @  r&   zXtest_partition_text_from_file_path_gets_last_modified_from_filesystem.<locals>.<genexpr>©Úpatchr   r   rf   )rt   r)   r   ry   r$   ÚEtest_partition_text_from_file_path_gets_last_modified_from_filesystem8  s    ÿr}   c                  C  sR   t tdƒdƒ�} t| d�}W d   ƒ n1 s.0    Y  tdd„ |D ƒƒsNJ ‚d S )Nr   r7   r8   c                 s  s   | ]}|j jd u V  qd S rQ   rx   ra   r   r   r$   rc   G  r&   zHtest_partition_text_from_file_gets_last_modified_None.<locals>.<genexpr>ri   rj   r   r   r$   Ú5test_partition_text_from_file_gets_last_modified_NoneC  s    (r~   c                  C  sX   t tdƒƒ�} |  ¡ }W d   ƒ n1 s*0    Y  t|d�}tdd„ |D ƒƒsTJ ‚d S )Nr   r   c                 s  s   | ]}|j jd u V  qd S rQ   rx   ra   r   r   r$   rc   P  r&   zHtest_partition_text_from_text_gets_last_modified_None.<locals>.<genexpr>©r;   r   rN   r   rf   ©r=   r   r)   r   r   r$   Ú5test_partition_text_from_text_gets_last_modified_NoneJ  s    &
r�   c                   sD   d}d‰ | j d|d� ttdƒˆ d�}t‡ fdd„|D ƒƒs@J ‚d S )	Nru   ú2020-07-05T09:24:28rv   rw   r   ©Úmetadata_last_modifiedc                 3  s   | ]}|j jˆ kV  qd S rQ   rx   ra   rƒ   r   r$   rc   ^  r&   zTtest_partition_text_from_file_path_prefers_metadata_last_modified.<locals>.<genexpr>r{   )rt   rz   r)   r   rƒ   r$   ÚAtest_partition_text_from_file_path_prefers_metadata_last_modifiedS  s    ÿÿr…   c                    s\   d‰ t tdƒdƒ�} t| ˆ d�}W d   ƒ n1 s40    Y  t‡ fdd„|D ƒƒsXJ ‚d S )Nr‚   r   r7   )r9   r„   c                 3  s   | ]}|j jˆ kV  qd S rQ   rx   ra   rƒ   r   r$   rc   f  r&   zOtest_partition_text_from_file_prefers_metadata_last_modified.<locals>.<genexpr>ri   rj   r   rƒ   r$   Ú<test_partition_text_from_file_prefers_metadata_last_modifieda  s    *r†   c                    sb   d‰ t tdƒƒ�} |  ¡ }W d   ƒ n1 s.0    Y  t|ˆ d�}t‡ fdd„|D ƒƒs^J ‚d S )Nr‚   r   )r   r„   c                 3  s   | ]}|j jˆ kV  qd S rQ   rx   ra   rƒ   r   r$   rc   p  r&   zOtest_partition_text_from_text_prefers_metadata_last_modified.<locals>.<genexpr>r   r€   r   rƒ   r$   Ú<test_partition_text_from_text_prefers_metadata_last_modifiedi  s
    &r‡   c                  C  s(   dd„ t dd�D ƒ} | g d¢ks$J ‚d S )Nc                 S  s   g | ]
}|j ‘qS r   )Úidr!   r   r   r$   Ú
<listcomp>w  r&   zYtest_Text_element_assigns_id_hashes_that_are_unique_and_deterministic.<locals>.<listcomp>úhello
hello
hellor   )Z 8657c0ec31a4cfc822f6cd4a5684cafdZ 72aefb4a12be063ad160931fdb380163Z ba8c1a216ca585aecdd365a72e6124f1r   )Úidsr   r   r$   ÚEtest_Text_element_assigns_id_hashes_that_are_unique_and_deterministicv  s    rŒ   c                  C  s<   t ddd�} | D ]&}tj|jdd�s(J ‚t | ¡ ¡ qd S )NrŠ   T)r   Zunique_element_idsé   )Úversion)r   ÚuuidÚUUIDrˆ   ÚjsonÚdumpsÚto_dictr-   r   r   r$   Ú>test_Text_element_assings_UUID_when_unique_element_ids_is_True  s    r”   )Ú	file_namer   z
str | Nonec                 C  s   t t| ƒ|d�}t|ƒ d S )Nr   )r   r   r   )r•   r   r)   r   r   r$   Útest_partition_text_with_json‰  s    	r–   c                  C  sB   t dƒ} t| d�}t| dd�}t|ƒ}||ks2J ‚||ks>J ‚d S )NrF   r/   Zby_title)Zchunking_strategy)r   r   r   )r   r)   Zchunk_elementsÚchunksr   r   r$   Ú,test_add_chunking_strategy_on_partition_text–  s    
r˜   c                  C  s&   t tdƒƒ} | d jjdgks"J ‚d S )NrX   r   rJ   rK   rL   r   r   r$   Ú2test_partition_text_element_metadata_has_languagesŸ  s    r™   c                  C  sD   t tdƒdd�} dd„ | D ƒ}|dgddgdgdgdggks@J ‚d S )Nzlanguage-docs/eng_spa_mult.txtT©Zdetect_language_per_elementc                 S  s   g | ]}|j j‘qS r   ©r    rI   r!   r   r   r$   r‰   ©  r&   zLtest_partition_text_respects_detect_language_per_element.<locals>.<listcomp>rJ   Zspa)r   r   ©r)   Zlangsr   r   r$   Ú8test_partition_text_respects_detect_language_per_element¤  s
    ÿr�   c                  C  s,   t tdƒdgd�} | d jjdgks(J ‚d S )NrX   Zdeu©rI   r   rK   rL   r   r   r$   Ú*test_partition_text_respects_languages_arg®  s    rŸ   c                   C  s>   t  t¡�  ttdƒdd� W d   ƒ n1 s00    Y  d S )NrX   rJ   rž   )r3   r4   Ú	TypeErrorr   r   r   r   r   r$   Ú5test_partition_text_element_metadata_raises_TypeError³  s    r¡   c                  C  s2   t tdƒdd�} dd„ | D ƒ}t|ƒdks.J ‚d S )Nz(language-docs/UDHR_first_article_all.txtTrš   c                 S  s    g | ]}|j jr|j jd  ‘qS )r   r›   ra   r   r   r$   r‰   ½  r&   zEtest_partition_text_detects_more_than_3_languages.<locals>.<listcomp>é
   )r   r   r'   rœ   r   r   r$   Ú1test_partition_text_detects_more_than_3_languages¸  s    þr£   )KÚ
__future__r   r‘   r�   Útypingr   r   r3   Zpytest_mockr   Ztest_unstructured.unit_utilsr   r   Zunstructured.chunking.titler   Zunstructured.cleaners.corer	   Zunstructured.documents.elementsr
   r   r   r   Zunstructured.file_utils.modelr   Zunstructured.partition.textr   Z&unstructured.partition.utils.constantsr   r(   ÚreplaceZMIN_MAX_TEXTZSHORT_PARAGRAPHSÚmarkZparametrizer*   r.   r0   ÚUnicodeDecodeErrorÚUnicodeErrorr5   r>   rA   rC   rD   rE   rM   rO   rP   rS   rT   rV   rW   r\   r_   rg   rk   rl   rm   rr   rs   r}   r~   r�   r…   r†   r‡   rŒ   r”   r–   r˜   r™   r�   rŸ   r¡   r£   r   r   r   r$   Ú<module>   s²   ú	ûþ
þ	þþ


þ	
þ
				
þ	
