a
    ¡˜ëj›+  ã                   @  sä  d Z ddlmZ ddlZddlZddlZddlZddlmZ ddl	m
Z
 ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ e e¡j ¡ Zej  !d¡Z"g d¢Z#ej  !d¡Z"dd„ Z$ej% &de#¡ddœdd„ƒZ'ej% &de#¡ddœdd„ƒZ(ej% &de#¡ddœdd„ƒZ)ej% &de#¡ddœdd„ƒZ*ej% &de#¡ddœdd„ƒZ+dd „ Z,d!d"„ Z-d#d$„ Z.d%d&„ Z/d'd(„ Z0d)d*œd+d,„Z1d-d.„ Z2d/d0„ Z3d)d*œd1d2„Z4d3d4„ Z5d5d6„ Z6d7d8„ Z7d9d:„ Z8dS );z6Test-suite for `unstructured.partition.ndjson` module.é    )ÚannotationsN)ÚMockFixture)Úexample_doc_path)ÚCompositeElement)ÚFileType)Úpartition_email)Úpartition_html©Úpartition_ndjson)Úpartition_text)Úpartition_xml)Úelements_to_ndjsonz/.dockerenv)úfake-text.txtzfake-html.htmlzeml/fake-email.emlc                  C  s<   t tdƒddd�} t| ƒdks"J ‚tdd„ | D ƒƒs8J ‚d S )Núspring-weather.html.ndjsonÚbasiciÜ  )Zchunking_strategyZmax_charactersé	   c                 s  s   | ]}t |tƒV  qd S ©N)Ú
isinstancer   )Ú.0Úch© r   úf/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/test_unstructured/partition/test_ndjson.pyÚ	<genexpr>+   ó    zPtest_it_chunks_elements_when_a_chunking_strategy_is_specified.<locals>.<genexpr>)r
   r   ÚlenÚall)Úchunksr   r   r   Ú=test_it_chunks_elements_when_a_chunking_strategy_is_specified#   s    ýr   ÚfilenameÚstr©r   c           	      C  sZ  t | ƒ}g }t tj |¡d ¡}|tjkr6t|d�}|tjkrJt	|d�}|tj
kr^t|d�}|tjkrrt|d�}t ¡ �D}tj | ¡}tj ||d ¡}t||d� t|d�}W d   ƒ n1 sÄ0    Y  t|ƒdksÞJ ‚tt|d ƒƒdksöJ ‚t|ƒt|ƒk�sJ ‚tt|ƒƒD ]<}|| || k�s2J ‚|| jj|  d¡d k�sJ ‚�qd S )Né   r    ú.ndjsonr   ú/éÿÿÿÿ)r   r   Úfrom_extensionÚosÚpathÚsplitextÚTXTr   ÚHTMLr   ÚXMLr   ÚEMLr   ÚtempfileÚTemporaryDirectoryÚbasenameÚjoinr   r
   r   r   ÚrangeÚmetadatar   Úsplit)	r   r'   ÚelementsÚfiletypeÚtmpdirÚ	_filenameÚ	test_pathÚtest_elementsÚir   r   r   Ú#test_partition_ndjson_from_filename.   s,    








(r;   c                 C  s  t | ƒ}g }t tj |¡d ¡}|tjkr6t|d�}|tjkrJt	|d�}|tj
kr^t|d�}|tjkrrt|d�}t ¡ �F}tj | ¡}tj ||d ¡}t||d� t|dd�}W d   ƒ n1 sÆ0    Y  t|ƒdksàJ ‚tt|d ƒƒdksøJ ‚tdd„ |D ƒƒ�sJ ‚d S )	Nr!   r    r"   Útest)r   Úmetadata_filenamer   c                 s  s   | ]}|j jd kV  qdS )r<   N)r2   r   )r   Úelementr   r   r   r   a   r   zMtest_partition_ndjson_from_filename_with_metadata_filename.<locals>.<genexpr>)r   r   r%   r&   r'   r(   r)   r   r*   r   r+   r   r,   r   r-   r.   r/   r0   r   r
   r   r   r   )r   r'   r4   r5   r6   r7   r8   r9   r   r   r   Ú:test_partition_ndjson_from_filename_with_metadata_filenameK   s&    








*r?   c           
   	   C  sˆ  t | ƒ}g }t tj |¡d ¡}|tjkr6t|d�}|tjkrJt	|d�}|tj
kr^t|d�}|tjkrrt|d�}t ¡ �n}tj | ¡}tj ||d ¡}t||d� t|dƒ�}t|d�}W d   ƒ n1 sÐ0    Y  W d   ƒ n1 sî0    Y  t|ƒdk�s
J ‚tt|d ƒƒdk�s$J ‚t|ƒt|ƒk�s:J ‚tt|ƒƒD ]<}	||	 ||	 k�s`J ‚||	 jj|  d¡d k�sFJ ‚�qFd S )	Nr!   r    r"   Úrb©Úfiler   r#   r$   )r   r   r%   r&   r'   r(   r)   r   r*   r   r+   r   r,   r   r-   r.   r/   r0   r   Úopenr
   r   r   r1   r2   r   r3   ©
r   r'   r4   r5   r6   r7   r8   Úfr9   r:   r   r   r   Útest_partition_ndjson_from_filed   s.    








FrF   c           
   	   C  s(  t | ƒ}g }t tj |¡d ¡}|tjkr6t|d�}|tjkrJt	|d�}|tj
kr^t|d�}|tjkrrt|d�}t ¡ �p}tj | ¡}tj ||d ¡}t||d� t|dƒ�}t|dd�}W d   ƒ n1 sÒ0    Y  W d   ƒ n1 sð0    Y  tt|ƒƒD ]}	||	 jjdk�sJ ‚�qd S )Nr!   r    r"   r@   r<   )rB   r=   )r   r   r%   r&   r'   r(   r)   r   r*   r   r+   r   r,   r   r-   r.   r/   r0   r   rC   r
   r1   r   r2   r   rD   r   r   r   Ú6test_partition_ndjson_from_file_with_metadata_filename�   s&    








HrG   c              	   C  sŽ  t | ƒ}g }t tj |¡d ¡}|tjkr6t|d�}|tjkrJt	|d�}|tj
kr^t|d�}|tjkrrt|d�}t ¡ �t}tj | ¡}tj ||d ¡}t||d� t|ƒ�}| ¡ }W d   ƒ n1 sÌ0    Y  t|d�}	W d   ƒ n1 sô0    Y  t|ƒdk�sJ ‚tt|d ƒƒdk�s*J ‚t|ƒt|	ƒk�s@J ‚tt|ƒƒD ]<}
||
 |	|
 k�sfJ ‚||
 jj|  d¡d k�sLJ ‚�qLd S )Nr!   r    r"   ©Útextr   r#   r$   )r   r   r%   r&   r'   r(   r)   r   r*   r   r+   r   r,   r   r-   r.   r/   r0   r   rC   Úreadr
   r   r   r1   r2   r   r3   )r   r'   r4   r5   r6   r7   r8   rE   rI   r9   r:   r   r   r   Útest_partition_ndjson_from_text™   s0    









&(rK   c                   C  s4   t  t¡� tƒ  W d   ƒ n1 s&0    Y  d S r   ©ÚpytestÚraisesÚ
ValueErrorr
   r   r   r   r   Ú.test_partition_json_raises_with_none_specified·   s    rP   c                   C  s   t dd�g ksJ ‚d S )NÚ rH   r	   r   r   r   r   Ú-test_partition_ndjson_works_with_empty_string¼   s    rR   c                   C  s8   t  t¡� tdd� W d   ƒ n1 s*0    Y  d S )Nz{}rH   rL   r   r   r   r   Ú+test_partition_ndjson_fails_with_empty_itemÀ   s    rS   c                   C  s8   t  t¡� tdd� W d   ƒ n1 s*0    Y  d S )Nz[]rH   rL   r   r   r   r   Ú+test_partition_ndjson_fails_with_empty_listÅ   s    rT   c               	   C  sÒ  t dƒ} g }t tj | ¡d ¡}|tjkr6t| d�}|tjkrJt	| d�}|tj
kr^t| d�}|tjkrrt| d�}t ¡ �b}tj |d¡}t||d� t|dƒ�}| ¡  d¡}W d   ƒ n1 sÄ0    Y  W d   ƒ n1 sâ0    Y  t t¡� t||d� W d   ƒ n1 �s0    Y  t t¡� t||d� W d   ƒ n1 �sR0    Y  t t¡� t||d	� W d   ƒ n1 �sŠ0    Y  t t¡� t|||d
� W d   ƒ n1 �sÄ0    Y  d S )Nr   r!   r    zfake-text.txt.ndjsonr@   zutf-8)r   rB   )r   rI   )rB   rI   )r   rB   rI   )r   r   r%   r&   r'   r(   r)   r   r*   r   r+   r   r,   r   r-   r.   r0   r   rC   rJ   ÚdecoderM   rN   rO   r
   )r'   r4   r5   r6   r8   rE   rI   r   r   r   Ú4test_partition_ndjson_raises_with_too_many_specifiedÊ   s0    








J,,,rV   r   )Úmockerc                   s<   d‰ | j dˆ d� ttdƒƒ}t‡ fdd„|D ƒƒs8J ‚d S )Nú2029-07-05T09:24:28ú4unstructured.partition.ndjson.get_last_modified_date©Zreturn_valuer   c                 3  s   | ]}|j jˆ kV  qd S r   ©r2   Zlast_modified©r   Úe©Úfilesystem_last_modifiedr   r   r   ö   r   zZtest_partition_ndjson_from_file_path_gets_last_modified_from_filesystem.<locals>.<genexpr>©Úpatchr
   r   r   )rW   r4   r   r^   r   ÚGtest_partition_ndjson_from_file_path_gets_last_modified_from_filesystemí   s    þrb   c                  C  sR   t tdƒdƒ�} t| d�}W d   ƒ n1 s.0    Y  tdd„ |D ƒƒsNJ ‚d S )Nr   r@   rA   c                 s  s   | ]}|j jd u V  qd S r   r[   r\   r   r   r   r   ý   r   zJtest_partition_ndjson_from_file_gets_last_modified_None.<locals>.<genexpr>©rC   r   r
   r   ©rE   r4   r   r   r   Ú7test_partition_ndjson_from_file_gets_last_modified_Noneù   s    (re   c                  C  sX   t tdƒƒ�} |  ¡ }W d   ƒ n1 s*0    Y  t|d�}tdd„ |D ƒƒsTJ ‚d S )Nr   rH   c                 s  s   | ]}|j jd u V  qd S r   r[   r\   r   r   r   r     r   zJtest_partition_ndjson_from_text_gets_last_modified_None.<locals>.<genexpr>©rC   r   rJ   r
   r   ©rE   rI   r4   r   r   r   Ú7test_partition_ndjson_from_text_gets_last_modified_None   s    &
rh   c                   sD   d}d‰ | j d|d� ttdƒˆ d�}t‡ fdd„|D ƒƒs@J ‚d S )	NrX   ú2020-07-05T09:24:28rY   rZ   r   ©Úmetadata_last_modifiedc                 3  s   | ]}|j jˆ kV  qd S r   r[   r\   rj   r   r   r     r   zVtest_partition_ndjson_from_file_path_prefers_metadata_last_modified.<locals>.<genexpr>r`   )rW   r_   r4   r   rj   r   ÚCtest_partition_ndjson_from_file_path_prefers_metadata_last_modified	  s    þþrl   c                    s\   d‰ t tdƒdƒ�} t| ˆ d�}W d   ƒ n1 s40    Y  t‡ fdd„|D ƒƒsXJ ‚d S )Nri   r   r@   )rB   rk   c                 3  s   | ]}|j jˆ kV  qd S r   r[   r\   rj   r   r   r     r   zQtest_partition_ndjson_from_file_prefers_metadata_last_modified.<locals>.<genexpr>rc   rd   r   rj   r   Ú>test_partition_ndjson_from_file_prefers_metadata_last_modified  s    *rm   c                    sb   d‰ t tdƒƒ�} |  ¡ }W d   ƒ n1 s.0    Y  t|ˆ d�}t‡ fdd„|D ƒƒs^J ‚d S )Nri   r   )rI   rk   c                 3  s   | ]}|j jˆ kV  qd S r   r[   r\   rj   r   r   r   (  r   zQtest_partition_ndjson_from_text_prefers_metadata_last_modified.<locals>.<genexpr>rf   rg   r   rj   r   Ú>test_partition_ndjson_from_text_prefers_metadata_last_modified!  s
    &rn   c                  C  s<   d} t  t¡� t| d� W d   ƒ n1 s.0    Y  d S )Nz{"invalid": "schema"}rH   rL   rH   r   r   r   Ú2test_partition_json_raises_with_unprocessable_json.  s    ro   c                  C  s<   d} t  t¡� t| d� W d   ƒ n1 s.0    Y  d S )Nz[{"hi": "there"}]]rH   rL   rH   r   r   r   Ú,test_partition_json_raises_with_invalid_json4  s    rp   )9Ú__doc__Ú
__future__r   r&   Úpathlibr-   rM   Zpytest_mockr   Ztest_unstructured.unit_utilsr   Zunstructured.documents.elementsr   Zunstructured.file_utils.modelr   Zunstructured.partition.emailr   Zunstructured.partition.htmlr   Zunstructured.partition.ndjsonr
   Zunstructured.partition.textr   Zunstructured.partition.xmlr   Zunstructured.staging.baser   ÚPathÚ__file__ÚparentÚresolveZ	DIRECTORYr'   ÚexistsZis_in_dockerZ
test_filesr   ÚmarkZparametrizer;   r?   rF   rG   rK   rP   rR   rS   rT   rV   rb   re   rh   rl   rm   rn   ro   rp   r   r   r   r   Ú<module>   sV   #	