a
    ¡˜ëj,  ã                   @  sì  d Z ddlmZ ddlZddlZddlZddlZddlmZ ddl	m
Z
 ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ e e¡j ¡ Zej  !d¡Z"g d¢Z#ej  !d¡Z"dd„ Z$ej% &de#¡ddœdd„ƒZ'ej% &de#¡ddœdd„ƒZ(ej% &de#¡ddœdd„ƒZ)ej% &de#¡ddœdd„ƒZ*ej% &de#¡ddœdd„ƒZ+dd „ Z,d!d"„ Z-d#d$„ Z.d%d&„ Z/d'd(„ Z0d)d*œd+d,„Z1d-d.„ Z2d/d0„ Z3d)d*œd1d2„Z4d3d4„ Z5d5d6„ Z6d7d8„ Z7d9d:„ Z8d;d<„ Z9dS )=z4Test-suite for `unstructured.partition.json` module.é    )ÚannotationsN)ÚMockFixture)Úexample_doc_path)ÚCompositeElement)ÚFileType)Úpartition_email)Úpartition_html©Úpartition_json)Úpartition_text)Úpartition_xml)Úelements_to_jsonz/.dockerenv)úfake-text.txtzfake-html.htmlzeml/fake-email.emlc                  C  s8   t dddd�} t| ƒdksJ ‚tdd„ | D ƒƒs4J ‚d S )Nú%example-docs/spring-weather.html.jsonÚbasiciÜ  )Zchunking_strategyZmax_charactersé	   c                 s  s   | ]}t |tƒV  qd S ©N)Ú
isinstancer   )Ú.0Úch© r   úd/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/test_unstructured/partition/test_json.pyÚ	<genexpr>)   ó    zPtest_it_chunks_elements_when_a_chunking_strategy_is_specified.<locals>.<genexpr>)r
   ÚlenÚall)Úchunksr   r   r   Ú=test_it_chunks_elements_when_a_chunking_strategy_is_specified#   s
    ÿr   ÚfilenameÚstr©r   c           	      C  s\  t | ƒ}g }t tj |¡d ¡}|tjkr6t|d�}|tjkrJt	|d�}|tj
kr^t|d�}|tjkrrt|d�}t ¡ �F}tj | ¡}tj ||d ¡}t||dd� t|d�}W d   ƒ n1 sÆ0    Y  t|ƒdksàJ ‚tt|d ƒƒdksøJ ‚t|ƒt|ƒk�sJ ‚tt|ƒƒD ]<}|| || k�s4J ‚|| jj|  d¡d k�sJ ‚�qd S )	Né   r    ú.jsoné   ©r   Úindentr   ú/éÿÿÿÿ)r   r   Úfrom_extensionÚosÚpathÚsplitextÚTXTr   ÚHTMLr   ÚXMLr   ÚEMLr   ÚtempfileÚTemporaryDirectoryÚbasenameÚjoinr   r
   r   r   ÚrangeÚmetadatar   Úsplit)	r   r*   ÚelementsÚfiletypeÚtmpdirÚ	_filenameÚ	test_pathÚtest_elementsÚir   r   r   Ú!test_partition_json_from_filename,   s,    








(r>   c                 C  s  t | ƒ}g }t tj |¡d ¡}|tjkr6t|d�}|tjkrJt	|d�}|tj
kr^t|d�}|tjkrrt|d�}t ¡ �H}tj | ¡}tj ||d ¡}t||dd� t|dd�}W d   ƒ n1 sÈ0    Y  t|ƒdksâJ ‚tt|d ƒƒdksúJ ‚td	d
„ |D ƒƒ�sJ ‚d S )Nr!   r    r"   r#   r$   Útest)r   Úmetadata_filenamer   c                 s  s   | ]}|j jd kV  qdS )r?   N)r5   r   )r   Úelementr   r   r   r   _   r   zKtest_partition_json_from_filename_with_metadata_filename.<locals>.<genexpr>)r   r   r(   r)   r*   r+   r,   r   r-   r   r.   r   r/   r   r0   r1   r2   r3   r   r
   r   r   r   )r   r*   r7   r8   r9   r:   r;   r<   r   r   r   Ú8test_partition_json_from_filename_with_metadata_filenameI   s&    








*rB   c           
   	   C  sŠ  t | ƒ}g }t tj |¡d ¡}|tjkr6t|d�}|tjkrJt	|d�}|tj
kr^t|d�}|tjkrrt|d�}t ¡ �p}tj | ¡}tj ||d ¡}t||dd� t|dƒ�}t|d�}W d   ƒ n1 sÒ0    Y  W d   ƒ n1 sð0    Y  t|ƒdk�sJ ‚tt|d ƒƒdk�s&J ‚t|ƒt|ƒk�s<J ‚tt|ƒƒD ]<}	||	 ||	 k�sbJ ‚||	 jj|  d	¡d
 k�sHJ ‚�qHd S )Nr!   r    r"   r#   r$   Úrb©Úfiler   r&   r'   )r   r   r(   r)   r*   r+   r,   r   r-   r   r.   r   r/   r   r0   r1   r2   r3   r   Úopenr
   r   r   r4   r5   r   r6   ©
r   r*   r7   r8   r9   r:   r;   Úfr<   r=   r   r   r   Útest_partition_json_from_fileb   s.    








FrI   c           
   	   C  s*  t | ƒ}g }t tj |¡d ¡}|tjkr6t|d�}|tjkrJt	|d�}|tj
kr^t|d�}|tjkrrt|d�}t ¡ �r}tj | ¡}tj ||d ¡}t||dd� t|dƒ�}t|dd�}W d   ƒ n1 sÔ0    Y  W d   ƒ n1 sò0    Y  tt|ƒƒD ]}	||	 jjdk�sJ ‚�qd S )	Nr!   r    r"   r#   r$   rC   r?   )rE   r@   )r   r   r(   r)   r*   r+   r,   r   r-   r   r.   r   r/   r   r0   r1   r2   r3   r   rF   r
   r4   r   r5   r   rG   r   r   r   Ú4test_partition_json_from_file_with_metadata_filename   s&    








HrJ   c              	   C  s�  t | ƒ}g }t tj |¡d ¡}|tjkr6t|d�}|tjkrJt	|d�}|tj
kr^t|d�}|tjkrrt|d�}t ¡ �v}tj | ¡}tj ||d ¡}t||dd� t|ƒ�}| ¡ }W d   ƒ n1 sÎ0    Y  t|d�}	W d   ƒ n1 sö0    Y  t|ƒdk�sJ ‚tt|d ƒƒdk�s,J ‚t|ƒt|	ƒk�sBJ ‚tt|ƒƒD ]<}
||
 |	|
 k�shJ ‚||
 jj|  d¡d	 k�sNJ ‚�qNd S )
Nr!   r    r"   r#   r$   ©Útextr   r&   r'   )r   r   r(   r)   r*   r+   r,   r   r-   r   r.   r   r/   r   r0   r1   r2   r3   r   rF   Úreadr
   r   r   r4   r5   r   r6   )r   r*   r7   r8   r9   r:   r;   rH   rL   r<   r=   r   r   r   Útest_partition_json_from_text—   s0    









&(rN   c                   C  s4   t  t¡� tƒ  W d   ƒ n1 s&0    Y  d S r   ©ÚpytestÚraisesÚ
ValueErrorr
   r   r   r   r   Ú.test_partition_json_raises_with_none_specifiedµ   s    rS   c                   C  s   t dd�g ksJ ‚d S )NÚ rK   r	   r   r   r   r   Ú+test_partition_json_works_with_empty_stringº   s    rU   c                   C  s8   t  t¡� tdd� W d   ƒ n1 s*0    Y  d S )Nz{}rK   rO   r   r   r   r   Ú)test_partition_json_fails_with_empty_item¾   s    rV   c                   C  s   t dd�g ksJ ‚d S )Nz[]rK   r	   r   r   r   r   Ú)test_partition_json_works_with_empty_listÃ   s    rW   c               	   C  sÔ  t dƒ} g }t tj | ¡d ¡}|tjkr6t| d�}|tjkrJt	| d�}|tj
kr^t| d�}|tjkrrt| d�}t ¡ �d}tj |d¡}t||dd� t|dƒ�}| ¡  d¡}W d   ƒ n1 sÆ0    Y  W d   ƒ n1 sä0    Y  t t¡� t||d	� W d   ƒ n1 �s0    Y  t t¡� t||d
� W d   ƒ n1 �sT0    Y  t t¡� t||d� W d   ƒ n1 �sŒ0    Y  t t¡� t|||d� W d   ƒ n1 �sÆ0    Y  d S )Nr   r!   r    zfake-text.txt.jsonr#   r$   rC   zutf-8)r   rE   )r   rL   )rE   rL   )r   rE   rL   )r   r   r(   r)   r*   r+   r,   r   r-   r   r.   r   r/   r   r0   r1   r3   r   rF   rM   ÚdecoderP   rQ   rR   r
   )r*   r7   r8   r9   r;   rH   rL   r   r   r   Ú2test_partition_json_raises_with_too_many_specifiedÇ   s0    








J,,,rY   r   )Úmockerc                   s<   d‰ | j dˆ d� ttdƒƒ}t‡ fdd„|D ƒƒs8J ‚d S )Nú2029-07-05T09:24:28ú2unstructured.partition.json.get_last_modified_date©Zreturn_valueúspring-weather.html.jsonc                 3  s   | ]}|j jˆ kV  qd S r   ©r5   Zlast_modified©r   Úe©Úfilesystem_last_modifiedr   r   r   ò   r   zXtest_partition_json_from_file_path_gets_last_modified_from_filesystem.<locals>.<genexpr>)Úpatchr
   r   r   )rZ   r7   r   rb   r   ÚEtest_partition_json_from_file_path_gets_last_modified_from_filesystemê   s    ÿre   c                  C  sN   t ddƒ�} t| d�}W d   ƒ n1 s*0    Y  tdd„ |D ƒƒsJJ ‚d S )Nr   rC   rD   c                 s  s   | ]}|j jd u V  qd S r   r_   r`   r   r   r   r   ù   r   zHtest_partition_json_from_file_gets_last_modified_None.<locals>.<genexpr>)rF   r
   r   ©rH   r7   r   r   r   Ú5test_partition_json_from_file_gets_last_modified_Noneõ   s    (rg   c                  C  sT   t dƒ�} |  ¡ }W d   ƒ n1 s&0    Y  t|d�}tdd„ |D ƒƒsPJ ‚d S )Nr   rK   c                 s  s   | ]}|j jd u V  qd S r   r_   r`   r   r   r   r     r   zHtest_partition_json_from_text_gets_last_modified_None.<locals>.<genexpr>©rF   rM   r
   r   ©rH   rL   r7   r   r   r   Ú5test_partition_json_from_text_gets_last_modified_Noneü   s    
&
rj   c                   s@   d}d‰ | j d|d� tdˆ d�}t‡ fdd„|D ƒƒs<J ‚d S )	Nr[   ú2020-07-05T09:24:28r\   r]   r   ©Úmetadata_last_modifiedc                 3  s   | ]}|j jˆ kV  qd S r   r_   r`   rl   r   r   r     r   zTtest_partition_json_from_file_path_prefers_metadata_last_modified.<locals>.<genexpr>)rd   r
   r   )rZ   rc   r7   r   rl   r   ÚAtest_partition_json_from_file_path_prefers_metadata_last_modified  s    ÿÿrn   c                    s\   d‰ t tdƒdƒ�} t| ˆ d�}W d   ƒ n1 s40    Y  t‡ fdd„|D ƒƒsXJ ‚d S )Nrk   r^   rC   )rE   rm   c                 3  s   | ]}|j jˆ kV  qd S r   r_   r`   rl   r   r   r     r   zOtest_partition_json_from_file_prefers_metadata_last_modified.<locals>.<genexpr>)rF   r   r
   r   rf   r   rl   r   Ú<test_partition_json_from_file_prefers_metadata_last_modified  s    *ro   c                    s^   d‰ t dƒ�} |  ¡ }W d   ƒ n1 s*0    Y  t|ˆ d�}t‡ fdd„|D ƒƒsZJ ‚d S )Nrk   r   )rL   rm   c                 3  s   | ]}|j jˆ kV  qd S r   r_   r`   rl   r   r   r   "  r   zOtest_partition_json_from_text_prefers_metadata_last_modified.<locals>.<genexpr>rh   ri   r   rl   r   Ú<test_partition_json_from_text_prefers_metadata_last_modified  s
    
&rp   c                  C  s<   d} t  t¡� t| d� W d   ƒ n1 s.0    Y  d S )Nz[{"invalid": "schema"}]rK   rO   rK   r   r   r   Ú8test_partition_json_raises_with_unprocessable_json_array(  s    rq   c                  C  s<   d} t  t¡� t| d� W d   ƒ n1 s.0    Y  d S )Nz{"hi": "there"}rK   rO   rK   r   r   r   Ú2test_partition_json_raises_with_unprocessable_json.  s    rr   c                  C  s<   d} t  t¡� t| d� W d   ƒ n1 s.0    Y  d S )Nz[{"hi": "there"}]]rK   rO   rK   r   r   r   Ú,test_partition_json_raises_with_invalid_json6  s    rs   ):Ú__doc__Ú
__future__r   r)   Úpathlibr0   rP   Zpytest_mockr   Ztest_unstructured.unit_utilsr   Zunstructured.documents.elementsr   Zunstructured.file_utils.modelr   Zunstructured.partition.emailr   Zunstructured.partition.htmlr   Zunstructured.partition.jsonr
   Zunstructured.partition.textr   Zunstructured.partition.xmlr   Zunstructured.staging.baser   ÚPathÚ__file__ÚparentÚresolveZ	DIRECTORYr*   ÚexistsZis_in_dockerZ
test_filesr   ÚmarkZparametrizer>   rB   rI   rJ   rN   rS   rU   rV   rW   rY   re   rg   rj   rn   ro   rp   rq   rr   rs   r   r   r   r   Ú<module>   sX   	#	