a
    ¡˜ëj†]  ã                   @   s2  d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZ d dl	Z	d dl
Z
d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZmZ d d
lmZmZmZmZmZ ddlmZm Z m!Z!m"Z" e #e$¡j% &¡ Z'dZ(e )dd¡ *¡ dvZ+e )dd¡ *¡ dkZ,e e-e dœdd„Z.e e-e dœdd„Z/e e-e e	j0dœdd„Z1dd„ Z2dd„ Z3e dœd d!„Z4e	j5j6e+ d"d#�e	j5j6e,d$d#�d%d&„ ƒƒZ7e	j5j6e+ d"d#�e	j5j6e,d$d#�d'd(„ ƒƒZ8e	j5j6e+ d"d#�e	j5j6e,d$d#�d)d*„ ƒƒZ9e	j5j6e+ d"d#�e	j5j6e,d$d#�d+d,„ ƒƒZ:d-d.„ Z;d/d0„ Z<d1d2„ Z=d3d4„ Z>d5d6„ Z?e dœd7d8„Z@e dœd9d:„ZAe dœd;d<„ZBe	j0e d=œd>d?„ZCd@dA„ ZDe dœdBdC„ZEdDdE„ ZFdFdG„ ZGdHdI„ ZHdJdK„ ZIe	j5j6e+ d"d#�e	j5j6e,d$d#�dLdM„ ƒƒZJe	j5j6e+ d"d#�dNdO„ ƒZKdPZLG dQdR„ dRƒZMG dSdT„ dTƒZNG dUdV„ dVƒZOe	 P¡ dWdX„ ƒZQdS )Yé    N)ÚAny)ÚMock)ÚGeneral)Úshared)ÚPartitionRequest)ÚPartitionParameters)Úretries)ÚElementTypeÚNarrativeText)Ú$DEFAULT_RETRIES_MAX_ELAPSED_TIME_SECÚ DEFAULT_RETRIES_MAX_INTERVAL_SECÚget_retries_configÚpartition_multiple_via_apiÚpartition_via_apié   )ÚANYÚFixtureRequestÚexample_doc_pathÚmethod_mockz1https://api.unstructuredapp.io/general/v0/generalÚCIÚ >   r   ÚfÚfalseÚ0ZGITHUB_REF_NAMEÚmain)ÚrequestÚexpected_call_c                 C   s„   t | tdtdd�d�}ttdƒd�}|j|Ž  t|jd jd tƒsHJ ‚t	|ƒdksXJ ‚|d t
d	ƒkslJ ‚|d jjd
ks€J ‚d S )NÚ	partitionéÈ   ©Ústatus_code©Zreturn_valueúeml/fake-email.eml©Úfilenamer   é   ú+This is a test email to use for unit tests.úmessage/rfc822)r   r   ÚFakeResponser   r   Úassert_called_once_withÚ
isinstanceÚcall_args_listÚargsÚlenr
   ÚmetadataÚfiletype)r   r   Úpartition_mock_Úelements© r2   úc/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/test_unstructured/partition/test_api.pyÚ8test_partition_via_api_with_filename_correctly_calls_sdk%   s    ÿ
r4   c                 C   sÎ   t | tdtdd�d�}ttdƒdƒ� }t|tdƒd�}W d   ƒ n1 sJ0    Y  |d d … }||d jj_|j	|Ž  t
|jd	 jd	 tƒs’J ‚t|ƒdks¢J ‚|d	 td
ƒks¶J ‚|d	 jjdksÊJ ‚d S )Nr   r   r   r!   r"   Úrb)ÚfileÚmetadata_filenamer%   r   r&   r'   )r   r   r(   Úopenr   r   Úpartition_parametersÚfilesÚcontentr)   r*   r+   r,   r-   r
   r.   r/   )r   r   r0   r   r1   Úmodified_expected_callr2   r2   r3   Ú4test_partition_via_api_with_file_correctly_calls_sdk5   s    ÿÿ$
r=   )r   r   Úcaplogc                 C   s˜   t | tdtdd�d�}ttdƒdƒ� }t|tdƒd� W d   ƒ n1 sJ0    Y  |d d … }||d jj_|j	|Ž  d	|j
v s†J ‚d
|j
v s”J ‚d S )Nr   r   r   r!   r"   r5   )r6   Úfile_filenamer%   ÚWARNINGz*The file_filename kwarg will be deprecated)r   r   r(   r8   r   r   r9   r:   r;   r)   Útext)r   r   r>   r0   r   r<   r2   r2   r3   ÚAtest_partition_via_api_warns_with_file_and_filename_and_calls_sdkL   s    ÿ.
rB   c               	   C   sn   t dƒ} t| dƒ�H}t t¡� t|| | d� W d   ƒ n1 sB0    Y  W d   ƒ n1 s`0    Y  d S )Nr"   r5   )r6   r?   r7   )r   r8   ÚpytestÚraisesÚ
ValueErrorr   )r$   r   r2   r2   r3   ÚKtest_partition_via_api_from_file_raises_with_metadata_and_file_and_filename_   s    rF   c               	   C   sf   t tdƒdƒ�D} t t¡� t| d� W d   ƒ n1 s:0    Y  W d   ƒ n1 sX0    Y  d S )Nr"   r5   )r6   )r8   r   rC   rD   rE   r   )r   r2   r2   r3   Ú8test_partition_via_api_from_file_raises_without_filenamef   s    rG   )r   c                 C   sZ   t | tdtdd�d�}t t¡� ttdƒd� W d   ƒ n1 sD0    Y  | ¡  d S )Nr   éô  r   r!   r"   r#   )	r   r   r(   rC   rD   rE   r   r   Zassert_called_once)r   r0   r2   r2   r3   Ú/test_partition_via_api_raises_with_bad_responsek   s    ÿ,rI   zSkipping test run outside of CI)Úreasonz(Skipping test run outside of main branchc                  C   sŠ   t dƒ} t| dtƒ tdgd�}t| dtƒ tdgd�}t| dtƒ tdgd�}t|ƒt|ƒks^J ‚t|ƒt|ƒksrJ ‚|d jjd u s†J ‚d S )Nzpdf/loremipsum-flat.pdfÚautoZpdf)r$   ÚstrategyÚapi_keyÚapi_urlÚskip_infer_table_typesÚhi_resÚfastr   )r   r   Úget_api_keyÚAPI_URLr-   r.   Úcoordinates)Z	test_fileZelements_no_strategyZelements_hi_resZelements_fast_resr2   r2   r3   Ú'test_partition_via_api_with_no_strategyu   s2    úúúrU   c                  C   s0   t tdƒddtƒ td�} | d jjd us,J ‚d S )Nzpdf/fake-memo.pdfrP   Útrue)r$   rL   rT   rM   rN   r   )r   r   rR   rS   r.   rT   ©r1   r2   r2   r3   ÚFtest_partition_via_api_with_image_hi_res_strategy_includes_coordinates�   s    ûrX   c                  C   st   t tdƒdddgtƒ td�} dd„ | D ƒ}|D ]@}|jjd usBJ ‚|jjd usRJ ‚t |jj¡}t	|t
ƒs.J ‚q.d S )Núpdf/embedded-images-tables.pdfrP   ÚimageÚtable)r$   rL   Úextract_image_block_typesrM   rN   c                 S   s   g | ]}|j tjkr|‘qS r2   )Úcategoryr	   ZIMAGE)Ú.0Úelr2   r2   r3   Ú
<listcomp>·   ó    zAtest_partition_via_api_image_block_extraction.<locals>.<listcomp>)r   r   rR   rS   r.   Zimage_base64Zimage_mime_typeÚbase64Ú	b64decoder*   Úbytes)r1   Zimage_elementsr_   Z
image_datar2   r2   r3   Ú-test_partition_via_api_image_block_extraction¬   s    úre   c                  C   s4   t tdƒdtƒ tdddddd�	} t| ƒd	ks0J ‚d S )
NrY   rQ   é   é   éd   Tg      ø?)	r$   rL   rM   rN   Úretries_initial_intervalÚretries_max_intervalÚretries_max_elapsed_timeÚretries_connection_errorsÚretries_exponentr   )r   r   rR   rS   r-   rW   r2   r2   r3   Ú%test_partition_via_api_retries_config¿   s    örn   c                  C   sh   t ƒ } tddddd| d�}|js$J ‚|jjdks4J ‚|jjdksDJ ‚|jjdksTJ ‚|jjdksdJ ‚d S )NTç      ü?é   iè  rh   ©rl   rm   ri   rk   rj   Úsdk)r   r   Úretry_connection_errorsÚbackoffÚexponentÚinitial_intervalÚmax_elapsed_timeÚmax_interval©rr   Úretries_configr2   r2   r3   Ú'test_retries_config_with_parameters_setÙ   s    ú	
r{   c                  C   s*   t ƒ } td d d d d | d�}|d u s&J ‚d S )Nrq   )r   r   ry   r2   r2   r3   Ú7test_retries_config_none_parameters_return_empty_configë   s    ú	r|   c                  C   sp   t ƒ } d | j_tdddd d | d�}|js,J ‚|jjdks<J ‚|jjdksLJ ‚|jjt	ks\J ‚|jj
tkslJ ‚d S )NTç®Gázþ?é¸  rq   )r   Úsdk_configurationÚretry_configr   rs   rt   ru   rv   rw   r   rx   r   ry   r2   r2   r3   Ú=test_retry_config_with_empty_sdk_retry_config_returns_defaultù   s    ú	
r�   c               	   C   sŠ   t  dt  dddd¡d¡} tƒ }| |j_tdd d d d |d�}|jsFJ ‚|jj	dksVJ ‚|jj
dksfJ ‚|jjdksvJ ‚|jjdks†J ‚d S )Nrt   r~   i€ü
 r}   i@w Trq   )r   ÚRetryConfigÚBackoffStrategyr   r   r€   r   rs   rt   ru   rv   rw   rx   ©r€   rr   rz   r2   r2   r3   Ú*test_retries_config_with_no_parameters_set  s$    ÿú	
r…   c               	   C   sŠ   t  dt  dddd ¡d¡} tƒ }| |j_tdddd d |d	�}|jrFJ ‚|jj	dksVJ ‚|jj
dksfJ ‚|jjtksvJ ‚|jjtks†J ‚d S )
Nrt   r~   r   r}   TFro   rp   rq   )r   r‚   rƒ   r   r   r€   r   rs   rt   ru   rv   rw   r   rx   r   r„   r2   r2   r3   Útest_retries_config_cascade"  s$    ÿú	
r†   c                 C   s†   t | tdtdd�d�}tdƒ}t|gd�}|jddtd	œi d
tdƒtd ffgd� |d d tdƒksjJ ‚|d d jj	dks‚J ‚d S )NÚpostr   r   r!   r"   ©Ú	filenamesú.https://api.unstructured.io/general/v0/generalúapplication/json©ZACCEPTzUNSTRUCTURED-API-KEYr:   ©ÚheadersÚdatar:   r   r&   r'   )
r   Úrequestsr(   r   r   r)   r   r
   r.   r/   )r   r0   r$   r1   r2   r2   r3   Ú4test_partition_multiple_via_api_with_single_filename:  s    ÿür‘   c              	   C   s¬   t | tdtdd�d�}tdƒtdƒg}t|d�}|jdd	td
œi dtdƒtd ffdtdƒtd ffgd� t|ƒdksxJ ‚|d d tdƒks�J ‚|d d j	j
dks¨J ‚d S )Nr‡   r   r   r!   r"   ú	fake.docxrˆ   rŠ   r‹   rŒ   r:   r�   r   r   r&   r'   )r   r�   ÚFakeMultipleResponser   r   r)   r   r-   r
   r.   r/   )r   r0   r‰   r1   r2   r2   r3   Ú.test_partition_multiple_via_api_from_filenamesL  s     ÿ
þü	r”   c              	      sè   t | tdtdd�d�}tdƒtdƒg}t ¡ �.‰ ‡ fdd„|D ƒ}t||d	�}W d   ƒ n1 sb0    Y  |jd
dtdœi dtdƒtd ffdtdƒtd ffgd� t	|ƒdks´J ‚|d d t
dƒksÌJ ‚|d d jjdksäJ ‚d S )Nr‡   r   r   r!   r"   r’   c                    s   g | ]}ˆ   t|d ƒ¡‘qS ©r5   ©Úenter_contextr8   ©r^   r$   ©Ústackr2   r3   r`   i  ra   z>test_partition_multiple_via_api_from_files.<locals>.<listcomp>)r:   Úmetadata_filenamesrŠ   r‹   rŒ   r:   r�   r   r   r&   r'   )r   r�   r“   r   Ú
contextlibÚ	ExitStackr   r)   r   r-   r
   r.   r/   )r   r0   r‰   r:   r1   r2   r™   r3   Ú*test_partition_multiple_via_api_from_filesb  s*    ÿ
þ$þü	rž   )r>   r   c              	      sÄ   t |tdtdd�d�}tdƒtdƒg}t ¡ �.‰ ‡ fdd„|D ƒ}t||d	� W d   ƒ n1 sb0    Y  |jd
dtdœi dtdƒtd ffdtdƒtd ffgd� d| j	v s²J ‚d| j	v sÀJ ‚d S )Nr‡   r   r   r!   r"   r’   c                    s   g | ]}ˆ   t|d ƒ¡‘qS r•   r–   r˜   r™   r2   r3   r`   †  ra   zLtest_partition_multiple_via_api_warns_with_file_filename.<locals>.<listcomp>)r:   Úfile_filenamesrŠ   r‹   rŒ   r:   r�   r@   z+The file_filenames kwarg will be deprecated)
r   r�   r“   r   rœ   r�   r   r)   r   rA   )r>   r   r0   r‰   r:   r2   r™   r3   Ú8test_partition_multiple_via_api_warns_with_file_filename}  s(    ÿ
þ$þü	r    c               	      s†   t dƒt dƒg} t ¡ �Z‰ ‡ fdd„| D ƒ}t t¡� t|| | d� W d   ƒ n1 sZ0    Y  W d   ƒ n1 sx0    Y  d S )Nr"   r’   c                    s   g | ]}ˆ   t|d ƒ¡‘qS r•   r–   r˜   r™   r2   r3   r`   �  ra   zZtest_partition_multiple_via_api_raises_with_file_and_metadata_filename.<locals>.<listcomp>)r:   r›   rŸ   ©r   rœ   r�   rC   rD   rE   r   ©r‰   r:   r2   r™   r3   ÚFtest_partition_multiple_via_api_raises_with_file_and_metadata_filename™  s    
ýr£   c              	   C   s–   t | tdtdd�d�}tdƒtdƒg}t t¡� t|d� W d   ƒ n1 sP0    Y  |jdd	t	d
œi dtdƒt	d ffdtdƒt	d ffgd� d S )Nr‡   rH   r   r!   r"   r’   rˆ   rŠ   r‹   rŒ   r:   r�   )
r   r�   r“   r   rC   rD   rE   r   r)   r   )r   r0   r‰   r2   r2   r3   Ú8test_partition_multiple_via_api_raises_with_bad_response¦  s    ÿ(þür¤   c                  C   sL   t dƒt dƒg} t t¡� t| dgd� W d   ƒ n1 s>0    Y  d S )Nr"   r’   ú
text/plain)r‰   Úcontent_types)r   rC   rD   rE   r   rˆ   r2   r2   r3   ÚGtest_partition_multiple_via_api_raises_with_content_types_size_mismatch¹  s    þr§   c               	      sˆ   t dƒt dƒg} t ¡ �\‰ ‡ fdd„| D ƒ}t t¡�  t|| dgd� W d   ƒ n1 s\0    Y  W d   ƒ n1 sz0    Y  d S )Nr"   r’   c                    s   g | ]}ˆ   t|d ƒ¡‘qS r•   r–   r˜   r™   r2   r3   r`   Ç  ra   zXtest_partition_multiple_via_api_from_files_raises_with_size_mismatch.<locals>.<listcomp>r¥   )r:   r›   r¦   r¡   r¢   r2   r™   r3   ÚDtest_partition_multiple_via_api_from_files_raises_with_size_mismatchÃ  s    
ýr¨   c               	      s‚   t dƒt dƒg} t ¡ �V‰ ‡ fdd„| D ƒ}t t¡� t|d� W d   ƒ n1 sV0    Y  W d   ƒ n1 st0    Y  d S )Nr"   r’   c                    s   g | ]}ˆ   t|d ƒ¡‘qS r•   r–   r˜   r™   r2   r3   r`   Ô  ra   zWtest_partition_multiple_via_api_from_files_raises_without_filenames.<locals>.<listcomp>)r:   r¡   r¢   r2   r™   r3   ÚCtest_partition_multiple_via_api_from_files_raises_without_filenamesÐ  s    
ÿr©   c                  C   s   t  d¡} | d u rtdƒ‚| S )NZUNS_API_KEYz(UNS_API_KEY environment variable not set)ÚosÚgetenvrE   )rM   r2   r2   r3   rR   Û  s    
rR   c                  C   sZ   t dƒt dƒg} t| dtƒ td�}t|ƒdks2J ‚t|d tƒsDJ ‚t|d tƒsVJ ‚d S )Nzfake-text.txtzfake-email.txtrQ   ©r‰   rL   rM   rN   r   r   r%   )r   r   rR   rS   r-   r*   Úlist)r‰   Zlist_of_lists_of_elementsr2   r2   r3   Ú9test_partition_multiple_via_api_valid_request_data_kwargsâ  s    þür®   c                  C   sP   t dƒt dƒg} t t¡�" t| dtƒ td� W d   ƒ n1 sB0    Y  d S )Nz pdf/layout-parser-paper-fast.pdfz img/layout-parser-paper-fast.jpgZnot_a_strategyr¬   )r   rC   rD   rE   r   rR   rS   rˆ   r2   r2   r3   Ú;test_partition_multiple_via_api_invalid_request_data_kwargsö  s    þûr¯   a  [
    {
        "element_id": "f49fbd614ddf5b72e06f59e554e6ae2b",
        "text": "This is a test email to use for unit tests.",
        "type": "NarrativeText",
        "metadata": {
            "sent_from": [
                "Matthew Robinson <mrobinson@unstructured.io>"
            ],
            "sent_to": [
                "Matthew Robinson <mrobinson@unstructured.io>"
            ],
            "subject": "Test Email",
            "filename": "fake-email.eml",
            "filetype": "message/rfc822"
        }
    }
]c                   @   s.   e Zd Zedœdd„Zdd„ Zedd„ ƒZdS )	r(   r   c                 C   s   || _ tƒ | _ddi| _d S )NzContent-Typer‹   )r    ÚFakeRawResponseZraw_responserŽ   ©Úselfr    r2   r2   r3   Ú__init__  s    zFakeResponse.__init__c                 C   s   t  | j¡S ©N©ÚjsonÚloadsrA   ©r²   r2   r2   r3   r¶   #  s    zFakeResponse.jsonc                 C   s   t S r´   )Ú	MOCK_TEXTr¸   r2   r2   r3   rA   &  s    zFakeResponse.textN©Ú__name__Ú
__module__Ú__qualname__Úintr³   r¶   ÚpropertyrA   r2   r2   r2   r3   r(     s   r(   c                   @   s   e Zd Zdd„ ZdS )r°   c                 C   s
   t | _d S r´   )r¹   rA   r¸   r2   r2   r3   r³   ,  s    zFakeRawResponse.__init__N)r»   r¼   r½   r³   r2   r2   r2   r3   r°   +  s   r°   c                   @   s.   e Zd Zedœdd„Zdd„ Zedd„ ƒZdS )	r“   r   c                 C   s
   || _ d S r´   r   r±   r2   r2   r3   r³   1  s    zFakeMultipleResponse.__init__c                 C   s   t  | j¡S r´   rµ   r¸   r2   r2   r3   r¶   4  s    zFakeMultipleResponse.jsonc                 C   s   dS )Na¸  [
    [
        {
            "element_id": "f49fbd614ddf5b72e06f59e554e6ae2b",
            "text": "This is a test email to use for unit tests.",
            "type": "NarrativeText",
            "metadata": {
                "sent_from": [
                    "Matthew Robinson <mrobinson@unstructured.io>"
                ],
                "sent_to": [
                    "Matthew Robinson <mrobinson@unstructured.io>"
                ],
                "subject": "Test Email",
                "filename": "fake-email.eml",
                "filetype": "message/rfc822"
            }
        }
    ],
    [
        {
            "element_id": "f49fbd614ddf5b72e06f59e554e6ae2b",
            "text": "This is a test email to use for unit tests.",
            "type": "NarrativeText",
            "metadata": {
                "sent_from": [
                    "Matthew Robinson <mrobinson@unstructured.io>"
                ],
                "sent_to": [
                    "Matthew Robinson <mrobinson@unstructured.io>"
                ],
                "subject": "Test Email",
                "filename": "fake-email.eml",
                "filetype": "message/rfc822"
            }
        }
    ]
]r2   r¸   r2   r2   r3   rA   7  s    zFakeMultipleResponse.textNrº   r2   r2   r2   r3   r“   0  s   r“   c                  C   s–   t tdƒdƒ�} |  ¡ }W d   ƒ n1 s,0    Y  ttttj|tdƒd�d d dd d d d d dd d dd d tjj	dddd d ddd tj
jddd�d	�d gS )
Nr"   r5   )r;   Ú	file_nameFTr   rf   )r:   Zchunking_strategyZcombine_under_n_charsrT   Úencodingr\   Zgz_uncompressed_content_typeZhi_res_model_nameZinclude_orig_elementsZinclude_page_breaksÚ	languagesZmax_charactersZmultipage_sectionsZnew_after_n_charsZocr_languagesZoutput_formatÚoverlapZoverlap_allZpdf_infer_table_structureZsimilarity_thresholdrO   Zsplit_pdf_concurrency_levelZsplit_pdf_pageZstarting_page_numberrL   Zunique_element_idsZxml_keep_tags)r9   )r8   r   Úreadr   r   r   r   ZFilesZOutputFormatZAPPLICATION_JSONZStrategyZHI_RES)r   Z
file_bytesr2   r2   r3   r   a  sN    &þâÿ"Ür   )Rrb   rœ   r¶   rª   ÚpathlibÚtypingr   Zunittest.mockr   rC   r�   Zunstructured_client.generalr   Zunstructured_client.modelsr   Z%unstructured_client.models.operationsr   Z!unstructured_client.models.sharedr   Zunstructured_client.utilsr   Zunstructured.documents.elementsr	   r
   Zunstructured.partition.apir   r   r   r   r   Z
unit_utilsr   r   r   r   ÚPathÚ__file__ÚparentÚresolveZ	DIRECTORYrS   r«   ÚlowerZis_in_ciZskip_not_on_mainr­   r4   r=   ZLogCaptureFixturerB   rF   rG   rI   ÚmarkZskipifrU   rX   re   rn   r{   r|   r�   r…   r†   r‘   r”   rž   r    r£   r¤   r§   r¨   r©   rR   r®   r¯   r¹   r(   r°   r“   Zfixturer   r2   r2   r2   r3   Ú<module>   sˆ   ÿÿÿ
&ÿ

1