a
    ¡˜ëjTB  ã                   @  s¦  d Z ddlmZ ddlZddlmZ ddlZddlmZ ddl	m
Z
mZmZmZmZmZmZ ddlmZ ddlmZmZmZmZ dd	lmZ dd
lmZmZ edd�edd�edd�edd�gZdd„ Zdd„ Zdd„ Z dd„ Z!dd„ Z"dd„ Z#dd„ Z$dd„ Z%d d!„ Z&d"d#„ Z'd$d%œd&d'„Z(d(d)„ Z)d*d+„ Z*d,d-„ Z+d.d/„ Z,d0d1„ Z-d2d3„ Z.d4d5„ Z/d6d7„ Z0d8d9„ Z1d:d;œd<d=„Z2d>d?„ Z3d@dA„ Z4dBdC„ Z5dDdE„ Z6G dFdG„ dGƒZ7dS )Hz3Test suite for `unstructured.partition.msg` module.é    )ÚannotationsN)ÚAny)ÚMessage)ÚFixtureRequestÚLogCaptureFixtureÚMockÚassert_round_trips_through_JSONÚexample_doc_pathÚfunction_mockÚproperty_mock)Úchunk_by_title)ÚElementMetadataÚListItemÚNarrativeTextÚText)ÚUnsupportedFileFormatError)ÚMsgPartitionerOptionsÚpartition_msgú+This is a test email to use for unit tests.©ÚtextzImportant points:zRoses are redzViolets are bluec                  C  sh   t dƒ} t| d�}|d jj}|tks*J ‚|d j ¡ td | dd d dgdgdd|d	gd
� ¡ ksdJ ‚d S )Núfake-email.msg©Úfilenamer   ú2023-03-28T17:00:31+00:00z."Matthew Robinson" <mrobinson@unstructured.io>zmrobinson@unstructured.ioz
Test Emailúapplication/vnd.ms-outlookÚeng)Zcoordinatesr   Úlast_modifiedZpage_numberÚurlÚ	sent_fromÚsent_toÚsubjectÚfiletypeÚ	parent_idÚ	languages)r	   r   Úmetadatar#   ÚEXPECTED_MSG_OUTPUTÚto_dictr   )r   Úelementsr#   © r)   úc/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/test_unstructured/partition/test_msg.pyÚ test_partition_msg_from_filename&   s(    
õÿÿr+   c                  C  s(   t dƒ} t| d�}t|d tƒs$J ‚d S )Nr   r   r   )r	   r   Ú
isinstancer   ©r   r(   r)   r)   r*   Ú5test_partition_msg_from_filename_returns_uns_elements>   s    
r.   c                  C  s.   t dƒ} t| dd�}tdd„ |D ƒƒs*J ‚d S )Nr   Útest)r   Úmetadata_filenamec                 s  s   | ]}|j jd kV  qdS )r/   N©r%   r   ©Ú.0Úelementr)   r)   r*   Ú	<genexpr>G   ó    zJtest_partition_msg_from_filename_with_metadata_filename.<locals>.<genexpr>)r	   r   Úallr-   r)   r)   r*   Ú7test_partition_msg_from_filename_with_metadata_filenameD   s    r8   c                  C  sV   t dƒ} t| d�}t|d ƒdks&J ‚|d jjdks:J ‚|d jjt dƒksRJ ‚d S )Nr   r   r   r   Ú )r	   r   Ústrr%   r   Úfile_directoryr-   r)   r)   r*   Ú2test_partition_msg_from_filename_with_text_contentJ   s
    
r<   c                  C  s@   t dƒ} t t¡� t| d� W d   ƒ n1 s20    Y  d S )Nzdoesnt-exist.msgr   )r	   ÚpytestÚraisesÚFileNotFoundErrorr   r   r)   r)   r*   Ú+test_partition_msg_raises_with_missing_fileT   s    r@   c                  C  sf   t dƒ} t| dƒ�}t|d�}W d   ƒ n1 s20    Y  |tksHJ ‚|D ]}|jjd u sLJ ‚qLd S )Nr   Úrb©Úfile©r	   Úopenr   r&   r%   r   ©r   Úfr(   r4   r)   r)   r*   Útest_partition_msg_from_fileZ   s    (rH   c                  C  sh   t dƒ} t| dƒ�}t|dd�}W d   ƒ n1 s40    Y  |tksJJ ‚|D ]}|jjdksNJ ‚qNd S )Nr   rA   r/   ©rC   r0   rD   rF   r)   r)   r*   Ú3test_partition_msg_from_file_with_metadata_filenamec   s    *rJ   c                  C  s&   t tdƒt d¡d�} | tks"J ‚d S )Nr   s   abcderB   )r   r	   ÚioÚBytesIOr&   ©r(   r)   r)   r*   Ú9test_partition_msg_uses_file_path_when_both_are_specifiedl   s    rN   c                   C  s4   t  t¡� tƒ  W d   ƒ n1 s&0    Y  d S ©N)r=   r>   Ú
ValueErrorr   r)   r)   r)   r*   Ú&test_partition_msg_raises_with_neitherq   s    rQ   c                  C  sò   t tdƒdd�} tdd„ | d d… D ƒƒs.J ‚tdd„ | dd… D ƒƒsLJ ‚td	d„ | dd
… D ƒƒsjJ ‚tdd„ | d
d … D ƒƒsˆJ ‚dd„ | d d… D ƒg d¢ksªJ ‚dd„ | D ƒd d… g d¢ksÌJ ‚dd„ | D ƒdd … g d¢ksîJ ‚d S )Nú#fake-email-multiple-attachments.msgT©Zprocess_attachmentsc                 s  s   | ]}|j jd kV  qdS )rR   Nr1   ©r3   Úer)   r)   r*   r5   ~   r6   z=test_partition_msg_can_process_attachments.<locals>.<genexpr>é   c                 s  s   | ]}|j jd kV  qdS )zunstructured_logo.pngNr1   rT   r)   r)   r*   r5      r6   é   c                 s  s   | ]}|j jd kV  qdS )zdense_doc.pdfNr1   rT   r)   r)   r*   r5   €   r6   iW  c                 s  s   | ]}|j jd kV  qdS )zEngineering Onboarding.pptxNr1   rT   r)   r)   r*   r5   �   r6   c                 S  s   g | ]
}|j ‘qS r)   r   rT   r)   r)   r*   Ú
<listcomp>‚   r6   z>test_partition_msg_can_process_attachments.<locals>.<listcomp>)úHere are those documents.ú--úMallori HarrellúUnstructured TechnologiesúData Scientistc                 S  s   g | ]}t |ƒj‘qS r)   ©ÚtypeÚ__name__rT   r)   r)   r*   rX   ‰   r6   é
   )
r   r   r   r   r   ZImager   r   ÚTitlerb   c                 S  s   g | ]}t |ƒj‘qS r)   r^   rT   r)   r)   r*   rX   •   r6   iöÿÿÿ)
rb   r   r   r   r   r   r   r   r   r   ©r   r	   r7   rM   r)   r)   r*   Ú*test_partition_msg_can_process_attachmentsy   s    ÿ""rd   r   ©Úrequestc                 C  sN   t | dtƒ d� ttdƒdd�}|tdƒtdƒtdƒtd	ƒtd
ƒgksJJ ‚d S )Nz%unstructured.partition.auto.partition)Zside_effectrR   TrS   rY   rZ   r[   r\   r]   )r
   r   r   r	   r   r   )rf   r(   r)   r)   r*   ÚAtest_partition_msg_silently_skips_attachments_it_cannot_partition£   s    ÿÿúrg   c                  C  s<   t tdƒƒ} tdd„ | D ƒƒs"J ‚tdd„ | D ƒƒs8J ‚d S )Nr   c                 s  s   | ]}|j jd kV  qdS )r   Nr1   rT   r)   r)   r*   r5   ¾   r6   zYtest_partition_msg_from_filename_gets_filename_metadata_from_file_path.<locals>.<genexpr>c                 s  s   | ]}|j jtd ƒkV  qdS )r9   N)r%   r;   r	   rT   r)   r)   r*   r5   ¿   r6   rc   rM   r)   r)   r*   ÚFtest_partition_msg_from_filename_gets_filename_metadata_from_file_path»   s    rh   c                  C  sh   t tdƒdƒ�} t| d�}W d   ƒ n1 s.0    Y  tdd„ |D ƒƒsNJ ‚tdd„ |D ƒƒsdJ ‚d S )Nr   rA   rB   c                 s  s   | ]}|j jd u V  qd S rO   r1   rT   r)   r)   r*   r5   Æ   r6   zKtest_partition_msg_from_file_gets_filename_metadata_None.<locals>.<genexpr>c                 s  s   | ]}|j jd u V  qd S rO   ©r%   r;   rT   r)   r)   r*   r5   Ç   r6   ©rE   r	   r   r7   ©rG   r(   r)   r)   r*   Ú8test_partition_msg_from_file_gets_filename_metadata_NoneÂ   s    (rl   c                  C  s@   t tdƒdd�} tdd„ | D ƒƒs&J ‚tdd„ | D ƒƒs<J ‚d S )Nr   ú	a/b/c.msg)r0   c                 s  s   | ]}|j jd kV  qdS )zc.msgNr1   rT   r)   r)   r*   r5   Í   r6   zMtest_partition_msg_from_filename_prefers_metadata_filename.<locals>.<genexpr>c                 s  s   | ]}|j jd kV  qdS )za/bNri   rT   r)   r)   r*   r5   Î   r6   rc   rM   r)   r)   r*   Ú:test_partition_msg_from_filename_prefers_metadata_filenameÊ   s    rn   c                  C  sj   t tdƒdƒ�} t| dd�}W d   ƒ n1 s00    Y  tdd„ |D ƒƒsPJ ‚tdd„ |D ƒƒsfJ ‚d S )Nr   rA   z	d/e/f.msgrI   c                 s  s   | ]}|j jd kV  qdS )zf.msgNr1   rT   r)   r)   r*   r5   Õ   r6   zItest_partition_msg_from_file_prefers_metadata_filename.<locals>.<genexpr>c                 s  s   | ]}|j jd kV  qdS )zd/eNri   rT   r)   r)   r*   r5   Ö   r6   rj   rk   r)   r)   r*   Ú6test_partition_msg_from_file_prefers_metadata_filenameÑ   s    *ro   c                    sJ   d‰ t tdƒƒ} t‡ fdd„| D ƒƒsFJ dˆ › dt| d jjƒ› �ƒ‚d S )Nr   r   c                 3  s   | ]}|j jˆ kV  qd S rO   )r%   r"   rT   ©ZMSG_MIME_TYPEr)   r*   r5   ß   r6   zQtest_partition_msg_gets_the_MSG_mime_type_in_metadata_filetype.<locals>.<genexpr>zExpected all elements to have 'z' as their filetype, but got: r   )r   r	   r7   Úreprr%   r"   rM   r)   rp   r*   Ú>test_partition_msg_gets_the_MSG_mime_type_in_metadata_filetypeÜ   s    ÿÿrr   c                  C  s&   t tdƒƒ} tdd„ | D ƒƒs"J ‚d S )Nr   c                 s  s   | ]}|j jd kV  qdS )r   N©r%   r   rT   r)   r)   r*   r5   ê   r6   zPtest_partition_msg_pulls_last_modified_from_message_sent_date.<locals>.<genexpr>rc   rM   r)   r)   r*   Ú=test_partition_msg_pulls_last_modified_from_message_sent_dateè   s    rt   c                  C  s,   d} t tdƒ| d�}|d jj| ks(J ‚d S )Nú2020-07-05T09:24:28r   ©Úmetadata_last_modifiedr   )r   r	   r%   r   )rw   r(   r)   r)   r*   Ú@test_partition_msg_from_file_path_prefers_metadata_last_modifiedí   s
    ÿrx   c                    s\   d‰ t tdƒdƒ�} t| ˆ d�}W d   ƒ n1 s40    Y  t‡ fdd„|D ƒƒsXJ ‚d S )Nru   r   rA   )rC   rw   c                 3  s   | ]}|j jˆ kV  qd S rO   rs   rT   rv   r)   r*   r5   ý   r6   zNtest_partition_msg_from_file_prefers_metadata_last_modified.<locals>.<genexpr>rj   rk   r)   rv   r*   Ú;test_partition_msg_from_file_prefers_metadata_last_modified÷   s    *ry   c                  C  s   t tdƒƒ} t| ƒ d S )Nr   )r   r	   r   rM   r)   r)   r*   Útest_partition_msg_with_json  s    rz   r   )Úcaplogc                 C  s8   t tdƒƒ}|g ksJ ‚d| jv s&J ‚d| jv s4J ‚d S )Núfake-encrypted.msgÚWARNINGzEncrypted email detected)r   r	   r   )r{   r(   r)   r)   r*   Ú-test_partition_msg_with_pgp_encrypted_message  s    r~   c                  C  sB   t dƒ} t| d�}t| dd�}t|ƒ}||ks2J ‚||ks>J ‚d S )Nr   r   Zby_title)Zchunking_strategy)r	   r   r   )r   r(   Zchunk_elementsÚchunksr)   r)   r*   Ú4test_add_chunking_strategy_by_title_on_partition_msg  s    
r€   c                  C  s(   d} t | d�}|d jjdgks$J ‚d S )Núexample-docs/fake-email.msgr   r   r   )r   r%   r$   r-   r)   r)   r*   Ú1test_partition_msg_element_metadata_has_languages  s    
r‚   c                  C  s,   d} t | dgd�}tdd„ |D ƒƒs(J ‚d S )Nr�   Údeu©r   r$   c                 s  s   | ]}|j jd gkV  qdS )rƒ   N)r%   r$   r2   r)   r)   r*   r5   '  r6   z<test_partition_msg_respects_languages_arg.<locals>.<genexpr>)r   r7   r-   r)   r)   r*   Ú)test_partition_msg_respects_languages_arg$  s    r…   c                  C  s>   t  t¡�  d} t| dd� W d   ƒ n1 s00    Y  d S )Nr�   r   r„   )r=   r>   Ú	TypeErrorr   r   r)   r)   r*   Ú9test_partition_msg_raises_TypeError_for_invalid_languages*  s    r‡   c                   @  sD  e Zd ZdZddœdd„Zej dddg¡d	d
ddœdd„ƒZddœdd„Z	ddœdd„Z
ddœdd„Zddœdd„Zddœdd„Zej dddg¡dddddœdd„ƒZddœd d!„Zddœd"d#„Zddœd$d%„Zej d&d'd(g¡d
dd)œd*d+„ƒZddœd,d-„Zejd.d/œd0d1„ƒZejd.d/œd2d3„ƒZejdd4œd5d6„ƒZdS )7ÚDescribeMsgPartitionerOptionszOUnit-test suite for `unstructured.partition.msg.MsgPartitionerOptions` objects.zdict[str, Any])Ú	opts_argsc                 C  s‚   t dƒ|d< tf i |¤Ž}|j}|jdgks0J ‚|jdgks@J ‚|jdksNJ ‚|jdgks^J ‚|jg d¢kspJ ‚|jdks~J ‚d S )	Nzfake-email-with-cc-and-bcc.msgÚ	file_pathúhello@unstructured.ioústeve@unstructured.ioz.14DDEF33-2BA7-4CDD-A4D8-E7C5873B37F2@gmail.comz""John" <johnjennings702@gmail.com>)zjohn-ctr@unstructured.iorŒ   r‹   z%Fake email with cc and bcc recipients)	r	   r   Zextra_msg_metadataZbcc_recipientZcc_recipientZemail_message_idr   r    r!   )Úselfr‰   ÚoptsÚmr)   r)   r*   Ú:it_provides_email_specific_metadata_to_add_to_each_element=  s    zXDescribeMsgPartitionerOptions.it_provides_email_specific_metadata_to_add_to_each_element)Ú	file_nameÚexpected_value)r|   T)r   Fr:   Úbool)r‘   r’   r‰   c                 C  s,   t |ƒ|d< tf i |¤Ž}|j|u s(J ‚d S )NrŠ   )r	   r   Úis_encrypted)r�   r‘   r’   r‰   rŽ   r)   r)   r*   Ú"it_knows_when_the_msg_is_encryptedO  s    z@DescribeMsgPartitionerOptions.it_knows_when_the_msg_is_encryptedc                 C  s0   d|d< d|d< t f i |¤Ž}|jdks,J ‚d S )Nz	x/y/z.msgrŠ   rm   Úmetadata_file_path©r   r–   ©r�   r‰   rŽ   r)   r)   r*   Ú0it_uses_the_metadata_file_path_arg_when_provided\  s    zNDescribeMsgPartitionerOptions.it_uses_the_metadata_file_path_arg_when_providedc                 C  s0   t dƒ}||d< tf i |¤Ž}|j|ks,J ‚d S ©Nr   rŠ   )r	   r   r–   )r�   r‰   rŠ   rŽ   r)   r)   r*   Ú8and_it_falls_back_to_the_MSG_file_path_arg_when_providedc  s    zVDescribeMsgPartitionerOptions.and_it_falls_back_to_the_MSG_file_path_arg_when_providedc                 C  s    t f i |¤Ž}|jd u sJ ‚d S rO   r—   r˜   r)   r)   r*   Ú2but_it_returns_None_when_neither_path_is_availablej  s    zPDescribeMsgPartitionerOptions.but_it_returns_None_when_neither_path_is_availablec                 C  s8   d}||d< t dƒ|d< tf i |¤Ž}|j|ks4J ‚d S )Nz2024-03-05T17:02:53rw   r   rŠ   ©r	   r   rw   )r�   r‰   rw   rŽ   r)   r)   r*   Ú:it_uses_metadata_last_modified_when_provided_by_the_callerq  s
    zXDescribeMsgPartitionerOptions.it_uses_metadata_last_modified_when_provided_by_the_callerc                 C  s,   t dƒ|d< tf i |¤Ž}|jdks(J ‚d S )Nr   rŠ   r   r�   r˜   r)   r)   r*   ÚOand_it_uses_the_message_Date_header_when_metadata_last_modified_is_not_providedy  s    zmDescribeMsgPartitionerOptions.and_it_uses_the_message_Date_header_when_metadata_last_modified_is_not_providedÚfilesystem_last_modifiedz2024-06-03T20:12:53Nz
str | Noner   )r‰   r    ÚMessage_sent_date_Ú_last_modified_prop_c                 C  s8   d |_ ||_ tdƒ|d< tf i |¤Ž}|j|ks4J ‚d S rš   )Zreturn_valuer	   r   rw   )r�   r‰   r    r¡   r¢   rŽ   r)   r)   r*   ÚYand_it_uses_the_last_modified_date_from_the_source_file_when_the_message_has_no_sent_date�  s
    zwDescribeMsgPartitionerOptions.and_it_uses_the_last_modified_date_from_the_source_file_when_the_message_has_no_sent_datec                 C  s.   t dƒ|d< tf i |¤Ž}t|jtƒs*J ‚d S rš   )r	   r   r,   Úmsgr   r˜   r)   r)   r*   Ú8it_loads_the_msg_document_from_a_file_path_when_provided’  s    zVDescribeMsgPartitionerOptions.it_loads_the_msg_document_from_a_file_path_when_providedc                 C  sb   t tdƒdƒ�"}t | ¡ ¡|d< W d   ƒ n1 s60    Y  tf i |¤Ž}t|jtƒs^J ‚d S )Nr   rA   rC   )	rE   r	   rK   rL   Úreadr   r,   r¤   r   )r�   r‰   rG   rŽ   r)   r)   r*   ÚCand_it_loads_the_msg_document_from_a_file_like_object_when_provided˜  s    0zaDescribeMsgPartitionerOptions.and_it_loads_the_msg_document_from_a_file_like_object_when_providedc                 C  sB   t jtdd��  tf i |¤Žj W d   ƒ n1 s40    Y  d S )Nz2one of `file` or `filename` arguments must be prov)Úmatch)r=   r>   rP   r   r¤   )r�   r‰   r)   r)   r*   Ú&but_it_raises_when_neither_is_provided¡  s    zDDescribeMsgPartitionerOptions.but_it_raises_when_neither_is_providedÚpartition_attachmentsTF)rª   r‰   c                 C  s4   t dƒ|d< ||d< tf i |¤Ž}|j|u s0J ‚d S )Nr   rŠ   rª   )r	   r   rª   )r�   rª   r‰   rŽ   r)   r)   r*   Ú7it_knows_whether_attachments_should_also_be_partitioned§  s    zUDescribeMsgPartitionerOptions.it_knows_whether_attachments_should_also_be_partitionedc                 C  s4   dddœ|d< t f i |¤Ž}|jdddœks0J ‚d S )Né*   Zbaz)ZfooÚbarÚkwargs)r   Zpartitioning_kwargsr˜   r)   r)   r*   ÚOit_provides_access_to_pass_through_kwargs_collected_by_the_partitioner_function³  s    zmDescribeMsgPartitionerOptions.it_provides_access_to_pass_through_kwargs_collected_by_the_partitioner_functionr   re   c                 C  s   t |tdƒS )NZ_last_modified)r   r   ©r�   rf   r)   r)   r*   r¢   ½  s    z2DescribeMsgPartitionerOptions._last_modified_prop_c                 C  s   t |tdƒS )NZ	sent_date)r   r   r°   r)   r)   r*   r¡   Á  s    z0DescribeMsgPartitionerOptions.Message_sent_date_)Úreturnc                 C  s   dddddi dœS )zÇAll default arguments for `MsgPartitionerOptions`.

        Individual argument values can be changed to suit each test. Makes construction of opts more
        compact for testing purposes.
        NF)rC   rŠ   r–   rw   rª   r®   r)   )r�   r)   r)   r*   r‰   Å  s    úz'DescribeMsgPartitionerOptions.opts_args)r`   Ú
__module__Ú__qualname__Ú__doc__r�   r=   ÚmarkZparametrizer•   r™   r›   rœ   rž   rŸ   r£   r¥   r§   r©   r«   r¯   Zfixturer¢   r¡   r‰   r)   r)   r)   r*   rˆ   8  s2   ÿ
	
rˆ   )8r´   Ú
__future__r   rK   Útypingr   r=   Zoxmsgr   Ztest_unstructured.unit_utilsr   r   r   r   r	   r
   r   Zunstructured.chunking.titler   Zunstructured.documents.elementsr   r   r   r   Zunstructured.partition.commonr   Zunstructured.partition.msgr   r   r&   r+   r.   r8   r<   r@   rH   rJ   rN   rQ   rd   rg   rh   rl   rn   ro   rr   rt   rx   ry   rz   r~   r€   r‚   r…   r‡   rˆ   r)   r)   r)   r*   Ú<module>   sR   $	ü
		*
