a
    ¡˜ëj0  ã                   @  sÊ  d dl mZ d dlZd dlZd dlmZ d dlmZmZm	Z	m
Z
mZmZmZ d dlmZmZmZmZmZ d dlmZ d dlmZ d dlmZ d d	lmZmZ d d
lmZ dZej  !dde
efdee	fdeefg¡ddddœdd„ƒZ"ej  !dddg¡ddœdd„ƒZ#dd„ Z$dd„ Z%ej  !dde
efdee	fg¡ddddœdd„ƒZ&d d!„ Z'd"d#œd$d%„Z(d"d#œd&d'„Z)d(d)„ Z*d*d+„ Z+ej  !d,ddg¡dd-œd.d/„ƒZ,d0d1„ Z-d2d3„ Z.d4d5„ Z/d6d7„ Z0G d8d9„ d9ƒZ1dS ):é    )ÚannotationsN)ÚMockFixture)ÚEXPECTED_TABLEÚ"EXPECTED_TABLE_SEMICOLON_DELIMITERÚEXPECTED_TABLE_WITH_EMOJIÚEXPECTED_TEXTÚ!EXPECTED_TEXT_SEMICOLON_DELIMITERÚEXPECTED_TEXT_WITH_EMOJIÚEXPECTED_TEXT_XLSX)ÚFixtureRequestÚMockÚassert_round_trips_through_JSONÚexample_doc_pathÚfunction_mock)Úchunk_by_title)Úclean_extra_whitespace)ÚTable)Ú_CsvPartitioningContextÚpartition_csv)Ú#UNSTRUCTURED_INCLUDE_DEBUG_METADATAztext/csv)ÚfilenameÚexpected_textÚexpected_tableústanley-cups.csvzstanley-cups-with-emoji.csvztable-semicolon-delimiter.csvÚstrc                 C  sj   d| › �}t |d�}t|d jƒ|ks*J ‚|d jj|ks>J ‚|d jjtksRJ ‚|d jj| ksfJ ‚d S )Núexample-docs/©r   r   )r   r   ÚtextÚmetadataÚtext_as_htmlÚfiletypeÚEXPECTED_FILETYPEr   )r   r   r   Úf_pathÚelements© r$   úc/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/test_unstructured/partition/test_csv.pyÚ test_partition_csv_from_filename#   s    

r&   Úinfer_table_structureTFÚbool)r'   c                 C  s@   d}t || d�}t|d jdƒo.|d jjd u}|| ks<J ‚d S )Núexample-docs/stanley-cups.csv)r   r'   r   r   )r   Úhasattrr   r   )r'   r"   r#   Z$table_element_has_text_as_html_fieldr$   r$   r%   Ú6test_partition_csv_from_filename_infer_table_structure9   s    þr+   c                  C  s>   t tdƒdd�} t| d jƒtks&J ‚| d jjdks:J ‚d S )Nr   Útest)Úmetadata_filenamer   )r   r   r   r   r   r   r   ©r#   r$   r$   r%   Ú7test_partition_csv_from_filename_with_metadata_filenameE   s    r/   c                  C  s*   t tdƒdd�} t| d jƒtks&J ‚d S )Nzstanley-cups-utf-16.csvzutf-16)Úencodingr   )r   r   r   r   r   r.   r$   r$   r%   Ú test_partition_csv_with_encodingL   s    r1   c                 C  sÂ   d| › �}t |dƒ�}t|d�}W d   ƒ n1 s40    Y  t|d jƒ|ksTJ ‚t|d tƒsfJ ‚|d jj|kszJ ‚|d jjt	ksŽJ ‚|d jj
d u s¢J ‚tr¾dd„ |D ƒdhks¾J ‚d S )Nr   Úrb©Úfiler   c                 S  s   h | ]}|j j’qS r$   )r   Zdetection_origin)Ú.0Úelementr$   r$   r%   Ú	<setcomp>c   ó    z/test_partition_csv_from_file.<locals>.<setcomp>Úcsv)Úopenr   r   r   Ú
isinstancer   r   r   r    r!   r   r   )r   r   r   r"   Úfr#   r$   r$   r%   Útest_partition_csv_from_fileR   s    
(r=   c                  C  sh   t tdƒdƒ�} t| dd�}W d   ƒ n1 s00    Y  t|d jƒtksPJ ‚|d jjdksdJ ‚d S )Nr   r2   r,   )r4   r-   r   )r:   r   r   r   r   r   r   r   ©r<   r#   r$   r$   r%   Ú3test_partition_csv_from_file_with_metadata_filenamef   s    *r?   r   )Úmockerc                 C  s6   d}| j d|d� ttdƒƒ}|d jj|ks2J ‚d S )Nú2029-07-05T09:24:28ú1unstructured.partition.csv.get_last_modified_date©Úreturn_valuer   r   ©Úpatchr   r   r   Úlast_modified)r@   Úfilesystem_last_modifiedr#   r$   r$   r%   ÚDtest_partition_csv_from_file_path_gets_last_modified_from_filesystemq   s    þrI   c                 C  s>   d}d}| j d|d� ttdƒ|d�}|d jj|ks:J ‚d S )NrA   ú2020-07-05T09:24:28rB   rC   r   )Úmetadata_last_modifiedr   rE   )r@   rH   rK   r#   r$   r$   r%   Ú@test_partition_csv_from_file_path_prefers_metadata_last_modified}   s    ÿÿrL   c                  C  sP   t tdƒdƒ�} t| d�}W d   ƒ n1 s.0    Y  |d jjd u sLJ ‚d S )Nr   r2   r3   r   ©r:   r   r   r   rG   r>   r$   r$   r%   Ú4test_partition_csv_from_file_gets_last_modified_NoneŒ   s    (rN   c                  C  sV   d} t tdƒdƒ�}t|| d�}W d   ƒ n1 s40    Y  |d jj| ksRJ ‚d S )NrJ   r   r2   )r4   rK   r   rM   )rK   r<   r#   r$   r$   r%   Ú;test_partition_csv_from_file_prefers_metadata_last_modified“   s    *rO   r   r   c                 C  s   t t| ƒd�}t|ƒ d S )Nr   )r   r   r   ©r   r#   r$   r$   r%   Útest_partition_csv_with_jsonŸ   s    rQ   c                  C  sJ   d} t | d�}t | ddddd�}t|ddd�}||ks:J ‚||ksFJ ‚d S )	Nr)   r   Zby_titleé	   r   F)Zchunking_strategyÚmax_charactersÚcombine_text_under_n_charsÚinclude_header)rS   rT   )r   r   )r   r#   Zchunk_elementsÚchunksr$   r$   r%   Ú7test_add_chunking_strategy_to_partition_csv_non_default¥   s    
ûrW   c                  C  s,   d} t | ddd�}|d jjdgks(J ‚d S )Nr)   ÚfastF)r   ÚstrategyrU   r   Zeng©r   r   Ú	languagesrP   r$   r$   r%   Ú1test_partition_csv_element_metadata_has_languages·   s    r\   c                  C  s0   d} t | ddgdd�}|d jjdgks,J ‚d S )Nr)   rX   ZdeuF)r   rY   r[   rU   r   rZ   rP   r$   r$   r%   Ú)test_partition_csv_respects_languages_arg½   s
    
ÿr]   c                  C  s@   t tdƒddd�} | d }|jdt ks,J ‚|jjd us<J ‚d S )Nr   rX   T)rY   rU   r   z#Stanley Cups Unnamed: 1 Unnamed: 2 )r   r   r   r
   r   r   )r#   Útabler$   r$   r%   Útest_partition_csv_headerÅ   s    
ÿr_   c                   @  s¸   e Zd ZdZdd„ Zdd„ Zej dddg¡d	d
œdd„ƒZ	dd„ Z
dd„ Zej dddg¡dddœdd„ƒZddœdd„Zdd„ Zdd„ Zd d!„ Zd"d#„ Ze ¡ d$dd%œd&d'„ƒZd(S ))ÚDescribe_CsvPartitioningContextzIUnit-test suite for `unstructured.partition.csv._CsvPartitioningContext`.c                 C  s*   t jtdƒd d ddd�}t|t ƒs&J ‚d S )Nr   T©Ú	file_pathr4   r0   rU   r'   )r   Úloadr   r;   ©ÚselfÚctxr$   r$   r%   Ú.it_provides_a_validating_alternate_constructorÙ   s    ûzNDescribe_CsvPartitioningContext.it_provides_a_validating_alternate_constructorc                 C  sF   t jtdd��$ tjd d d ddd� W d   ƒ n1 s80    Y  d S )Nú1either file-path or file-like object must be prov©ÚmatchTra   )ÚpytestÚraisesÚ
ValueErrorr   rc   ©re   r$   r$   r%   Ú;and_the_validating_constructor_raises_on_an_invalid_contextã   s    ûz[Describe_CsvPartitioningContext.and_the_validating_constructor_raises_on_an_invalid_contextÚ	file_namer   zcsv-with-long-lines.csvr   )rp   c                 C  s   t t|ƒƒ}|jdksJ ‚d S )Nú,©r   r   Ú	delimiter)re   rp   rf   r$   r$   r%   Ú<it_auto_detects_the_delimiter_for_a_comma_delimited_CSV_fileï   s    
z\Describe_CsvPartitioningContext.it_auto_detects_the_delimiter_for_a_comma_delimited_CSV_filec                 C  s   t tdƒƒ}|jdksJ ‚d S )Nzsemicolon-delimited.csvú;rr   rd   r$   r$   r%   ÚDand_it_auto_detects_the_delimiter_for_a_semicolon_delimited_CSV_fileü   s    zdDescribe_CsvPartitioningContext.and_it_auto_detects_the_delimiter_for_a_semicolon_delimited_CSV_filec                 C  s   t tdƒƒ}|jd u sJ ‚d S )Nzsingle-column.csvrr   rd   r$   r$   r%   ÚAbut_it_returns_None_as_the_delimiter_for_a_single_column_CSV_file   s    zaDescribe_CsvPartitioningContext.but_it_returns_None_as_the_delimiter_for_a_single_column_CSV_file)rU   Úexpected_value)FN)Tr   r(   z
int | Nonec                 C  s   t |d�j|ksJ ‚d S )N)rU   )r   Úheader)re   rU   rx   r$   r$   r%   Ú8it_identifies_the_header_row_based_on_include_header_arg  s    zXDescribe_CsvPartitioningContext.it_identifies_the_header_row_based_on_include_header_argr   )Úget_last_modified_date_c                 C  s4   d}||_ tdd�}|j}| d¡ ||ks0J ‚d S )Nz2024-08-04T02:23:53za/b/document.csv)rb   )rD   r   rG   Zassert_called_once_with)re   r{   rH   rf   rG   r$   r$   r%   ÚAit_gets_last_modified_from_the_filesystem_when_a_path_is_provided  s    

zaDescribe_CsvPartitioningContext.it_gets_last_modified_from_the_filesystem_when_a_path_is_providedc                 C  s*   t  d¡}t|d�}|j}|d u s&J ‚d S )Ns   abcdefgr3   )ÚioÚBytesIOr   rG   )re   r4   rf   rG   r$   r$   r%   ÚSand_it_falls_back_to_None_for_the_last_modified_date_when_file_path_is_not_provided  s    

zsDescribe_CsvPartitioningContext.and_it_falls_back_to_None_for_the_last_modified_date_when_file_path_is_not_providedc              	   C  sº   t tdƒdƒ�˜}| ¡  t|d�}|  ¡ �N}||u s8J ‚| ¡ dksHJ ‚| d¡dksZJ ‚| ¡ dksjJ ‚W d   ƒ n1 s~0    Y  | ¡ dks˜J ‚W d   ƒ n1 s¬0    Y  d S )Nr   r2   r3   r   é   ó   Stanley Cups,,)r:   r   Úreadr   Útell)re   r<   rf   r4   r$   r$   r%   ÚOit_provides_transparent_access_to_the_source_file_when_it_is_a_file_like_object$  s    

.zoDescribe_CsvPartitioningContext.it_provides_transparent_access_to_the_source_file_when_it_is_a_file_like_objectc                 C  sJ   t tdƒƒ}| ¡ �"}| d¡dks(J ‚W d   ƒ n1 s<0    Y  d S )Nr   r€   r�   )r   r   r:   r‚   )re   rf   r4   r$   r$   r%   ÚHit_provides_transparent_access_to_the_source_file_when_it_is_a_file_path3  s    
zhDescribe_CsvPartitioningContext.it_provides_transparent_access_to_the_source_file_when_it_is_a_file_pathc                 C  s<   t jtdd�� tƒ  ¡  W d   ƒ n1 s.0    Y  d S )Nrh   ri   )rk   rl   rm   r   Ú	_validatern   r$   r$   r%   Ú5it_raises_when_neither_file_path_nor_file_is_provided:  s    zUDescribe_CsvPartitioningContext.it_raises_when_neither_file_path_nor_file_is_providedr   )ÚrequestÚreturnc                 C  s
   t |dƒS )NrB   )r   )re   rˆ   r$   r$   r%   r{   @  s    z7Describe_CsvPartitioningContext.get_last_modified_date_N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__rg   ro   rk   ÚmarkÚparametrizert   rv   rw   rz   r|   r   r„   r…   r‡   Zfixturer{   r$   r$   r$   r%   r`   Ô   s*   
üþ	
r`   )2Ú
__future__r   r}   rk   Zpytest_mockr   Z*test_unstructured.partition.test_constantsr   r   r   r   r   r	   r
   Ztest_unstructured.unit_utilsr   r   r   r   r   Zunstructured.chunking.titler   Zunstructured.cleaners.corer   Zunstructured.documents.elementsr   Zunstructured.partition.csvr   r   Z&unstructured.partition.utils.constantsr   r!   rŽ   r�   r&   r+   r/   r1   r=   r?   rI   rL   rN   rO   rQ   rW   r\   r]   r_   r`   r$   r$   r$   r%   Ú<module>   sZ   $	ýýþ
þþ