a
    DBìj0  ã                   @   s  d Z ddlmZ ddlZddlZddlZddlZddlm	  m
Z ddlmZ ddlmZ ddlmZ ddlmZ ej d¡Zejjejjej dd	d
g¡ej dddg¡dd„ ƒƒƒƒZejjejjdd„ ƒƒZejdd„ ƒZejjej d¡e  ¡ G dd„ dƒƒƒƒZ!dS )zl
Tests parsers ability to read and parse non-local files
and hence require a network connection to be read.
é    )ÚBytesION)Ú	DataFrame)Úread_feather©Úread_csvz=ignore:Passing a BlockManager to DataFrame:DeprecationWarningÚmodeÚexplicitÚinferÚengineÚpythonÚcc                 C   sœ   |dkrt  d¡ || }t|dddd| ƒdƒ� }| j| ¡ d� W d   ƒ n1 sX0    Y  | jd	 | }	|d
kr||}t|	d||d�}
t |
|¡ d S )NÚtarz5TODO: Add tar salaraies.csv to pandas/io/parsers/dataÚioÚparserÚdatazsalaries.csvÚrb©Úcontentz/salaries.csvr   ú	)ÚsepÚcompressionr
   )	ÚpytestÚskipÚopenÚserve_contentÚreadÚurlr   ÚtmÚassert_frame_equal)Ú
httpserverÚdatapathZsalaries_tabler   r
   Zcompression_onlyZcompression_to_extensionÚ	extensionÚfr   Z	url_table© r#   úb/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/pandas/tests/io/parser/test_network.pyÚtest_compressed_urls   s    
.r%   c                 C   sj   t |ddddƒdƒ�0}| j| ¡ d� t| jddd	�}W d  ƒ n1 sJ0    Y  |jd
 dksfJ ‚dS )zN
    read_csv should honor the requested encoding for URLs.

    GH 10424
    r   r   r   zunicode_series.csvr   r   zlatin-1N)ÚencodingÚheader)é   é   u$   Ã� kÃ¶ldum klaka (Cold Fever) (1994))r   r   r   r   r   Úloc)r   r    r"   Údfr#   r#   r$   Útest_url_encoding_csv8   s    .r,   c                 C   s   t | ddddƒƒS )z DataFrame with the tips dataset.r   r   Úcsvútips.csvr   )r    r#   r#   r$   Útips_dfF   s    r/   Zs3_resourcec                   @   sÌ   e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zd	d
„ Zdd„ Zdd„ Z	dd„ Z
dd„ Zdd„ Zdd„ Zdd„ Zejjddd�dd„ ƒZejjddd�dd„ ƒZejjd d!„ ƒZejjd"d#„ ƒZd$d%„ Zd&d'„ Zd(S ))ÚTestS3c                 C   s\   t  d¡ dD ]H\}}td|j› d�| ||d�}t|tƒs@J ‚|jrJJ ‚t ||¡ qd S )NÚs3fs©)Ú N)ú.gzÚgzip)ú.bz2Úbz2ús3://ú	/tips.csv)r   Ústorage_options©	r   Úimportorskipr   ÚnameÚ
isinstancer   Úemptyr   r   ©ÚselfÚs3_public_bucket_with_datar/   Ús3soÚextÚcompr+   r#   r#   r$   Útest_parse_public_s3_bucketP   s    
ý
z"TestS3.test_parse_public_s3_bucketc                 C   sH   t  d¡ td|j› d�|d�}t|tƒs.J ‚|jr8J ‚t ||¡ d S )Nr1   r8   r9   ©r:   r;   )rA   Zs3_private_bucket_with_datar/   rC   r+   r#   r#   r$   Útest_parse_private_s3_bucket^   s    
ÿ
z#TestS3.test_parse_private_s3_bucketc                 C   sJ   t d|j› d�d|d�}t|tƒs&J ‚|jr0J ‚t |jd d… |¡ d S )Nzs3n://r9   é
   ©Únrowsr:   ©r   r=   r>   r   r?   r   r   Úiloc©rA   rB   r/   rC   r+   r#   r#   r$   Útest_parse_public_s3n_bucketh   s    ý
z#TestS3.test_parse_public_s3n_bucketc                 C   sJ   t d|j› d�d|d�}t|tƒs&J ‚|jr0J ‚t |jd d… |¡ d S )Nzs3a://r9   rI   rJ   rL   rN   r#   r#   r$   Útest_parse_public_s3a_buckets   s    ý
z#TestS3.test_parse_public_s3a_bucketc                 C   s^   dD ]T\}}t d|j› d�| d||d�}t|tƒs8J ‚|jrBJ ‚t |jd d… |¡ qd S )Nr2   r8   r9   rI   )rK   r   r:   rL   r@   r#   r#   r$   Ú!test_parse_public_s3_bucket_nrows~   s    ü
z(TestS3.test_parse_public_s3_bucket_nrowsc              	   C   s²   d}dD ]¤\}}t d|j› d�| |||d��n}|j|ks>J ‚dD ]J}| ¡ }	t|	tƒs\J ‚|	jrfJ ‚|j|| ||d  … }
t 	|
|	¡ qBW d   ƒ q1 s¢0    Y  qd S )Né   r2   r8   r9   )Ú	chunksizer   r:   ©r   r)   é   r)   ©
r   r=   rS   Z	get_chunkr>   r   r?   rM   r   r   ©rA   rB   r/   rC   rS   rD   rE   Z	df_readerZi_chunkr+   Ztrue_dfr#   r#   r$   Ú#test_parse_public_s3_bucket_chunkedŒ   s$    ü
ÿz*TestS3.test_parse_public_s3_bucket_chunkedc              	   C   s´   d}dD ]¦\}}t d|j› d�| ||d|d��n}|j|ks@J ‚dD ]J}| ¡ }	t|	tƒs^J ‚|	jrhJ ‚|j|| ||d  … }
t 	|
|	¡ qDW d   ƒ q1 s¤0    Y  qd S )	NrR   r2   r8   r9   r   )rS   r   r
   r:   rT   r)   rV   rW   r#   r#   r$   Ú*test_parse_public_s3_bucket_chunked_python¤   s&    û
ÿz1TestS3.test_parse_public_s3_bucket_chunked_pythonc                 C   sT   dD ]J\}}t d|j› d�| d||d�}t|tƒs8J ‚|jrBJ ‚t ||¡ qd S )Nr2   r8   r9   r   ©r
   r   r:   ©r   r=   r>   r   r?   r   r   r@   r#   r#   r$   Ú"test_parse_public_s3_bucket_python¼   s    ü
z)TestS3.test_parse_public_s3_bucket_pythonc                 C   sP   dD ]F}t d|j› d�| dd|d�}t|tƒs4J ‚|jr>J ‚t ||¡ qd S )N)r3   r4   r6   r8   r9   r   r	   rZ   r[   )rA   rB   r/   rC   rD   r+   r#   r#   r$   Útest_infer_s3_compressionÊ   s    ü
z TestS3.test_infer_s3_compressionc                 C   s`   dD ]V\}}t d|j› d�| dd||d�}t|tƒs:J ‚|jrDJ ‚t |jd d… |¡ qd S )Nr2   r8   r9   r   rI   )r
   rK   r   r:   rL   r@   r#   r#   r$   Ú(test_parse_public_s3_bucket_nrows_pythonÖ   s    û
z/TestS3.test_parse_public_s3_bucket_nrows_pythonc                 C   sB   d}t jt|d�� td|d� W d   ƒ n1 s40    Y  d S )Nú#The specified bucket does not exist©Úmatchzs3://nyqpug/asdf.csvrG   )r   ÚraisesÚOSErrorr   )rA   rC   Úmsgr#   r#   r$   Útest_read_s3_failså   s    zTestS3.test_read_s3_failsc                 C   sH   d}t jt|d��" td|j› d�ƒ W d   ƒ n1 s:0    Y  d S )Nr_   r`   r8   z	/file.csv)r   rb   rc   r   r=   )rA   Zs3_private_bucketrC   rd   r#   r#   r$   Útest_read_s3_fails_privateê   s    z!TestS3.test_read_s3_fails_privatezGH#39155 s3fs upgradeF)ÚreasonÚstrictc                 C   sT   dd l }t|jjf}tj|dd�� |jd|d� W d   ƒ n1 sF0    Y  d S )Nr   r_   r`   z/s3://an_s3_bucket_data_doesnt_exit/not_real.csvrG   )ÚbotocoreÚFileNotFoundErrorÚ
exceptionsÚClientErrorr   rb   Úto_csv©rA   r/   rC   ri   Úerrorr#   r#   r$   Útest_write_s3_csv_failsñ   s    ÿzTestS3.test_write_s3_csv_failsc                 C   s^   t  d¡ dd l}t|jjf}t j|dd�� |jd|d� W d   ƒ n1 sP0    Y  d S )NÚpyarrowr   r_   r`   z3s3://an_s3_bucket_data_doesnt_exit/not_real.parquetrG   )r   r<   ri   rj   rk   rl   rb   Z
to_parquetrn   r#   r#   r$   Útest_write_s3_parquet_fails  s    
þz"TestS3.test_write_s3_parquet_failsc                 C   sz   |  d¡}t| ¡ d  ¡ ƒ�}t|dd�}W d   ƒ n1 s@0    Y  t|tƒsXJ ‚|jrbJ ‚t|ƒ}t 	||¡ d S )Nr.   ÚBodyÚutf8)r&   )
ZObjectr   Úgetr   r   r>   r   r?   r   r   )rA   rB   Z	tips_fileZ	s3_objectÚbufferÚresultÚexpectedr#   r#   r$   Ú$test_read_csv_handles_boto_s3_object  s    
*
z+TestS3.test_read_csv_handles_boto_s3_objectc           
   	   C   sî   t t d¡tdƒd�}t| ¡  d¡ƒ�²}|jd|d� |j› d�}t	 
d|› d	�¡}|jtjd
d��V td|› �d|d� |jD ]*}t	 ||¡ }	r‚t|	 d¡ƒdk s‚J ‚q‚W d   ƒ n1 sÂ0    Y  W d   ƒ n1 sà0    Y  d S )N)i † é   Úabcd)Úcolumnszutf-8zlarge-file.csv)ÚKeyrs   z/large-file.csvz^Fetch: z, 0-(?P<stop>\d+)$r1   )Úloggerr8   rR   rJ   Ústopi z )r   ÚnpZzerosÚlistr   rm   ÚencodeZ
put_objectr=   ÚreÚcompileZat_levelÚloggingÚDEBUGr   Úmessagesra   ÚintÚgroup)
rA   Zs3_public_bucketZcaplogrC   r+   ÚbufÚuriZmatch_reÚlogra   r#   r#   r$   Útest_read_csv_chunked_download#  s    ý
z%TestS3.test_read_csv_chunked_downloadc                 C   s&   t d|j› d�|d�}t ||¡ d S )Nr8   z/tips#1.csvrG   )r   r=   r   r   )rA   rB   r/   rC   rw   r#   r#   r$   Útest_read_s3_with_hash_in_key6  s    ÿz$TestS3.test_read_s3_with_hash_in_keyc                 C   s8   t  d¡ t|ƒ}td|j› d�|d�}t ||¡ d S )Nrq   r8   z/simple_dataset.featherrG   )r   r<   r   r=   r   r   )rA   rB   Zfeather_filerC   rx   Úresr#   r#   r$   Útest_read_feather_s3_file_path=  s    
þz%TestS3.test_read_feather_s3_file_pathN)Ú__name__Ú
__module__Ú__qualname__rF   rH   rO   rP   rQ   rX   rY   r\   r]   r^   re   rf   r   ÚmarkZxfailrp   rr   Ú
single_cpury   r�   rŽ   r�   r#   r#   r#   r$   r0   L   s,   




r0   )"Ú__doc__r   r   r…   rƒ   Únumpyr€   r   Zpandas.util._test_decoratorsÚutilZ_test_decoratorsÚtdZpandasr   Zpandas._testingZ_testingr   Zpandas.io.feather_formatr   Zpandas.io.parsersr   r”   ÚfilterwarningsZ
pytestmarkÚnetworkr•   Zparametrizer%   r,   Zfixturer/   ZusefixturesZskip_if_not_us_localer0   r#   r#   r#   r$   Ú<module>   s6   ÿ

