a
    DBìj¼W  ã                   @   s¸  d Z ddlmZ ddlZddlZddlmZ ddlm	Z	m
Z
mZ ddlmZ ej d¡Zej d¡Zej d¡Zd	d
„ Zdd„ Zej dddgddgddgdgdgdgdgg¡ej dddg¡dd„ ƒƒZdd„ Zej dddgg¡dd„ ƒZdd„ Zdd„ Zej d dge	ejgd!gd"gd#œe
dgd$d%�d&�fdd'ge	ejgd"gd(œejd)gd$d*gd+�d&�fd$d*ge	ejgd"gd(œejd)gd$d*gd+�d&�fg¡d,d-„ ƒZej d.i e	d$d/ejd0d1ejd2gg d3¢d4d5d6ejd7ejd8gd9œƒfg g d:œd;d<œe	g d=¢g d3¢g d>¢d9œƒfd$gd;d<œe	ejd/d?d0d1d@d2gg d3¢g d>¢d9œƒfdg g d:œie	d$d/ejd0d1ejd2gg d3¢d4d5d6ejd7ejd8gd9œƒfg¡dAdB„ ƒZ dCdD„ Z!dEdF„ Z"dGdH„ Z#ej dIdJdKg¡dLdM„ ƒZ$ej dNdOd!dPgejejgdQdRggfd;dSdPgd@dTgdUdRggfg¡dVdW„ ƒZ%edXdY„ ƒZ&ej dZd!ejd[gd[ejggfd'd!d\œd]d[gejejggfg¡d^d_„ ƒZ'd`da„ Z(dbdc„ Z)ej dde*deƒdf e*dgƒ ddegie	e*deƒe*dgƒgƒfe*deƒdh di i e	e*deƒd!gd?d'ggƒfe*deƒdj ddegie	ejd!gƒfg¡dkdl„ ƒZ+dmdn„ Z,ej dod;d?dpgfdOejdqgfg¡drds„ ƒZ-dtdu„ Z.eej dvdOd;g¡dwdx„ ƒƒZ/eej dydzd{d|d}d~dgfd~d$difg¡d€d�„ ƒƒZ0ed‚dƒ„ ƒZ1d„d…„ Z2ed†d‡„ ƒZ3dˆd‰„ Z4dŠd‹„ Z5dS )Œzg
Tests that NA values are properly handled during
parsing for all of the parsers defined in parsers.py
é    )ÚStringION)ÚSTR_NA_VALUES)Ú	DataFrameÚIndexÚ
MultiIndexz=ignore:Passing a BlockManager to DataFrame:DeprecationWarningZpyarrow_xfailZpyarrow_skipc                 C   sn   | }d}|  t|ƒ¡}tg d¢dtjdgtjddggg d¢d�}|jd	kr^d |jd
< d |jd< t ||¡ d S )NzA,B,C
a,b,c
d,,f
,g,h
©ÚaÚbÚcÚdÚfÚgÚh©ÚAÚBÚC©ÚcolumnsÚpyarrow)é   r   )é   r   )	Úread_csvr   r   ÚnpÚnanÚengineÚlocÚtmÚassert_frame_equal©Úall_parsersÚparserÚdataÚresultÚexpected© r%   úd/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/pandas/tests/io/parser/test_na_values.pyÚtest_string_nas   s    þ


r'   c                 C   st   | }d}t ddgtjdgtjtjggddgd�}|jdkrVd |jd	d
gdf< d |jd< | t|ƒ¡}t ||¡ d S )NzA,B
foo,bar
NA,baz
NaN,nan
ÚfooÚbarÚbazr   r   r   r   r   r   ©r   r   )	r   r   r   r   r   r   r   r   r   )r    r!   r"   r$   r#   r%   r%   r&   Útest_detect_string_na-   s     ÿ

r,   Ú	na_valuesz-999.0z-999iüÿÿg     8�Àr"   zA,B
-999,1.2
2,-999
3,4.5
z"A,B
-999,1.200
2,-999.000
3,4.500
c           	      C   sÚ   | }t tjdgdtjgddggddgd�}|jdkrŽtd	d
„ |D ƒƒsŽd}tjt|d��" |jt	|ƒ|d� W d   ƒ n1 s€0    Y  d S |jdkr¸d|v r¸tj
jdd�}| |¡ |jt	|ƒ|d�}t ||¡ d S )Ng333333ó?ç       @g      @g      @r   r   r   r   c                 s   s   | ]}t |tƒV  qd S )N)Ú
isinstanceÚstr)Ú.0Úxr%   r%   r&   Ú	<genexpr>_   ó    z,test_non_string_na_values.<locals>.<genexpr>ú9The 'pyarrow' engine requires all na_values to be strings©Úmatch©r-   z-999.000z4pyarrow engined does not recognize equivalent floats©Úreason)r   r   r   r   ÚallÚpytestÚraisesÚ	TypeErrorr   r   ÚmarkÚxfailÚapplymarkerr   r   )	r    r"   r-   Úrequestr!   r$   Úmsgr?   r#   r%   r%   r&   Útest_non_string_na_values>   s    &0ÿ
rD   c                    s‚   h d£}|t ksJ ‚| }t|ƒ‰‡fdd„‰ td ‡ fdd„t|ƒD ƒ¡ƒ}ttjtˆƒtˆƒd�}|j	|d d�}t
 ||¡ d S )	N>   Ú z-1.#QNANz-NaNz#NAÚNaNz<NA>r   ÚNonez-1.#INDz1.#QNANZNAZNULLzN/Az#N/AÚnullzn/az1.#INDz#N/A N/Az-nanc                    sf   | dkrd}n| dkr&d  dg|  ¡}|› |› �}| ˆ d k rbd  dgˆ |  d  ¡}|› |› �}|S )Nr   rE   ú,r   )Újoin)ÚiÚvÚbufZjoined)Únvr%   r&   r   Œ   s    z!test_default_na_values.<locals>.fÚ
c                    s   g | ]\}}ˆ ||ƒ‘qS r%   r%   )r1   rK   rL   )r   r%   r&   Ú
<listcomp>š   r4   z*test_default_na_values.<locals>.<listcomp>)r   Úindex)Úheader)r   Úlenr   rJ   Ú	enumerater   r   r   Úranger   r   r   )r    Z
_NA_VALUESr!   r"   r$   r#   r%   )r   rN   r&   Útest_default_na_valuesq   s     rV   r*   c                 C   s´   | }d}t dtjdgtjdtjgddtjggg d¢d�}|jd	krŽd
}tjt|d��& |jt|ƒ|dgd� W d   ƒ n1 s€0    Y  d S |jt|ƒ|dgd�}t	 
||¡ d S )Nz3A,B,C
ignore,this,row
1,NA,3
-1.#IND,5,baz
7,8,NaN
ç      ð?é   é   é   é   r   r   r   z@skiprows argument must be an integer when using engine='pyarrow'r6   r   )r-   Zskiprows©r   r   r   r   r<   r=   Ú
ValueErrorr   r   r   r   )r    r-   r!   r"   r$   rC   r#   r%   r%   r&   Útest_custom_na_values¡   s    (ÿ
4r^   c                 C   s|   d}| }|  t|ƒ¡}ttjdtjdgtd�tjddtjgtd�g d¢dœƒ}|jdkrld |jd< d |jd	< t	 
||¡ d S )
Nz1A,B,C
True,False,True
NA,True,False
False,NA,TrueTF©Údtype)TFTr   r   )r   r   r+   )r   r   r   r   Úarrayr   Úobjectr   r   r   r   ©r    r"   r!   r#   r$   r%   r%   r&   Útest_bool_na_values·   s    ýÿ


rd   c                 C   sÄ   d}| }|j dkrdd}tjt|d��, |jt|ƒdgdgdœd� W d   ƒ n1 sV0    Y  d S |jt|ƒdgdgdœd�}ttjdtjdgtjdtjdgtjdtjdgd	œƒ}t	 
||¡ d S )
Nz3A,B,C
foo,bar,NA
bar,foo,foo
foo,bar,NA
bar,foo,foor   z;pyarrow engine doesn't support passing a dict for na_valuesr6   r(   r)   )r   r   r8   r   ©r   r<   r=   r]   r   r   r   r   r   r   r   ©r    r"   r!   rC   Zdfr$   r%   r%   r&   Útest_na_value_dictË   s    
:ýÿrg   zindex_col,expectedr   rY   )r	   r
   r   r   ©Úname©rQ   r   )r	   r   )r   r   r
   )Únamesc                 C   s.   d}| }|j t|ƒtƒ |d�}t ||¡ d S )Nza,b,c,d
0,NA,1,5
)r-   Ú	index_col)r   r   Úsetr   r   )r    rl   r$   r"   r!   r#   r%   r%   r&   Útest_na_value_dict_multi_indexä   s    rn   zkwargs,expectedr	   r   Úer   ©r   r   rX   é   rY   é   rZ   ÚoneÚtwoÚthreeÚfiveÚsevenr   )r   r   F©r-   Úkeep_default_na©r   r	   rE   r   ro   r   r   )rs   rt   ru   r   rv   rE   rw   rE   r   c           
      C   sº   d}| }|j dkr”d|v rtt|d tƒrtd}tjt|d��& |jt|ƒfi |¤Ž W d   ƒ n1 sf0    Y  d S |r€d|v r”tj 	¡ }| 
|¡ |jt|ƒfi |¤Ž}	t |	|¡ d S )NzAA,B,C
a,1,one
b,2,two
,3,three
d,4,nan
e,5,five
nan,6,
g,7,seven
r   r-   ú?The pyarrow engine doesn't support passing a dict for na_valuesr6   )r   r/   Údictr<   r=   r]   r   r   r?   r@   rA   r   r   )
r    Úkwargsr$   rB   Úusing_infer_stringr"   r!   rC   r?   r#   r%   r%   r&   Útest_na_values_keep_default  s    0

4

r   c                 C   sF   d}| }|j t|ƒdd�}tg d¢g d¢g d¢dœƒ}t ||¡ d S )NzAA,B,C
a,1,None
b,2,two
,3,None
d,4,nan
e,5,five
nan,6,
g,7,seven
F)ry   rz   rp   )rG   rt   rG   r   rv   rE   rw   r   )r   r   r   r   r   rc   r%   r%   r&   Ú!test_no_na_values_no_keep_defaultN  s    
ýÿr€   c                 C   s    d}| }|j dkrbd}tjt|d��* |jt|ƒddgidd� W d   ƒ n1 sT0    Y  d S |jt|ƒddgidd�}td	gtjgd
œƒ}t	 
||¡ d S )Nza,b
,2r   r{   r6   r	   Ú2Frx   rE   ©r   r	   re   )r    r"   r!   rC   r#   r$   r%   r%   r&   Ú&test_no_keep_default_na_dict_na_valuesh  s    
ÿ$ÿrƒ   c                 C   sœ   d}| }|j dkr`d}tjt|d��( |jt|ƒddidd� W d   ƒ n1 sR0    Y  d S |jt|ƒddidd�}td	gtjgd
œƒ}t	 
||¡ d S )Nza,b
1,2r   r{   r6   r	   r   Frx   r   r‚   re   rf   r%   r%   r&   Ú-test_no_keep_default_na_dict_na_scalar_values|  s    
6r„   Úcol_zero_na_valuesiå¹ Z113125c              
   C   sÔ   d}| }t tjdgtjdgdtjgddgddgd	d
gtjdgdœƒ}|jdkr¤d}tjt|d��0 |jt|ƒd dd
dd|dœd� W d   ƒ n1 s–0    Y  d S |jt|ƒd dd
dd|dœd�}t	 
||¡ d S )Nz_113125,"blah","/blaha",kjsdkj,412.166,225.874,214.008
729639,"qwer","",asdfkj,466.681,,252.373
g    ND&AZqwerz/blahaZkjsdkjZasdfkjg-²�ï§Ây@g7‰A`å*}@z225.874rE   g-²�ï‹o@)r   r   r   rX   rq   rY   rr   r   r{   r6   Fz214.008Zblah)r   rr   r   r   )rR   ry   r-   r\   )r    r…   r"   r!   r$   rC   r#   r%   r%   r&   Ú1test_no_keep_default_na_dict_na_values_diff_reprsŽ  s:    ùÿ
ü$ür†   zna_filter,row_dataTr   rX   r   Ú1r   Ú3c           
      C   sh   |}|j dkr.|r|s.tjjdd�}|  |¡ d}|jt|ƒdg|d�}t|ddgd�}	t 	||	¡ d S )	Nr   zpyarrow doesn't support this.r9   zA,B
1,A
nan,B
3,C
r   )r-   Ú	na_filterr   r   )
r   r<   r?   r@   rA   r   r   r   r   r   )
rB   r    r‰   Úrow_datar~   r!   r?   r"   r#   r$   r%   r%   r&   Ú!test_na_values_na_filter_override¶  s    


r‹   c              
   C   sf   | }d}|  t|ƒ¡}tdddddtjtjtjgdddd	d
tjtjtjggg d¢d�}t ||¡ d S )NzlDate,Currency,Symbol,Type,Units,UnitPrice,Cost,Tax
2012-03-14,USD,AAPL,BUY,1000
2012-05-12,USD,SBUX,SELL,500z
2012-03-14ÚUSDZAAPLZBUYiè  z
2012-05-12ZSBUXZSELLiô  )ÚDateZCurrencyÚSymbolÚTypeZUnitsZ	UnitPriceZCostZTaxr   )r   r   r   r   r   r   r   r   r%   r%   r&   Útest_na_trailing_columnsÒ  s    þûr�   zna_values,row_datar.   r‚   rW   c           
      C   s
  | }ddg}d}|j dkr†t|tƒr†t|tƒr8t}d}nt}d}tj||d��$ |jt|ƒ||d� W d   ƒ n1 sx0    Y  d S |j dkrÚd}tjt|d��$ |jt|ƒ||d� W d   ƒ n1 sÌ0    Y  d S |jt|ƒ||d�}t	||d	�}	t
 ||	¡ d S )
Nr   r	   ú1,2
2,1r   r{   r5   r6   ©rk   r-   r   )r   r/   r|   r]   r>   r<   r=   r   r   r   r   r   )
r    r-   rŠ   r!   rk   r"   ÚerrrC   r#   r$   r%   r%   r&   Útest_na_values_scalarî  s(    	
2
2r”   c           	      C   sÂ   | }dddœ}|  ¡ }ddg}d}tddgtjtjgg|d	�}|jd
kr’d}tjt|d��$ |jt	|ƒ||d� W d   ƒ n1 s„0    Y  d S |jt	|ƒ||d�}t
 ||¡ t
 ||¡ d S )Nr   r   r‚   r   r	   r‘   rW   r.   r   r   r{   r6   r’   )Úcopyr   r   r   r   r<   r=   r]   r   r   r   r   Zassert_dict_equal)	r    r!   r-   Zna_values_copyrk   r"   r$   rC   r#   r%   r%   r&   Útest_na_values_dict_aliasing  s    

2r–   c                 C   s–   d}| }ddi}|j dkrbd}tjt|d��" |jt|ƒ|d� W d   ƒ n1 sT0    Y  d S |jt|ƒ|d�}tdtjd	giƒ}t	 
||¡ d S )
Nza
foo
1r   r(   r   r{   r6   r8   r   r   re   )r    r"   r!   r-   rC   r#   r$   r%   r%   r&   Útest_na_values_dict_col_index&  s    
0r—   zdata,kwargs,expectedl            rO   l           z,1z
,2z
1c           	      C   s²   | }|j dkrfd|v rfd}tjt|d��* |jt|ƒfdd i|¤Ž W d   ƒ n1 sX0    Y  d S |j dkrˆtjjdd�}| |¡ |jt|ƒfdd i|¤Ž}t	 
||¡ d S )Nr   r-   r5   r6   rR   z!Returns float64 instead of objectr9   )r   r<   r=   r>   r   r   r?   r@   rA   r   r   )	r    r"   r}   r$   rB   r!   rC   r?   r#   r%   r%   r&   Útest_na_values_uint647  s    8

r˜   c                 C   sH   d}| }t ddgitdgdd�d�}|jt|ƒdd	d
�}t ||¡ d S )Nza,1
b,2r‡   r   r	   r   rh   rj   r   F)rl   ry   )r   r   r   r   r   r   )r    r"   r!   r$   r#   r%   r%   r&   Ú*test_empty_na_values_no_default_with_indexT  s
    r™   zna_filter,index_dataÚ5g      @c           	      C   sz   | }d}|j dkr2|du r2tjjdd�}| |¡ tddgdd	gd
œt|dd�d�}|jt|ƒdg|d�}t	 
||¡ d S )Núa,b,c
1,,3
4,5,6r   Fzmismatched index resultr9   r   rq   rX   rr   )r   r
   r	   rh   rj   )rl   r‰   )r   r<   r?   r@   rA   r   r   r   r   r   r   )	r    r‰   Z
index_datarB   r!   r"   r?   r$   r#   r%   r%   r&   Útest_no_na_filter_on_index^  s    
"rœ   c                 C   s\   | }d}|j t|ƒdgddgd�}tdtjgdtjgdœtd	d
gdd�d�}t ||¡ d S )Nzidx,col1,col2
1,3,4
2,inf,-infr   Úinfz-inf)rl   r-   rX   rq   )Úcol1Úcol2r   r   Úidxrh   rj   )r   r   r   r   r   r   r   r   )r    r!   r"   Úoutr$   r%   r%   r&   Ú!test_inf_na_values_with_int_indexq  s    "ÿr¢   r‰   c                 C   sV   | }d}|rt jnd}tddg|dgddgdœƒ}|jt|ƒ|td	�}t ||¡ d S )
Nr›   rE   r‡   Ú4rš   rˆ   Ú6r   )r‰   r`   )r   r   r   r   r   r0   r   r   )r    r‰   r!   r"   Úemptyr$   r#   r%   r%   r&   Ú+test_na_values_with_dtype_str_and_na_filter~  s    r¦   zdata, na_values)zfalse,1
,1
trueN)zfalse,1
null,1
trueN)zfalse,1
nan,1
trueN)úfalse,1
foo,1
truer(   r§   r(   c                 C   sd   | }d  ddg¡}tjt|d��0 |jt|ƒd ddgddi|d� W d   ƒ n1 sV0    Y  d S )	Nú|z(Bool column has NA values in column [0a]zRcannot safely convert passed user dtype of bool for object dtyped data in column 0r6   r   r	   Úbool)rR   rk   r`   r-   )rJ   r<   r=   r]   r   r   )r    r"   r-   r!   rC   r%   r%   r&   Ú!test_cast_NA_to_bool_raises_errorŽ  s    þÿûrª   c                 C   sb   | }d}|j t|ƒd g d¢tttdœd� ¡ }tddgddgdd	gdœd
dgd�}t ||¡ d S )NzDFile: small.csv,,
10010010233,0123,654
foo,,bar
01001000155,4530,898)rž   rŸ   Zcol3)rR   rk   r`   Z10010010233Z01001000155Z0123Z4530Z654Z898r   rX   rj   )r   r   r0   Zdropnar   r   r   r   r%   r%   r&   Útest_str_nan_dropped°  s     
ü
ýú	r«   c                 C   s°   | }d}|j dkrhd}tjt|d��0 |jt|ƒttdƒƒddid� W d   ƒ n1 sZ0    Y  d S |jt|ƒttdƒƒddid�}td	gdgt	j
gd
œƒ}t ||¡ d S )NzA,B,B
X,Y,Z
1,2,infr   r{   r6   r   ©r   ÚZr�   )rR   r-   r   ))r   ÚX)r   ÚYr¬   )r   r<   r=   r]   r   r   ÚlistrU   r   r   r   r   r   )r    r!   r"   rC   r#   r$   r%   r%   r&   Útest_nan_multi_indexÍ  s&    
ÿ$ÿýÿr±   c                 C   sL   | }d}t jtdd��" |jt|ƒdd� W d   ƒ n1 s>0    Y  d S )Nú0
NaN
True
False
z	NA valuesr6   r©   r_   ©r<   r=   r]   r   r   ©r    r!   r"   r%   r%   r&   Útest_bool_and_nan_to_boolé  s    rµ   c                 C   sL   | }d}t jtdd��" |jt|ƒdd� W d   ƒ n1 s>0    Y  d S )Nr²   zconvert|NoneTyper6   Úintr_   r³   r´   r%   r%   r&   Útest_bool_and_nan_to_intö  s    r·   c                 C   s@   | }d}|j t|ƒdd�}t dtjddgi¡}t ||¡ d S )Nr²   Úfloatr_   Ú0rW   g        )r   r   r   Ú	from_dictr   r   r   r   r   r%   r%   r&   Útest_bool_and_nan_to_float  s
    r»   )6Ú__doc__Úior   Únumpyr   r<   Zpandas._libs.parsersr   Zpandasr   r   r   Zpandas._testingZ_testingr   r?   ÚfilterwarningsZ
pytestmarkZusefixturesZxfail_pyarrowZskip_pyarrowr'   r,   ZparametrizerD   rV   r^   rd   rg   r   Úfrom_tuplesrn   r   r€   rƒ   r„   r†   r‹   r�   r”   r–   r—   r0   r˜   r™   rœ   r¢   r¦   rª   r«   r±   rµ   r·   r»   r%   r%   r%   r&   Ú<module>   s6  ÿùþúþ0
$þþþþþôþ

ýÿþýÿþ
ýÿþýÿþáþ-

'þþ

þþ
ý( ùþ

ÿ

úþ

