a
    DBìjÏJ  ã                   @  s  d dl mZ d dlmZmZ d dlZd dlZd dlm	Z	m
ZmZmZ d dlmZ d dlmZ d dlmZ d dlmZmZ d d	lmZmZmZ d d
lmZ d dlmZm Z  d dl!m"Z"m#Z#m$Z$ d dl%m&Z& d dl'm(Z( d dl)m*Z*m+Z+ d dl,m-Z-m.Z. e�r>d dl/m0Z0 d dl1m2Z2m3Z3m4Z4m5Z5m6Z6 d dl7m8Z8 d dl)m9Z9m:Z: dddddœdd„Z;dddddœdd„Z<dd dd!d"œd#d$„Z=d%d&dd'œd(d)„Z>d*d+d,d-œd.d/„Z?d!d0d1œd2d3„Z@d%d4d5d5d,d6œd7d8„ZAG d9d:„ d:ƒZBd;dd<d=œd>d?„ZCd&dd@œdAdB„ZDdCdDdEœdFdG„ZEd;ddEœdHdI„ZFdS )Jé    )Úannotations)ÚTYPE_CHECKINGÚcastN)ÚNaTÚalgosÚ	internalsÚlib)ÚNA)Úcache_readonly)Úfind_stack_level)Úensure_dtype_can_hold_naÚfind_common_type)Úis_1d_only_ea_dtypeÚ	is_scalarÚneeds_i8_conversion)Úconcat_compat)ÚExtensionDtypeÚSparseDtype)Úis_valid_na_for_dtypeÚisnaÚisna_all)Úensure_wrapped_if_datetimelike)ÚArrayManager)Úensure_block_shapeÚnew_block_2d)ÚBlockManagerÚmake_na_array)ÚSequence)Ú	ArrayLikeÚAxisIntÚDtypeObjÚ	Manager2DÚShape)ÚIndex)ÚBlockÚBlockPlacementzlist[ArrayManager]zlist[Index]r   r!   )ÚmgrsÚaxesÚconcat_axisÚreturnc                 C  s8   |dkr| d   | |¡S |dks$J ‚| d  | |¡S dS )zí
    Concatenate array managers into one.

    Parameters
    ----------
    mgrs_indexers : list of (ArrayManager, {axis: indexer,...}) tuples
    axes : list of Index
    concat_axis : int

    Returns
    -------
    ArrayManager
    é   r   N)Zconcat_verticalÚconcat_horizontal)r&   r'   r(   © r,   ú[/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/pandas/core/internals/concat.pyÚ_concatenate_array_managersF   s    r.   Úbool)r'   r(   Úcopyr)   c                   s  |o
|dk}t | d d tƒr6t|| |ƒ}t|||ƒS |dkrZt|| |ƒ}|d  ||¡S t| ƒdkrè| d d jdkrè| d d jd j‰ ˆ t	j
t	jfv rèt‡ fdd„| D ƒƒrèt| ƒdkrètdd„ |D ƒƒ}t| |ˆ ƒ}t|f|ƒS t|| |ƒ}t|ƒdk�r |d }|jdd�}	||	_|	S t|ƒ}
g }|
D ]Ð\}}|d }|j}t|ƒ�rÂdd	„ |D ƒ}|j�svt	j|dd
�}n4t|jƒ�ržt|ddd�}t|dd�}nt|dd
�}t|ƒ}|jj|jk}nt||d�}d}|�rè|j||d�}nt||d�}| |¡ �q0tt|ƒ|ƒS )zý
    Concatenate block managers into one.

    Parameters
    ----------
    mgrs_indexers : list of (BlockManager, {axis: indexer,...}) tuples
    axes : list of Index
    concat_axis : int
    copy : bool

    Returns
    -------
    BlockManager
    r   c                 3  s   | ]\}}t |ˆ ƒV  qd S ©N)Ú_is_homogeneous_mgr©Ú.0ÚmgrÚ_©Úfirst_dtyper,   r-   Ú	<genexpr>Œ   ó    z'concatenate_managers.<locals>.<genexpr>r*   c                 s  s   | ]}t |ƒV  qd S r1   )Úlen©r4   Úxr,   r,   r-   r9   ‘   r:   F)Údeepc                 S  s   g | ]}|j j‘qS r,   )ÚblockÚvalues©r4   Zjur,   r,   r-   Ú
<listcomp>§   r:   z(concatenate_managers.<locals>.<listcomp>©ÚaxisT©rD   Zea_compat_axisé   )Úndim)r0   )Ú	placement) Ú
isinstancer   Ú_maybe_reindex_columns_na_proxyr.   r+   r;   ÚnblocksÚblocksÚdtypeÚnpÚfloat64Zfloat32ÚallÚtupleÚ_concat_homogeneous_fastpathr   r0   r'   Ú_get_combined_planr?   Ú_is_uniform_join_unitsÚis_extensionÚconcatenater   r   r   r   r@   Ú_concatenate_join_unitsZmake_block_same_classr   Úappend)Úmgrs_indexersr'   r(   r0   Ú
needs_copyr&   ÚshapeÚnbr5   ÚoutZconcat_planrL   rH   Ú
join_unitsÚunitÚblkÚvalsr@   ZfastpathÚbr,   r7   r-   Úconcatenate_managers^   s^    ÿ
ÿ
þ
rc   z0list[tuple[BlockManager, dict[int, np.ndarray]]]zlist[BlockManager])r'   rY   rZ   r)   c                 C  sb   g }|D ]T\}}|  ¡ D ](\}}|j| | || |ddddd�}q|rR|sR| ¡ }| |¡ q|S )zô
    Reindex along columns so that all of the BlockManagers being concatenated
    have matching columns.

    Columns added in this reindexing have dtype=np.void, indicating they
    should be ignored when choosing a column's final dtype.
    FT)rD   r0   Z
only_sliceZ
allow_dupsZuse_na_proxy)ÚitemsZreindex_indexerr0   rX   )r'   rY   rZ   Znew_mgrsr5   ÚindexersÚiÚindexerr,   r,   r-   rJ   Ê   s     ù	rJ   r   r    )r5   r8   r)   c                 C  s<   | j dkrdS | jd }|jjr.|jjjdks2dS |j|kS )zC
    Check if this Manager can be treated as a single ndarray.
    r*   Fr   )rK   rL   Úmgr_locsÚis_slice_likeÚas_sliceÚsteprM   )r5   r8   r`   r,   r,   r-   r2   ì   s    

r2   r"   znp.dtyper$   )r[   r8   r)   c                 C  s  t dd„ | D ƒƒrLdd„ | D ƒ}t |¡j}t t|d ƒ¡}t||ƒ}|S tj||d�}|tj	krlt
j}nt
j}d}| D ]p\}	}
|	jd }|| }d|
v rÆ||	jd j|
d |dd…||…f ƒ n|	jd j|dd…||…f< ||7 }qzt t|d ƒ¡}t||ƒ}|S )	zl
    With single-Block managers with homogeneous dtypes (that can already hold nan),
    we avoid [...]
    c                 s  s   | ]\}}| V  qd S r1   r,   )r4   r6   re   r,   r,   r-   r9     r:   z/_concat_homogeneous_fastpath.<locals>.<genexpr>c                 S  s   g | ]\}}|j d  jj‘qS )r   )rL   r@   ÚTr3   r,   r,   r-   rB     r:   z0_concat_homogeneous_fastpath.<locals>.<listcomp>r   ©rM   r*   N)rP   rN   rV   rl   Úlibinternalsr%   Úslicer   ÚemptyrO   ÚlibalgosZtake_2d_axis0_float64_float64Ztake_2d_axis0_float32_float32r[   rL   r@   )rY   r[   r8   ZarrsZarrÚbpr\   Z	take_funcÚstartr5   re   Zmgr_lenÚendr,   r,   r-   rR   ù   s2    




ý

rR   z+list[tuple[BlockPlacement, list[JoinUnit]]])r&   r)   c                 C  s’   g }| d j d }dd„ | D ƒ}t |¡}t|ƒD ]Z\}\}}g }t| ƒD ]2\}	}
||	 }t|
|||d�}t|ƒ}| |¡ qJ| ||f¡ q2|S )Nr   c                 S  s   g | ]
}|j ‘qS r,   )Úblknos)r4   r5   r,   r,   r-   rB   /  r:   z&_get_combined_plan.<locals>.<listcomp>)Úmax_len)r[   rn   Zget_concat_blkno_indexersÚ	enumerateÚ_get_block_for_concat_planÚJoinUnitrX   )r&   Zplanrv   Zblknos_listÚpairsÚindru   rr   Zunits_for_bpÚkr5   Úblknor\   r_   r,   r,   r-   rS   (  s    
rS   r%   Úint)r5   rr   r}   rv   r)   c                C  st   | j | }t|ƒt|jƒkr8|jjr8|jjjdkr8|}n8| j|j }t 	||¡}t
|tƒrf| |¡}n
| |¡}|S )Nr*   )rL   r;   rh   ri   rj   rk   Zblklocsrg   r   Zmaybe_indices_to_slicerI   ro   Zslice_block_columnsZtake_block_columns)r5   rr   r}   rv   r`   r\   Zax0_blk_indexerZslcr,   r,   r-   rx   B  s    
ÿÿ

rx   c                   @  sn   e Zd Zdddœdd„Zddœdd	„Zd
ddœdd„Zeddœdd„ƒZeddœdd„ƒZd
ddœdd„Z	dS )ry   r$   ÚNone)r?   r)   c                 C  s
   || _ d S r1   )r?   )Úselfr?   r,   r,   r-   Ú__init___  s    zJoinUnit.__init__Ústr)r)   c                 C  s   t | ƒj› dt| jƒ› d�S )Nú(ú))ÚtypeÚ__name__Úreprr?   )r€   r,   r,   r-   Ú__repr__b  s    zJoinUnit.__repr__r    r/   )rM   r)   c                   sˆ   | j s
dS | j}|jjdkr dS |jtkrN|j}t‡ fdd„|jdd�D ƒƒS |j}|t	u rj|jˆ krjdS |t
u r~tˆ ƒr~dS t|ˆ ƒS )z¯
        Check that we are all-NA of a type/dtype that is compatible with this dtype.
        Augments `self.is_na` with an additional check of the type of NA values.
        FÚVTc                 3  s   | ]}t |ˆ ƒV  qd S r1   )r   r<   rm   r,   r-   r9   s  r:   z,JoinUnit._is_valid_na_for.<locals>.<genexpr>ÚK)Úorder)Úis_nar?   rM   ÚkindÚobjectr@   rP   ZravelÚ
fill_valuer   r	   r   r   )r€   rM   r`   r@   Úna_valuer,   rm   r-   Ú_is_valid_na_fore  s    
zJoinUnit._is_valid_na_forc                 C  s¨   | j }|jjdkrdS |js dS |j}|jdkr4dS t|jtƒrDdS |jdkrr|d }t	|ƒrft
|ƒsjdS t|ƒS |d d }t	|ƒrŽt
|ƒs’dS tdd„ |D ƒƒS d S )Nr‰   TFr   r*   c                 s  s   | ]}t |ƒV  qd S r1   )r   )r4   Úrowr,   r,   r-   r9   ¢  r:   z!JoinUnit.is_na.<locals>.<genexpr>)r?   rM   r�   Z_can_hold_nar@   ÚsizerI   r   rG   r   r   r   rP   )r€   r`   r@   Úvalr,   r,   r-   rŒ   …  s&    

zJoinUnit.is_nac                 C  s   | j }|jjdkrdS dS )z{
        Will self.is_na be True after values.size == 0 deprecation and isna_all
        deprecation are enforced?
        r‰   TF©r?   rM   r�   )r€   r`   r,   r,   r-   Ú)is_na_after_size_and_isna_all_deprecation¤  s    z2JoinUnit.is_na_after_size_and_isna_all_deprecationr   )Úempty_dtyper)   c                 C  s„   |d u r| j jjdkr| j jS |}|  |¡rx| j j}|t d¡krhttj| j jƒ}|jrh|d d u rhd }t	|| j j
|ƒS | j jS d S )Nr‰   rŽ   )r   r   )r?   rM   r�   r@   r‘   rN   r   Zndarrayr“   r   r[   )r€   r—   Úupcasted_nar�   Z	blk_dtyper@   r,   r,   r-   Úget_reindexed_values¯  s    
zJoinUnit.get_reindexed_valuesN)
r†   Ú
__module__Ú__qualname__r�   rˆ   r‘   r
   rŒ   r–   r™   r,   r,   r,   r-   ry   ^  s    
ry   zlist[JoinUnit]r   )r^   r0   r)   c                   sª   t | ƒ\‰ }tdd„ | D ƒƒ}tˆ |ƒ‰‡ ‡fdd„| D ƒ}tdd„ |D ƒƒrvdd„ |D ƒ}t|ddd	�}t|d
ƒ}nt|dd�}ˆ |kr¦ˆ |jkr¦tjdtt	ƒ d� |S )zB
    Concatenate values from several join units along axis=1.
    c                 s  s   | ]}|j jjd kV  qdS ©r‰   Nr•   ©r4   r_   r,   r,   r-   r9   Ï  r:   z*_concatenate_join_units.<locals>.<genexpr>c                   s   g | ]}|j ˆ ˆd �‘qS )©r—   r˜   )r™   rA   rž   r,   r-   rB   Ò  s   ÿz+_concatenate_join_units.<locals>.<listcomp>c                 s  s   | ]}t |jƒV  qd S r1   ©r   rM   ©r4   Útr,   r,   r-   r9   ×  r:   c                 S  s*   g | ]"}t |jƒr|n|d dd…f ‘qS )r   NrŸ   r    r,   r,   r-   rB   Ü  s   þÿr   TrE   rF   r*   rC   a  The behavior of DataFrame concatenation with empty or all-NA entries is deprecated. In a future version, this will no longer exclude empty or all-NA columns when determining the result dtypes. To retain the old behavior, exclude the relevant entries before the concat operation.)Ú
stacklevel)
Ú_get_empty_dtypeÚanyÚ_dtype_to_na_valuer   r   rM   ÚwarningsÚwarnÚFutureWarningr   )r^   r0   Zempty_dtype_futureÚhas_none_blocksZ	to_concatZconcat_valuesr,   rž   r-   rW   É  s*    
þü
ù	rW   ©rM   r©   c                 C  sv   t | tƒr| jS | jdv r$|  d¡S | jdv r8|  d¡S | jdkrFdS | jdv r^|sXdS tjS | jdkrntjS t‚dS )	z2
    Find the NA value to go with this dtype.
    ZmMr   ÚfcÚNaNrb   NÚiuÚO)rI   r   r�   r�   r…   rN   ÚnanÚNotImplementedErrorrª   r,   r,   r-   r¥   ÷  s    







r¥   zSequence[JoinUnit]ztuple[DtypeObj, DtypeObj])r^   r)   c                 C  sÖ   t  dd„ | D ƒ¡r(| d jj}||fS tdd„ | D ƒƒ}dd„ | D ƒ}t|ƒs^dd„ | D ƒ}t|ƒ}|rrt|ƒ}|}t|ƒt| ƒkrÎdd„ | D ƒ}t|ƒsªd	d„ | D ƒ}t|ƒt|ƒkrÎt|ƒ}|rÎt|ƒ}||fS )
zÂ
    Return dtype and N/A values to use when concatenating specified units.

    Returned N/A value may be None which means there was no casting involved.

    Returns
    -------
    dtype
    c                 S  s   g | ]}|j j‘qS r,   )r?   rM   rA   r,   r,   r-   rB     r:   z$_get_empty_dtype.<locals>.<listcomp>r   c                 s  s   | ]}|j jjd kV  qdS rœ   r•   r�   r,   r,   r-   r9     r:   z#_get_empty_dtype.<locals>.<genexpr>c                 S  s   g | ]}|j s|jj‘qS r,   )rŒ   r?   rM   r�   r,   r,   r-   rB     r:   c                 S  s"   g | ]}|j jjd kr|j j‘qS ©r‰   r•   r�   r,   r,   r-   rB      s   c                 S  s   g | ]}|j s|jj‘qS r,   )r–   r?   rM   r�   r,   r,   r-   rB   *  s   þc                 S  s"   g | ]}|j jjd kr|j j‘qS r±   r•   r�   r,   r,   r-   rB   0  s   )r   Zdtypes_all_equalr?   rM   r¤   r;   r   r   )r^   r—   r©   ZdtypesrM   Zdtype_futureZdtypes_futurer,   r,   r-   r£     s4    
ÿþÿr£   c                   sX   | d j ‰ ˆ jjdkrdS t‡ fdd„| D ƒƒoVt‡ fdd„| D ƒƒoVtdd„ | D ƒƒS )zÏ
    Check if the join units consist of blocks of uniform type that can
    be concatenated using Block.concat_same_type instead of the generic
    _concatenate_join_units (which uses `concat_compat`).

    r   r‰   Fc                 3  s    | ]}t |jƒt ˆ ƒu V  qd S r1   )r…   r?   rA   ©Úfirstr,   r-   r9   H  r:   z)_is_uniform_join_units.<locals>.<genexpr>c                 3  s*   | ]"}|j jˆ jkp |j jjd v V  qdS )ZiubNr•   rA   r²   r,   r-   r9   K  s   üc                 s  s   | ]}|j  p|jjV  qd S r1   )rŒ   r?   rU   rA   r,   r,   r-   r9   U  r:   )r?   rM   r�   rP   )r^   r,   r²   r-   rT   <  s    
ûýñrT   )GÚ
__future__r   Útypingr   r   r¦   ÚnumpyrN   Zpandas._libsr   r   rq   r   rn   r   Zpandas._libs.missingr	   Zpandas.util._decoratorsr
   Zpandas.util._exceptionsr   Zpandas.core.dtypes.castr   r   Zpandas.core.dtypes.commonr   r   r   Zpandas.core.dtypes.concatr   Zpandas.core.dtypes.dtypesr   r   Zpandas.core.dtypes.missingr   r   r   Zpandas.core.constructionr   Z#pandas.core.internals.array_managerr   Zpandas.core.internals.blocksr   r   Zpandas.core.internals.managersr   r   Úcollections.abcr   Zpandas._typingr   r   r    r!   r"   Zpandasr#   r$   r%   r.   rc   rJ   r2   rR   rS   rx   ry   rW   r¥   r£   rT   r,   r,   r,   r-   Ú<module>   sB   l"/k..