a
    DBìjÛ  ã                   @   s¬   d Z ddlmZ ddlmZ ddlmZ ddlZddl	Z	ddl
Zddl
mZ ddlmZ ddlmZ e	j d¡Ze	jje	jjgZe	j d	¡d
d„ ƒZdd„ Zedd„ ƒZdS )zd
Tests multithreading behaviour for reading and
parsing files for each parser defined in parsers.py
é    )Ú	ExitStack)ÚBytesIO)Ú
ThreadPoolN)Ú	DataFrame)ÚVersionZpyarrow_xfailz0ignore:Passing a BlockManager:DeprecationWarningc                    sÒ   | }|j dkr>t d¡}t|jƒtdƒk r>| tjjdd�¡ d‰ d}‡ fdd„t|ƒD ƒ}t	ƒ �\‰‡fd	d
„|D ƒ}ˆ 
tdƒ¡}| |j|¡}|d }	|D ]}
t |	|
¡ qžW d   ƒ n1 sÄ0    Y  d S )NZpyarrowz16.0z+# ValueError: Found non-unique column index)Úreasonéd   é
   c                 3   s*   | ]"}d   dd„ tˆ ƒD ƒ¡ ¡ V  qdS )Ú
c                 S   s&   g | ]}|d ›d|d ›d|d ›�‘qS )Údú,© ©Ú.0Úir   r   úg/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/pandas/tests/io/parser/test_multi_thread.pyÚ
<listcomp>)   ó    zBtest_multi_thread_string_io_read_csv.<locals>.<genexpr>.<listcomp>N)ÚjoinÚrangeÚencode)r   Ú_)Úmax_row_ranger   r   Ú	<genexpr>(   s   ÿz7test_multi_thread_string_io_read_csv.<locals>.<genexpr>c                    s   g | ]}ˆ   t|ƒ¡‘qS r   )Úenter_contextr   )r   Úb)Ústackr   r   r   /   r   z8test_multi_thread_string_io_read_csv.<locals>.<listcomp>é   r   )ZengineÚpytestZimportorskipr   Ú__version__ZapplymarkerÚmarkZxfailr   r   r   r   ÚmapÚread_csvÚtmÚassert_frame_equal)Úall_parsersÚrequestÚparserÚpaZ	num_filesZbytes_to_dfÚfilesÚpoolÚresultsZfirst_resultÚresultr   )r   r   r   Ú$test_multi_thread_string_io_read_csv   s&    

ÿ
þr-   c                    sŒ   ‡‡fdd„}‡ ‡fdd„t ˆƒD ƒ}tˆd��}| ||¡}W d  ƒ n1 sR0    Y  |d j}|dd… D ]
}	||	_qrt |¡}
|
S )	aš  
    Generate a DataFrame via multi-thread.

    Parameters
    ----------
    parser : BaseParser
        The parser object to use for reading the data.
    path : str
        The location of the CSV file to read.
    num_rows : int
        The number of rows to read per task.
    num_tasks : int
        The number of tasks to use for reading this DataFrame.

    Returns
    -------
    df : DataFrame
    c                    sB   | \}}|s"ˆ j ˆdd|dgd�S ˆ j ˆddt|ƒd |dgd�S )aj  
        Create a reader for part of the CSV.

        Parameters
        ----------
        arg : tuple
            A tuple of the following:

            * start : int
                The starting row to start for parsing CSV
            * nrows : int
                The number of rows to read.

        Returns
        -------
        df : DataFrame
        r   Údate)Ú	index_colÚheaderÚnrowsÚparse_datesNé   é	   )r/   r0   Zskiprowsr1   r2   )r"   Úint)ÚargÚstartr1   )r'   Úpathr   r   ÚreaderN   s    ÿ
úz0_generate_multi_thread_dataframe.<locals>.readerc                    s    g | ]}ˆ | ˆ ˆ ˆ f‘qS r   r   r   )Únum_rowsÚ	num_tasksr   r   r   p   s   z4_generate_multi_thread_dataframe.<locals>.<listcomp>)Z	processesNr   r3   )r   r   r!   ÚcolumnsÚpdÚconcat)r'   r8   r:   r;   r9   Útasksr*   r+   r0   ÚrÚfinal_dataframer   )r:   r;   r'   r8   r   Ú _generate_multi_thread_dataframe:   s    "ÿ*

rB   c                 C   sî   d}d}| }d}t tj d¡ |¡tj d¡ |¡tj d¡ |¡tj d¡ |¡tj d¡ |¡dg| dg| dg| tjd|d	d
�tj|dd�dœ
ƒ}t |¡�4}| 	|¡ t
||||ƒ}t ||¡ W d   ƒ n1 sà0    Y  d S )Né   é0   z__thread_pool_reader__.csvé   ÚfooÚbarÚbazz20000101 09:00:00Ús)ZperiodsÚfreqZint64)Zdtype)
Úar   Úcr   ÚerF   rG   rH   r.   r5   )r   ÚnpÚrandomZdefault_rngr=   Z
date_rangeZaranger#   Zensure_cleanZto_csvrB   r$   )r%   r;   r:   r'   Ú	file_nameZdfr8   rA   r   r   r   Ú)test_multi_thread_path_multipart_read_csv€   s.    öÿ
ÿrQ   )Ú__doc__Ú
contextlibr   Úior   Zmultiprocessing.poolr   ÚnumpyrN   r   Zpandasr=   r   Zpandas._testingZ_testingr#   Zpandas.util.versionr   r    ZusefixturesZxfail_pyarrowZ
single_cpuZslowZ
pytestmarkÚfilterwarningsr-   rB   rQ   r   r   r   r   Ú<module>   s$   þ

F