a
    Ÿ˜ëj”Y  ã                   @   sð  d dl mZmZmZ d dlZd dlZd dlmZ d dlmZ d dl	m
Z
 d dlZddlmZmZmZmZ ddlmZ dd	lmZmZ d
dlmZmZ d
dlmZmZmZ e d¡Ze d¡Zd dlm Z m!Z!m"Z" d dl#mZ d dl$m%Z%m&Z&m'Z' ej(ddd�dd„ ƒZ)dd„ Z*didd„Z+dd„ Z,dd„ Z-d d!„ Z.ej/ 0d"dd#g¡ej/ 0d$eeg¡d%d&„ ƒƒZ1eej/ 0d"dd#g¡ej/ 0d'd(d)g¡ej/ 0d$eeg¡d*d+„ ƒƒƒƒZ2eej/ 0d$eeg¡d,d-„ ƒƒZ3d.d/„ Z4d0d1„ Z5ej/ 0d2dd3g¡d4d5„ ƒZ6d6d7„ Z7G d8d9„ d9e8ƒZ9djd:d;„Z:d<d=„ Z;d>d?„ Z<ej/ 0d@e=e>dƒƒ¡dAdB„ ƒZ?dCdD„ Z@dEdF„ ZAdGdH„ ZBdIdJ„ ZCdKdL„ ZDdMdN„ ZEdOdP„ ZFdQdR„ ZGdSdT„ ZHdUdV„ ZIdWdX„ ZJej/ 0dYdZd[g¡ej/jKejLd\k�o¸ejLd]kd^d_�d`da„ ƒƒZMdbdc„ ZNej/ 0dddedfg¡dgdh„ ƒZOdS )ké    )Úabsolute_importÚdivisionÚprint_functionN©Úrandom)Úsleep)Úuuid4é   )ÚParallelÚdelayedÚparallel_backendÚparallel_config)ÚDaskDistributedBackend)ÚAutoBatchingMixinÚThreadingBackendé   )ÚnpÚ
with_numpy)Ú_recursive_backend_infoÚ_test_deadlock_with_generatorÚ8_test_parallel_unordered_generator_returns_fastest_firstÚdistributedÚdask)ÚClientÚLocalClusterÚ
get_client)Útime)ÚcleanupÚclusterÚincÚfunctionT)ZscopeZautousec                 c   sZ   ddl m} dd„ |jD ƒ}d V  | ¡ D ]*\}}|d u rJtj |d ¡ q*|tj|< q*d S )Nr   )ÚParallelBackendBasec                 S   s   i | ]}|t j |¡“qS © )ÚosÚenvironÚget)Ú.0Úkr"   r"   úT/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/joblib/test/test_dask.pyÚ
<dictcomp>(   ó    z(avoid_dask_env_leaks.<locals>.<dictcomp>)Zjoblib._parallel_backendsr!   ZMAX_NUM_THREADS_VARSÚitemsr#   r$   Úpop)Ztmp_pathr!   Ú	old_valuer'   Úvr"   r"   r(   Úavoid_dask_env_leaks!   s    r/   c                  O   s   d S ©Nr"   )ÚargsÚkwargsr"   r"   r(   Únoop3   s    r3   çš™™™™™©?c                 C   s   t |ƒ | rtdƒ‚d S )Nzcondition evaluated to True)r   Ú
ValueError)Ú	conditionÚdurationr"   r"   r(   Úslow_raise_value_error7   s    r8   c                    sF   |  dd„ ¡}i }| ¡ D ]&\}}t‡ fdd„t|ƒD ƒƒ||< q|S )Nc                 S   s   | j S r0   )Úlog)Zdask_workerr"   r"   r(   Ú<lambda>>   r*   zcount_events.<locals>.<lambda>c                    s   g | ]}|d  ˆ kr|‘qS ©r   r"   )r&   Úevent©Ú
event_namer"   r(   Ú
<listcomp>B   r*   z count_events.<locals>.<listcomp>)Úrunr+   ÚlenÚlist)r>   ÚclientZworker_eventsZevent_countsÚwÚeventsr"   r=   r(   Úcount_events=   s    ÿ
rF   c                 C   s8  t ƒ ��\}\}}t|d | d��à}tdd��¶ tƒ dd„ tdƒD ƒƒ}|dd	„ tdƒD ƒksbJ ‚t t¡�( tƒ d
d„ tdƒD ƒƒ W d   ƒ n1 sš0    Y  tƒ dd„ tdƒD ƒƒ}|dd	„ tdƒD ƒksÖJ ‚W d   ƒ n1 sê0    Y  W d   ƒ n1 �s
0    Y  W d   ƒ n1 �s*0    Y  d S )NÚaddress©Úloopr   ©Úbackendc                 s   s   | ]}t tƒ|ƒV  qd S r0   ©r   r   ©r&   Úir"   r"   r(   Ú	<genexpr>K   r*   ztest_simple.<locals>.<genexpr>é
   c                 S   s   g | ]}t |ƒ‘qS r"   ©r   rM   r"   r"   r(   r?   L   r*   ztest_simple.<locals>.<listcomp>c                 s   s   | ]}t tƒ|d kƒV  qdS )é   N)r   r8   rM   r"   r"   r(   rO   O   s   c                 s   s   | ]}t tƒ|ƒV  qd S r0   rL   rM   r"   r"   r(   rO   S   r*   c                 S   s   g | ]}t |ƒ‘qS r"   rQ   rM   r"   r"   r(   r?   T   r*   )r   r   r   r
   ÚrangeÚpytestÚraisesr5   ©rI   ÚsÚaÚbrC   Úseqr"   r"   r(   Útest_simpleG   s    
ÿ&r[   c                 C   s  t jtju sJ ‚tƒ �ð\}\}}t|d | d��¸}tdd��Ž tƒ �h}|j}t|t ƒsZJ ‚|j	|u shJ ‚|j
dksvJ ‚|dd„ ttdƒƒD ƒƒ |j
d	ksžJ ‚W d   ƒ n1 s²0    Y  W d   ƒ n1 sÐ0    Y  W d   ƒ n1 sî0    Y  W d   ƒ n1 �s0    Y  d S )
NrG   rH   r   rJ   r   c                 s   s   | ]}t d d„ ƒƒ V  qdS )c                   S   s   d S r0   r"   r"   r"   r"   r(   r:   j   r*   z?test_dask_backend_uses_autobatching.<locals>.<genexpr>.<lambda>N©r   ©r&   Ú_r"   r"   r(   rO   j   r*   z6test_dask_backend_uses_autobatching.<locals>.<genexpr>ç     ˆÃ@rP   )r   Zcompute_batch_sizer   r   r   r   r
   Ú_backendÚ
isinstanceÚparallelZ_effective_batch_sizerS   Úint)rI   rW   rX   rY   rC   rb   rK   r"   r"   r(   Ú#test_dask_backend_uses_autobatchingW   s    ÿÿrd   Ún_jobséÿÿÿÿÚcontextc              	   C   sd   t jddd��B |dƒ� td | ƒ W d   ƒ n1 s80    Y  W d   ƒ n1 sV0    Y  d S ©Nr	   ©Ú	n_workersÚthreads_per_workerr   )r   r   r   )re   rg   r"   r"   r(   ÚAtest_parallel_unordered_generator_returns_fastest_first_with_daskn   s    rl   Ú	return_asÚ	generatorZgenerator_unorderedc              	   C   sf   t jddd��D | dƒ� td ||ƒ W d   ƒ n1 s:0    Y  W d   ƒ n1 sX0    Y  d S rh   )r   r   r   )rg   rm   re   r"   r"   r(   Ú%test_deadlock_with_generator_and_dasku   s    ro   c              
   C   s   t jddd��Þ tjtdƒtjd�}tdƒD ]\}| dƒ� t|d�}W d   ƒ n1 sX0    Y  t|ƒdksrJ ‚t	dd	„ |D ƒƒs,J ‚q,| dƒ� tƒ }W d   ƒ n1 s®0    Y  t|ƒdksÈJ ‚t	d
d	„ |D ƒƒsÞJ ‚W d   ƒ n1 sò0    Y  d S )Nr	   ri   g    ÐcA©Zdtyper   ©Údataé   c                 s   s   | ]\}}|d kV  qdS ©r   Nr"   ©r&   Únamer^   r"   r"   r(   rO   ˆ   s   z4test_nested_parallelism_with_dask.<locals>.<genexpr>c                 s   s   | ]\}}|d kV  qdS rt   r"   ru   r"   r"   r(   rO   �   s   )
r   r   r   Úonesrc   Úuint8rS   r   rA   Úall)rg   rr   rN   Zbackend_types_and_levelsr"   r"   r(   Ú!test_nested_parallelism_with_dask~   s    
(ÿ
$ÿrz   c                   C   s   t ƒ S r0   r   r"   r"   r"   r(   Úrandom2•   s    r{   c              
   C   s´   t ƒ �š\}\}}t|d | d��b}tdd��8 tƒ dd„ tdƒD ƒƒ\}}||ksVJ ‚W d   ƒ n1 sj0    Y  W d   ƒ n1 sˆ0    Y  W d   ƒ n1 s¦0    Y  d S )NrG   rH   r   rJ   c                 s   s   | ]}t tƒƒ V  qd S r0   )r   r{   rM   r"   r"   r(   rO   �   r*   z3test_dont_assume_function_purity.<locals>.<genexpr>r	   ©r   r   r   r
   rS   )rI   rW   rX   rY   rC   ÚxÚyr"   r"   r(   Ú test_dont_assume_function_purity™   s
    r   ÚmixedFc              
   C   s$  ddl m} |s(dd„ tdƒD ƒ}d}ndd„ tdƒD ƒ}d}t||ƒƒ|ksRJ ‚tƒ �¶\}\}}t|d	 | d
��~}tdd��  tddd�|ƒ}	W d   ƒ n1 s¤0    Y  dd„ }
| dd¡}| 	|
¡}t
dd„ |D ƒƒsâJ ‚W d   ƒ n1 sö0    Y  W d   ƒ n1 �s0    Y  d S )Nr   )ÚBatchc                 S   s   g | ]}t tƒ|ƒ‘qS r"   rL   rM   r"   r"   r(   r?   ¦   r*   z&test_dask_funcname.<locals>.<listcomp>rs   Zbatch_of_inc_4_callsc                 S   s,   g | ]$}|d  rt tƒ|ƒn
t tƒ|ƒ‘qS )r	   )r   Úabsr   rM   r"   r"   r(   r?   ©   r*   Zmixed_batch_of_inc_4_callsrG   rH   r   rJ   r	   ry   )Ú
batch_sizeZpre_dispatchc                 S   s
   t | jƒS r0   )rB   Ztransition_log)Zdask_schedulerr"   r"   r(   Úf³   s    ztest_dask_funcname.<locals>.fÚ4Ú2c                 s   s   | ]}d |d v V  qdS )Zbatch_of_incr   Nr"   )r&   Útupr"   r"   r(   rO   ¸   r*   z%test_dask_funcname.<locals>.<genexpr>)Zjoblib._daskr�   rS   Úreprr   r   r   r
   ÚreplaceZrun_on_schedulerry   )rI   r€   r�   ÚtasksZ
batch_reprrW   rX   rY   rC   r^   r„   r9   r"   r"   r(   Útest_dask_funcname¡   s    .
r‹   c               	      sš  dd„ t dƒD ƒ} t d¡‰ˆ tdƒ¡‰ d‡ ‡fdd„	‰tdd	d
�}t|ƒ}�z*tdd��( tƒ ‡fdd„| D ƒƒ}W d   ƒ n1 sŠ0    Y  | dd„ t dƒD ƒks®J ‚t	d|ƒ}t
| ¡ ƒdksÌJ ‚tdd„ |D ƒƒsâJ ‚tdd��* tƒ ‡ ‡fdd„| D ƒƒ}W d   ƒ n1 �s0    Y  t	d|ƒ}t
| ¡ ƒdk�sHJ ‚tdd„ |D ƒƒ�s`J ‚W |jdd� |jdd� n|jdd� |jdd� 0 d S )Nc                 S   s   g | ]}g ‘qS r"   r"   r]   r"   r"   r(   r?   Æ   r*   z;test_no_undesired_distributed_cache_hit.<locals>.<listcomp>éd   Únumpyg    €„.Ac                    s(   |d urˆj  |ˆ ¡ |  tƒ j¡ | S r0   )ZtestingZassert_array_equalÚappendr   Úhex)Úlist_rr   )ÚXr   r"   r(   Úisolated_operationÊ   s    zCtest_no_undesired_distributed_cache_hit.<locals>.isolated_operationr   r	   ri   r   rJ   c                 3   s   | ]}t ˆ ƒ|ƒV  qd S r0   r\   ©r&   r�   )r’   r"   r(   rO   Õ   r*   z:test_no_undesired_distributed_cache_hit.<locals>.<genexpr>c                 S   s   g | ]}g ‘qS r"   r"   r]   r"   r"   r(   r?   Ù   r*   úreceive-from-scatterr   c                 S   s   g | ]}t |ƒd k‘qS r;   ©rA   ©r&   Úrr"   r"   r(   r?   à   r*   c                 3   s   | ]}t ˆƒ|ˆ d �V  qdS )rq   Nr\   r“   )r‘   r’   r"   r(   rO   å   s   c                 S   s   g | ]}t |ƒd k‘qS r;   r•   r–   r"   r"   r(   r?   ì   r*   é   ©Útimeout)N)rS   rT   ÚimportorskipZarangerc   r   r   r   r
   rF   ÚsumÚvaluesry   Úclose)Úlistsr   rC   ÚresÚcountsr"   )r‘   r’   r   r(   Ú'test_no_undesired_distributed_cache_hit»   s0    
6
ÿ(
ÿr¢   c                   @   s(   e Zd Zdd„ Zdd„ ZeZdd„ ZdS )ÚCountSerializedc                 C   s   || _ d| _d S )Nr   )r}   Úcount)Úselfr}   r"   r"   r(   Ú__init__ó   s    zCountSerialized.__init__c                 C   s   | j t|d|ƒ S )Nr}   )r}   Úgetattr)r¥   Úotherr"   r"   r(   Ú__add__÷   s    zCountSerialized.__add__c                 C   s   |  j d7  _ t| jffS )Nr   )r¤   r£   r}   )r¥   r"   r"   r(   Ú
__reduce__ü   s    zCountSerialized.__reduce__N)Ú__name__Ú
__module__Ú__qualname__r¦   r©   Ú__radd__rª   r"   r"   r"   r(   r£   ò   s   r£   c                 C   s   | | | | | S r0   r"   )rX   rY   ÚcÚdÚer"   r"   r(   Úadd5  s    r²   c                    s   dd„ t dƒD ƒ\}‰‰‰ttƒ‰‡‡‡‡fdd„t dƒD ƒ}|ˆˆˆˆddd�ˆˆˆˆˆdd�ˆˆˆˆˆˆd�g7 }d	d„ |D ƒ}tƒ �î\}}t|d
 | d��¸‰ td|ˆˆgd��* tdd�|ƒ}||ksÐJ ‚W d   ƒ n1 sä0    Y  t t	¡�@ td| dd�� W d   ƒ n1 �s 0    Y  W d   ƒ n1 �s@0    Y  W d   ƒ n1 �s`0    Y  W d   ƒ n1 �s€0    Y  |j
}ˆj
|k�s J ‚ˆj
|k�s°J ‚ˆj
}|ˆˆˆfD ]}	d|	_
�qÂtƒ �–\}}t|d
 | d��`‰ tƒ ‰|ˆˆfD ]}
ˆ j|
dd�ˆt|
ƒ< �qþ‡ ‡fdd„|D ƒ}||k�s>J ‚W d   ƒ n1 �sT0    Y  W d   ƒ n1 �st0    Y  |j
}ˆj
|k�s”J ‚ˆj
|k�s¤J ‚||k�s²J ‚tdd„ tj d¡D ƒƒ}|dk �rìˆj
|d k�süJ ‚nˆj
|k�süJ ‚d S )Nc                 s   s   | ]}t |ƒV  qd S r0   )r£   rM   r"   r"   r(   rO     r*   z&test_manual_scatter.<locals>.<genexpr>rs   c              	      s   g | ]}ˆ ˆˆˆd dd�‘qS )rs   é   ©r°   r±   r"   r]   )r„   r}   r~   Úzr"   r(   r?     r*   z'test_manual_scatter.<locals>.<listcomp>rP   r³   r´   c                 S   s    g | ]\}}}||i |¤Ž‘qS r"   r"   ©r&   Úfuncr1   r2   r"   r"   r(   r?     r*   rG   rH   r   )rK   Úscatterr   )rƒ   )rK   rI   r¸   r   T)Ú	broadcastc              	      s`   g | ]X\}}}ˆ j |g‡fd d„|D ƒ¢R i t‡fdd„| ¡ D ƒƒ¤dttƒ ƒi¤Ž ¡ ‘qS )c                 3   s   | ]}ˆ   t|ƒ|¡V  qd S r0   ©r%   Úid)r&   Úarg©Ú	scatteredr"   r(   rO   :  r*   z1test_manual_scatter.<locals>.<listcomp>.<genexpr>c                 3   s&   | ]\}}|ˆ   t|ƒ|¡fV  qd S r0   rº   )r&   ÚkeyÚvaluer½   r"   r(   rO   ;  s   ÿr¿   )ÚsubmitÚdictr+   Ústrr   Úresultr¶   )rC   r¾   r"   r(   r?   7  s   
÷ÿþþýùc                 s   s   | ]}t |ƒV  qd S r0   )rc   )r&   r.   r"   r"   r(   rO   M  r*   Ú.)iç  rs   )rS   r   r²   r   r   r   r
   rT   rU   Ú	TypeErrorr¤   rÂ   r¸   r»   Útupler   Ú__version__Úsplit)rI   rD   rŠ   ÚexpectedrW   r^   Zresults_parallelZ%n_serialization_scatter_with_parallelZn_serialization_with_parallelÚvarÚobjZresults_nativeZn_serialization_scatter_nativeZdistributed_versionr"   )rC   r„   r¾   r}   r~   rµ   r(   Útest_manual_scatter  sP    
ý*€

öN
rÍ   c           	   
      sö  t  d¡}|jtdƒ|jd�‰ |jtdƒ|jd�}ˆ gd |gd  }tƒ �´\}\}}t|d | d��|}tdd��( tƒ d	d
„ t	|ƒD ƒƒ W d   ƒ n1 s 0    Y  t
d|ƒ}||d  ||d   dksÔJ ‚W d   ƒ n1 sè0    Y  W d   ƒ n1 �s0    Y  tƒ �È\}\}}t|d | d��Ž}tdd��, tƒ ‡ fdd
„tdƒD ƒƒ W d   ƒ n1 �sr0    Y  t
d|ƒ}||d  dk�sœJ ‚||d  dk�s²J ‚W d   ƒ n1 �sÈ0    Y  W d   ƒ n1 �sè0    Y  d S )Nr�   r_   rp   rR   rG   rH   r   rJ   c                 s   s&   | ]\}}t tƒ||||d �V  qdS ))ÚoptN©r   r3   )r&   rN   rr   r"   r"   r(   rO   f  s   ÿz$test_auto_scatter.<locals>.<genexpr>r”   r	   c                 3   s$   | ]}t tƒˆ d d… |ƒV  qd S )NrR   rÏ   rM   ©Zdata1r"   r(   rO   s  r*   r³   r   )rT   r›   rw   rc   rx   r   r   r   r
   Ú	enumeraterF   rS   )	Úloop_in_threadr   Zdata2Zdata_to_processrW   rX   rY   rC   r¡   r"   rÐ   r(   Útest_auto_scatter[  s&    

þ&
^<
rÓ   Úretry_noc              
      sà   t  d¡‰d‰ d}‡fdd„‰‡ ‡fdd„‰tƒ �š\}\}}t|d | d��b}td	d
��8 ˆ d¡‰tƒ ‡‡fdd„t|ƒD ƒƒ}W d   ƒ n1 s–0    Y  W d   ƒ n1 s´0    Y  W d   ƒ n1 sÒ0    Y  d S )Nr�   rP   c                    s
   ˆ   | ¡S r0   )rœ   )r}   rN   Új)r   r"   r(   Úmy_sum‚  s    z#test_nested_scatter.<locals>.my_sumc                    sX   t ƒ }tdd��0 tƒ ‡ ‡‡fdd„tˆƒD ƒƒ}W d   ƒ n1 sF0    Y  t|ƒS )Nr   rJ   c                 3   s&   | ]}t ˆƒˆ |d … ˆ|ƒV  qd S r0   r\   )r&   rÕ   )ÚarrayrN   rÖ   r"   r(   rO   ˆ  s   zEtest_nested_scatter.<locals>.outer_function_joblib.<locals>.<genexpr>)r   r   r
   rS   rœ   )r×   rN   rC   Úresults)ÚNUM_INNER_TASKSrÖ   )r×   rN   r(   Úouter_function_joblib…  s    ÿ&z2test_nested_scatter.<locals>.outer_function_joblibrG   rH   r   rJ   i'  c                 3   s$   | ]}t ˆƒˆ |d … |ƒV  qd S r0   r\   rM   )Úmy_arrayrÚ   r"   r(   rO   ‘  s   ÿz&test_nested_scatter.<locals>.<genexpr>)rT   r›   r   r   r   rw   r
   rS   )rI   rÔ   ZNUM_OUTER_TASKSrW   rX   rY   r^   r"   )rÙ   rÛ   rÖ   r   rÚ   r(   Útest_nested_scatter{  s    

þrÜ   c              
      sr  dd„ ‰ t ƒ ��L\}\}}t|d | d��x}tdd��N tdd�‡ fd	d
„tdƒD ƒƒ}|D ]}tt|ƒƒdks\J ‚q\W d   ƒ n1 sŠ0    Y  W d   ƒ n1 s¨0    Y  t|d | d��z}tdd��N tdd�‡ fdd
„tdƒD ƒƒ}|D ]}tt|ƒƒdksôJ ‚qôW d   ƒ n1 �s$0    Y  W d   ƒ n1 �sD0    Y  W d   ƒ n1 �sd0    Y  d S )Nc                  S   sH   t tdd�dd„ tdƒD ƒƒƒ} | t tdd�dd„ tdƒD ƒƒƒO } | S )Nr	   ©re   c                 s   s   | ]}t tjƒƒ V  qd S r0   ©r   r#   Úgetpidr]   r"   r"   r(   rO   ™  r*   zOtest_nested_backend_context_manager.<locals>.get_nested_pids.<locals>.<genexpr>c                 s   s   | ]}t tjƒƒ V  qd S r0   rÞ   r]   r"   r"   r(   rO   š  r*   )Úsetr
   rS   )Zpidsr"   r"   r(   Úget_nested_pids˜  s     $z<test_nested_backend_context_manager.<locals>.get_nested_pidsrG   rH   r   rJ   r	   rÝ   c                 3   s   | ]}t ˆ ƒƒ V  qd S r0   r\   r]   ©rá   r"   r(   rO      s   z6test_nested_backend_context_manager.<locals>.<genexpr>rP   c                 3   s   | ]}t ˆ ƒƒ V  qd S r0   r\   r]   râ   r"   r(   rO   ©  s   )r   r   r   r
   rS   rA   rà   )rÒ   rW   rX   rY   rC   Z
pid_groupsZ	pid_groupr"   râ   r(   Ú#test_nested_backend_context_manager—  s     ÿRÿrã   c           	         s&  dd„ ‰ ‡ fdd„‰t ƒ �ö\}\}}t|d | d��¾}tdd��” tƒ �H}ˆ |ƒd	ksZJ ‚|jd
kshJ ‚|‡fdd„tdƒD ƒƒ}W d   ƒ n1 s–0    Y  |D ] \}}|d	ks¸J ‚|d
ks¤J ‚q¤W d   ƒ n1 sÚ0    Y  W d   ƒ n1 sø0    Y  W d   ƒ n1 �s0    Y  d S )Nc                 S   s
   | j jjS r0   )r`   Ú	__class__r«   ©Úpr"   r"   r(   Ú_backend_type´  s    zJtest_nested_backend_context_manager_implicit_n_jobs.<locals>._backend_typec                     s8   t ƒ �} ˆ | ƒ| jfW  d   ƒ S 1 s*0    Y  d S r0   )r
   re   rå   )rç   r"   r(   Úget_nested_implicit_n_jobs·  s    zWtest_nested_backend_context_manager_implicit_n_jobs.<locals>.get_nested_implicit_n_jobsrG   rH   r   rJ   r   rf   c                 3   s   | ]}t ˆ ƒƒ V  qd S r0   r\   r]   )rè   r"   r(   rO   Á  s   zFtest_nested_backend_context_manager_implicit_n_jobs.<locals>.<genexpr>r	   )r   r   r   r
   re   rS   )	rI   rW   rX   rY   rC   ræ   Zall_nested_n_jobsZbackend_typeZnested_n_jobsr"   )rç   rè   r(   Ú3test_nested_backend_context_manager_implicit_n_jobs°  s    ÿ&ré   c              	   C   sn   t  t¡�:}tdd�� W d   ƒ n1 s,0    Y  W d   ƒ n1 sJ0    Y  dt|jƒ ¡ v sjJ ‚d S )Nr   rJ   zcreate a dask client)rT   rU   r5   r   rÃ   rÀ   Úlower)rI   Úinfor"   r"   r(   Útest_errorsÉ  s    <rì   c              
   C   s&  t ƒ ��\}\}}t|d | d��Ð}tdd��F tdd�dd„ td	ƒD ƒƒ}t|d
 d
 d
 tƒsfJ ‚W d   ƒ n1 sz0    Y  tdd��F tdd�dd„ td	ƒD ƒƒ}t|d
 d
 d
 tƒsÆJ ‚W d   ƒ n1 sÚ0    Y  W d   ƒ n1 sø0    Y  W d   ƒ n1 �s0    Y  d S )NrG   rH   r   rJ   r	   rÝ   c                 s   s   | ]}t tƒd d�V  qd S )N©Znested_require©r   Úouterr]   r"   r"   r(   rO   Ö  s   z.test_correct_nested_backend.<locals>.<genexpr>r   r   c                 s   s   | ]}t tƒd d�V  qdS )Z	sharedmemrí   Nrî   r]   r"   r"   r(   rO   Ý  s   )r   r   r   r
   rS   ra   r   r   )rI   rW   rX   rY   rC   rÄ   r"   r"   r(   Útest_correct_nested_backendÑ  s    ÿ8ÿrð   c                    s"   t ddd�‡ fdd„tdƒD ƒƒS )Nr	   Úthreads)re   Zpreferc                 3   s   | ]}t tƒˆ ƒV  qd S r0   )r   Úmiddler]   rí   r"   r(   rO   ä  s   zouter.<locals>.<genexpr>r   ©r
   rS   rí   r"   rí   r(   rï   ã  s    ÿrï   c                 C   s   t d| d�dd„ tdƒD ƒƒS )Nr	   )re   Úrequirec                 s   s   | ]}t tƒƒ V  qd S r0   )r   Úinnerr]   r"   r"   r(   rO   ê  r*   zmiddle.<locals>.<genexpr>r   ró   )rô   r"   r"   r(   rò   é  s    rò   c                   C   s   t ƒ jS r0   )r
   r`   r"   r"   r"   r(   rõ   í  s    rõ   c              	   C   sx   t | ddd��V tdd��, tdd�dd	„ td
ƒD ƒƒ W d   ƒ n1 sL0    Y  W d   ƒ n1 sj0    Y  d S )NFT)rI   Ú	processesZset_as_defaultr   rJ   rs   rÝ   c                 s   s   | ]}t tƒ|ƒV  qd S r0   )r   r»   rM   r"   r"   r(   rO   õ  r*   z0test_secede_with_no_processes.<locals>.<genexpr>r	   )r   r   r
   rS   rH   r"   r"   r(   Útest_secede_with_no_processesñ  s    r÷   c                 C   s   ddl m} |ƒ jS )Nr   )Ú
get_worker)r   rø   rG   )r^   rø   r"   r"   r(   Ú_worker_addressø  s    rù   c              
   C   s"  t ƒ ��\}\}}t|d | d��Ì}td|d d��> tƒ dd„ tdƒD ƒƒ}||d gd ksdJ ‚W d   ƒ n1 sx0    Y  td|d d��> tƒ dd„ tdƒD ƒƒ}||d gd ksÂJ ‚W d   ƒ n1 sÖ0    Y  W d   ƒ n1 sô0    Y  W d   ƒ n1 �s0    Y  d S )	NrG   rH   r   )rK   Úworkersc                 s   s   | ]}t tƒ|ƒV  qd S r0   ©r   rù   rM   r"   r"   r(   rO     r*   z-test_dask_backend_keywords.<locals>.<genexpr>rP   c                 s   s   | ]}t tƒ|ƒV  qd S r0   rû   rM   r"   r"   r(   rO     r*   r|   rV   r"   r"   r(   Útest_dask_backend_keywordsþ  s    4rü   c              	   C   s¨   t d| d��ˆ}tdd��( tƒ dd„ tdƒD ƒƒ W d   ƒ n1 sF0    Y  tƒ }|jjjr|tdƒ tƒ |d	 k sVJ ‚qV|j	r†J ‚W d   ƒ n1 sš0    Y  d S )
NF)rö   rI   r   rJ   c                 s   s   | ]}t tƒ|ƒV  qd S r0   rL   rM   r"   r"   r(   rO     r*   z/test_scheduler_tasks_cleanup.<locals>.<genexpr>rP   g{®Gáz„?r³   )
r   r   r
   rS   r   r   Z	schedulerrŠ   r   Zfutures)rI   rC   Ústartr"   r"   r(   Útest_scheduler_tasks_cleanup
  s    6
rþ   Úcluster_strategyÚadaptiveÚlate_scalingz2.1.1z1.28.0z?distributed bug - https://github.com/dask/distributed/pull/2841)Úreasonc                 C   s®   t dddd�}t|ƒ}| dkr.|jddd� n| dkr@| d¡ zVtdd	��( tƒ d
d„ tdƒD ƒƒ W d   ƒ n1 sz0    Y  W | ¡  | ¡  n| ¡  | ¡  0 d S )Nr   Fr	   ©rj   rö   rk   r   )ZminimumÚmaximumr  r   rJ   c                 s   s   | ]}t tƒ|ƒV  qd S r0   rL   rM   r"   r"   r(   rO   +  r*   z(test_wait_for_workers.<locals>.<genexpr>rP   )r   r   ZadaptZscaler   r
   rS   rž   )rÿ   r   rC   r"   r"   r(   Útest_wait_for_workers  s    
8
ÿr  c               	   C   s0  t dddd�} t| ƒ}�z tddd��Z d}tjt|d	��( tƒ d
d„ tdƒD ƒƒ W d   ƒ n1 sh0    Y  W d   ƒ n1 s†0    Y  tddd��Z d}tjt|d	��( tƒ dd„ tdƒD ƒƒ W d   ƒ n1 sÞ0    Y  W d   ƒ n1 sü0    Y  W | 	¡  |  	¡  n| 	¡  |  	¡  0 d S )Nr   Fr	   r  r   gš™™™™™¹?)rK   Zwait_for_workers_timeoutz7DaskDistributedBackend has no worker after 0.1 seconds.)Úmatchc                 s   s   | ]}t tƒ|ƒV  qd S r0   rL   rM   r"   r"   r(   rO   :  r*   z0test_wait_for_workers_timeout.<locals>.<genexpr>rP   z+DaskDistributedBackend has no active workerc                 s   s   | ]}t tƒ|ƒV  qd S r0   rL   rM   r"   r"   r(   rO   @  r*   )
r   r   r   rT   rU   ÚTimeoutErrorr
   rS   ÚRuntimeErrorrž   )r   rC   Úmsgr"   r"   r(   Útest_wait_for_workers_timeout1  s    TV
ÿr
  rK   ZlokyÚmultiprocessingc              	      s¤   t dd�}t|ƒ}zr‡ fdd„}| |¡}| ¡ }t|ƒdksBJ ‚|d j}t|tƒsZJ ‚dt|ƒv sjJ ‚W |j	dd	� |j	dd	� n|j	dd	� |j	dd	� 0 d S )
Nr	   )rj   c                     sN   t jdd��.} tdˆ d�dd„ tdƒD ƒƒ W d   ƒ n1 s@0    Y  | S )NT©Úrecordr	   )re   rK   c                 s   s   | ]}t tƒ|ƒV  qd S r0   rL   rM   r"   r"   r(   rO   R  r*   zftest_joblib_warning_inside_dask_daemonic_worker.<locals>.func_using_joblib_parallel.<locals>.<genexpr>rP   )ÚwarningsÚcatch_warningsr
   rS   r  rJ   r"   r(   Úfunc_using_joblib_parallelL  s    <zStest_joblib_warning_inside_dask_daemonic_worker.<locals>.func_using_joblib_parallelr   r   zdistributed.worker.daemonr˜   r™   )
r   r   rÁ   rÄ   rA   Úmessagera   ÚUserWarningrÃ   rž   )rK   r   rC   r  Zfutr  Úwarningr"   rJ   r(   Ú/test_joblib_warning_inside_dask_daemonic_workerF  s    



ÿr  )r4   )r   r   )PÚ
__future__r   r   r   r#   r  r   r   r   Úuuidr   rT   Ú r
   r   r   r   Z_daskr   rb   r   r   Úcommonr   r   Ztest_parallelr   r   r   r›   r   r   r   r   r   Zdistributed.metricsZdistributed.utils_testr   r   r   Zfixturer/   r3   r8   rF   r[   rd   ÚmarkZparametrizerl   ro   rz   r{   r   r‹   r¢   Úobjectr£   r²   rÍ   rÓ   rB   rS   rÜ   rã   ré   rì   rð   rï   rò   rõ   r÷   rù   rü   rþ   ZskipifrÈ   r  r
  r  r"   r"   r"   r(   Ú<module>   s‚   





7
V 
þ