a
    Ÿ˜ëj1 ã                   @   sþ  d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl	m
Z
 ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZdd	lmZmZmZ dd
lmZ ddlmZmZmZmZ ddl m!Z!m"Z"m#Z#m$Z$m%Z% edu�rddl&m'Z' ddl(m)Z) zddl*Z*W n e+�y6   dZ*Y n0 zddl,m-Z- W n e+�y`   dZ-Y n0 zddl.Z.W n e+�y†   dZ.Y n0 ddl/m0Z0m1Z1m2Z2m3Z3m4Z4 ddl5m6Z6m7Z7m8Z8m9Z9m:Z:mZm;Z;m<Z<m=Z= e6 >¡ Z?e? @dd¡ dgeAe6 B¡ ƒ ZCeCdd„ e6D ƒ7 ZCedu �rg ZDnddgZDeDdg ZEeFedƒ�rFeC Ge Hd¡¡ dd„ ZIdd„ ZJdd„ ZKd d!„ ZLG d"d#„ d#eMƒZN�dTd%d&„ZOd'd(„ ZP�dUd)d*„ZQd+d,„ ZRd-d.„ ZSd/d0„ ZTd1d2„ ZUe"d3e<e;gƒejVj"d4d5d6e:d6d7�fd8gg d9¢d:�ed;d<„ ƒƒƒZWe"d=eCƒe"d>g d?¢ƒe"d@g dA¢ƒdBdC„ ƒƒƒZXe"d=eCƒe"d>ddDgƒdEdF„ ƒƒZYe"d=eCƒdGdH„ ƒZZdIdJ„ Z[ee"dKg dL¢ƒdMdN„ ƒƒZ\ee"d=g dO¢ƒdPdQ„ ƒƒZ]dRdS„ Z^e"dTe6ƒe"dUe6ƒdVdW„ ƒƒZ_dXdY„ Z`edZd[„ ƒZad\d]„ Zbe"d>g d^¢ƒd_d`„ ƒZce"d=eEƒdadb„ ƒZdedcdd„ ƒZeeee"deg df¢ƒe"dgddhgƒdidj„ ƒƒƒƒZfe"d=eEƒdkdl„ ƒZgee"d=eEƒdmdn„ ƒƒZhee"d=eie?ƒdoh ƒe"dpdqdrgƒdsdt„ ƒƒƒZjee"d=eDƒdudv„ ƒƒZkee"d=e6ƒdwdx„ ƒƒZldydz„ Zme"d=e6ƒe"d{dg d|¢fd}g d~¢fgƒdd€„ ƒƒZnee"d=eEƒd�d‚„ ƒƒZodƒd„„ Zpee"d=eDƒd…d†„ ƒƒZqd‡dˆ„ Zrd‰dŠ„ Zsd‹dŒ„ Ztee"d=eEƒejVjud�dŽ�d�d�„ ƒƒƒZvG d‘d’„ d’e3ƒZwd“d”„ Zxe"d=eCƒd•d–„ ƒZyee"d=eEƒe"d>g d—¢ƒd˜d™„ ƒƒƒZzdšd›„ Z{dœd�„ Z|e$edudždŽ�dŸd „ ƒZ}d¡d¢„ Z~eEdd… Ze €d£d„ e�d¤ƒD ƒ¡ ee"d=eƒe"d3e;e<gƒd¥d¦„ ƒƒƒZ‚G d§d¨„ d¨e3ƒZƒe"d3e<e;gƒd©dª„ ƒZ„e"d3e<e;gƒd«d¬„ ƒZ…d­d®„ Z†d¯d°„ Z‡G d±d²„ d²ejˆj0ƒZ‰e=d³e‰ƒ ee"d=g d´¢ƒe"d3e<e;gƒdµd¶„ ƒƒƒZŠee"d>g d·¢ƒe"d=eEƒe"d3e<e;gƒd¸d¹„ ƒƒƒƒZ‹dºd»„ ZŒee"d¼eEƒe"d½eEƒe"d3e<e;gƒd¾d¿„ ƒƒƒƒZ�ee"d3e<e;gƒe"dÀdÁd$gƒdÂdÃ„ ƒƒƒZŽe"dÄg dÅ¢ƒdÆdÇ„ ƒZ�e"dÈg dÉ¢ƒdÊdË„ ƒZ�edÌdÍ„ ƒZ‘eee"d=eDƒdÎdÏ„ ƒƒƒZ’dÐZ“ee"d=eDƒdÑdÒ„ ƒƒZ”dÓZ•dÔZ–dÕZ—dÖZ˜ee"d=eDedu �rÖg ndg ƒe"d×e–e—e˜gƒe"dØg dÙ¢ƒdÚdÛ„ ƒƒƒƒZ™dÜjšej› œej› œej�¡¡dÝ�ZžedÞdß„ ƒZŸdàjšej› œej› œej�¡¡dÝ�Z edádâ„ ƒZ¡dãdä„ Z¢dådæ„ Z£eee"d=eDƒdçdè„ ƒƒƒZ¤dédê„ Z¥eedëdì„ ƒƒZ¦dídî„ Z§dïdð„ Z¨ejV "d>g dñ¢¡dòdó„ ƒZ©dôdõ„ Zªdöd÷„ Z«ejV "d>dDd}g¡ee"d=eie?ƒdoh ƒdødù„ ƒƒƒZ¬e"d=eCƒe"d>g dú¢ƒdûdü„ ƒƒZ­dýdþ„ Z®dÿ�d „ Z¯ee"d=e?ƒe"dpdqdrgƒe"d>g dú¢ƒ�d�d„ ƒƒƒƒZ°e"d=e?ƒe"dpdqdrgƒe"d>g dú¢ƒ�d�d„ ƒƒƒZ±e"d=e?ƒe"dpdqdrgƒe"d>g dú¢ƒ�d�d„ ƒƒƒZ²e"d=e?ƒe"�ddqdrgƒe"�ddqdrgƒe"d>g dú¢ƒ�d	�d
„ ƒƒƒƒZ³e"�dg �d¢ƒe"�ddqdrgƒe"�ddqdrgƒ�d�d„ ƒƒƒZ´eee"d=eDƒ�d�d„ ƒƒƒZµe"d=edu �
rŒddgng �d¢ƒ�d�d„ ƒZ¶ee"d=eDƒ�d�d„ ƒƒZ·ee"d3e<e;gƒ�d�d„ ƒƒZ¸e"d>ddDgƒe"�dg �d¢ƒ�d�d„ ƒƒZ¹e"d3e<e;gƒ�d�d„ ƒZº�d�d„ Z»�dV�d �d!„Z¼ee"d=ddgƒe"d3e<e;gƒ�d"�d#„ ƒƒƒZ½�dW�d$�d%„Z¾ejVj¿e"d3e<e;gƒe"d=edu �r dgnddgƒ�d&�d'„ ƒƒƒZÀ�d(�d)„ ZÁe"d=edu�rÜddgndgƒe$e-du �d*dŽ��d+�d,„ ƒƒZÂe"d3e<e;gƒe$e.du�d-dŽ��d.�d/„ ƒƒZÃe"d3e<e;gƒ�d0�d1„ ƒZÄ�d2�d3„ ZÅ�d4�d5„ ZÆ�d6�d7„ ZÇ�d8�d9„ ZÈeee"d>g �d:¢ƒ�d;�d<„ ƒƒƒZÉeee"�d=g �d>¢ƒe"d>dDd6gƒe"d3e<e;gƒ�d?�d@„ ƒƒƒƒƒZÊee"d>dDd6gƒe"�dAg �dB¢ƒe"d3e<e;gƒ�dC�dD„ ƒƒƒƒZËee"d>g �dE¢ƒ�dF�dG„ ƒƒZÌ�dH�dI„ ZÍ�dX�dJ�dK„ZÎee"d>dDd}gƒe"d=eDƒe"d3e<e;gƒ�dL�dM„ ƒƒƒƒZÏee"d>dDd}gƒe"d=eDƒ�dN�dO„ ƒƒƒZÐeejV "d>dDd}g¡�dP�dQ„ ƒƒZÑeejV "d>dDd}g¡�dR�dS„ ƒƒZÒdS (Y  z
Test the parallel module.
é    N)Únullcontext©Úsqrt©ÚTimeoutError)ÚPicklingError)Úsleep)Úformat_exception)ÚdumpÚloadÚparallel)Úmp)ÚIS_GIL_DISABLEDÚnpÚwith_multiprocessingÚ
with_numpy)Úcheck_subprocess_callÚparametrizeÚraisesÚskipifÚwarns)Úget_reusable_executor)ÚQueueé   )Úparallel_sum)ÚLokyBackendÚMultiprocessingBackendÚParallelBackendBaseÚSequentialBackendÚThreadingBackend)	ÚBACKENDSÚParallelÚ	cpu_countÚdelayedÚeffective_n_jobsr   Úparallel_backendÚparallel_configÚregister_parallel_backendÚmultiprocessingc                 C   s   g | ]}t | ƒ ‘qS © )r    )Ú.0Zbackend_strr)   r)   úX/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/joblib/test/test_parallel.pyÚ
<listcomp>R   ó    r,   ÚlokyÚ	threadingÚget_contextÚspawnc                   C   s
   t tj S ©N)r    r   ÚDEFAULT_BACKENDr)   r)   r)   r+   Úget_default_backend_instance^   s    r4   c                 C   s   t | dt | dd ƒƒS )NÚ_poolZ_workers)Úgetattr©Úbackendr)   r)   r+   Úget_workersf   s    r9   c                 C   s   | | S r2   r)   )ÚxÚyr)   r)   r+   Údivisionj   s    r<   c                 C   s   | d S ©Né   r)   ©r:   r)   r)   r+   Úsquaren   s    r@   c                   @   s   e Zd ZdZdd„ ZdS )ÚMyExceptionWithFinickyInitz,An exception class with non trivial __init__c                 C   s   d S r2   r)   )ÚselfÚaÚbÚcÚdr)   r)   r+   Ú__init__u   s    z#MyExceptionWithFinickyInit.__init__N©Ú__name__Ú
__module__Ú__qualname__Ú__doc__rG   r)   r)   r)   r+   rA   r   s   rA   Fc                 C   s"   | dkr|rt ddddƒnt‚| S )Né   rC   rD   rE   rF   )rA   Ú
ValueError)r:   Úcustom_exceptionr)   r)   r+   Úexception_raisery   s    ÿýrP   c                 C   s   t  d¡ t‚d S )Ngš™™™™™©?)Útimer   ÚKeyboardInterruptr?   r)   r)   r+   Úinterrupt_raiserƒ   s    
rS   c                 C   s   | d | | S )zNA module-level function so that it can be spawn with
    multiprocessing.
    r>   r)   )r:   r;   Úzr)   r)   r+   Úfˆ   s    rU   c                   C   s   t t ¡ d ƒS ©Nr   )Útyper   Úget_active_backendr)   r)   r)   r+   Ú_active_backend_type�   s    rY   c                 C   s   t | |d�dd„ tdƒD ƒƒS )N©Ún_jobsr8   c                 s   s   | ]}t tƒ|ƒV  qd S r2   ©r#   r@   ©r*   Úir)   r)   r+   Ú	<genexpr>”   s   z parallel_func.<locals>.<genexpr>é   ©r!   Úrange)Úinner_n_jobsr8   r)   r)   r+   Úparallel_func“   s    ÿrd   c                   C   s   t ƒ dksJ ‚d S rV   )r"   r)   r)   r)   r+   Útest_cpu_countš   s    re   c                   C   s   t ƒ dksJ ‚d S rV   ©r$   r)   r)   r)   r+   Útest_effective_n_jobsž   s    rg   Úcontextzbackend_n_jobs, expected_n_jobs)r`   r`   éÿÿÿÿ©r[   ©Nr   )zpositive-intznegative-intÚNone)Úidsc                 C   sT   | d|d��" t d d�|ks J ‚W d   ƒ n1 s40    Y  t d d�dksPJ ‚d S )Nr/   rj   r   rf   )rh   Zbackend_n_jobsZexpected_n_jobsr)   r)   r+   Útest_effective_n_jobs_None¢   s    
0rn   r8   r[   )r   r>   ri   éþÿÿÿÚverbose)r>   é   éd   c                 C   s:   dd„ t dƒD ƒt|| |d�dd„ t dƒD ƒƒks6J ‚d S )Nc                 S   s   g | ]}t |ƒ‘qS r)   ©r@   ©r*   r:   r)   r)   r+   r,   ¼   r-   z(test_simple_parallel.<locals>.<listcomp>é   ©r[   r8   rp   c                 s   s   | ]}t tƒ|ƒV  qd S r2   r\   rt   r)   r)   r+   r_   ¾   r-   z'test_simple_parallel.<locals>.<genexpr>©rb   r!   )r8   r[   rp   r)   r)   r+   Útest_simple_parallel¸   s
    ÿþrx   r>   c                 C   s�   d}t  d¡}G dd„ dtƒ}||| dd�}|dd„ t|ƒD ƒƒ tƒ }|jD ],}| |¡ }rN| ¡ \}	}
| |
|	 ¡ qNt	|ƒd	ksŒJ ‚d S )
Nrr   z(Done\s+\d+ out of \d+ \|)c                   @   s   e Zd Zg Zdd„ ZdS )z/test_parallel_pretty_print.<locals>.ParallelLogc                 S   s   | j  |¡ d S r2   )ÚmessagesÚappend)rB   Úmsgr)   r)   r+   Ú_printÊ   s    z6test_parallel_pretty_print.<locals>.ParallelLog._printN)rI   rJ   rK   ry   r|   r)   r)   r)   r+   ÚParallelLogÇ   s   r}   i'  rv   c                 S   s   g | ]}t tƒ|ƒ‘qS r)   ©r#   rU   r]   r)   r)   r+   r,   Î   r-   z.test_parallel_pretty_print.<locals>.<listcomp>r   )
ÚreÚcompiler!   rb   Úsetry   ÚsearchÚspanÚaddÚlen)r8   r[   Ún_tasksÚpatternr}   ÚexecutorZlensÚmessageÚsrC   rD   r)   r)   r+   Útest_parallel_pretty_printÁ   s    

r‹   c                    s¸   |j t ¡ ddd� tjdd��>}td| d�dd	„ td
ƒD ƒƒ}|g d¢ksPJ ‚W d   ƒ n1 sd0    Y  dd„ |D ƒ}| dv sŽt| tƒr¤d‰ ‡ fdd„|D ƒ}t	|ƒdks´J ‚d S )NÚnameZ!some_new_name_for_the_main_thread)ÚtargetrŒ   ÚvalueT©Úrecordr>   rZ   c                 s   s   | ]}t tƒ|ƒV  qd S r2   r\   rt   r)   r)   r+   r_   ä   s   z6test_main_thread_renamed_no_warning.<locals>.<genexpr>r`   )r   r   é   c                 S   s*   g | ]"}d t |jƒvrt|jtƒs|‘qS )zworker timeout)Ústrr‰   Ú
isinstanceÚDeprecationWarning©r*   Úwr)   r)   r+   r,   ì   s   ýz7test_main_thread_renamed_no_warning.<locals>.<listcomp>)Nr(   z3multi-threaded, use of fork() may lead to deadlocksc                    s   g | ]}ˆ t |jƒvr|‘qS r)   )r’   r‰   r•   ©Úmessage_partr)   r+   r,   ú   r-   r   )
Úsetattrr/   Úcurrent_threadÚwarningsÚcatch_warningsr!   rb   r“   r   r…   )r8   ÚmonkeypatchÚwarninfoÚresultsr)   r—   r+   Ú#test_main_thread_renamed_no_warning×   s&    ýÿ.þ

ÿr    c                 C   s¢   t jdd��&}t  d¡ t| |d� W d   ƒ n1 s80    Y  dd„ |D ƒ}|r’|rŽtdd„ |D ƒƒ}trzt|ƒd	kn
t|ƒd	k}|oŒ|S d
S |ršJ ‚dS d S )NTr�   Úalways)r8   rc   c                 S   s   g | ]
}|j ‘qS r)   )r‰   r•   r)   r)   r+   r,     r-   z*_assert_warning_nested.<locals>.<listcomp>c                 s   s   | ]}d |j d v V  qdS )zbacked parallel loops cannotr   N)Úargs)r*   Zeachr)   r)   r+   r_   
  s   z)_assert_warning_nested.<locals>.<genexpr>r   F)r›   rœ   Úsimplefilterrd   Úallr   r…   )r8   rc   Úexpectedrž   Zwarnings_are_correctZwarnings_have_the_right_lengthr)   r)   r+   Ú_assert_warning_nested  s    
*ÿÿr¦   z%parent_backend,child_backend,expected))r.   r(   T)r.   r.   F)r(   r(   T)r(   r.   T)r/   r(   T)r/   r.   Tc                    sl   t d| d�‡ fdd„tdƒD ƒƒ t d| d�‡ ‡fdd„tdƒD ƒƒ}| dkr\t|ƒshJ ‚nt|ƒshJ ‚d S )Nr>   rZ   c                 3   s    | ]}t tƒˆ d dd�V  qdS )r   F©r8   rc   r¥   N©r#   r¦   ©r*   Ú_©Úchild_backendr)   r+   r_   (  s   ýÿz0test_nested_parallel_warnings.<locals>.<genexpr>ru   c                 3   s    | ]}t tƒˆ d ˆd�V  qdS )r>   r§   Nr¨   r©   ©r¬   r¥   r)   r+   r_   0  s   ýÿr/   )r!   rb   Úanyr¤   )Úparent_backendr¬   r¥   Úresr)   r­   r+   Útest_nested_parallel_warnings  s    üü	r±   )r.   r(   r/   c                 C   s>   dg}dd„ }t j||fd�}| ¡  | ¡  |d s:J ‚d S )NFc                 S   sh   t jdd��,}tdd�dd„ tdƒD ƒƒ W d   ƒ n1 s>0    Y  tt|ƒƒ t|ƒdk| d< d S )	NTr�   r>   rj   c                 s   s   | ]}t tƒd ƒV  qdS )çš™™™™™¹?N©r#   r   r©   r)   r)   r+   r_   F  r-   zPtest_background_thread_parallelism.<locals>.background_thread.<locals>.<genexpr>r‘   r   )r›   rœ   r!   rb   Úprintr…   )Úis_run_parallelrž   r)   r)   r+   Úbackground_threadD  s    :z=test_background_thread_parallelism.<locals>.background_thread)r�   r¢   r   )r/   ÚThreadÚstartÚjoin)r8   rµ   r¶   Útr)   r)   r+   Ú"test_background_thread_parallelism?  s    r»   c                 C   s"   t d| d�dd„ tdƒD ƒƒ d S )Nr>   rZ   c                 s   s   | ]}t tƒd ƒV  qdS ©ç{®Gáz„?Nr\   r©   r)   r)   r+   r_   Q  r-   znested_loop.<locals>.<genexpr>ra   r7   r)   r)   r+   Únested_loopP  s    r¾   r¬   r¯   c                    s&   t d| d�‡ fdd„tdƒD ƒƒ d S )Nr>   rZ   c                 3   s   | ]}t tƒˆ ƒV  qd S r2   )r#   r¾   r©   r«   r)   r+   r_   W  s   z#test_nested_loop.<locals>.<genexpr>ra   )r¯   r¬   r)   r«   r+   Útest_nested_loopT  s    ÿr¿   c                 C   s   t ‚d S r2   )rN   r7   r)   r)   r+   Úraise_exception\  s    rÀ   c               	   C   st   t tƒ�X tddd��,} | ttƒdƒttƒdƒgƒ W d   ƒ n1 sH0    Y  W d   ƒ n1 sf0    Y  d S )Nr>   r.   rZ   )r   rN   r!   r#   r¾   rÀ   )r   r)   r)   r+   Ú)test_nested_loop_with_exception_with_loky`  s    
rÁ   c                      s<   t dd�‰ tddd�‡ fdd„tdƒD ƒƒ ˆ  ¡ s8J ‚dS )	z1Input is mutable when using the threading backendru   )Úmaxsizer>   r/   rZ   c                 3   s   | ]}t ˆ jƒd ƒV  qdS ©r   N)r#   Úputr©   ©Úqr)   r+   r_   j  r-   z1test_mutate_input_with_threads.<locals>.<genexpr>N)r   r!   rb   Úfullr)   r)   rÅ   r+   Útest_mutate_input_with_threadsg  s    
"rÈ   )r   r>   r`   c                 C   s6   t dƒ}dd„ |D ƒt| d�dd„ |D ƒƒks2J ‚dS )z.Check the keyword argument processing of pmap.é
   c                 S   s   g | ]}t |d d�‘qS ©r   ©r;   ©rU   rt   r)   r)   r+   r,   r  r-   z(test_parallel_kwargs.<locals>.<listcomp>rj   c                 s   s   | ]}t tƒ|d d�V  qdS ©r   rË   Nr~   rt   r)   r)   r+   r_   r  s   z'test_parallel_kwargs.<locals>.<genexpr>Nrw   )r[   Úlstr)   r)   r+   Útest_parallel_kwargsn  s    ÿrÏ   c                 C   sì   t dƒ}dd„ |D ƒ}td| d��h}|j}||dd„ |D ƒƒksDJ ‚||dd„ |D ƒƒks^J ‚td ur|t|ƒt|jƒu s|J ‚W d   ƒ n1 s�0    Y  td ur´t|jƒd u s´J ‚||d	d„ |D ƒƒksÎJ ‚td urèt|jƒd u sèJ ‚d S )
NrÉ   c                 S   s   g | ]}t |d d�‘qS rÊ   rÌ   rt   r)   r)   r+   r,   z  r-   z4test_parallel_as_context_manager.<locals>.<listcomp>r‘   rZ   c                 s   s   | ]}t tƒ|d d�V  qdS rÍ   r~   rt   r)   r)   r+   r_   ƒ  r-   z3test_parallel_as_context_manager.<locals>.<genexpr>c                 s   s   | ]}t tƒ|d d�V  qdS rÍ   r~   rt   r)   r)   r+   r_   „  r-   c                 s   s   | ]}t tƒ|d d�V  qdS rÍ   r~   rt   r)   r)   r+   r_   �  r-   )rb   r!   Ú_backendr   r9   )r8   rÎ   r¥   ÚpZmanaged_backendr)   r)   r+   Ú test_parallel_as_context_managerw  s    4rÒ   c                      sb   G dd„ dt ƒ‰ ttdd��2 tddd�‡ fdd	„td
ƒD ƒƒ W d  ƒ n1 sT0    Y  dS )z`Check that pmap captures the errors when it is passed an object
    that cannot be pickled.
    c                   @   s   e Zd Zdd„ ZdS )z1test_parallel_pickling.<locals>.UnpicklableObjectc                 S   s   t dƒ‚d S )NZ123)ÚRuntimeError©rB   r)   r)   r+   Ú
__reduce__œ  s    z<test_parallel_pickling.<locals>.UnpicklableObject.__reduce__N)rI   rJ   rK   rÕ   r)   r)   r)   r+   ÚUnpicklableObject›  s   rÖ   zthe task to send©Úmatchr>   r.   rZ   c                 3   s   | ]}t tƒˆ ƒ ƒV  qd S r2   ©r#   Úidr©   ©rÖ   r)   r+   r_      s   z)test_parallel_pickling.<locals>.<genexpr>rÉ   N)Úobjectr   r   r!   rb   r)   r)   rÛ   r+   Útest_parallel_pickling•  s
    ÿrÝ   Ú	byteorder)ú<ú>ú=Ú
max_nbytesZ1Mc                    sŽ   dd„ ‰ t  d¡ d¡ | › d�¡‰ˆjj}tdd|d�‡ ‡fd	d
„tdƒD ƒƒ}|D ]2\}}||ksjJ ‚||jjkszJ ‚t j 	ˆ|¡ qVd S )Nc                 S   s   | | j jfS r2   )ÚdtyperÞ   r?   r)   r)   r+   Úinspect_byteorderª  s    z=test_parallel_byteorder_corruption.<locals>.inspect_byteorderé   )r>   r`   Úi4r>   r.   )r[   r8   râ   c                 3   s   | ]}t ˆ ƒˆƒV  qd S r2   ©r#   r©   ©rä   r:   r)   r+   r_   ±  s   z5test_parallel_byteorder_corruption.<locals>.<genexpr>r`   )
r   ZarangeZreshapeÚviewrã   rÞ   r!   rb   ÚtestingÚassert_array_equal)rÞ   râ   Zinitial_np_byteorderÚresultZ
x_returnedZbyteorder_in_workerr)   rè   r+   Ú"test_parallel_byteorder_corruption¥  s    ÿrí   c                 C   s0   t td| dd�dd„ tdƒD ƒƒƒdks,J ‚d S )Nr>   é   ©r[   r8   Útimeoutc                 s   s   | ]}t tƒd ƒV  qdS )gü©ñÒMbP?Nr³   rt   r)   r)   r+   r_   À  s   z0test_parallel_timeout_success.<locals>.<genexpr>rÉ   )r…   r!   rb   r7   r)   r)   r+   Útest_parallel_timeout_success»  s    ÿÿûÿrñ   c                 C   sL   t tƒ�0 td| dd�dd„ tdƒD ƒƒ W d   ƒ n1 s>0    Y  d S )Nr>   r½   rï   c                 s   s   | ]}t tƒd ƒV  qdS ©rÉ   Nr³   rt   r)   r)   r+   r_   Í  s   z-test_parallel_timeout_fail.<locals>.<genexpr>rÉ   )r   r   r!   rb   r7   r)   r)   r+   Útest_parallel_timeout_failÈ  s    
ÿró   Ú
sequentialÚ	return_asÚ	generatorÚgenerator_unorderedc                 C   sx   t tƒ�6 ttd| |dd�dd„ tdƒD ƒƒƒ W d   ƒ n1 sD0    Y  ttd| |dd�dd„ tdƒD ƒƒƒ d S )Nr>   r²   )r[   r8   rõ   rð   c                 s   s   | ]}t tƒd ƒV  qdS rò   r³   rt   r)   r)   r+   r_   Ú  s   z<test_parallel_timeout_fail_with_generator.<locals>.<genexpr>rÉ   c                 s   s   | ]}t tƒd ƒV  qdS r¼   r³   rt   r)   r)   r+   r_   á  s   )r   r   Úlistr!   rb   ©r8   rõ   r)   r)   r+   Ú)test_parallel_timeout_fail_with_generatorÒ  s    
ÿÿ"ÿÿrú   c              	   C   s  t d u�r<ttƒ�0 td| d�dd„ tddƒD ƒƒ W d   ƒ n1 sH0    Y  ttƒ�* td| d�dd„ dD ƒƒ W d   ƒ n1 sŠ0    Y  td| d���r}t|jƒd us¶J ‚t|jƒ}ttƒ�( |dd„ tddƒD ƒƒ W d   ƒ n1 sö0    Y  t|jƒd u�sJ ‚t|jƒ|u�s(J ‚d	d„ td
ƒD ƒ|dd„ td
ƒD ƒƒk�sVJ ‚t|jƒ}ttƒ�" |dd„ dD ƒƒ W d   ƒ n1 �s’0    Y  t|jƒd u�s°J ‚t|jƒ|u�sÄJ ‚dd„ td
ƒD ƒ|dd„ td
ƒD ƒƒk�sJ |j	|j
|j|jfƒ‚W d   ƒ n1 �s0    Y  t|jƒd u �s~J ‚nBttƒ�( tdd�dd„ dD ƒƒ W d   ƒ n1 �st0    Y  ttƒ�. tdd�dd„ tddƒD ƒƒ W d   ƒ n1 �s¼0    Y  ttƒ�. tddd�dd„ tdƒD ƒƒ W d   ƒ n1 �s0    Y  d S )Nr>   rZ   c                 S   s   g | ]\}}t tƒ||ƒ‘qS r)   ©r#   r<   ©r*   r:   r;   r)   r)   r+   r,   ï  r-   z&test_error_capture.<locals>.<listcomp>)r   r   )r   r   c                 S   s   g | ]}t tƒ|ƒ‘qS r)   ©r#   rS   rt   r)   r)   r+   r,   ô  r-   c                 S   s   g | ]\}}t tƒ||ƒ‘qS r)   rû   rü   r)   r)   r+   r,   ý  r-   c                 S   s   g | ]}t |d d�‘qS rÊ   rÌ   rt   r)   r)   r+   r,     r-   rÉ   c                 s   s   | ]}t tƒ|d d�V  qdS rÍ   r~   rt   r)   r)   r+   r_     s   z%test_error_capture.<locals>.<genexpr>c                 S   s   g | ]}t tƒ|ƒ‘qS r)   rý   rt   r)   r)   r+   r,     r-   c                 S   s   g | ]}t |d d�‘qS rÊ   rÌ   rt   r)   r)   r+   r,     r-   c                 s   s   | ]}t tƒ|d d�V  qdS rÍ   r~   rt   r)   r)   r+   r_     s   rj   c                 S   s   g | ]}t tƒ|ƒ‘qS r)   rý   rt   r)   r)   r+   r,   "  r-   c                 S   s   g | ]\}}t tƒ||ƒ‘qS r)   rû   rü   r)   r)   r+   r,   '  r-   r   )r[   rp   c                 s   s   | ]}t tƒ|d d�V  qdS )T)rO   N©r#   rP   r]   r)   r)   r+   r_   +  r-   rî   )r   r   ÚZeroDivisionErrorr!   ÚziprR   r9   rÐ   rb   Ú
_iteratingZn_completed_tasksZn_dispatched_tasksÚ	_abortingrA   )r8   r   Zoriginal_workersr)   r)   r+   Útest_error_captureç  sT    


ÿ"

ÿ"

6ÿ

2ÿüþ$
8
>

ÿr  c              	   C   s  ddd„}t d| d��Ü}ttdd��( |dd	„ |dd
�D ƒƒ W d   ƒ n1 sR0    Y  ttdd��( |dd	„ |dd
�D ƒƒ W d   ƒ n1 s–0    Y  ttdd��( |dd	„ |dd
�D ƒƒ W d   ƒ n1 sÚ0    Y  W d   ƒ n1 sø0    Y  d S )Nr   c                 s   s(   t dƒD ]}|| krtdƒ‚|V  qd S )Né   úIterator Raising Error)rb   rN   )Úraise_atr^   r)   r)   r+   Úmy_generator2  s    z1test_error_in_task_iterator.<locals>.my_generatorr>   rZ   r  r×   c                 s   s   | ]}t tƒ|ƒV  qd S r2   r\   r]   r)   r)   r+   r_   ;  r-   z.test_error_in_task_iterator.<locals>.<genexpr>)r  c                 s   s   | ]}t tƒ|ƒV  qd S r2   r\   r]   r)   r)   r+   r_   @  r-   ru   c                 s   s   | ]}t tƒ|ƒV  qd S r2   r\   r]   r)   r)   r+   r_   D  r-   é   )r   )r!   r   rN   )r8   r  rÑ   r)   r)   r+   Útest_error_in_task_iterator/  s    
66r	  c                 C   s   |   d| ¡ d S )NzConsumed %s)rz   )ÚqueueÚitemr)   r)   r+   ÚconsumerG  s    r  zbatch_size, expected_queue)ú
Produced 0ú
Consumed 0ú
Produced 1ú
Consumed 1ú
Produced 2ú
Consumed 2ú
Produced 3ú
Consumed 3ú
Produced 4ú
Consumed 4ú
Produced 5ú
Consumed 5r‘   )r  r  r  r  r  r  r  r  r  r  r  r  c                    sT   t ƒ ‰ ‡ fdd„}td|| d�‡ fdd„|ƒ D ƒƒ ˆ |ks@J ‚tˆ ƒdksPJ ‚dS )	z=Test that with only one job, Parallel does act as a iterator.c                  3   s&   t dƒD ]} ˆ  d|  ¡ | V  qd S ©Nrå   zProduced %i©rb   rz   ©r^   ©r
  r)   r+   Úproducerx  s    z'test_dispatch_one_job.<locals>.producerr   ©r[   Ú
batch_sizer8   c                 3   s   | ]}t tƒˆ |ƒV  qd S r2   ©r#   r  rt   r  r)   r+   r_   }  s   z(test_dispatch_one_job.<locals>.<genexpr>é   N)rø   r!   r…   )r8   r  Zexpected_queuer  r)   r  r+   Útest_dispatch_one_jobK  s    +ÿr"  c                    s    t  ¡ }| ¡ ‰ ‡ fdd„}tddd| d�‡ fdd„|ƒ D ƒƒ tˆ ƒ}|d	 d
ksXJ ‚|dd…  d¡}|dksvJ ‚| d¡}||ksŒJ ‚tˆ ƒdksœJ ‚dS )zRCheck that using pre_dispatch Parallel does indeed dispatch items
    lazily.
    c                  3   s&   t dƒD ]} ˆ  d|  ¡ | V  qd S r  r  r  r  r)   r+   r  �  s    z/test_dispatch_multiprocessing.<locals>.producerr>   r   r`   )r[   r  Úpre_dispatchr8   c                 3   s   | ]}t tƒˆ d ƒV  qdS )r®   Nr   r©   r  r)   r+   r_   ’  s   z0test_dispatch_multiprocessing.<locals>.<genexpr>r   r  Nr‘   zConsumed anyri   r  r!  )r   ÚManagerrø   r!   Úindexr…   )r8   Úmanagerr  Zqueue_contentsZfirst_consumption_indexZproduced_3_indexr)   r  r+   Útest_dispatch_multiprocessing„  s    ÿ
r'  c                  C   sZ   t dddd��8} | dd„ tdƒD ƒƒ | j ¡ dks8J ‚W d   ƒ n1 sL0    Y  d S )	Nr>   Úautor/   r  c                 s   s   | ]}t tƒ|ƒV  qd S r2   rÙ   r]   r)   r)   r+   r_   ª  r-   z/test_batching_auto_threading.<locals>.<genexpr>éˆ  r   ©r!   rb   rÐ   Zcompute_batch_size)rÑ   r)   r)   r+   Útest_batching_auto_threading¤  s    r+  c                 C   sZ   t dd| d��8}|dd„ tdƒD ƒƒ |j ¡ dks8J ‚W d   ƒ n1 sL0    Y  d S )Nr>   r(  r  c                 s   s   | ]}t tƒ|ƒV  qd S r2   rÙ   r]   r)   r)   r+   r_   ²  r-   z2test_batching_auto_subprocesses.<locals>.<genexpr>r)  r   r*  )r8   rÑ   r)   r)   r+   Útest_batching_auto_subprocesses®  s    r,  c                   C   sL   t tƒ�0 tdddd�dd„ tdƒD ƒƒ W d  ƒ n1 s>0    Y  dS )	zCMake sure that exception raised during dispatch are indeed capturedr>   é   r   )r[   r#  rp   c                 s   s   | ]}t tƒ|ƒV  qd S r2   rþ   r]   r)   r)   r+   r_   ½  s   z*test_exception_dispatch.<locals>.<genexpr>rî   N)r   rN   r!   rb   r)   r)   r)   r+   Útest_exception_dispatchº  s    
ÿr.  c                 C   s    t dd�dd„ tdƒD ƒƒ d S )Nr>   rj   c                 s   s   | ]}t tƒ|ƒV  qd S r2   rþ   ©r*   Újr)   r)   r+   r_   Ã  r-   z(nested_function_inner.<locals>.<genexpr>rî   ra   r  r)   r)   r+   Únested_function_innerÂ  s    r1  c                 C   s    t dd�dd„ tdƒD ƒƒ d S )Nr>   rj   c                 s   s   | ]}t tƒ|ƒV  qd S r2   )r#   r1  r/  r)   r)   r+   r_   Ç  r-   z(nested_function_outer.<locals>.<genexpr>rî   ra   r  r)   r)   r+   Únested_function_outerÆ  s    r2  z'https://github.com/joblib/loky/pull/255©Úreasonc                 C   sœ   t tƒ�.}td| d�dd„ tdƒD ƒƒ W d  ƒ n1 s<0    Y  t|j|j|jƒ}d |¡}d|v snJ ‚d	|v szJ ‚d
|v s†J ‚t|jƒtu s˜J ‚dS )z¯Ensure errors for nested joblib cases gets propagated

    We rely on the Python 3 built-in __cause__ system that already
    report this kind of information to the user.
    r>   rZ   c                 s   s   | ]}t tƒ|ƒV  qd S r2   )r#   r2  r]   r)   r)   r+   r_   Ô  s   z1test_nested_exception_dispatch.<locals>.<genexpr>rî   NÚ r2  r1  rP   )	r   rN   r!   rb   r	   rW   rŽ   Útbr¹   )r8   ÚexcinfoZreport_linesÚreportr)   r)   r+   Útest_nested_exception_dispatchÊ  s    	
ÿ&
r9  c                   @   s$   e Zd ZdZddd„Zd	dd„ZdS )
ÚFakeParallelBackendz8Pretends to run concurrently while running sequentially.r   Nc                 K   s   |   |¡| _|| _|S r2   )r$   r[   r   )rB   r[   r   Zbackend_argsr)   r)   r+   Ú	configureæ  s    zFakeParallelBackend.configurec                 C   s"   |dk rt t ¡ d | dƒ}|S )Nr   r   )Úmaxr   r"   ©rB   r[   r)   r)   r+   r$   ë  s    z$FakeParallelBackend.effective_n_jobs)r   N)r   )rI   rJ   rK   rL   r;  r$   r)   r)   r)   r+   r:  ã  s   
r:  c                	   C   sæ   t tdd�� tdd� W d   ƒ n1 s,0    Y  t tdd��: tdd�� W d   ƒ n1 sd0    Y  W d   ƒ n1 s‚0    Y  t tdd��: tdd�� W d   ƒ n1 sº0    Y  W d   ƒ n1 sØ0    Y  d S )NzInvalid backend:r×   zunit-testingr7   )r   rN   r!   r&   r)   r)   r)   r+   Útest_invalid_backendñ  s    (<r>  c                 C   s0  t tƒ� }td| d� ¡  W d   ƒ n1 s.0    Y  dt|jƒv sJJ ‚t tƒ� }td| d� ¡  W d   ƒ n1 sx0    Y  dt|jƒv s”J ‚t tƒ� }td| d� ¡  W d   ƒ n1 sÂ0    Y  dt|jƒv sÞJ ‚t tƒ� }td| d� ¡  W d   ƒ n1 �s0    Y  dt|jƒv �s,J ‚d S )Nr   rZ   z&n_jobs == 0 in Parallel has no meaningç      à?z2.3z$n_jobs could not be converted to intZinvalid_str)r   rN   r!   Z_initialize_backendr’   rŽ   )r8   r7  r)   r)   r+   Útest_invalid_njobsþ  s    
.
.
.
0r@  )Ú2gffffff@r>   c                 C   sP   t || d�}| ¡ dksJ ‚|dd„ tdƒD ƒƒ}tdd„ t|ƒD ƒƒsLJ ‚d S )NrZ   r>   c                 s   s   | ]}t tƒ|ƒV  qd S r2   r\   r]   r)   r)   r+   r_     r-   z.test_njobs_converted_to_int.<locals>.<genexpr>rÉ   c                 s   s   | ]\}}|t |ƒkV  qd S r2   rs   )r*   r^   Úrr)   r)   r+   r_     r-   )r!   Ú_effective_n_jobsrb   r¤   Ú	enumerate)r8   r[   rÑ   r°   r)   r)   r+   Útest_njobs_converted_to_int  s    rE  c                   C   s>   z0t dtƒ dtv sJ ‚td tks(J ‚W td= ntd= 0 d S )NZtest_backend)r'   r:  r    r)   r)   r)   r+   Útest_register_parallel_backend  s
    
rF  c                  C   s^   t j} tƒ tƒ ksJ ‚z*tdtd dd� tƒ tks8J ‚W | t _n| t _0 tƒ tƒ ksZJ ‚d S )Nr/   T)Zmake_default)r   r3   rY   r4   r'   r    r   )Zdefault_backend_origr)   r)   r+   Útest_overwrite_default_backend%  s    rG  zOnly without multiprocessingc                   C   sŽ   t tdd��, tdd�dd„ tdƒD ƒƒ W d   ƒ n1 s>0    Y  tdd��( tƒ dd„ tdƒD ƒƒ W d   ƒ n1 s€0    Y  d S )	Nz)joblib backend '.*' is not available on.*r×   r.   r7   c                 s   s   | ]}t tƒ|ƒV  qd S r2   r\   r]   r)   r)   r+   r_   4  r-   z2test_backend_no_multiprocessing.<locals>.<genexpr>r`   c                 s   s   | ]}t tƒ|ƒV  qd S r2   r\   r]   r)   r)   r+   r_   8  r-   )r   ÚUserWarningr!   rb   r&   r)   r)   r)   r+   Útest_backend_no_multiprocessing1  s    :rI  c                 C   s  | |dd��ü t  ¡ \}}|dks&J ‚tdƒdks6J ‚tƒ }|jdksJJ ‚|dkrvt|ƒtu sbJ ‚t|jƒtu súJ ‚n„|dkr¢t|ƒtu sŽJ ‚t|jƒtu súJ ‚nX|dkrÎt|ƒt	u sºJ ‚t|jƒt	u súJ ‚n,| 
d¡rút|ƒtu sèJ ‚t|jƒtu súJ ‚W d   ƒ n1 �s0    Y  d S )Nr`   rj   r(   r.   r/   Ztest_)r   rX   r$   r!   r[   rW   r   rÐ   r   r   Ú
startswithr:  )rh   Úbackend_nameÚactive_backendÚactive_n_jobsrÑ   r)   r)   r+   Úcheck_backend_context_manager;  s$    
rN  c                 C   s   g | ]}d | ‘qS )ztest_backend_%dr)   r]   r)   r)   r+   r,   Q  r-   r`   c                    sr   |t vr|  t |t¡ tƒ tƒ ks&J ‚tˆ |ƒ tƒ tƒ ks@J ‚tddd�‡ fdd„tD ƒƒ tƒ tƒ ksnJ ‚d S )Nr>   r/   rZ   c                 3   s    | ]}|st tƒˆ |ƒV  qd S r2   )r#   rN  )r*   rD   ©rh   r)   r+   r_   c  s   þz/test_backend_context_manager.<locals>.<genexpr>)r    Úsetitemr:  rY   r4   rN  r!   Ú all_backends_for_context_manager)r�   r8   rh   r)   rO  r+   Útest_backend_context_managerT  s    
þrR  c                   @   s   e Zd ZdZddd„ZdS )ÚParameterizedParallelBackendz9Pretends to run conncurrently while running sequentially.Nc                 C   s   |d u rt dƒ‚|| _d S )Nzparam should not be None)rN   Úparam)rB   rT  r)   r)   r+   rG   p  s    z%ParameterizedParallelBackend.__init__)NrH   r)   r)   r)   r+   rS  m  s   rS  c                 C   sè   |   tdt¡ tƒ tƒ ksJ ‚|dddd��~ t ¡ \}}t|ƒtu sJJ ‚|jdksXJ ‚|dksdJ ‚t	ƒ }|j
dksxJ ‚|j|u s†J ‚|dd„ tdƒD ƒƒ}W d   ƒ n1 s°0    Y  |dd	„ tdƒD ƒksÔJ ‚tƒ tƒ ksäJ ‚d S )
NZparam_backendé*   r`   )rT  r[   c                 s   s   | ]}t tƒ|ƒV  qd S r2   ©r#   r   r]   r)   r)   r+   r_   ƒ  r-   z=test_parameterized_backend_context_manager.<locals>.<genexpr>ru   c                 S   s   g | ]}t |ƒ‘qS r)   r   r]   r)   r)   r+   r,   „  r-   z>test_parameterized_backend_context_manager.<locals>.<listcomp>)rP  r    rS  rY   r4   r   rX   rW   rT  r!   r[   rÐ   rb   )r�   rh   rL  rM  rÑ   rŸ   r)   r)   r+   Ú*test_parameterized_backend_context_managerv  s    4rW  c                 C   sÞ   t ƒ tƒ ksJ ‚| tdd�dd��~ t ¡ \}}t|ƒtu s@J ‚|jdksNJ ‚|dksZJ ‚tƒ }|jdksnJ ‚|j	|u s|J ‚|dd„ t
dƒD ƒƒ}W d   ƒ n1 s¦0    Y  |dd„ t
dƒD ƒksÊJ ‚t ƒ tƒ ksÚJ ‚d S )	Né+   )rT  ru   rj   c                 s   s   | ]}t tƒ|ƒV  qd S r2   rV  r]   r)   r)   r+   r_   ˜  r-   zFtest_directly_parameterized_backend_context_manager.<locals>.<genexpr>c                 S   s   g | ]}t |ƒ‘qS r)   r   r]   r)   r)   r+   r,   ™  r-   zGtest_directly_parameterized_backend_context_manager.<locals>.<listcomp>)rY   r4   rS  r   rX   rW   rT  r!   r[   rÐ   rb   )rh   rL  rM  rÑ   rŸ   r)   r)   r+   Ú3test_directly_parameterized_backend_context_managerŠ  s    4rY  c                   C   s   t dƒ t ¡ S )Nr²   )r   ÚosÚgetpidr)   r)   r)   r+   Úsleep_and_return_pidŸ  s    r\  c                   C   s<   t ƒ tksJ ‚tƒ  ¡ dks J ‚tdd�dd„ tdƒD ƒƒS )Nr   r>   rj   c                 s   s   | ]}t tƒƒ V  qd S r2   )r#   r\  r©   r)   r)   r+   r_   «  r-   z"get_nested_pids.<locals>.<genexpr>)rY   r   r!   rC  rb   r)   r)   r)   r+   Úget_nested_pids¤  s    r]  c                       s    e Zd ZdZ‡ fdd„Z‡  ZS )Ú	MyBackendz:Backend to test backward compatibility with older backendsc                    s   t t| ƒ ¡ d S rV   )Úsuperr^  Úget_nested_backendrÔ   ©Ú	__class__r)   r+   r`  ±  s    zMyBackend.get_nested_backend)rI   rJ   rK   rL   r`  Ú__classcell__r)   r)   ra  r+   r^  ®  s   r^  Úback_compat_backend)r/   r.   r(   rd  c                 C   sf   | |ƒ�J t dd�dd„ tdƒD ƒƒ}|D ]}tt|ƒƒdks*J ‚q*W d   ƒ n1 sX0    Y  d S )Nr>   rj   c                 s   s   | ]}t tƒƒ V  qd S r2   )r#   r]  r©   r)   r)   r+   r_   Ã  r-   z6test_nested_backend_context_manager.<locals>.<genexpr>rÉ   r   )r!   rb   r…   r�   )rh   r8   Z
pid_groupsZ	pid_groupr)   r)   r+   Ú#test_nested_backend_context_manager»  s    
re  )r>   ri   Nc                    s|   dd„ ‰t dd�‡fdd„tdƒD ƒƒ |ˆ ˆd��4 t dd�‡ ‡‡fdd„tdƒD ƒƒ W d   ƒ n1 sn0    Y  d S )	Nc                 S   s0   t ƒ t|  ksJ ‚t|ƒ}tƒ  ¡ |ks,J ‚d S r2   )rY   r    r$   r!   rC  )Zexpected_backend_typeZexpected_n_jobr)   r)   r+   Úcheck_nested_backendÐ  s    z?test_nested_backend_in_sequential.<locals>.check_nested_backendr   rj   c                 3   s   | ]}t ˆ ƒtjd ƒV  qdS rÃ   )r#   r   r3   r©   )rf  r)   r+   r_   Ú  s   z4test_nested_backend_in_sequential.<locals>.<genexpr>rÉ   c                 3   s   | ]}t ˆƒˆ ˆƒV  qd S r2   rç   r©   ©r8   rf  r[   r)   r+   r_   ß  s   ra   )r8   r[   rh   r)   rg  r+   Ú!test_nested_backend_in_sequentialÈ  s    
ÿÿrh  c                 C   sZ   | |ƒ�>}| t u r|d }| tu r*|d }|j|ks8J ‚W d   ƒ n1 sL0    Y  d S )Nr8   r   )r&   r%   Únesting_level)rh   Úinner_backendZexpected_levelÚctxr8   r)   r)   r+   Úcheck_nesting_levelä  s    
rl  Úouter_backendrj  c                    s~   t ˆ |dƒ td|d�‡ ‡fdd„tdƒD ƒƒ ˆ ˆdd��. tƒ ‡ ‡fdd„tdƒD ƒƒ W d   ƒ n1 sp0    Y  d S )	Nr   r>   rZ   c                 3   s   | ]}t tƒˆ ˆd ƒV  qdS rÃ   ©r#   rl  r©   ©rh   rj  r)   r+   r_   õ  s   z-test_backend_nesting_level.<locals>.<genexpr>rÉ   rj   c                 3   s   | ]}t tƒˆ ˆd ƒV  qdS rÃ   rn  r©   ro  r)   r+   r_   ú  s   )rl  r!   rb   )rh   rm  rj  r)   ro  r+   Útest_backend_nesting_levelí  s    ÿÿrp  Úwith_retrieve_callbackTc                    s°   dd l ‰ G ‡ ‡fdd„dtƒ}td|ƒ dd„ ‰| dƒ�d}tdd�‡fd	d
„tdƒD ƒƒ | tu rt|d jdkstJ ‚| tu rŽ|d jdksŽJ ‚W d   ƒ n1 s¢0    Y  d S )Nr   c                       s"   e Zd ZdZ”Z” jdd„ ƒZdS )z)test_retrieval_context.<locals>.MyBackendr   c                 s   s   |  j d7  _ d V  d S rk   r  rÔ   r)   r)   r+   Úretrieval_context	  s    z;test_retrieval_context.<locals>.MyBackend.retrieval_contextN)rI   rJ   rK   r^   Zsupports_retrieve_callbackÚcontextmanagerrr  r)   )Ú
contextlibrq  r)   r+   r^    s   r^  Z	retrievalc                 S   s   t dd�dd„ t| ƒD ƒƒS )Nr>   rj   c                 s   s   | ]}t tƒ|ƒV  qd S r2   rÙ   r]   r)   r)   r+   r_     r-   z>test_retrieval_context.<locals>.nested_call.<locals>.<genexpr>ra   )Únr)   r)   r+   Únested_call  s    z+test_retrieval_context.<locals>.nested_callr>   rj   c                 3   s   | ]}t ˆ ƒ|ƒV  qd S r2   rç   r]   )rv  r)   r+   r_     r-   z)test_retrieval_context.<locals>.<genexpr>ru   r8   r   )rt  r   r'   r!   rb   r&   r^   r%   )rh   rq  r^  rk  r)   )rt  rv  rq  r+   Útest_retrieval_contextÿ  s    	

 rw  r  )r   ri   g¸…ëQ¸ö?c                 C   s6   t tƒ� t| d� W d   ƒ n1 s(0    Y  d S )N©r  )r   rN   r!   rx  r)   r)   r+   Útest_invalid_batch_size  s    
ry  z)n_tasks, n_jobs, pre_dispatch, batch_size))r>   r>   r¤   r(  )r>   r>   r[   r(  ©rÉ   r>   r[   r(  )i  r>   r[   r(  rz  )rÉ   r‘   r[   r(  )éÈ   r!  r[   r(  )é   r!  ú
2 * n_jobsr   )éú   r!  r¤   r   )r~  r!  r}  rM   )r{  r!  r}  r(  c                 C   sN   |||dœ}dd„ t | ƒD ƒ}tf i |¤Ždd„ t | ƒD ƒƒ}||ksJJ ‚d S )N)r[   r#  r  c                 S   s   g | ]}t |ƒ‘qS r)   rs   r]   r)   r)   r+   r,   :  r-   z0test_dispatch_race_condition.<locals>.<listcomp>c                 s   s   | ]}t tƒ|ƒV  qd S r2   r\   r]   r)   r)   r+   r_   ;  r-   z/test_dispatch_race_condition.<locals>.<genexpr>rw   )r†   r[   r#  r  Úparamsr¥   rŸ   r)   r)   r+   Útest_dispatch_race_condition%  s     r€  c                  C   s8   t  ¡ } tddd�}|j d¡}| ¡ }|| ks4J ‚d S )Nr>   r(   rZ   rh   )r   Úget_start_methodr!   Z_backend_kwargsÚget)Zmp_start_methodrÑ   rh   Zstart_methodr)   r)   r+   Útest_default_mp_context?  s
    rƒ  c                    s^   | dkrt  d¡} tj d¡}| dd¡‰ t ˆ ˆ j¡ td| d�‡ fdd„t	dƒD ƒƒ d S )	Nr(   r1   rU  iè  r>   rZ   c                 3   s    | ]}t tjƒˆ ˆ jƒV  qd S r2   )r#   r   ÚdotÚTr]   ©rC   r)   r+   r_   ^  r-   zAtest_no_blas_crash_or_freeze_with_subprocesses.<locals>.<genexpr>)
r   r0   r   ÚrandomZRandomStateÚrandnr„  r…  r!   rb   )r8   Úrngr)   r†  r+   Ú.test_no_blas_crash_or_freeze_with_subprocessesH  s    
rŠ  a  from joblib import Parallel, delayed

def square(x):
    return x ** 2

backend = "{}"
if backend == "spawn":
    from multiprocessing import get_context
    backend = get_context(backend)

print(Parallel(n_jobs=2, backend=backend)(
      delayed(square)(i) for i in range(5)))
c                 C   sB   | dkrt  ¡ dkrt d¡ t | ¡}ttjd|gddd� d S )Nr(   ÚforkzVRequire fork start method to use interactively defined functions with multiprocessing.z-crÉ   ú\[0, 1, 4, 9, 16\]©rð   Ústdout_regex)	r   r�  ÚpytestÚskipÚ,UNPICKLABLE_CALLABLE_SCRIPT_TEMPLATE_NO_MAINÚformatr   ÚsysÚ
executable)r8   Úcoder)   r)   r+   Ú2test_parallel_with_interactively_defined_functionsq  s    ÿ
ÿr–  a¼  import sys
# Make sure that joblib is importable in the subprocess launching this
# script. This is needed in case we run the tests from the joblib root
# folder without having installed joblib
sys.path.insert(0, {joblib_root_folder!r})

from joblib import Parallel, delayed

def run(f, x):
    return f(x)

{define_func}

if __name__ == "__main__":
    backend = "{backend}"
    if backend == "spawn":
        from multiprocessing import get_context
        backend = get_context(backend)

    callable_position = "{callable_position}"
    if callable_position == "delayed":
        print(Parallel(n_jobs=2, backend=backend)(
                delayed(square)(i) for i in range(5)))
    elif callable_position == "args":
        print(Parallel(n_jobs=2, backend=backend)(
                delayed(run)(square, i) for i in range(5)))
    else:
        print(Parallel(n_jobs=2, backend=backend)(
                delayed(run)(f=square, x=i) for i in range(5)))
z!def square(x):
    return x ** 2
zcdef gen_square():
    def square(x):
        return x ** 2
    return square
square = gen_square()
zsquare = lambda x: x ** 2
Údefine_funcÚcallable_position)r#   r¢   Úkwargsc              	   C   sv   | dv r$|t kstjdkr$t d¡ tj|| |tj 	tj 	t
j¡¡d�}| d¡}| |¡ ttj|jgddd� d S )	N)r(   r1   Úwin32zNot picklable with pickle)r—  r8   r˜  Újoblib_root_folderzunpicklable_func_script.pyrÉ   rŒ  r�  )ÚSQUARE_MAINr“  Úplatformr�  r�  Ú)UNPICKLABLE_CALLABLE_SCRIPT_TEMPLATE_MAINr’  rZ  ÚpathÚdirnameÚjoblibÚ__file__r¹   Úwriter   r”  Ústrpath)r8   r—  r˜  Útmpdirr•  Z	code_filer)   r)   r+   Ú0test_parallel_with_unpicklable_functions_in_args±  s&    ÿÿ
ü


ýr¦  aý  import sys
import faulthandler
# Make sure that joblib is importable in the subprocess launching this
# script. This is needed in case we run the tests from the joblib root
# folder without having installed joblib
sys.path.insert(0, {joblib_root_folder!r})

from joblib import Parallel, delayed
from functools import partial

class MyClass:
    '''Class defined in the __main__ namespace'''
    def __init__(self, value):
        self.value = value


def square(x, ignored=None, ignored2=None):
    '''Function defined in the __main__ namespace'''
    return x.value ** 2


square2 = partial(square, ignored2='something')

# Here, we do not need the `if __name__ == "__main__":` safeguard when
# using the default `loky` backend (even on Windows).

# To make debugging easier
faulthandler.dump_traceback_later(30, exit=True)

# The following baroque function call is meant to check that joblib
# introspection rightfully uses cloudpickle instead of the (faster) pickle
# module of the standard library when necessary. In particular cloudpickle is
# necessary for functions and instances of classes interactively defined in the
# __main__ module.

print(Parallel(backend="loky", n_jobs=2)(
    delayed(square2)(MyClass(i), ignored=[dict(a=MyClass(1))])
    for i in range(5)
))
)r›  c                 C   s.   |   d¡}| t¡ ttj|jgdd d� d S )Nz(joblib_interactively_defined_function.pyrŒ  )rŽ  rð   )r¹   r£  Ú5INTERACTIVE_DEFINED_FUNCTION_AND_CLASS_SCRIPT_CONTENTr   r“  r”  r¤  ©r¥  Úscriptr)   r)   r+   Ú7test_parallel_with_interactively_defined_functions_lokyö  s    


ýrª  a…  import sys
# Make sure that joblib is importable in the subprocess launching this
# script. This is needed in case we run the tests from the joblib root
# folder without having installed joblib
sys.path.insert(0, {joblib_root_folder!r})

from joblib import Parallel, delayed, hash
import multiprocessing as mp
mp.util.log_to_stderr(5)

class MyList(list):
    '''MyList is interactively defined by MyList.append is a built-in'''
    def __hash__(self):
        # XXX: workaround limitation in cloudpickle
        return hash(self).__hash__()

l = MyList()

print(Parallel(backend="loky", n_jobs=2)(
    delayed(l.append)(i) for i in range(3)
))
c                 C   s0   |   d¡}| t¡ ttj|jgdddd� d S )Nz)joblib_interactive_bound_method_script.pyz\[None, None, None\]ZLokyProcessé   )rŽ  Zstderr_regexrð   )r¹   r£  Ú9INTERACTIVELY_DEFINED_SUBCLASS_WITH_METHOD_SCRIPT_CONTENTr   r“  r”  r¤  r¨  r)   r)   r+   Ú:test_parallel_with_interactively_defined_bound_method_loky  s    


ür­  c                  C   s"   t g ƒ} tdd�| ƒg ksJ ‚d S )Nr>   rj   )Úiterr!   )Zexhausted_iteratorr)   r)   r+   Ú%test_parallel_with_exhausted_iterator)  s    r¯  c                 C   s"   t | tjƒstdt| ƒƒ‚|  ¡ S )Nz#Expected np.memmap instance, got %r)r“   r   ÚmemmapÚ	TypeErrorrW   Úcopyr†  r)   r)   r+   Úcheck_memmap.  s    r³  c                 C   s    dd„ }t dd| d�dd„ |dƒD ƒƒ}t||t|ƒƒƒD ]\}}tj ||¡ q:t d	d| d�d
d„ |dƒD ƒƒ}t||t|ƒƒƒD ]\}}tj ||¡ q„d S )Nc                 s   s(   t | ƒD ]}tjdtjd�| V  qd S )NrÉ   ©rã   )rb   r   ÚonesÚfloat32)ru  r^   r)   r)   r+   Úgenerate_arrays<  s    zBtest_auto_memmap_on_arrays_from_generator.<locals>.generate_arraysr>   r   ©r[   râ   r8   c                 s   s   | ]}t tƒ|ƒV  qd S r2   ©r#   r³  ©r*   rC   r)   r)   r+   r_   B  s   z<test_auto_memmap_on_arrays_from_generator.<locals>.<genexpr>rr   r‘   c                 s   s   | ]}t tƒ|ƒV  qd S r2   r¹  rº  r)   r)   r+   r_   K  s   )r!   r   r…   r   rê   rë   )r8   r·  rŸ   rì   r¥   r)   r)   r+   Ú)test_auto_memmap_on_arrays_from_generator4  s    ÿÿr»  c                 C   s   | S r2   r)   )Úargr)   r)   r+   ÚidentityR  s    r½  c                    sš   |   d¡j}tj}tj|dd�tj|dd�g}t||ƒ t|dd�‰ t	dd�‡ fdd	„d
D ƒƒ\}t
ˆ d tjƒsvJ ‚ˆ d j|ksˆJ ‚tj ||¡ d S )Nz	test.mmapZuint8r´  rB  )Z	mmap_moder>   rj   c                 3   s   | ]}t tƒˆ ƒV  qd S r2   )r#   r½  r©   ©r°  r)   r+   r_   ^  r-   z.test_memmap_with_big_offset.<locals>.<genexpr>)r   r   )r¹   r¤  ÚmmapZALLOCATIONGRANULARITYr   Úzerosrµ  r
   r   r!   r“   r°  Úoffsetrê   rë   )r¥  ÚfnameÚsizeÚobjrì   r)   r¾  r+   Útest_memmap_with_big_offsetV  s    
rÅ  c                  C   sˆ   t jdd��.} tddd�dd„ tdƒD ƒƒ W d   ƒ n1 s@0    Y  t| ƒdksZJ ‚| d }t|jtƒsrJ ‚t|jƒd	ks„J ‚d S )
NTr�   r   )r[   rð   c                 s   s   | ]}t tƒ|ƒV  qd S r2   r\   r]   r)   r)   r+   r_   f  r-   zFtest_warning_about_timeout_not_supported_by_backend.<locals>.<genexpr>é2   r   z’The backend class 'SequentialBackend' does not support timeout. You have set 'timeout=1' in Parallel but the 'timeout' parameter will not be used.)	r›   rœ   r!   rb   r…   r“   r‰   rH  r’   )rž   r–   r)   r)   r+   Ú3test_warning_about_timeout_not_supported_by_backendd  s    <ÿrÇ  c                 C   s   || |< |S r2   r)   )Ú
input_listr%  rŽ   r)   r)   r+   Úset_list_valueq  s    rÉ  )r   r>   r‘   c                    sV   dgd ‰ t | ddd�‡ fdd„tdƒD ƒƒ}t|ƒ}tdd„ tˆ |ƒD ƒƒsRJ ‚d S )	Nr   ru   rö   r/   ©r[   rõ   r8   c                 3   s   | ]}t tƒˆ ||ƒV  qd S r2   )r#   rÉ  r]   ©rÈ  r)   r+   r_   |  s   zPtest_parallel_return_order_with_return_as_generator_parameter.<locals>.<genexpr>c                 s   s   | ]\}}||kV  qd S r2   r)   ©r*   ÚvrB  r)   r)   r+   r_   ƒ  r-   )r!   rb   rø   r¤   r   )r[   rì   r)   rË  r+   Ú=test_parallel_return_order_with_return_as_generator_parameterv  s    
ÿrÎ  c                 C   s   |rt dƒ t| ƒS )Nrî   )r   r   )ÚeÚdelayr)   r)   r+   Ú_sqrt_with_delay†  s    rÑ  c                    sp   t |d| d�dd„ tdƒD ƒƒ‰ t‡ fdd„tdƒD ƒƒ}dgttd	dƒƒ }td
d„ t||ƒD ƒƒsjJ ‚Š d S )Nr÷   rÊ  c                 s   s$   | ]}t tƒ|d  |dkƒV  qdS )r>   r   N)r#   rÑ  r]   r)   r)   r+   r_   ”  s   zK_test_parallel_unordered_generator_returns_fastest_first.<locals>.<genexpr>rÉ   c                 3   s   | ]}t ˆ ƒV  qd S r2   )Únextr©   ©rì   r)   r+   r_   ˜  r-   é	   r   r>   c                 s   s   | ]\}}||kV  qd S r2   r)   rÌ  r)   r)   r+   r_   œ  r-   )r!   rb   Úsortedrø   r¤   r   )r8   r[   Zquickly_returnedZexpected_quickly_returnedr)   rÓ  r+   Ú8_test_parallel_unordered_generator_returns_fastest_first�  s    ÿrÖ  c                 C   s   t | |ƒ d S r2   )rÖ  ©r8   r[   r)   r)   r+   Ú7test_parallel_unordered_generator_returns_fastest_first¡  s    rØ  )r   r>   ro   ri   c                 C   sv   dgdgd  }t tƒ�2 t ¡ }t| |d�dd„ |D ƒƒ W d   ƒ n1 sP0    Y  t ¡ | }|dk srJ ‚d S )NrC   rÉ   rr   rZ   c                 s   s   | ]}t tjƒ|ƒV  qd S r2   ©r#   rQ   r   r]   r)   r)   r+   r_   ±  r-   z%test_abort_backend.<locals>.<genexpr>r  )r   r±  rQ   r!   )r[   r8   ZdelaysÚt_startÚdtr)   r)   r+   Útest_abort_backend«  s    
8rÜ  c                 C   s   t jtdƒtd�}d|d< |S )Ng    €„Ar´  Fr   )r   rµ  ÚintÚbool)r¼  rì   r)   r)   r+   Úget_large_object¶  s    rß  c                 C   sZ   t || |d��8}|dd„ tdƒD ƒƒ}t|ƒ t|ƒ ~W d   ƒ n1 sL0    Y  d S )N)r[   r8   rõ   c                 s   s   | ]}t tƒ|ƒV  qd S r2   )r#   rß  r]   r)   r)   r+   r_   Ä  r-   z0_test_deadlock_with_generator.<locals>.<genexpr>rÉ   )r!   rb   rÒ  )r8   rõ   r[   r   rì   r)   r)   r+   Ú_test_deadlock_with_generatorÀ  s
    rà  c                 C   s   t | ||ƒ d S r2   )rà  )r8   rõ   r[   r)   r)   r+   Útest_deadlock_with_generatorÊ  s    rá  c                 C   sŒ   t tdd��R t|| |d�}|dd„ tdƒD ƒƒ}t ¡ }|dd„ tdƒD ƒƒ}W d   ƒ n1 sd0    Y  t ¡ | d	k s†J d
ƒ‚~d S )Nú)This Parallel instance is already runningr×   rù   c                 s   s   | ]}t tƒd ƒV  qdS rÃ   r³   r©   r)   r)   r+   r_   Û  r-   z/test_multiple_generator_call.<locals>.<genexpr>rÉ   c                 s   s   | ]}t tƒ|ƒV  qd S r2   rÙ   r]   r)   r)   r+   r_   Ý  r-   rr   r>   ú[The error should be raised immediately when submitting a new task but it took more than 2s.)r   rÓ   r!   rb   rQ   )r8   rõ   r[   r   ÚgrÚ  Zgen2r)   r)   r+   Útest_multiple_generator_callÒ  s    4ÿrå  c              	   C   s¬   t || |d��ˆ}|dd„ tdƒD ƒƒ}t ¡ }ttdd��& |dd„ tdƒD ƒƒ}W d   ƒ n1 sf0    Y  t ¡ | d	k sˆJ d
ƒ‚W d   ƒ n1 sœ0    Y  ~d S )Nrù   c                 s   s   | ]}t tƒd ƒV  qdS rò   r³   r©   r)   r)   r+   r_   ð  r-   z7test_multiple_generator_call_managed.<locals>.<genexpr>rÉ   râ  r×   c                 s   s   | ]}t tƒ|ƒV  qd S r2   rÙ   r]   r)   r)   r+   r_   ó  r-   rr   r>   rã  )r!   rb   rQ   r   rÓ   )r8   rõ   r[   r   rä  rÚ  Úg2r)   r)   r+   Ú$test_multiple_generator_call_managedè  s    4ÿ"rç  Úreturn_as_1Úreturn_as_2c                 C   s¨   t || |d�dd„ tdƒD ƒƒ}t || |d�dd„ tddƒD ƒƒ}|dkrRt|ƒ}|dkrbt|ƒ}tdd„ t|tdƒƒD ƒƒs‚J ‚td	d„ t|tddƒƒD ƒƒs¤J ‚d S )
Nrù   c                 s   s   | ]}t tƒ|d  ƒV  qdS ©r>   NrV  r]   r)   r)   r+   r_     s   z9test_multiple_generator_call_separated.<locals>.<genexpr>rÉ   c                 s   s   | ]}t tƒ|d  ƒV  qdS rê  rV  r]   r)   r)   r+   r_     s   r  r÷   c                 s   s   | ]\}}||kV  qd S r2   r)   ©r*   r°   r^   r)   r)   r+   r_     r-   c                 s   s   | ]\}}||kV  qd S r2   r)   rë  r)   r)   r+   r_     r-   )r!   rb   rÕ  r¤   r   )r8   rè  ré  r[   rä  ræ  r)   r)   r+   Ú&test_multiple_generator_call_separatedþ  s    ÿÿ rì  zbackend, error))r.   T)r/   F)rô   Fc           
      C   sV  | dkrt d u rt d¡ td| |d�}|dd„ tdƒD ƒƒ}t |dd	„ ¡}|r^ttd
d�nt	ƒ }|�l t
 
¡ }td| |d�dd„ tddƒD ƒƒ}|dkr¤t|ƒ}tdd„ t|tddƒƒD ƒƒsÆJ ‚W d   ƒ n1 sÚ0    Y  t
 
¡ | dk søJ ‚d}	|j�r"|	dk �r"|	d7 }	t
 d¡ qüt
 
¡ | dk �s8J ‚| ¡ dk�rR|j�sRJ ‚d S )Nr.   zRequires multiprocessingr>   rù   c                 s   s   | ]}t tƒd ƒV  qdS rò   r³   r]   r)   r)   r+   r_   &  r-   z<test_multiple_generator_call_separated_gc.<locals>.<genexpr>rÉ   c                   S   s   t dƒS )NzGenerator collected)r´   r)   r)   r)   r+   Ú<lambda>'  r-   z;test_multiple_generator_call_separated_gc.<locals>.<lambda>z The executor underlying Parallelr×   c                 s   s   | ]}t tƒ|d  ƒV  qdS rê  rV  r]   r)   r)   r+   r_   3  s   r  r÷   c                 s   s   | ]\}}||kV  qd S r2   r)   rë  r)   r)   r+   r_   :  r-   ru   r   r`   r   r?  )r   r�  r�  r!   rb   ÚweakrefÚfinalizer   rÓ   r   rQ   rÕ  r¤   r   Úaliver   rC  r  )
r8   rè  ré  Úerrorr   rä  Zg_wrrk  rÚ  Úretryr)   r)   r+   Ú)test_multiple_generator_call_separated_gc  s2    
ÿýÿ@ró  c                 C   sö   |j }tdd| |d��F}|dd„ tj d¡gd D ƒƒ tt |¡ƒdksNJ ‚W d   ƒ n1 sb0    Y  tdƒD ]}t |¡s† q˜td	ƒ qtt	d
ƒ‚tdd| d�}|dd„ tj d¡gd D ƒƒ tdƒD ]}t |¡sà qòtd	ƒ qÎt	d
ƒ‚d S )Nr>   r   )r[   râ   r8   Ztemp_folderc                 s   s   | ]}t tƒ|ƒV  qd S r2   r¹  rº  r)   r)   r+   r_   V  r-   z(test_memmapping_leaks.<locals>.<genexpr>rÉ   r   rr   r²   z/temporary directory of Parallel was not removedr¸  c                 s   s   | ]}t tƒ|ƒV  qd S r2   r¹  rº  r)   r)   r+   r_   f  r-   )
r¤  r!   r   r‡  r…   rZ  Úlistdirrb   r   ÚAssertionError)r8   r¥  rÑ   rª   r)   r)   r+   Útest_memmapping_leaksK  s      4

 

rö  )Nr.   r/   c                 C   s<   t d| d�dd„ tdƒD ƒƒ}|dd„ tdƒD ƒks8J ‚d S )Nr>   rZ   c                 s   s   | ]}t d d„ ƒ|ƒV  qdS )c                 S   s   | d S r=   r)   r?   r)   r)   r+   rí  v  r-   z2test_lambda_expression.<locals>.<genexpr>.<lambda>Nrç   r]   r)   r)   r+   r_   u  s   z)test_lambda_expression.<locals>.<genexpr>rÉ   c                 S   s   g | ]}|d  ‘qS ©r>   r)   r]   r)   r)   r+   r,   x  r-   z*test_lambda_expression.<locals>.<listcomp>ra   )r8   rŸ   r)   r)   r+   Útest_lambda_expressionp  s    ÿrø  c                    s¦   d}d}d| ‰ t d|| d�}|‡ fdd„t|ƒD ƒƒ |jj|jjksLJ ‚|jj|jjks`J ‚|‡ fdd„t|ƒD ƒƒ |jj|jjksŽJ ‚|jj|jjks¢J ‚d	S )
zCTest that a parallel backend correctly resets its batch statistics.r>   iô  g       @rÉ   )rp   r[   r8   c                 3   s   | ]}t tjƒˆ ƒV  qd S r2   rÙ  r]   ©Z	task_timer)   r+   r_   „  r-   z6test_backend_batch_statistics_reset.<locals>.<genexpr>c                 3   s   | ]}t tjƒˆ ƒV  qd S r2   rÙ  r]   rù  r)   r+   r_   ‹  r-   N)r!   rb   rÐ   Z_effective_batch_sizeZ_DEFAULT_EFFECTIVE_BATCH_SIZEZ_smoothed_batch_durationZ _DEFAULT_SMOOTHED_BATCH_DURATION)r8   r[   Zn_inputsrÑ   r)   rù  r+   Ú#test_backend_batch_statistics_reset{  s     ÿÿÿÿrú  c                 C   s  dD ]x}t t|d�jƒtƒ u s"J ‚t|dd�}t |jƒtu s@J ‚t|dd�}t |jƒtu s^J ‚t|dd�}t |jƒtu sJ ‚qtdd	dd
�}t |jƒtu sžJ ‚| d	dd��: tdd�}t |jƒtu sÈJ ‚|jdksÖJ ‚W d   ƒ n1 sê0    Y  | d	dd��@ tddd�}t |jƒtu �s"J ‚|jdk�s2J ‚W d   ƒ n1 �sH0    Y  | d	dd��> tdd�}t |jƒtu �s~J ‚|jdk�sŽJ ‚W d   ƒ n1 �s¤0    Y  | d	dd��@ tddd�}t |jƒtu �sÜJ ‚|jdk�sìJ ‚W d   ƒ n1 �s0    Y  d S )N)r   r>   ri   rj   Úthreads©r[   ÚpreferÚ	processesÚ	sharedmem©r[   Úrequirer>   r.   )r[   r8   rý  ©rý  r`   ©r  r   )rW   r!   rÐ   r4   r   r   r[   )rh   r[   rÑ   r)   r)   r+   Ú$test_backend_hinting_and_constraints“  s4    
,0
0r  rý  )Nrþ  rû  c                 C   s�   dd„ t dƒD ƒ}t| |d�dd„ t dƒD ƒƒ}||ks<J ‚t|| d��( tƒ dd„ t dƒD ƒƒ}W d   ƒ n1 sv0    Y  ||ksŒJ ‚d S )	Nc                 S   s   g | ]}|d  ‘qS r÷  r)   r]   r)   r)   r+   r,   È  r-   z7test_backend_hinting_always_running.<locals>.<listcomp>rÉ   rü  c                 s   s   | ]}t tƒ|ƒV  qd S r2   r\   r]   r)   r)   r+   r_   Ê  s   z6test_backend_hinting_always_running.<locals>.<genexpr>)rý  r[   c                 s   s   | ]}t tƒ|ƒV  qd S r2   r\   r]   r)   r)   r+   r_   Ð  r-   )rb   r!   r&   )r[   rý  Zexpected_resultsrŸ   r)   r)   r+   Ú#test_backend_hinting_always_runningÃ  s    ÿ6r  c                 C   s€  G dd„ dt ƒ}||ƒ ƒ�L tddd�}t|jƒ|u s:J ‚tddd�}t|jƒ|u sXJ ‚W d   ƒ n1 sl0    Y  G dd	„ d	t ƒ}||ƒ ƒ�¢ tddd�}t|jƒ|u s°J ‚|  ¡ \}}|d
ksÈJ ‚|d
ksÔJ ‚tdddd�}t|jƒtu sôJ ‚|  ¡ \}}d}| ¡ |k�sJ ‚|d
k�s$J ‚W d   ƒ n1 �s:0    Y  ttƒ� t|ƒ dd� W d   ƒ n1 �sr0    Y  d S )Nc                   @   s$   e Zd ZdZdZdd„ Zdd„ ZdS )z[test_backend_hinting_and_constraints_with_custom_backends.<locals>.MyCustomThreadingBackendTc                 S   s   d S r2   r)   rÔ   r)   r)   r+   Úapply_asyncÜ  s    zgtest_backend_hinting_and_constraints_with_custom_backends.<locals>.MyCustomThreadingBackend.apply_asyncc                 S   s   |S r2   r)   r=  r)   r)   r+   r$   ß  s    zltest_backend_hinting_and_constraints_with_custom_backends.<locals>.MyCustomThreadingBackend.effective_n_jobsN©rI   rJ   rK   Zsupports_sharedmemZuse_threadsr  r$   r)   r)   r)   r+   ÚMyCustomThreadingBackendØ  s   r  r>   rþ  rü  rÿ  r   c                   @   s$   e Zd ZdZdZdd„ Zdd„ ZdS )z\test_backend_hinting_and_constraints_with_custom_backends.<locals>.MyCustomProcessingBackendFc                 S   s   d S r2   r)   rÔ   r)   r)   r+   r  í  s    zhtest_backend_hinting_and_constraints_with_custom_backends.<locals>.MyCustomProcessingBackend.apply_asyncc                 S   s   |S r2   r)   r=  r)   r)   r+   r$   ð  s    zmtest_backend_hinting_and_constraints_with_custom_backends.<locals>.MyCustomProcessingBackend.effective_n_jobsNr  r)   r)   r)   r+   ÚMyCustomProcessingBackendé  s   r	  r5  rÉ   )r[   r  rp   z…Using ThreadingBackend as joblib backend instead of MyCustomProcessingBackend as the latter does not provide shared memory semantics.©r8   r  )	r   r!   rW   rÐ   Z
readouterrr   Ústripr   rN   )Zcapsysrh   r  rÑ   r	  ÚoutÚerrr¥   r)   r)   r+   Ú9test_backend_hinting_and_constraints_with_custom_backendsÔ  s,    
0
ÿ.
r  c                   C   s  t tƒ� tdd� W d   ƒ n1 s(0    Y  t tƒ� tdd� W d   ƒ n1 sZ0    Y  t tƒ� tddd� W d   ƒ n1 sŽ0    Y  td u�rt tƒ� tddd� W d   ƒ n1 sÌ0    Y  t tƒ� td	dd� W d   ƒ n1 �s0    Y  d S )
NÚinvalidr  r  rþ  rÿ  )rý  r  r.   r
  r(   )r   rN   r!   r   r)   r)   r)   r+   Ú,test_invalid_backend_hinting_and_constraints  s    
(
(
*

*
r  c                    s†   t dd��h}t|jƒj|jjfg}ˆdkr<|W  d  ƒ S |‡ ‡fdd„tdƒD ƒƒ}||d  W  d  ƒ S 1 sx0    Y  dS )zCPerform nested parallel calls and introspect the backend on the wayr>   rj   r   Nc                 3   s(   | ] }t tƒf d ˆd iˆ ¤ŽV  qdS )Úlimitr   N)r#   Ú_recursive_backend_infor]   ©r™  r  r)   r+   r_   '  s   ÿz*_recursive_backend_info.<locals>.<genexpr>r   )r!   rW   rÐ   rI   ri  rb   )r  r™  rÑ   Z
this_levelrŸ   r)   r  r+   r     s    þr  c                 C   s^   | |dd�� t ƒ }W d   ƒ n1 s(0    Y  | ¡ d }|dfdddg}||ksZJ ‚d S )Nr>   rj   ZBackendr   )r   r   )r   r>   )r  Útitle)rh   r8   Zbackend_types_and_levelsZtop_level_backend_typeZexpected_types_and_levelsr)   r)   r+   Útest_nested_parallelism_limit.  s    $ür  c                 C   s   t ƒ dd„ tdƒD ƒƒS )z6A horrible function that does recursive parallel callsc                 s   s   | ]}t tƒƒ V  qd S r2   )r#   Ú_recursive_parallelr]   r)   r)   r+   r_   A  r-   z&_recursive_parallel.<locals>.<genexpr>r>   ra   )Znesting_limitr)   r)   r+   r  ?  s    r  c              	   C   sž   | |dd��> t tƒ�}tƒ  W d   ƒ n1 s20    Y  W d   ƒ n1 sP0    Y  |j}|dkrŒddlm} t||tfƒrŒt 	d¡ t|t
ƒsšJ ‚d S )Nr>   rj   r.   r   )ÚTerminatedWorkerErrorz1Loky worker crash when serializing RecursionError)r   ÚBaseExceptionr  rŽ   Z&joblib.externals.loky.process_executorr  r“   r   r�  ÚxfailÚRecursionError)rh   r8   r7  Úexcr  r)   r)   r+   Útest_thread_bomb_mitigationD  s    
B
r  c                  C   s*   i } dD ]}t j |¡| |< q| tdƒfS )N)ÚOMP_NUM_THREADSÚOPENBLAS_NUM_THREADSÚMKL_NUM_THREADSZVECLIB_MAXIMUM_THREADSZNUMEXPR_NUM_THREADSZNUMBA_NUM_THREADSÚ
ENABLE_IPCrr   )rZ  Úenvironr‚  r   )Zenv_varsÚvarr)   r)   r+   Ú_run_parallel_sum`  s    	r#  zNeed OpenMP helper compiledc                 C   s’   t d| d�dd„ tdƒD ƒƒ}ttƒ d dƒ}|D ]Z\}}||ksFJ ‚| ¡ D ]<\}}| d¡rr|t|ƒksŠJ ‚qN|dks~J ‚|dksNJ ‚qNq2d S )	Nr>   rZ   c                 s   s   | ]}t tƒƒ V  qd S r2   )r#   r#  r©   r)   r)   r+   r_   r  s   z-test_parallel_thread_limit.<locals>.<genexpr>r   Z_THREADSr   Ú1)r!   rb   r<  r"   ÚitemsÚendswithr’   )r8   rŸ   Zexpected_num_threadsZworker_env_varsZomp_num_threadsrŒ   rŽ   r)   r)   r+   Útest_parallel_thread_limito  s    ÿ
r'  zThis test requires daskc                 C   s8   t tdd�� | dƒ W d   ƒ n1 s*0    Y  d S )NzPlease install daskr×   Zdask)r   rN   rO  r)   r)   r+   Ú)test_dask_backend_when_dask_not_installed€  s    r(  c              	   C   sŒ   G dd„ dt ƒ}d}| |ƒ ƒ�Z tjt|d��, tdd�dd„ tdƒD ƒƒ W d   ƒ n1 s`0    Y  W d   ƒ n1 s~0    Y  d S )	Nc                   @   s&   e Zd Zdd„ Zddd„Zdd„ ZdS )	z3test_zero_worker_backend.<locals>.ZeroWorkerBackendc                 _   s   dS rV   r)   )rB   r¢   r™  r)   r)   r+   r;  Œ  s    z=test_zero_worker_backend.<locals>.ZeroWorkerBackend.configureNc                 S   s   t dƒ‚d S )NzNo worker availabler   )rB   ÚfuncÚcallbackr)   r)   r+   r  �  s    z?test_zero_worker_backend.<locals>.ZeroWorkerBackend.apply_asyncc                 S   s   dS rV   r)   r=  r)   r)   r+   r$   ’  s    zDtest_zero_worker_backend.<locals>.ZeroWorkerBackend.effective_n_jobs)N)rI   rJ   rK   r;  r  r$   r)   r)   r)   r+   ÚZeroWorkerBackend‹  s   
r+  z&ZeroWorkerBackend has no active workerr×   r>   rj   c                 s   s   | ]}t tƒ|ƒV  qd S r2   rÙ   r]   r)   r)   r+   r_   ˜  r-   z+test_zero_worker_backend.<locals>.<genexpr>)r   r�  r   rÓ   r!   rb   )rh   r+  Zexpected_msgr)   r)   r+   Útest_zero_worker_backend‡  s
    
r,  c                     s”   da dd„ ‰ ˆ ƒ dksJ ‚tdd�‡ fdd„tdƒD ƒƒ} t| ƒdhksLJ ‚da ˆ ƒ dks^J ‚tdd�‡ fd	d„tdƒD ƒƒ} t| ƒdhks�J ‚d S )
Nzoriginal valuec                   S   s   t S r2   )ÚMY_GLOBAL_VARIABLEr)   r)   r)   r+   Úcheck_globals£  s    z@test_globals_update_at_each_parallel_call.<locals>.check_globalsr>   rj   c                 3   s   | ]}t ˆ ƒƒ V  qd S r2   rç   r]   ©r.  r)   r+   r_   ©  s   z<test_globals_update_at_each_parallel_call.<locals>.<genexpr>zchanged valuec                 3   s   | ]}t ˆ ƒƒ V  qd S r2   rç   r]   r/  r)   r+   r_   ³  s   )r-  r!   rb   r�   )Zworkers_global_variabler)   r/  r+   Ú)test_globals_update_at_each_parallel_call›  s    ÿÿr0  c                  C   s4   dd l } | j dd¡}|  ||¡ t d¡}| ¡ S )Nr   rr   Úthreadpoolctl)Únumpyr‡  rˆ  r„  r�  ZimportorskipZthreadpool_info)r   rC   r1  r)   r)   r+   Ú_check_numpy_threadpool_limits¿  s
    
r3  c                 C   s8   |D ] }|d | d kr|d   S qt d | ¡ƒ‚d S )NÚfilepathÚnum_threadsz,An unexpected module was loaded in child:
{})rN   r’  )Úchild_moduleÚparent_infoÚparent_moduler)   r)   r+   Ú_parent_max_num_threads_forË  s    ÿr9  c                 C   s@   | D ]6}|D ],}t ||ƒ}t||ƒ|h}|d |v sJ ‚qqd S )Nr5  )r9  Úmin)Zworkers_infor7  r5  Zchild_threadpool_infor6  Zparent_max_num_threadsr¥   r)   r)   r+   Úcheck_child_num_threadsÔ  s    ÿr;  )r>   r‘   ro   ri   c                 C   sz   t ƒ }t|ƒdkrtjdd� td| d�dd„ tdƒD ƒƒ}t| ƒ} | d	krZ|d d
 }nttƒ |  d	ƒ}t	|||ƒ d S )Nr   ú&Need a version of numpy linked to BLASr3  r.   r×  c                 s   s   | ]}t tƒƒ V  qd S r2   ©r#   r3  r]   r)   r)   r+   r_   í  s   z;test_threadpool_limitation_in_child_loky.<locals>.<genexpr>r>   r   r5  ©
r3  r…   r�  r�  r!   rb   r$   r<  r"   r;  )r[   r7  Úworkers_threadpool_infosÚexpected_child_num_threadsr)   r)   r+   Ú(test_threadpool_limitation_in_child_lokyá  s    ÿÿrA  Úinner_max_num_threads)r   r>   r‘   Nc                 C   s²   t ƒ }t|ƒdkrtjdd� | d|d��, t|d�dd„ td	ƒD ƒƒ}W d   ƒ n1 s\0    Y  t|ƒ}|d
kr„|d d }n|d u ržttƒ | d
ƒ}n|}t	|||ƒ d S )Nr   r<  r3  r.   ©rB  rj   c                 s   s   | ]}t tƒƒ V  qd S r2   r=  r]   r)   r)   r+   r_     s   z>test_threadpool_limitation_in_child_context.<locals>.<genexpr>r>   r   r5  r>  )rh   r[   rB  r7  r?  r@  r)   r)   r+   Ú+test_threadpool_limitation_in_child_contextü  s     
ÿ&ÿrD  Úvar_name)r  r  r  c              	      s  t |ƒdkrt d¡ tdd� ¡  dd„ ‰ tj ˆ¡}zºdtjˆ< t|d�‡ ‡fd	d
„t	dƒD ƒƒ}|ddgksvJ ‚| ddd��2 t|d�‡ ‡fdd
„t	dƒD ƒƒ}W d   ƒ n1 sº0    Y  |ddgksÔJ ‚W |d u rètjˆ= n
|tjˆ< n |d u �rtjˆ= n
|tjˆ< 0 d S )Nr   zSkip test when n_jobs == 1T©Zreusec                 S   s   t j | ¡S r2   )rZ  r!  r‚  )rE  r)   r)   r+   Ú_get_env,  s    z>test_threadpool_limitation_in_child_override.<locals>._get_envÚ4rj   c                 3   s   | ]}t ˆ ƒˆƒV  qd S r2   rç   r]   ©rG  rE  r)   r+   r_   3  r-   z?test_threadpool_limitation_in_child_override.<locals>.<genexpr>r>   r.   rC  c                 3   s   | ]}t ˆ ƒˆƒV  qd S r2   rç   r]   rI  r)   r+   r_   7  s   r$  )
r$   r�  r�  r   ÚshutdownrZ  r!  r‚  r!   rb   )rh   r[   rE  Zoriginal_var_valuerŸ   r)   rI  r+   Ú,test_threadpool_limitation_in_child_override  s(    	

"ÿ&
ý

rK  )r>   r‘   ri   c                 C   sJ   dd„ }|| ƒ t dd�}tdƒD ]"}|| ƒ t dd�}||ks"J ‚q"d S )Nc                    s,   t dƒ‰ t| d�‡ fdd„t dƒD ƒƒ d S )NrÉ   rj   c                 3   s   | ]}t tƒˆ ƒV  qd S r2   )r#   Úsumr]   r?   r)   r+   r_   K  r-   zAtest_loky_reuse_workers.<locals>.parallel_call.<locals>.<genexpr>rw   rj   r)   r?   r+   Úparallel_callI  s    z.test_loky_reuse_workers.<locals>.parallel_callTrF  rÉ   )r   rb   )r[   rM  Zfirst_executorrª   rˆ   r)   r)   r+   Útest_loky_reuse_workersC  s    

rN  c                 C   s   d| t  ¡ < d S )NÚinitialized)rZ  r[  )Ústatusr)   r)   r+   Ú_set_initializedY  s    rQ  c                 C   sª   t  ¡ }|  |d ¡}|dv s*J d|› �ƒ‚|s2d S d| |< t ¡ d }tdd„ |  ¡ D ƒƒ}t ¡ |k r’||k r’t d¡ tdd„ |  ¡ D ƒƒ}q\t ¡ |kr¦td	ƒ‚|S )
N)rO  Ústartedz2worker should have been in initialized state, got rR  rî   c                 S   s   g | ]\}}|d kr|‘qS ©rR  r)   ©r*   ÚpidrÍ  r)   r)   r+   r,   i  r-   z!_check_status.<locals>.<listcomp>r²   c                 S   s   g | ]\}}|d kr|‘qS rS  r)   rT  r)   r)   r+   r,   l  r-   z-Waited more than 30s to start all the workers)rZ  r[  r‚  rQ   r…   r%  r   r   )rP  r[   Úwait_workersrU  ÚstateÚdeadlineZ	n_startedr)   r)   r+   Ú_check_status]  s     
ÿ
rY  c                    sd   t  ¡ }| ¡ ‰||ˆ tˆfd��. tƒ ‡ ‡fdd„tdƒD ƒƒ W d   ƒ n1 sV0    Y  d S )N©r8   r[   ZinitializerZinitargsc                 3   s   | ]}t tƒˆˆ ƒV  qd S r2   ©r#   rY  r]   ©r[   rP  r)   r+   r_   „  r-   z+test_initializer_context.<locals>.<genexpr>rr   )r   r$  ÚdictrQ  r!   rb   )r[   r8   rh   r&  r)   r\  r+   Útest_initializer_contextt  s    ür^  c                    s>   t  ¡ }| ¡ ‰t|ˆ tˆfd�‡ ‡fdd„tdƒD ƒƒ d S )NrZ  c                 3   s   | ]}t tƒˆˆ ƒV  qd S r2   r[  r]   r\  r)   r+   r_   •  r-   z,test_initializer_parallel.<locals>.<genexpr>rr   )r   r$  r]  r!   rQ  rb   )r[   r8   r&  r)   r\  r+   Útest_initializer_parallel‡  s    üûr_  c                    sx   d}t  ¡ }| ¡ ‰tƒ }t|ƒD ]<}tdˆ tˆfd�‡ ‡fdd„tˆ ƒD ƒƒ}| t|ƒ¡}q"t|ƒˆ kstJ dƒ‚d S )Nr`   r.   rZ  c                 3   s    | ]}t tƒˆˆ d d�V  qdS ©T)rV  Nr[  r]   r\  r)   r+   r_   ¨  s   ÿz*test_initializer_reused.<locals>.<genexpr>z=The workers should be reused when the initializer is the same)	r   r$  r]  r�   rb   r!   rQ  Úunionr…   ©r[   Zn_repetitionsr&  Zpidsr^   rŸ   r)   r\  r+   Útest_initializer_reused˜  s&    üþû	ÿrc  c                    s|   d}t  ¡ }tƒ }t|ƒD ]D}| ¡ ‰tdˆ tˆfd�‡ ‡fdd„tˆ ƒD ƒƒ}| t|ƒ¡}qt|ƒ|ˆ  ksxJ dƒ‚d S )Nr`   r.   rZ  c                 3   s    | ]}t tƒˆˆ d d�V  qdS r`  r[  r]   r\  r)   r+   r_   Ã  s   ÿz.test_initializer_not_reused.<locals>.<genexpr>zFThe workers should not be reused when the initializer arguments change)	r   r$  r�   rb   r]  r!   rQ  ra  r…   rb  r)   r\  r+   Útest_initializer_not_reused²  s&    üþû	ÿrd  )F)r   r   )r`   )N)F)ÓrL   r¿  rZ  r   r“  r/   rQ   r›   rî  rt  r   Úmathr   r(   r   Úpickler   r   Ú	tracebackr	   r�  r¡  r
   r   r   Zjoblib._multiprocessing_helpersr   Zjoblib.test.commonr   r   r   r   Zjoblib.testingr   r   r   r   r   Zjoblib.externals.lokyr   r
  r   ÚposixÚImportErrorZ _openmp_test_helper.parallel_sumr   ZdistributedZjoblib._parallel_backendsr   r   r   r   r   Zjoblib.parallelr    r!   r"   r#   r$   r%   r&   r'   r²  ZRETURN_GENERATOR_BACKENDSÚpoprÕ  ÚkeysZALL_VALID_BACKENDSZPROCESS_BACKENDSZPARALLEL_BACKENDSÚhasattrrz   r0   r4   r9   r<   r@   Ú	ExceptionrA   rP   rS   rU   rY   rd   re   rg   Úmarkrn   rx   r‹   r    r¦   r±   r»   r¾   r¿   rÀ   rÁ   rÈ   rÏ   rÒ   rÝ   rí   rñ   ró   r�   rú   r  r	  r  r"  r'  r+  r,  r.  r1  r2  r  r9  r:  r>  r@  rE  rF  rG  rI  rN  rQ  Úextendrb   rR  rS  rW  rY  r\  r]  Z_parallel_backendsr^  re  rh  rl  rp  rw  ry  r€  rƒ  rŠ  r‘  r–  rž  rœ  ZSQUARE_LOCALZSQUARE_LAMBDAr¦  r’  rŸ  r   r¢  r§  rª  r¬  r­  r¯  r³  r»  r½  rÅ  rÇ  rÉ  rÎ  rÑ  rÖ  rØ  rÜ  rß  rà  rá  rå  rç  rì  ró  rö  rø  rú  r  r  r  r  r  r  r  Zno_coverr  r#  r'  r(  r,  r0  r3  r9  r;  rA  rD  rK  rN  rQ  rY  r^  r_  rc  rd  r)   r)   r)   r+   Ú<module>   sÌ  



,


ý
*þ




Fþþîþ(


	
		





	
þ


 (Ø+
ê


	


þ,"ÿ.6$	#