U
    NvfF                     @   s"  d dl mZmZmZ d dlZd dlZd dlZd dlmZ d dlm	Z	 d dl
mZ ddlmZmZmZ ddlmZmZ dd	lmZ ed
ZedZd dlmZmZmZ d dlm
Z
 d dlmZmZ dd ZdVddZ dd Z!dd Z"dd Z#dd Z$dd Z%ej&'ddd gd!d" Z(d#d$ Z)G d%d& d&e*Z+dWd'd(Z,d)d* Z-d+d, Z.ej&'d-e/e0dd.d/ Z1d0d1 Z2d2d3 Z3d4d5 Z4d6d7 Z5d8d9 Z6d:d; Z7d<d= Z8d>d? Z9d@dA Z:dBdC Z;dDdE Z<ej&'dFdGdHgej&j=ej>dIkoej>dJkdKdLdMdN Z?dOdP Z@ej&'dQdRdSgdTdU ZAdS )X    )print_functiondivisionabsolute_importNrandom)uuid4)sleep   )Paralleldelayedparallel_config)ThreadingBackendAutoBatchingMixin)DaskDistributedBackenddistributeddask)ClientLocalCluster
get_client)time)clusterincc                  O   s   d S N )argskwargsr   r   9/tmp/pip-unpacked-wheel-dylwa62s/joblib/test/test_dask.pynoop   s    r   皙?c                 C   s   t | | rtdd S )Nzcondition evaluated to True)r   
ValueError)	conditiondurationr   r   r   slow_raise_value_error   s    r"   c                    sF   | dd }i }| D ]&\}}t fddt|D ||< q|S )Nc                 S   s   | j S r   )log)Zdask_workerr   r   r   <lambda>"       zcount_events.<locals>.<lambda>c                    s   g | ]}|d   kr|qS    r   ).0event
event_namer   r   
<listcomp>%   s    z count_events.<locals>.<listcomp>)runitemslenlist)r+   clientZworker_eventsZevent_countsweventsr   r*   r   count_events!   s
     r4   c                 C   s   t  \}\}}t|d | d}tdd t dd tdD }|dd	 tdD ks`ttt t d
d tdD  W 5 Q R X t dd tdD }|dd	 tdD kstW 5 Q R X W 5 Q R X W 5 Q R X d S )Naddressloopr   backendc                 s   s   | ]}t t|V  qd S r   r   r   r(   ir   r   r   	<genexpr>.   s     ztest_simple.<locals>.<genexpr>
   c                 S   s   g | ]}t |qS r   r   r;   r   r   r   r,   /   s     ztest_simple.<locals>.<listcomp>c                 s   s   | ]}t t|d kV  qdS )   N)r   r"   r;   r   r   r   r=   2   s   c                 s   s   | ]}t t|V  qd S r   r:   r;   r   r   r   r=   5   s     c                 S   s   g | ]}t |qS r   r?   r;   r   r   r   r,   6   s     )	r   r   r   r
   rangeAssertionErrorpytestraisesr   r7   sabr1   seqr   r   r   test_simple*   s    
rJ   c                 C   s   t jtjkstt \}\}}t|d | d}tddp t ^}|j}t	|t sZt|j
|ksht|jdksvt|dd ttdD  |jd	kstW 5 Q R X W 5 Q R X W 5 Q R X W 5 Q R X d S )
Nr5   r6   r   r8   r'   c                 s   s   | ]}t d d  V  qdS )c                   S   s   d S r   r   r   r   r   r   r$   K   r%   z?test_dask_backend_uses_autobatching.<locals>.<genexpr>.<lambda>Nr   r(   _r   r   r   r=   J   s   z6test_dask_backend_uses_autobatching.<locals>.<genexpr>     @r>   )r   Zcompute_batch_sizer   rB   r   r   r   r
   _backend
isinstanceparallelZ_effective_batch_sizerA   int)r7   rF   rG   rH   r1   rQ   r9   r   r   r   #test_dask_backend_uses_autobatching9   s    
rS   c                   C   s   t  S r   r   r   r   r   r   random2Q   s    rT   c                 C   sx   t  h\}\}}t|d | dD}tdd. t dd tdD \}}||ksVtW 5 Q R X W 5 Q R X W 5 Q R X d S )Nr5   r6   r   r8   c                 s   s   | ]}t t V  qd S r   )r   rT   r;   r   r   r   r=   Y   s     z3test_dont_assume_function_purity.<locals>.<genexpr>r	   r   r   r   r
   rA   rB   )r7   rF   rG   rH   r1   xyr   r   r    test_dont_assume_function_purityU   s
    rX   mixedTFc                 C   s   ddl m} |s(dd tdD }d}ndd tdD }d}t|||ksRtt \}\}}t|d	 | d
`}tdd tddd|}	W 5 Q R X dd }
|	dd}|
|
}tdd |D stW 5 Q R X W 5 Q R X d S )Nr   )Batchc                 S   s   g | ]}t t|qS r   r:   r;   r   r   r   r,   a   s     z&test_dask_funcname.<locals>.<listcomp>   Zbatch_of_inc_4_callsc                 S   s,   g | ]$}|d  rt t|n
t t|qS )r	   )r   absr   r;   r   r   r   r,   d   s    Zmixed_batch_of_inc_4_callsr5   r6   r   r8   r	   all)Z
batch_sizeZpre_dispatchc                 S   s
   t | jS r   )r0   Ztransition_log)Zdask_schedulerr   r   r   fp   s    ztest_dask_funcname.<locals>.f42c                 s   s   | ]}d |d kV  qdS )Zbatch_of_incr   Nr   )r(   tupr   r   r   r=   t   s     z%test_dask_funcname.<locals>.<genexpr>)Zjoblib._daskrZ   rA   reprrB   r   r   r   r
   replaceZrun_on_schedulerr]   )r7   rY   rZ   tasksZ
batch_reprrF   rG   rH   r1   rM   r^   r#   r   r   r   test_dask_funcname]   s"    
re   c              	      sN  dd t dD }tdtd d fdd	tdd	d
}t|}ztdd t	 fdd|D }W 5 Q R X |dd t dD kst
td|}t| dkst
tdd |D st
tdd  t	  fdd|D }W 5 Q R X td|}t| dkst
tdd |D s4t
W 5 |  |  X d S )Nc                 S   s   g | ]}g qS r   r   rL   r   r   r   r,      s     z;test_no_undesired_distributed_cache_hit.<locals>.<listcomp>d   numpyg    .Ac                    s(   |d k	rj |  | t j | S r   )ZtestingZassert_array_equalappendr   hex)list_data)Xnpr   r   isolated_operation   s    zCtest_no_undesired_distributed_cache_hit.<locals>.isolated_operationr'   r	   )	n_workersthreads_per_workerr   r8   c                 3   s   | ]}t  |V  qd S r   rK   r(   rj   )rn   r   r   r=      s    z:test_no_undesired_distributed_cache_hit.<locals>.<genexpr>c                 S   s   g | ]}g qS r   r   rL   r   r   r   r,      s     receive-from-scatterr   c                 S   s   g | ]}t |d kqS r&   r/   r(   rr   r   r   r,      s     c                 3   s   | ]}t | d V  qdS ))rk   NrK   rq   )rl   rn   r   r   r=      s    c                 S   s   g | ]}t |d kqS r&   rs   rt   r   r   r   r,      s     )N)rA   rC   importorskipZarangerR   r   r   closer   r
   rB   r4   sumvaluesr]   )r7   listsr   r1   rescountsr   )rl   rn   rm   r   'test_no_undesired_distributed_cache_hitw   s0    


r}   c                   @   s(   e Zd Zdd Zdd ZeZdd ZdS )CountSerializedc                 C   s   || _ d| _d S )Nr   )rV   count)selfrV   r   r   r   __init__   s    zCountSerialized.__init__c                 C   s   | j t|d| S )NrV   )rV   getattr)r   otherr   r   r   __add__   s    zCountSerialized.__add__c                 C   s   |  j d7  _ t| jffS )Nr'   )r   r~   rV   )r   r   r   r   
__reduce__   s    zCountSerialized.__reduce__N)__name__
__module____qualname__r   r   __radd__r   r   r   r   r   r~      s   r~   c                 C   s   | | | | | S r   r   )rG   rH   cder   r   r   add5   s    r   c                 C   s@  t d}t d}t d}t \}\}}t|d | d}td||gdj tt}||||dd	d
||||d	dd
|||||d	d
||||||d
g}	dd |	D }
t |	}W 5 Q R X tt	  td| dd W 5 Q R X W 5 Q R X W 5 Q R X W 5 Q R X ||
kst
|jdkst
|jdks,t
|jdks<t
d S )Nr'   r	   r@   r5   r6   r   )r9   scatterr[      )r   r   c                 S   s   g | ]\}}}|||qS r   r   )r(   funcr   r   r   r   r   r,      s   z'test_manual_scatter.<locals>.<listcomp>)r9   r7   r   )r[      )r~   r   r   r   r   r   r
   rC   rD   	TypeErrorrB   r   )r7   rV   rW   zrF   rG   rH   r1   r^   rd   expectedresultsr   r   r   test_manual_scatter   s.    (r   c           	         sv  t d}|jtd|jd |jtd|jd} gd |gd  }t \}\}}t|d | d^}tdd t d	d
 t	|D  W 5 Q R X t
d|}||d  ||d   dkstW 5 Q R X W 5 Q R X t \}\}}t|d | dn}tdd" t  fdd
tdD  W 5 Q R X t
d|}||d  dksHt||d  dks^tW 5 Q R X W 5 Q R X d S )Nrg   rN   )Zdtyper@   r5   r6   r   r8   c                 s   s&   | ]\}}t t||||d V  qdS ))optNr   r   )r(   r<   rk   r   r   r   r=      s   z$test_auto_scatter.<locals>.<genexpr>rr   r	   c                 3   s$   | ]}t t d d |V  qd S )Nr@   r   r;   Zdata1r   r   r=      s     r   r   )rC   rv   onesrR   Zuint8r   r   r   r
   	enumerater4   rB   rA   )	loop_in_threadrm   Zdata2Zdata_to_processrF   rG   rH   r1   r|   r   r   r   test_auto_scatter   s&    


4&
r   retry_noc                    s   t dd d}fdd fddt h\}\}}t|d | dD}td	d
. dt fddt|D }W 5 Q R X W 5 Q R X W 5 Q R X d S )Nrg   r>   c                    s
     | S r   )rx   )rV   r<   j)rm   r   r   my_sum  s    z#test_nested_scatter.<locals>.my_sumc              	      sD   t  }tdd& t  fddtD }W 5 Q R X t|S )Nr   r8   c                 3   s&   | ]}t  |d  |V  qd S r   rK   )r(   r   )arrayr<   r   r   r   r=     s    zEtest_nested_scatter.<locals>.outer_function_joblib.<locals>.<genexpr>)r   r   r
   rA   rx   )r   r<   r1   r   )NUM_INNER_TASKSr   )r   r<   r   outer_function_joblib  s    z2test_nested_scatter.<locals>.outer_function_joblibr5   r6   r   r8   i'  c                 3   s$   | ]}t  |d  |V  qd S r   rK   r;   )my_arrayr   r   r   r=     s
   
 z&test_nested_scatter.<locals>.<genexpr>)rC   rv   r   r   r   r   r
   rA   )r7   r   ZNUM_OUTER_TASKSrF   rG   rH   rM   r   )r   r   r   rm   r   r   test_nested_scatter  s    
	
r   c                    s  dd  t  \}\}}t|d | dZ}tddD tdd fd	d
tdD }|D ]}tt|dksZtqZW 5 Q R X W 5 Q R X t|d | dZ}tddD tdd fdd
tdD }|D ]}tt|dkstqW 5 Q R X W 5 Q R X W 5 Q R X d S )Nc                  S   sH   t tdddd tdD } | t tdddd tdD O } | S )Nr	   n_jobsc                 s   s   | ]}t tj V  qd S r   r   osgetpidrL   r   r   r   r=   &  s     zOtest_nested_backend_context_manager.<locals>.get_nested_pids.<locals>.<genexpr>c                 s   s   | ]}t tj V  qd S r   r   rL   r   r   r   r=   '  s     )setr
   rA   )Zpidsr   r   r   get_nested_pids%  s     $z<test_nested_backend_context_manager.<locals>.get_nested_pidsr5   r6   r   r8   r	   r   c                 3   s   | ]}t   V  qd S r   rK   rL   r   r   r   r=   -  s   z6test_nested_backend_context_manager.<locals>.<genexpr>r>   c                 3   s   | ]}t   V  qd S r   rK   rL   r   r   r   r=   7  s   )r   r   r   r
   rA   r/   r   rB   )r   rF   rG   rH   r1   Z
pid_groupsZ	pid_groupr   r   r   #test_nested_backend_context_manager$  s     *r   c           	         s   dd   fddt  \}\}}t|d | d}tddv t >} |d	ksZt|jd
ksht|fddtdD }W 5 Q R X |D ] \}}|d	kst|d
kstqW 5 Q R X W 5 Q R X W 5 Q R X d S )Nc                 S   s
   | j jjS r   )rO   	__class__r   pr   r   r   _backend_typeC  s    zJtest_nested_backend_context_manager_implicit_n_jobs.<locals>._backend_typec               
      s,   t  }  | | jfW  5 Q R  S Q R X d S r   )r
   r   r   )r   r   r   get_nested_implicit_n_jobsF  s    zWtest_nested_backend_context_manager_implicit_n_jobs.<locals>.get_nested_implicit_n_jobsr5   r6   r   r8   r   c                 3   s   | ]}t   V  qd S r   rK   rL   )r   r   r   r=   P  s   zFtest_nested_backend_context_manager_implicit_n_jobs.<locals>.<genexpr>r	   )r   r   r   r
   rB   r   rA   )	r7   rF   rG   rH   r1   r   Zall_nested_n_jobsZbackend_typeZnested_n_jobsr   )r   r   r   3test_nested_backend_context_manager_implicit_n_jobs?  s    r   c              
   C   sF   t t}tdd W 5 Q R X W 5 Q R X dt|j ksBtd S )Nr   r8   zcreate a dask client)rC   rD   r   r   strvaluelowerrB   )r7   infor   r   r   test_errorsY  s    r   c                 C   s   t  \}\}}t|d | d}tdd< tdddd td	D }t|d
 d
 d
 tsdtW 5 Q R X tdd< tdddd td	D }t|d
 d
 d
 tstW 5 Q R X W 5 Q R X W 5 Q R X d S )Nr5   r6   r   r8   r	   r   c                 s   s   | ]}t td dV  qd S )NZnested_requirer   outerrL   r   r   r   r=   f  s    z.test_correct_nested_backend.<locals>.<genexpr>r'   r   c                 s   s   | ]}t td dV  qdS )Z	sharedmemr   Nr   rL   r   r   r   r=   l  s   )	r   r   r   r
   rA   rP   r   rB   r   )r7   rF   rG   rH   r1   resultr   r   r   test_correct_nested_backenda  s    $r   c                    s"   t ddd fddtdD S )Nr	   threads)r   Zpreferc                 3   s   | ]}t t V  qd S r   )r   middlerL   r   r   r   r=   s  s    zouter.<locals>.<genexpr>r'   r
   rA   r   r   r   r   r   r  s    r   c                 C   s   t d| ddd tdD S )Nr	   )r   requirec                 s   s   | ]}t t V  qd S r   )r   innerrL   r   r   r   r=   y  s    zmiddle.<locals>.<genexpr>r'   r   )r   r   r   r   r   x  s    r   c                   C   s   t  jS r   )r
   rO   r   r   r   r   r   ~  s    r   c              
   C   sP   t | ddd8 tdd" tdddd	 td
D  W 5 Q R X W 5 Q R X d S )NFT)r7   	processesZset_as_defaultr   r8   r[   r   c                 s   s   | ]}t t|V  qd S r   )r   idr;   r   r   r   r=     s     z0test_secede_with_no_processes.<locals>.<genexpr>r	   )r   r   r
   rA   r6   r   r   r   test_secede_with_no_processes  s    r   c                 C   s   ddl m} | jS )Nr   )
get_worker)r   r   r5   )rM   r   r   r   r   _worker_address  s    r   c                 C   s   t  \}\}}t|d | d}td|d d4 t dd tdD }||d gd ksbtW 5 Q R X td|d d4 t dd tdD }||d gd kstW 5 Q R X W 5 Q R X W 5 Q R X d S )	Nr5   r6   r   )r9   workersc                 s   s   | ]}t t|V  qd S r   r   r   r;   r   r   r   r=     s    z-test_dask_backend_keywords.<locals>.<genexpr>r>   c                 s   s   | ]}t t|V  qd S r   r   r;   r   r   r   r=     s    rU   rE   r   r   r   test_dask_backend_keywords  s    
 
r   c              
   C   s   t d| dj}tdd t dd tdD  W 5 Q R X t }|jjjrhtd t |d	 k sBt	qB|j
rrt	W 5 Q R X d S )
NF)r   r7   r   r8   c                 s   s   | ]}t t|V  qd S r   r:   r;   r   r   r   r=     s     ztest_cleanup.<locals>.<genexpr>r>   g{Gz?r   )r   r   r
   rA   r   r   Z	schedulerrd   r   rB   Zfutures)r7   r1   startr   r   r   test_cleanup  s    "
r   cluster_strategyadaptivelate_scalingz2.1.1z1.28.0z?distributed bug - https://github.com/dask/distributed/pull/2841)reasonc              	   C   s   t dddd}t|}| dkr.|jddd n| dkr@|d z2tdd	 t d
d tdD  W 5 Q R X W 5 |  |  X d S )Nr   Fr	   ro   r   rp   r   )Zminimummaximumr   r   r8   c                 s   s   | ]}t t|V  qd S r   r:   r;   r   r   r   r=     s     z(test_wait_for_workers.<locals>.<genexpr>r>   )r   r   ZadaptZscalerw   r   r
   rA   )r   r   r1   r   r   r   test_wait_for_workers  s    
&r   c               
   C   s   t dddd} t| }ztddd< d}tjt|d	 t d
d tdD  W 5 Q R X W 5 Q R X tddd< d}tjt	|d	 t dd tdD  W 5 Q R X W 5 Q R X W 5 |  |   X d S )Nr   Fr	   r   r   g?)r9   Zwait_for_workers_timeoutz7DaskDistributedBackend has no worker after 0.1 seconds.)matchc                 s   s   | ]}t t|V  qd S r   r:   r;   r   r   r   r=     s     z0test_wait_for_workers_timeout.<locals>.<genexpr>r>   z+DaskDistributedBackend has no active workerc                 s   s   | ]}t t|V  qd S r   r:   r;   r   r   r   r=     s     )
r   r   rw   r   rC   rD   TimeoutErrorr
   rA   RuntimeError)r   r1   msgr   r   r   test_wait_for_workers_timeout  s    ,0r   r9   Zlokymultiprocessingc                    sl   t dd}t|} fdd}||}| }t|dks@t|d j}t|tsXtdt	|kshtd S )Nr	   )ro   c               	      s:   t jdd$} td ddd tdD  W 5 Q R X | S )NTrecordr	   )r   r9   c                 s   s   | ]}t t|V  qd S r   r:   r;   r   r   r   r=     s    zftest_joblib_warning_inside_dask_daemonic_worker.<locals>.func_using_joblib_parallel.<locals>.<genexpr>r>   )warningscatch_warningsr
   rA   r   r8   r   r   func_using_joblib_parallel  s
    zStest_joblib_warning_inside_dask_daemonic_worker.<locals>.func_using_joblib_parallelr'   r   zdistributed.worker.daemon)
r   r   Zsubmitr   r/   rB   messagerP   UserWarningr   )r9   r   r1   r   Zfutr   warningr   r8   r   /test_joblib_warning_inside_dask_daemonic_worker  s    


r   )r   )r   r   )B
__future__r   r   r   r   r   rC   r   uuidr   r   r    r
   r   r   rQ   r   r   Z_daskr   rv   r   r   r   r   r   Zdistributed.metricsZdistributed.utils_testr   r   r   r"   r4   rJ   rS   rT   rX   markZparametrizere   r}   objectr~   r   r   r   r0   rA   r   r   r   r   r   r   r   r   r   r   r   r   Zskipif__version__r   r   r   r   r   r   r   <module>   sd   


	
9
%
