3
dB                 @   s  d dl mZmZmZ d dlZd dlZd dlmZ d dlmZ d dl	m
Z
 ddlmZmZmZ ddlmZmZ dd	lmZ ejd
Zd dlmZmZmZ d dlm	Z	 d dlmZmZ dd ZdPddZdd Zdd Z dd Z!dd Z"dd Z#ej$j%dddgd d! Z&d"d# Z'G d$d% d%e(Z)dQd&d'Z*d(d) Z+d*d+ Z,ej$j%d,e-e.dd-d. Z/d/d0 Z0d1d2 Z1d3d4 Z2d5d6 Z3d7d8 Z4d9d: Z5d;d< Z6d=d> Z7d?d@ Z8dAdB Z9dCdD Z:ej$j%dEdFdGgej$j;ej<dHkoej<dIkdJdKdLdM Z=dNdO Z>dS )R    )print_functiondivisionabsolute_importN)random)uuid4)sleep   )Paralleldelayedparallel_backend)ThreadingBackendAutoBatchingMixin)DaskDistributedBackenddistributed)ClientLocalCluster
get_client)time)clusterincc              O   s   d S )N )argskwargsr   r   7/tmp/pip-build-v9q4h5k9/joblib/joblib/test/test_dask.pynoop   s    r   皙?c             C   s   t | | rtdd S )Nzcondition evaluated to True)r   
ValueError)	conditiondurationr   r   r   slow_raise_value_error   s    r   c                sJ   |j dd }i }x2|j D ]&\}}t fddt|D ||< qW |S )Nc             S   s   | j S )N)log)Zdask_workerr   r   r   <lambda>   s    zcount_events.<locals>.<lambda>c                s   g | ]}|d   kr|qS )   r   ).0event)
event_namer   r   
<listcomp>!   s    z count_events.<locals>.<listcomp>)runitemslenlist)r%   clientZworker_eventsZevent_countsweventsr   )r%   r   count_events   s
    "r.   c              C   s   t  \}\}}t|d | d}td\}}t dd tdD }|dd tdD ksbttjt t d	d tdD  W d Q R X t d
d tdD }|dd tdD kstW d Q R X W d Q R X W d Q R X d S )Naddress)loopdaskc             s   s   | ]}t t|V  qd S )N)r
   r   )r#   ir   r   r   	<genexpr>*   s    ztest_simple.<locals>.<genexpr>
   c             S   s   g | ]}t |qS r   )r   )r#   r2   r   r   r   r&   +   s    ztest_simple.<locals>.<listcomp>c             s   s   | ]}t t|d kV  qdS )   N)r
   r   )r#   r2   r   r   r   r3   .   s   c             s   s   | ]}t t|V  qd S )N)r
   r   )r#   r2   r   r   r   r3   1   s    c             S   s   g | ]}t |qS r   )r   )r#   r2   r   r   r   r&   2   s    )	r   r   r   r	   rangeAssertionErrorpytestraisesr   )r0   sabr+   ba_seqr   r   r   test_simple&   s    
r@   c       	   !   C   s   t jtjkstt \}\}}t|d | d}tdt\}}t ^}|j}t	|t s\t|j
|ksjt|jdksxt|dd ttdD  |jdkstW d Q R X W d Q R X W d Q R X W d Q R X d S )	Nr/   )r0   r1   r"   c             s   s   | ]}t d d  V  qdS )c               S   s   d S )Nr   r   r   r   r   r!   G   s    z?test_dask_backend_uses_autobatching.<locals>.<genexpr>.<lambda>N)r
   )r#   r>   r   r   r   r3   G   s   z6test_dask_backend_uses_autobatching.<locals>.<genexpr>g     @r4   )r   Zcompute_batch_sizer   r7   r   r   r   r	   _backend
isinstanceparallelZ_effective_batch_sizer6   int)	r0   r:   r;   r<   r+   r=   r>   rC   backendr   r   r   #test_dask_backend_uses_autobatching5   s    rF   c               C   s   t  S )N)r   r   r   r   r   random2M   s    rG   c       	      C   sz   t  j\}\}}t|d | dF}td2\}}t dd tdD \}}||ksXtW d Q R X W d Q R X W d Q R X d S )Nr/   )r0   r1   c             s   s   | ]}t t V  qd S )N)r
   rG   )r#   r2   r   r   r   r3   U   s    z3test_dont_assume_function_purity.<locals>.<genexpr>r   )r   r   r   r	   r6   r7   )	r0   r:   r;   r<   r+   r=   r>   xyr   r   r    test_dont_assume_function_purityQ   s
    rJ   mixedTFc             C   s   ddl m} |s(dd tdD }d}ndd tdD }d}t|||ksRtt \}\}}t|d	 | d
b}td\}	}
tddd|}
W d Q R X dd }|j	dd}|j
|}tdd |D stW d Q R X W d Q R X d S )Nr   )Batchc             S   s   g | ]}t t|qS r   )r
   r   )r#   r2   r   r   r   r&   ]   s    z&test_dask_funcname.<locals>.<listcomp>   Zbatch_of_inc_4_callsc             S   s,   g | ]$}|d  rt t|n
t t|qS )r   )r
   absr   )r#   r2   r   r   r   r&   a   s    Zmixed_batch_of_inc_4_callsr/   )r0   r1   r   all)Z
batch_sizeZpre_dispatchc             S   s
   t | jS )N)r*   Ztransition_log)Zdask_schedulerr   r   r   fl   s    ztest_dask_funcname.<locals>.f42c             s   s   | ]}d |d kV  qdS )Zbatch_of_incr   Nr   )r#   tupr   r   r   r3   p   s    z%test_dask_funcname.<locals>.<genexpr>)Zjoblib._daskrL   r6   reprr7   r   r   r   r	   replaceZrun_on_schedulerrO   )r0   rK   rL   tasksZ
batch_reprr:   r;   r<   r+   r=   r>   rP   r    r   r   r   test_dask_funcnameY   s    
rW   c                sR  dd t dD }tjdjtd d fdd	tdd	d
}t|}ztd"\}}t fdd|D }W d Q R X |dd t dD kst	t
d|}t|j dkst	tdd |D st	td$\}}t  fdd|D }W d Q R X t
d|}t|j dks t	tdd |D s8t	W d |j  |j  X d S )Nc             S   s   g | ]}g qS r   r   )r#   r>   r   r   r   r&   ~   s    z;test_no_undesired_distributed_cache_hit.<locals>.<listcomp>d   numpyg    .Ac                s(   |d k	rj j|  | jt j | S )N)ZtestingZassert_array_equalappendr   hex)list_data)Xnpr   r   isolated_operation   s    zCtest_no_undesired_distributed_cache_hit.<locals>.isolated_operationr"   r   )	n_workersthreads_per_workerr1   c             3   s   | ]}t  |V  qd S )N)r
   )r#   r\   )r`   r   r   r3      s    z:test_no_undesired_distributed_cache_hit.<locals>.<genexpr>c             S   s   g | ]}g qS r   r   )r#   r>   r   r   r   r&      s    zreceive-from-scatterr   c             S   s   g | ]}t |d kqS )r"   )r)   )r#   rr   r   r   r&      s    c             3   s   | ]}t | d V  qdS ))r]   N)r
   )r#   r\   )r^   r`   r   r   r3      s    c             S   s   g | ]}t |d kqS )r"   )r)   )r#   rc   r   r   r   r&      s    )N)r6   r8   importorskipZarangerD   r   r   r   r	   r7   r.   sumvaluesrO   close)r0   listsr   r+   r=   r>   rescountsr   )r^   r`   r_   r   'test_no_undesired_distributed_cache_hits   s,    

 
rk   c               @   s(   e Zd Zdd Zdd ZeZdd ZdS )CountSerializedc             C   s   || _ d| _d S )Nr   )rH   count)selfrH   r   r   r   __init__   s    zCountSerialized.__init__c             C   s   | j t|d| S )NrH   )rH   getattr)rn   otherr   r   r   __add__   s    zCountSerialized.__add__c             C   s   |  j d7  _ t| jffS )Nr"   )rm   rl   rH   )rn   r   r   r   
__reduce__   s    zCountSerialized.__reduce__N)__name__
__module____qualname__ro   rr   __radd__rs   r   r   r   r   rl      s   rl   c             C   s   | | | | | S )Nr   )r;   r<   cder   r   r   add5   s    r{   c          &   C   sD  t d}t d}t d}t \}\}}t|d | d}td||gdn\}}	tt}
|
|||dd	d
|
|||d	dd
|
||||d	d
|
|||||d
g}dd |D }t |}W d Q R X tjt	  td| dd W d Q R X W d Q R X W d Q R X W d Q R X ||kst
|jdks t
|jdks0t
|jdks@t
d S )Nr"   r   r5   r/   )r0   r1   )scatterrM      )ry   rz   c             S   s   g | ]\}}}|||qS r   r   )r#   funcr   r   r   r   r   r&      s   z'test_manual_scatter.<locals>.<listcomp>)r0   r|      )rM   r   )rl   r   r   r   r
   r{   r	   r8   r9   	TypeErrorr7   rm   )r0   rH   rI   zr:   r;   r<   r+   r=   r>   rP   rV   expectedresultsr   r   r   test_manual_scatter   s*    (r   c          .      sz  t jd}|jtd|jd |jtd|jd} gd |gd  }t \}\}}t|d | d`}td"\}}	t dd	 t	|D  W d Q R X t
d
|}
|
|d  |
|d   dkstW d Q R X W d Q R X t \}\}}t|d | dp}td&\}}	t  fdd	tdD  W d Q R X t
d
|}
|
|d  dksLt|
|d  dksbtW d Q R X W d Q R X d S )NrY   g     @)Zdtyper5   r/   )r0   r1   c             s   s&   | ]\}}t t||||d V  qdS ))optN)r
   r   )r#   r2   r]   r   r   r   r3      s   z$test_auto_scatter.<locals>.<genexpr>zreceive-from-scatterr   c             3   s$   | ]}t t d d |V  qd S )Nr5   )r
   r   )r#   r2   )data1r   r   r3      s    r}   r   )r8   rd   onesrD   Zuint8r   r   r   r	   	enumerater.   r7   r6   )loop_in_threadr_   Zdata2Zdata_to_processr:   r;   r<   r+   r=   r>   rj   r   )r   r   test_auto_scatter   s$    


4&
r   retry_noc                s   t jdd d}fdd fddt f\}\}}t|d | dB}td	. jd
t fddt|D }W d Q R X W d Q R X W d Q R X d S )NrY   r4   c                s
    j | S )N)re   )rH   r2   j)r_   r   r   my_sum
  s    z#test_nested_scatter.<locals>.my_sumc                sB   t  }td& t  fddtD }W d Q R X t|S )Nr1   c             3   s&   | ]}t  |d  |V  qd S )N)r
   )r#   r   )arrayr2   r   r   r   r3     s    zEtest_nested_scatter.<locals>.outer_function_joblib.<locals>.<genexpr>)r   r   r	   r6   re   )r   r2   r+   r   )NUM_INNER_TASKSr   )r   r2   r   outer_function_joblib  s    
z2test_nested_scatter.<locals>.outer_function_joblibr/   )r0   r1   i'  c             3   s$   | ]}t  |d  |V  qd S )N)r
   )r#   r2   )my_arrayr   r   r   r3     s   z&test_nested_scatter.<locals>.<genexpr>)r8   rd   r   r   r   r   r	   r6   )r0   r   ZNUM_OUTER_TASKSr:   r;   r<   r>   r   )r   r   r   r_   r   r   test_nested_scatter  s    
	

r   c       	   '      s  dd  t  \}\}}t|d | d`}tdL\}}tdd fdd	td
D }x |D ]}tt|dks^tq^W W d Q R X W d Q R X t|d | d`}tdL\}}tdd fdd	td
D }x |D ]}tt|dkstqW W d Q R X W d Q R X W d Q R X d S )Nc              S   sH   t tdddd tdD } | t tdddd tdD O } | S )Nr   )n_jobsc             s   s   | ]}t tj V  qd S )N)r
   osgetpid)r#   r>   r   r   r   r3   "  s    zOtest_nested_backend_context_manager.<locals>.get_nested_pids.<locals>.<genexpr>c             s   s   | ]}t tj V  qd S )N)r
   r   r   )r#   r>   r   r   r   r3   #  s    )setr	   r6   )Zpidsr   r   r   get_nested_pids!  s     $z<test_nested_backend_context_manager.<locals>.get_nested_pidsr/   )r0   r1   r   )r   c             3   s   | ]}t   V  qd S )N)r
   )r#   r>   )r   r   r   r3   *  s   z6test_nested_backend_context_manager.<locals>.<genexpr>r4   c             3   s   | ]}t   V  qd S )N)r
   )r#   r>   )r   r   r   r3   4  s   )r   r   r   r	   r6   r)   r   r7   )	r   r:   r;   r<   r+   r=   r>   Z
pid_groupsZ	pid_groupr   )r   r   #test_nested_backend_context_manager   s     

,

r   c                 s   dd   fddt  \}\}}t|d | d}td~\}}t >} |dks\t|jdksjt|fd
dtdD }W d Q R X x(|D ] \}	}
|	dkst|
dkstqW W d Q R X W d Q R X W d Q R X d S )Nc             S   s
   | j jjS )N)rA   	__class__rt   )pr   r   r   _backend_type?  s    zJtest_nested_backend_context_manager_implicit_n_jobs.<locals>._backend_typec           	      s    t  }  | | jfS Q R X d S )N)r	   r   )r   )r   r   r   get_nested_implicit_n_jobsB  s    zWtest_nested_backend_context_manager_implicit_n_jobs.<locals>.get_nested_implicit_n_jobsr/   )r0   r1   r   r"   c             3   s   | ]}t   V  qd S )N)r
   )r#   r>   )r   r   r   r3   M  s   zFtest_nested_backend_context_manager_implicit_n_jobs.<locals>.<genexpr>r   r   )r   r   r   r	   r7   r   r6   )r0   r:   r;   r<   r+   r=   r>   r   Zall_nested_n_jobsZbackend_typeZnested_n_jobsr   )r   r   r   3test_nested_backend_context_manager_implicit_n_jobs;  s    
r   c             C   sD   t jt}td W d Q R X W d Q R X dt|jj ks@td S )Nr1   zcreate a dask client)r8   r9   r   r   strvaluelowerr7   )r0   infor   r   r   test_errorsU  s    
r   c              C   s   t  \}\}}t|d | d}td@\}}tdddd tdD }t|d	 d	 d	 tsftW d Q R X td@\}}tddd
d tdD }t|d	 d	 d	 tstW d Q R X W d Q R X W d Q R X d S )Nr/   )r0   r1   r   )r   c             s   s   | ]}t td dV  qd S )N)nested_require)r
   outer)r#   r>   r   r   r   r3   c  s    z.test_correct_nested_backend.<locals>.<genexpr>r"   r   c             s   s   | ]}t td dV  qdS )Z	sharedmem)r   N)r
   r   )r#   r>   r   r   r   r3   i  s   )	r   r   r   r	   r6   rB   r   r7   r   )r0   r:   r;   r<   r+   r=   r>   resultr   r   r   test_correct_nested_backend]  s    $r   c                s"   t ddd fddtdD S )Nr   threads)r   Zpreferc             3   s   | ]}t t V  qd S )N)r
   middle)r#   r>   )r   r   r   r3   p  s    zouter.<locals>.<genexpr>r"   )r	   r6   )r   r   )r   r   r   n  s    
r   c             C   s   t d| ddd tdD S )Nr   )r   requirec             s   s   | ]}t t V  qd S )N)r
   inner)r#   r>   r   r   r   r3   v  s    zmiddle.<locals>.<genexpr>r"   )r	   r6   )r   r   r   r   r   t  s    
r   c               C   s   t  jS )N)r	   rA   r   r   r   r   r   z  s    r   c             C   sN   t | ddd6 td" tdddd td	D  W d Q R X W d Q R X d S )
NFT)r0   	processesZset_as_defaultr1   rM   )r   c             s   s   | ]}t t|V  qd S )N)r
   id)r#   r2   r   r   r   r3     s    z0test_secede_with_no_processes.<locals>.<genexpr>r   )r   r   r	   r6   )r0   r   r   r   test_secede_with_no_processes~  s    
r   c             C   s   ddl m} | jS )Nr   )
get_worker)r   r   r/   )r>   r   r   r   r   _worker_address  s    r   c              C   s   t  \}\}}t|d | d}td|d d8\}}t dd tdD }||d gd ksftW d Q R X td|d d8\}}t dd tdD }||d gd kstW d Q R X W d Q R X W d Q R X d S )	Nr/   )r0   r1   )workersc             s   s   | ]}t t|V  qd S )N)r
   r   )r#   r2   r   r   r   r3     s    z-test_dask_backend_keywords.<locals>.<genexpr>r4   c             s   s   | ]}t t|V  qd S )N)r
   r   )r#   r2   r   r   r   r3     s    )r   r   r   r	   r6   r7   )r0   r:   r;   r<   r+   r=   r>   r?   r   r   r   test_dask_backend_keywords  s     r   c             C   s   t d| dn}td t dd tdD  W d Q R X t }x(|jjjrhtd t |d k sBt	qBW |j
 svt	W d Q R X d S )	NF)r   r0   r1   c             s   s   | ]}t t|V  qd S )N)r
   r   )r#   r2   r   r   r   r3     s    ztest_cleanup.<locals>.<genexpr>r4   g{Gz?r}   )r   r   r	   r6   r   r   Z	schedulerrV   r   r7   Zfutures)r0   r+   startr   r   r   test_cleanup  s    
"r   cluster_strategyadaptivelate_scalingz2.1.1z1.28.0z?distributed bug - https://github.com/dask/distributed/pull/2841)reasonc             C   s   t dddd}t|}| dkr.|jddd n| dkr@|jd z0td t d	d
 tdD  W d Q R X W d |j  |j  X d S )Nr   Fr   )ra   r   rb   r   )Zminimummaximumr   r1   c             s   s   | ]}t t|V  qd S )N)r
   r   )r#   r2   r   r   r   r3     s    z(test_wait_for_workers.<locals>.<genexpr>r4   )r   r   ZadaptZscaler   r	   r6   rg   )r   r   r+   r   r   r   test_wait_for_workers  s    

&r   c           &   C   s   t dddd} t| }ztddd< d}tjt|d	 t d
d tdD  W d Q R X W d Q R X tddd< d}tjt|d	 t dd tdD  W d Q R X W d Q R X W d |j	  | j	  X d S )Nr   Fr   )ra   r   rb   r1   g?)Zwait_for_workers_timeoutz7DaskDistributedBackend has no worker after 0.1 seconds.)matchc             s   s   | ]}t t|V  qd S )N)r
   r   )r#   r2   r   r   r   r3     s    z0test_wait_for_workers_timeout.<locals>.<genexpr>r4   z+DaskDistributedBackend has no active workerc             s   s   | ]}t t|V  qd S )N)r
   r   )r#   r2   r   r   r   r3     s    )
r   r   r   r8   r9   TimeoutErrorr	   r6   RuntimeErrorrg   )r   r+   msgr   r   r   test_wait_for_workers_timeout  s    ,0r   )r   )r   r   )?
__future__r   r   r   r   r8   r   uuidr   r   r    r	   r
   r   rC   r   r   Z_daskr   rd   r   r   r   r   Zdistributed.metricsZdistributed.utils_testr   r   r   r   r.   r@   rF   rG   rJ   markZparametrizerW   rk   objectrl   r{   r   r   r*   r6   r   r   r   r   r   r   r   r   r   r   r   r   Zskipif__version__r   r   r   r   r   r   <module>   sT   

	9
%