3
d>                 @   s  d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlmZ ddlmZ ddlZddlmZ ddlmZ ddlmZ ddlmZmZmZmZ dd	lmZmZ d
d Zdd Zdd Z G dd de!Z"G dd de!Z#dddddndodddpdqdgddgddiddiddidej$dgj%e&dre&dtdddddddidgZ'ede'ede'dd  Z(d!d" Z)ed#d$ed%d& Z*d'd( Z+d)d* Z,eed+d,d-gd.d/ Z-ed0d1 Z.eed2d3d4gd5d6 Z/eeej0d7kd8d9d:d; Z1d<d= Z2d>d? Z3ed@dA Z4edBdC Z5dDdE Z6dFdG Z7dHdI Z8dJdK Z9edLdM Z:edNdudvdydWdX e;dYD dZfd[dzdd]gd^d_fgd`da Z<edbdc Z=eddde Z>edfdg Z?edhdi Z@djdk ZAdldm ZBdS ){z
Test the hashing module.
    N)ProcessPoolExecutor)Decimal)hash)filter_args)Memory)raisesskipiffixtureparametrize)np
with_numpyc             C   s   | S )N )sr   r   :/tmp/pip-build-v9q4h5k9/joblib/joblib/test/test_hashing.pyunicode   s    r   c             G   sB   t  }x2tdD ]&}tj }| |  |jtj |  qW t|S )z" Time function func on *args.
       )listrangetimeappendmin)funcargstimes_t1r   r   r   	time_func$   s    r   c             G   s8   t | f| }t |f| }dt|| ||   }|S )zP Return the relative time between func1 and func2 applied on
        *args.
    g      ?)r   abs)Zfunc1Zfunc2r   Z
time_func1Z
time_func2relative_diffr   r   r   relative_time/   s
    r   c               @   s   e Zd Zdd ZdS )Klassc             C   s   |S )Nr   )selfxr   r   r   f<   s    zKlass.fN)__name__
__module____qualname__r#   r   r   r   r   r    :   s   r    c               @   s   e Zd Zdd Zdd ZdS )KlassWithCachedMethodc             C   s   t |d}|j| j| _d S )N)cachedir)r   cacher#   )r!   r(   Zmemr   r   r   __init__B   s    
zKlassWithCachedMethod.__init__c             C   s   |S )Nr   )r!   r"   r   r   r   r#   F   s    zKlassWithCachedMethod.fN)r$   r%   r&   r*   r#   r   r   r   r   r'   @   s   r'         g      ?g       @y              ?ab)r-   r+   )r-   r+   dobj1obj2c             C   s(   t | t |k}| |k}||ks$tdS )z!Smoke test hash on various types.N)r   AssertionError)r0   r1   are_hashes_equalZare_objs_identicalr   r   r   test_trivial_hash\   s    r4   c              C   s^   t jtd} t| jt| jks&ttjtd}tjtd}t|j	t|j	ksZtd S )Nr-   
   	   )
ioStringIOr   r   flushr2   collectionsdequer   extend)r-   Za1Za2r   r   r   test_hash_methodsf   s
    r=   function)Zscopec              C   s@   t jjd} | jd}|j }|j }|d  d7  < |||fS )Nr   r5   r+   )r5   r5   )r   randomRandomStaterandom_samplecopy)rndarr1arr2arr3r   r   r   three_np_arrayso   s    
rG   c             C   sj   | \}}}xDt j| ddD ]2\}}t|t|k}tj||k}||kstqW t|t|jksftd S )Nr,   )repeat)	itertoolsproductr   r   allr2   T)rG   rD   rE   rF   r0   r1   r3   Zare_arrays_equalr   r   r   test_hash_numpy_arraysz   s    
rM   c             C   sT   | \}}}||d}||d}||d}t |t |ks<tt |t |ksPtd S )N)r+   r,   )r   r2   )rG   rD   rE   rF   Zd1Zd2Zd3r   r   r   test_hash_numpy_dict_of_arrays   s    



rN   dtypezdatetime64[s]ztimedelta64[D]c             C   s2   t tjd}tjdd| d}t ||ks.td S )Nr5   r   )rO   )r   r   aranger2   )rO   Za_hasharrayr   r   r   test_numpy_datetime_array   s    rR   c              C   sp   t jt jdjdddd d d dd d f } t j| }t| t|ksNtt j| }t| t|ksltd S )	Nip    r,   r   F)orderr+   )rS   r,   r   )r   asarrayrP   reshapeZascontiguousarrayr   r2   asfortranarray)r-   r.   cr   r   r   test_hash_numpy_noncontiguous   s    

rZ   coerce_mmapTFc             C   sj   | j dj}zBtj|ddd}tj|}t||dt||dk}||ksLtW ddt krd~tj	  X dS )	zECheck that memmap and arrays hash identically if coerce_mmap is True.Zmemmap_tempr5   zw+)shapemode)r[   Nm)r5   r5   )
joinstrpathr   ZmemmaprV   r   r2   localsgccollect)tmpdirr[   filenamer^   r-   r3   r   r   r   test_hash_memmap   s    


rf   win32z5This test is not stable under windows for some reason)reasonc              C   s|   t jjd} | jd}dd }t|t|}|dk s6tdt|| }tt|||f}dt|| ||   }|dk sxtdS )	a   Check the performance of hashing numpy arrays:

        In [22]: a = np.random.random(1000000)

        In [23]: %timeit hashlib.md5(a).hexdigest()
        100 loops, best of 3: 20.7 ms per loop

        In [24]: %timeit hashlib.md5(pickle.dumps(a, protocol=2)).hexdigest()
        1 loops, best of 3: 73.1 ms per loop

        In [25]: %timeit hashlib.md5(cPickle.dumps(a, protocol=2)).hexdigest()
        10 loops, best of 3: 53.9 ms per loop

        In [26]: %timeit hash(a)
        100 loops, best of 3: 20.8 ms per loop
    r   i@B c             S   s   t jt| j S )N)hashlibmd5
memoryview	hexdigest)r"   r   r   r   md5_hash   s    z-test_hash_numpy_performance.<locals>.md5_hashg333333?r   g      ?N)	r   r?   r@   rA   r   r   r2   r   r   )rC   r-   rm   r   Ztime_hashlibZ	time_hashr   r   r   test_hash_numpy_performance   s    
rn   c              C   s8   t  } t  }tt| jg dtt|jg dks4tdS )z~ Make sure that calling the same method on two different instances
    of the same class does resolve to the same hashes.
    r+   N)r+   )r+   )r    r   r   r#   r2   )r-   r.   r   r   r   test_bound_methods_hash   s    ro   c             C   sD   t | j}t | j}tt|jjg dtt|jjg dks@tdS )z Make sure that calling the same _cached_ method on two different
    instances of the same class does resolve to the same hashes.
    r+   N)r+   )r+   )r'   r`   r   r   r#   r   r2   )rd   r-   r.   r   r   r   test_bound_cached_methods_hash   s    

rp   c              C   sP   t jdd tdD td} t jdd tdD td}t| t|ksLtdS )z< Make sure that ndarrays with dtype `object' hash correctly.c             S   s   g | ]}t j|qS r   )r   rP   ).0ir   r   r   
<listcomp>   s    z*test_hash_object_dtype.<locals>.<listcomp>   )rO   c             S   s   g | ]}t j|qS r   )r   rP   )rq   rr   r   r   r   rs      s    N)r   rQ   r   objectr   r2   )r-   r.   r   r   r   test_hash_object_dtype   s    rv   c              C   s,   t jd} t jd}t| t|ks(td S )Ng       @g      @)r   float64r   r2   )r-   r.   r   r   r   test_numpy_scalar   s    

rx   c             C   sp   t | j}dgdgdgdgdgdgdgdgdgdgdgdgdgd}|j|}|j|}t|t|ksltd S )N!   )z#s12069__c_maps.nii.gzz#s12158__c_maps.nii.gzz#s12258__c_maps.nii.gzz#s12277__c_maps.nii.gzz#s12300__c_maps.nii.gzz#s12401__c_maps.nii.gzz#s12430__c_maps.nii.gzz#s13817__c_maps.nii.gzz#s13903__c_maps.nii.gzz#s13916__c_maps.nii.gzz#s13981__c_maps.nii.gzz#s13982__c_maps.nii.gzz#s13983__c_maps.nii.gz)r'   r`   r#   r   r2   )rd   kr/   r-   r.   r   r   r   test_dict_hash  s"    



r{   c             C   sX   t | j}tddddddddd	d
dddg}|j|}|j|}t|t|ksTtd S )Nz#s12069__c_maps.nii.gzz#s12158__c_maps.nii.gzz#s12258__c_maps.nii.gzz#s12277__c_maps.nii.gzz#s12300__c_maps.nii.gzz#s12401__c_maps.nii.gzz#s12430__c_maps.nii.gzz#s13817__c_maps.nii.gzz#s13903__c_maps.nii.gzz#s13916__c_maps.nii.gzz#s13981__c_maps.nii.gzz#s13982__c_maps.nii.gzz#s13983__c_maps.nii.gz)r'   r`   setr#   r   r2   )rd   rz   r   r-   r.   r   r   r   test_set_hash  s"    


r}   c               C   s8   t ttdtdgt ttdtdgks4td S )Nr   NaN)r   r|   r   r2   r   r   r   r   test_set_decimal_hash8  s    r   c              C   sD   d} | di}| di}t jt j|}t||gt||gks@td S )Nfoobar)pickleloadsdumpsr   r2   )stringr-   r.   rY   r   r   r   test_string?  s
    r   c              C   s`  t jd} t jd}| |ks tt| t|ks4ttjtj| }| |k	sPtt| t|ksdtt| | gt||gkstt| | gt| |gkstt jdt jdfdt jdfg}t jdt jdfdt jdfg}t|t|ksttjtj|}||k	s
tt|t|ks tt||gt||gks>tt||gt||gks\td S )NZf4name   Zgradesr,   )r,   )r,   )	r   rO   r2   r   r   r   r   Zstr_rw   )Zdt1Zdt2Zdt1_roundtrippedZcomplex_dt1Zcomplex_dt2Zcomplex_dt1_roundtrippedr   r   r   test_numpy_dtype_picklingI  s*    	

r   zto_hash,expectedThis is a string to hash 71b3f47df22cb19431d85d92d0b230b2   C'est lété 2d8d189e9b2b0b2e384d93c868c0e576@ 1  í  e205227dd82250871fa25aa0ec690aa3c             C   s   g | ]}t jd j  qS )*   )r?   Random)rq   r   r   r   r   rs   }  s    rs      Z a11ffad81f9682a7d901e6edc3d16c84{   i'  r   )ZabcdeZsadfasZ aeda150553d4bb5c69f0e69d51b0e2efc             C   s   t | |kstd S )N)r   r2   )Zto_hashexpectedr   r   r   test_hashes_stay_the_samev  s    r   c              C   s8   t jjd} | jd}t j|}t|t|ks4td S )Nr   r5   )r5   r5   )r   r?   r@   rA   rX   r   r2   )rngZarr_cZarr_fr   r   r   Atest_hashes_are_different_between_c_and_fortran_contiguous_arrays  s    

r   c               C   s   t tjd d S )Nr   )r   r   rQ   r   r   r   r   test_0d_array  s    r   c               C   s&   t tjdt tjdgks"td S )Nr   )r   r   rQ   r2   r   r   r   r   )test_0d_and_1d_array_hashing_is_different  s    r   c        
   
   C   s   dd } |  }|  }t dd}t dd}zdx^t||D ]P\}}|jt|j }|jt|j }||ksjt|jt|j }	||	ks6tq6W W d |j  |j  X d S )Nc           
      s   t jjd  jddddjdt fddtdD  fd	d
tdD d jddjdd jdddjd jdjdgit jdddj	dt j
t jdddj	dt jdddj	dd d d df g} | S )Nr   i  2   )highsizez<i8c             3   s   | ]} j d jdV  qdS )r   z<f4N)randnastype)rq   r   )r   r   r   	<genexpr>  s    z_test_hashes_stay_the_same_with_numpy_objects.<locals>.create_objects_to_hash.<locals>.<genexpr>r   c                s   g | ]} j d jdqS )r   z<f4)r   r   )rq   r   )r   r   r   rs     s    z`test_hashes_stay_the_same_with_numpy_objects.<locals>.create_objects_to_hash.<locals>.<listcomp>i  r   z<f4r   r5      )r   d   )rO   r,   ii)r5   r5   )r5   r5   )r5   r5   )r   r?   r@   randintr   tupler   r   rP   rW   rX   )Zto_hash_listr   )r   r   create_objects_to_hash  s    &zLtest_hashes_stay_the_same_with_numpy_objects.<locals>.create_objects_to_hashr+   )max_workers)r   zipZsubmitr   resultr2   shutdown)
r   Zto_hash_list_oneZto_hash_list_twoZe1e2Zobj_1Zobj_2Zhash_1Zhash_2Zhash_3r   r   r   ,test_hashes_stay_the_same_with_numpy_objects  s    

r   c           	   C   s4   dd } t tj}t|  W d Q R X |jd d S )Nc               S   s   dS )Nr   r   r   r   r   r   non_picklable  s    z2test_hashing_pickling_error.<locals>.non_picklablezPicklingError while hashing)r   r   PicklingErrorr   match)r   Zexcinfor   r   r   test_hashing_pickling_error  s    r   c              C   s4   d} t t| d ddi}t|dd W d Q R X d S )Nz!Valid options for 'hash_name' are)r   r   r   invalid)	hash_name)r   
ValueErrorr   )msgdatar   r   r   test_wrong_hash_name  s    r   y      ?      ?y       @      ?)r+   )r+   r+   )r-   r+   r-   r+   )r-   r+   r   )r   r   )r   r   3~r   r   r   )r   r   i)C__doc__r   ri   sysosrb   r7   r:   rI   r   r?   concurrent.futuresr   decimalr   ZpytestZjoblib.hashingr   Zjoblib.func_inspectr   Zjoblib.memoryr   Zjoblib.testingr   r   r	   r
   Zjoblib.test.commonr   r   r   r   r   ru   r    r'   rc   r   r|   Z
input_listr4   r=   rG   rM   rN   rR   rZ   rf   platformrn   ro   rp   rv   rx   r{   r}   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   <module>   s   		
	%


	
-   

F	