3
d                 @   s  d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlmZ d dl	Z	d dl
Z
d dlmZmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZmZmZmZmZ d d	lmZ d d
lmZmZ d dlmZ d dlm Z! d dl"m#Z# d dl"m$Z$ d dl"m%Z% d dl"m&Z& d dl"m'Z' d dl"m(Z( d dl"j)Z*dd Z+dd Z,dd Z-dd Z.dd Z/eedd Z0eejdkd d!d"d# Z1eed$d% Z2ed&d' Z3eeed(ee!j4gd)d*gd+d,d- Z5eeed(ee!j4gd)d*gd+d.d/ Z6eed0d)d*gd1d2 Z7eed0d)d*gd3d4 Z8eeed0d)d*gd5d6 Z9eeed0d)d*gd7d8 Z:eed9d: Z;eed;d< Z<eed=d> Z=eeed0d)d*gd?d@ Z>eeed0d)d*gdAdB Z?eeed0d)ed*edCgdDdE Z@eeed(ee!j4gd)d*gd+dFdG ZAeeed0d)d*gdHdI ZBeeed(ee!j4gd)d*gd+dJdK ZCeeeed(ee!j4gd)d*gd+dLdM ZDeeeed(ee!j4gd)d*gd+dNdO ZEeeed(ee!j4gd)d*gd+dPdQ ZFdRdS ZGeeed(ee!j4gd)d*gd+dTdU ZHdVdW ZIeeedXeJejKee!j4geLdYdZd[ ejKd)d*geMeNeLdYD d+d\d] ZOd^d_ ZPeeejdkd`d!edadbdcgddde ZQedfdg ZRdhdi ZSeedjdk ZTdS )l    N)sleep)
with_numpynp)setup_autokill)teardown_autokill)with_multiprocessing)with_dev_shm)raisesparametrizeskipifxfailparam)make_memmap)Paralleldelayed)MemmappingPool)_TestingMemmappingExecutor)has_shareable_memory)ArrayMemmapForwardReducer)_strided_from_memmap)_get_temp_dir)_WeakArrayKeyMap)_get_backing_memmapc               C   s   t tdd d S )Ni,  )timeout)r   __name__ r   r   =/tmp/pip-build-v9q4h5k9/joblib/joblib/test/test_memmapping.pysetup_module    s    r   c               C   s   t t d S )N)r   r   r   r   r   r   teardown_module$   s    r   c             C   s   t | d k	st| S )N)r   AssertionError)arrayr   r   r   check_memmap_and_send_back(   s    r!   c             C   s    | \}}}t jj|| | dS )zDummy helper function to be executed in subprocesses

    Check that the provided array has the expected values in the provided
    range.

    N)r   testingassert_array_equal)argsdatapositionexpectedr   r   r   check_array-   s    
r(   c             C   sD   | \}}}|| |kst ||  d9  < tjj|| d|  dS )zDummy helper function to be executed in subprocesses


    Check that the input array has the right values in the provided range
    and perform an inplace modification to double the values in the range by
    two.

       N)r   r   r"   r#   )r$   r%   r&   r'   r   r   r   inplace_double8   s    	
r*   c                s$  t jj}| jdj}t j|t jddd}dt j|jd |j	d |dd< |j
  t j|t jddddd}t jdj|j|dd< |ddddddf }t j|}|j}td| jdd  fdd}||}	t|	stt|	t jst||	| ||}
t|
st||
| ||}t|t j s4tt|sBt||| ||}t|t j shtt|svt||| |d	 }t| st||}t| stt|t j st|||d	  t j|}t| st||}t|t jstt| st||| dS )z9Check that it is possible to reduce a memmap backed arrayz	test.mmapi  zw+)dtypeshapemodeg      ?r   )r+   N         zr+F)r+   r,   r-   orderoffset<      r)   cTc                s    | \}}|| S )Nr   )xconsr$   )reducerr   r   reconstruct_array_or_memmaph   s    zEtest_memmap_based_array_reducing.<locals>.reconstruct_array_or_memmapg      )r.   r/   r0   r;   )r   r"   r#   joinstrpathmemmapfloat64aranger,   r+   flushreshapeasarrayTr   r   r   
isinstancendarray)tmpdirr#   filenamebufferabr6   dr:   a_reconstructedb_reconstructedc_reconstructedd_reconstructeda3Za3_reconstructedZb3Zb3_reconstructedr   )r9   r    test_memmap_based_array_reducingG   sP    "





rR   win32z2PermissionError only easily triggerable on Windows)reasonc             C   s   | j dj}dj|d}tjtjd|gtjtjd}|j  |j	 \}}|j
dksVt|dksbtdj|}||j ks|td S )	Nz	test.mmapa  if 1:
    import os
    import numpy as np
    import time
    from joblib.externals.loky.backend import resource_tracker
    resource_tracker.VERBOSE = 1

    # Start the resource tracker
    resource_tracker.ensure_running()
    time.sleep(1)

    # Create a file containing numpy data
    memmap = np.memmap(r"{filename}", dtype=np.float64, shape=10, mode='w+')
    memmap[:] = np.arange(10).astype(np.int8).data
    memmap.flush()
    assert os.path.exists(r"{filename}")
    del memmap

    # Create a np.memmap backed by this file
    memmap = np.memmap(r"{filename}", dtype=np.float64, shape=10, mode='w+')
    resource_tracker.register(r"{filename}", "file")

    # Ask the resource_tracker to delete the file backing the np.memmap , this
    # should raise PermissionError that the resource_tracker will log.
    resource_tracker.maybe_unlink(r"{filename}", "file")

    # Wait for the resource_tracker to process the maybe_unlink before cleaning
    # up the memmap
    time.sleep(2)
    )rH   z-c)stderrstdoutr       z'tried to unlink {}, got PermissionError)r<   r=   format
subprocessPopensys
executablePIPEwaitcommunicate
returncoder   decode)rG   rH   cmdpouterrmsgr   r   r   2test_resource_tracker_retries_when_permissionerror   s    

rg   c                sX  t jj}| jdj}t j|t jddd}t jdj|j	|d d < |dd }|d d d	df }|d d d d d d df }|d
dd }t
d | jdd  fdd}||}	t|	stt|	t jst||	| ||}
t|
st||
| ||}t|s
t||| ||}t|s*t||| ||}t|sJt||| d S )Nz	test.mmapd      r.   zw+)r+   r,   r-   r   
   r/   r5   r0   r6   Tc                s    | \}}|| S )Nr   )r7   r8   r$   )r9   r   r   r:      s    zNtest_high_dimension_memmap_array_reducing.<locals>.reconstruct_array_or_memmap)rh   ri   ri   r.   i  iW  i )r   r"   r#   r<   r=   r>   r?   r@   rB   r,   r   r   r   rE   )rG   r#   rH   rJ   rK   r6   rL   er:   rM   rN   rO   rP   Ze_reconstructedr   )r9   r   )test_high_dimension_memmap_array_reducing   s6    



rl   c             C   s   | j dj}dtj }tjd }tj|d|| d}t|dd|d|d d d	d
	}t|tjs^t|j	|ksltt|dd|d|d fd|d	d
	}t
|j	|kstd S )Nz	test.mmapr/   r5   zw+)r-   r,   uint8rCF)r+   r-   r3   r2   r,   stridesZtotal_buffer_lenZunlink_on_gc_collectr)   )r)   )r<   r=   mmapALLOCATIONGRANULARITYr   r>   r   rE   r   r3   r   )rG   fnamesizer3   Z
memmap_objZmemmap_backed_objr   r   r   test__strided_from_memmap   s    


ru   factorymultiprocessingloky)idsc                s  t jj}|jdj}| dd|d}zh|jdj}t j|t jddd	  jd
 |j	t
 fddt jd D  | dt j j  t j|t jddd	|j	t
fddtjd D  tj|g kst| dt j j  |dt jj  t j|t jddddtt* |j	tfddtjd D  W dQ R X tttf* |j	t
fddtjd D  W dQ R X W d|j  ~X dS )z@Check that subprocess can access and update shared memory memmappoolrj   r)   )
max_nbytestemp_folderz	test.mmapr.   r/   zw+)r+   r,   r-   g      ?c                s.   g | ]&}t  jd  D ]} ||fdfqqS )r5   g      ?)ranger,   ).0ij)rJ   r   r   
<listcomp>  s   z)test_pool_with_memmap.<locals>.<listcomp>r   r6   c                s.   g | ]&}t  jd  D ]} ||fdfqqS )r5   g       @)r}   r,   )r~   r   r   )rK   r   r   r   !  s   rn   r0   )r+   r,   r-   r3   c                s   g | ]} |d fqS )g      @r   )r~   r   )r6   r   r   r   2  s    Nc                s   g | ]} |d fqS )g       @r   )r~   r   )r6   r   r   r   7  s    )r.   r/   )r/   r.   )rj      )r   r"   r#   mkdirr=   r<   r>   float32fillmapr*   r}   r,   onesoslistdirr   r	   r(   RuntimeError
ValueError	terminate)rv   rG   r#   pool_temp_folderrc   rH   r   )rJ   rK   r6   r   test_pool_with_memmap  s0    

.2r   c                s   t jj}|jdj}| dd|d}z|jdj}t j|t jddd	  jd
 t j	 t
t j sjttsvt|jt fddt jd D  | dt j j  |dt j j  tj|g kstW d|j  ~X dS )z?Check that subprocess can access and update shared memory arrayrz   rj   r)   )r{   r|   z	test.mmapr.   r/   zw+)r+   r,   r-   g      ?c                s.   g | ]&}t  jd  D ]}||fdfqqS )r5   g      ?)r}   r,   )r~   r   r   )rJ   a_viewr   r   r   T  s   z4test_pool_with_memmap_array_view.<locals>.<listcomp>r   N)r.   r/   )r   r"   r#   r   r=   r<   r>   r   r   rC   rE   r   r   r   r*   r}   r,   r   r   r   r   )rv   rG   r#   r   rc   rH   r   )rJ   r   r    test_pool_with_memmap_array_view>  s"    

r   backendc             C   sb   dj | d}tjtjd|gtjtjd}|j  |j \}}|jdks^t	|j
 d |j
  d S )Na  if 1:
        import numpy as np
        from joblib import Parallel, delayed


        data = np.random.rand(int(2e6)).reshape((int(1e6), 2))

        # Build a complex cyclic reference that is likely to delay garbage
        # collection of the memmapped array in the worker processes.
        first_list = current_list = [data]
        for i in range(10):
            current_list = [current_list]
        first_list.append(current_list)

        if __name__ == "__main__":
            results = Parallel(n_jobs=2, backend="{b}")(
                delayed(len)(current_list) for i in range(10))
            assert results == [1] * 10
    )rK   z-c)rU   rV   r   z

)rX   rY   rZ   r[   r\   r]   r^   r_   r`   r   ra   )r   rb   rc   rd   re   r   r   r   -test_permission_error_windows_reference_cyclee  s    
r   c             C   s   dj | d}xtdD ]}tjj }tjjt|d< tj	t
jd|gtjtj|d}|j  |j \}}|jdksxt||dkstt
jd d dkrd|kstqW d S )Nak  if 1:
        import os
        import time

        import numpy as np

        from joblib import Parallel, delayed
        from testutils import return_slice_of_data

        data = np.ones(int(2e6))

        if __name__ == '__main__':
            # warm-up call to launch the workers and start the resource_tracker
            _ = Parallel(n_jobs=2, verbose=5, backend='{b}')(
                delayed(id)(i) for i in range(20))

            time.sleep(0.5)

            slice_of_data = Parallel(n_jobs=2, verbose=5, backend='{b}')(
                delayed(return_slice_of_data)(data, 0, 20) for _ in range(10))
    )rK   r.   
PYTHONPATHz-c)rU   rV   envr   rW      r5   s   resource_trackerr.   r   r   r.   r   r5   )r   r   )rX   r}   r   environcopypathdirname__file__rY   rZ   r[   r\   r]   r^   r_   r`   r   version_info)r   rb   _r   rc   rd   re   r   r   r   3test_permission_error_windows_memmap_sent_to_parent  s    
r   c                sz   t jtd td| dd fddtdD \}td| dd fddtdD \}tjj|tjj|ksvtd S )	Ng      Y@r)   rj   )n_jobsr   r{   c             3   s   | ]}t t d V  qdS )rH   N)r   getattr)r~   r   )r    r   r   	<genexpr>  s    z6test_parallel_isolated_temp_folders.<locals>.<genexpr>r5   c             3   s   | ]}t t d V  qdS )rH   N)r   r   )r~   r   )r    r   r   r     s    )	r   r@   intr   r}   r   r   r   r   )r   
filename_1
filename_2r   )r    r   #test_parallel_isolated_temp_folders  s    r   c                s   t jtd td| dd>}| fddtdD \}| fddtdD \}W d Q R X tjj|tjj|ks|td S )	Ng      Y@r)   rj   )r   r   r{   c             3   s   | ]}t t d V  qdS )rH   N)r   r   )r~   r   )r    r   r   r     s    z9test_managed_backend_reuse_temp_folder.<locals>.<genexpr>r5   c             3   s   | ]}t t d V  qdS )rH   N)r   r   )r~   r   )r    r   r   r     s    )	r   r@   r   r   r}   r   r   r   r   )r   rc   r   r   r   )r    r   &test_managed_backend_reuse_temp_folder  s    $r   c              C   s   t jtd} t }t }dd }tj|| |fd}tj|| |fd}|j  |j  |j  |j  t|dksvt	t|dkst	||kst	d S )Ng      Y@c                s^   t ddddF}x>tdD ]2}| fddtdD \}|jtjj| qW W d Q R X d S )Nrx   r)   rj   )r   r   r{   c             3   s   | ]}t t d V  qdS )rH   N)r   r   )r~   r   )r    r   r   r     s    z]test_memmapping_temp_folder_thread_safety.<locals>.concurrent_get_filename.<locals>.<genexpr>r5   )r   r}   addr   r   r   )r    Z	temp_dirsrc   r   rH   r   )r    r   concurrent_get_filename  s
    zJtest_memmapping_temp_folder_thread_safety.<locals>.concurrent_get_filename)targetr$   r5   )
r   r@   r   set	threadingThreadstartr<   lenr   )r    Ztemp_dirs_thread_1Ztemp_dirs_thread_2r   t1t2r   r   r   )test_memmapping_temp_folder_thread_safety  s    r   c              C   s   d} d
ddg}xv|D ]n\}}}t jtjd| j||dgt jt jd}|j  |j \}}|j|ksnt	|j
 d	|kst	|j
 qW d S )Na>  if 1:
        import os
        import numpy as np
        from joblib import Parallel, delayed
        from joblib.externals.loky.backend import resource_tracker
        from concurrent.futures import ThreadPoolExecutor, wait

        resource_tracker.VERBOSE = 0

        array = np.arange(int(1e2))

        temp_dirs_thread_1 = set()
        temp_dirs_thread_2 = set()


        def raise_error(array):
            raise ValueError


        def parallel_get_filename(array, temp_dirs):
            with Parallel(backend="loky", n_jobs=2, max_nbytes=10) as p:
                for i in range(10):
                    [filename] = p(
                        delayed(getattr)(array, "filename") for _ in range(1)
                    )
                    temp_dirs.add(os.path.dirname(filename))


        def parallel_raise(array, temp_dirs):
            with Parallel(backend="loky", n_jobs=2, max_nbytes=10) as p:
                for i in range(10):
                    [filename] = p(
                        delayed(raise_error)(array) for _ in range(1)
                    )
                    temp_dirs.add(os.path.dirname(filename))


        executor = ThreadPoolExecutor(max_workers=2)

        # both function calls will use the same loky executor, but with a
        # different Parallel object.
        future_1 = executor.submit({f1}, array, temp_dirs_thread_1)
        future_2 = executor.submit({f2}, array, temp_dirs_thread_2)

        # Wait for both threads to terminate their backend
        wait([future_1, future_2])

        future_1.result()
        future_2.result()
    parallel_get_filenamer   parallel_raiser5   z-c)f1f2)rU   rV   s   resource_tracker)r   r   r   )r   r   r5   )r   r   r5   )rY   rZ   r[   r\   rX   r]   r^   r_   r`   r   ra   )rb   Zfunctions_and_returncodesr   r   r`   rc   rd   re   r   r   r   ?test_multithreaded_parallel_termination_resource_tracker_silent   s    8r   c              C   sb   d} t jtjd| gt jt jd}|j  |j \}}|jdksJt|j	 d|ks^t|j	 d S )Na  if 1:
        from joblib import Parallel, delayed


        def raise_error(i):
            raise ValueError


        def nested_loop(f):
            Parallel(backend="loky", n_jobs=2)(
                delayed(f)(i) for i in range(10)
            )


        if __name__ == "__main__":
            Parallel(backend="loky", n_jobs=2)(
                delayed(nested_loop)(func) for func in [raise_error]
            )
    z-c)rU   rV   r5   s   resource_tracker)
rY   rZ   r[   r\   r]   r^   r_   r`   r   ra   )rb   rc   rd   re   r   r   r   <test_nested_loop_error_in_grandchild_resource_tracker_silentH  s    r   c             C   s   dj | d}xtdD ]}tjj }tjjt|d< tj	t
jd|gtjtj|d}|j  |j \}}|jdksxt||dkstt
jd d dkrd|kstqW d S )Na  if 1:
        import os
        import time

        import numpy as np

        from joblib import Parallel, delayed
        from testutils import return_slice_of_data

        data = np.ones(100)

        if __name__ == '__main__':
            for i in range(5):
                slice_of_data = Parallel(
                    n_jobs=2, max_nbytes=1, backend='{b}')(
                        delayed(return_slice_of_data)(data, 0, 20)
                        for _ in range(10)
                    )
                slice_of_data = Parallel(
                    n_jobs=2, max_nbytes=1, backend='{b}')(
                        delayed(return_slice_of_data)(data, 0, 20)
                        for _ in range(10)
                    )
    )rK   r.   r   z-c)rU   rV   r   r   rW   r   r5   s   resource_trackerr.   r   r   r.   r   r5   )r   r   )rX   r}   r   r   r   r   r   r   rY   rZ   r[   r\   r]   r^   r_   r`   r   r   )r   rb   r   r   rc   rd   re   r   r   r   'test_many_parallel_calls_on_same_objecti  s     
r   c                sH   t jtd td| dd fddtdD \}t|d ksDtd S )Ng     @@r)   rh   )r   r   r{   c             3   s   | ]}t t V  qd S )N)r   r!   )r~   r   )r%   r   r   r     s    z8test_memmap_returned_as_regular_array.<locals>.<genexpr>r5   )r   r   r   r   r}   r   r   )r   resultr   )r%   r   %test_memmap_returned_as_regular_array  s    r   )Zmarksc             C   sj   dj | d}tjtjd|gtjtjd}|j  |j \}}|jdksRt	|j
 d|ksft	|j
 d S )Na  if 1:
        import numpy as np
        from joblib import Parallel, delayed


        data = np.random.rand(int(2e6)).reshape((int(1e6), 2))

        # Build a complex cyclic reference that is likely to delay garbage
        # collection of the memmapped array in the worker processes.
        first_list = current_list = [data]
        for i in range(10):
            current_list = [current_list]
        first_list.append(current_list)

        if __name__ == "__main__":
            results = Parallel(n_jobs=2, backend="{b}")(
                delayed(len)(current_list) for i in range(10))
            assert results == [1] * 10
    )rK   z-c)rU   rV   r   s   resource_tracker)rX   rY   rZ   r[   r\   r]   r^   r_   r`   r   ra   )r   rb   rc   rd   re   r   r   r   2test_resource_tracker_silent_when_reference_cycles  s     
r   c                s  t j|jg kst| dd|jdd}z&t j|jg ks>tt jj|j sRttjdtj	dj
dkspt|jtfdd	tjd
 D  t j|jg ksttjdtjd  j
dkst|jt fdd	t jd
 D  t jj|jstt j|j}t|dksttjdgd dd}|jt|g}|d
  sLtW d|j  x:tdD ] }td t jj|jsbP qbW tdj|~X dS )z0Check that large arrays are not copied in memoryr.   (   r)   )r{   r|   verboser/   )r+   r   c                s   g | ]} |d fqS )g      ?r   )r~   r   )smallr   r   r     s    z9test_memmapping_pool_for_large_arrays.<locals>.<listcomp>r   rh   i   c                s   g | ]} |d fqS )g      ?r   )r~   r   )larger   r   r     s    r5   abcobjectNrj   g?z&temporary folder of {} was not deleted)r   r   r=   r   r   exists_temp_folderr   r   r   nbytesr   r(   r}   r,   r?   isdirr   r    r   r   r   rX   )rv   rG   rc   Zdumped_filenamesZobjectsresultsr   r   )r   r   r   %test_memmapping_pool_for_large_arrays  s4    $$r   c             C   s   dj | d}tjj }tjjt|d< tjt	j
d|gtjtj|d}|j  |j \}}|j |j  }}|jdd }|jdkst||dksttjj| std S )	Na}  if 1:
        import os

        import numpy as np
        from joblib import Parallel, delayed
        from testutils import print_filename_and_raise

        data = np.random.rand(1000)


        def get_temp_folder(parallel_obj, backend):
            if "{b}" == "loky":
                return p._backend._workers._temp_folder
            else:
                return p._backend._pool._temp_folder


        if __name__ == "__main__":
            try:
                with Parallel(n_jobs=2, backend="{b}", max_nbytes=100) as p:
                    temp_folder = get_temp_folder(p, "{b}")
                    p(delayed(print_filename_and_raise)(data)
                              for i in range(1))
            except ValueError:
                # the temporary folder should be deleted by the end of this
                # call
                assert not os.path.exists(temp_folder)
    )rK   r   z-c)rU   rV   r   
r    )rX   r   r   r   r   r   r   rY   rZ   r[   r\   r]   r^   r_   ra   splitr`   r   r   )r   rb   r   rc   rd   re   rH   r   r   r   &test_child_raises_parent_exits_cleanly	  s    -
r   c                s   | dd|j d}zntj|j g ks&ttjdtjd  jdksDt|jt	 fddt
 jd	 D  tj|j g ks|tW d|j  ~X dS )
z2Check that large arrays memmapping can be disabledr.   N)r{   r|   rh   )r+   i   c                s   g | ]} |d fqS )g      ?r   )r~   r   )r   r   r   r   T  s    zBtest_memmapping_pool_for_large_arrays_disabled.<locals>.<listcomp>r   )r=   r   r   r   r   r   r?   r   r   r(   r}   r,   r   )rv   rG   rc   r   )r   r   .test_memmapping_pool_for_large_arrays_disabledD  s    $r   c             C   s,  t j}ztdt _| ddd}z|j}d}|j|s:ttjj|sJtt	j
dt	jd}|jdksht|jt|gd  ttj|d	kstt	j
dt	jdd
 }|jdkst|jt|gd  ttj|d
kstW d|j  ~X x0tdD ]}tjj|sP td qW tdW d|t _X dS )z1Check that memmapping uses /dev/shm when possibleg    ~Ar.   rj   )r{   z"/dev/shm/joblib_memmapping_folder_rh   )r+   i   r5   r)   Ng?z(temporary folder of pool was not deleted)jmrSYSTEM_SHARED_MEM_FS_MIN_SIZEr   r   
startswithr   r   r   r   r   r   r?   r   r   idr   r   r   r}   r   )rv   	orig_sizerc   r   Zfolder_prefixrJ   rK   r   r   r   r   'test_memmapping_on_large_enough_dev_shm_  s2    
r   c             C   sh   t j}zTtdt _| ddd}z|j}|jd s6tW d |j  ~X tjj	| sXtW d |t _X d S )Ng@6Dr.   rj   )r{   z/dev/shm)
r   r   r   r   r   r   r   r   r   r   )rv   r   rc   r   r   r   r   $test_memmapping_on_too_small_dev_shm  s    
r   c          
   C   sd   t jj}| dd|jd}z:|jt jdd}|j }t| s@t||t jd W d|j	  ~X dS )z:Check that large arrays are not copied in memory in returnr.   rj   )r{   r|     )r$   N)r   )
r   r"   r#   r=   apply_asyncr   getr   r   r   )rv   rG   r#   rc   resr   r   r   r   /test_memmapping_pool_for_large_arrays_in_return  s    	r   c             C   s   t | st| | S )z4Multiplication function to be executed by subprocess)r   r   )rJ   Zn_timesr   r   r   _worker_multiply  s    r   c             C   s   t jj}| dd|jd}zft jt jdjdddd	d	d	d
d	d	f }|jt|dfdj	 }t
| snt||d|  W d	|j  ~X d	S )zCheck that memmaps with a bad buffer are returned as regular arrays

    Unary operations and ufuncs on memmap instances return a new memmap
    instance with an in-memory buffer (probably a numpy bug).
    r.   rj   )r{   r|   ip    r)   r1   )r2   Nr5   )r$   )r   r)   r.   )r   r"   r#   r=   rC   r@   rB   r   r   r   r   r   r   )rv   rG   r#   rc   rJ   rK   r   r   r   6test_workaround_against_bad_memmap_with_copied_buffers  s    
r   c             C   s   | S )Nr   )argr   r   r   identity  s    r   zfactory,retry_nor.   c             C   s   g | ]\}}d j ||qS )z{}, {})rX   )r~   r7   yr   r   r   r     s    r   c       	      C   s   |j dj}dtj }tjd }t|d|d|d}| d|jd}|jt|fd	j }t|t	j
sdt|j|ksrtt	jj|| |j  d S )
Nz	test.mmapr/   r5   zw+rm   )r-   r,   r+   r3   r)   )r|   )r$   )r<   r=   rq   rr   r   r   r   r   rE   r   r>   r   r3   r"   r#   r   )	rv   Zretry_norG   rs   rt   r3   objrc   r   r   r   r    test_pool_memmap_with_big_offset  s    


r   c             C   sn   d}t || j\}}|dks t|| jdjks4tt |d d\}}tjjdr\|dks\t|j|sjtd S )Nztest.tmpdirF)r|   win)r   r=   r   r<   r[   platformr   endswith)rG   Zpool_folder_nameZpool_folderZ
shared_memr   r   r   test_pool_get_temp_dir  s    r   z1This test fails with a PermissionError on Windows	mmap_modezr+zw+c                sf   dd  dd t dD }t| ddd fd	d
t|D }x"t|D ]\}}tjj|| qHW d S )Nc             S   s   || d d < | S )Nr   )arrvaluer   r   r   func  s    z4test_numpy_arrays_use_different_memory.<locals>.funcc             S   s   g | ]}t jdddqS )rj   r?   )r+   )rj   rj   )r   zeros)r~   r   r   r   r   r   #  s    z:test_numpy_arrays_use_different_memory.<locals>.<listcomp>rj   r   r)   )r   r{   r   c             3   s    | ]\}}t  ||V  qd S )N)r   )r~   r   r   )r   r   r   r   &  s    z9test_numpy_arrays_use_different_memory.<locals>.<genexpr>)r}   r   	enumerater   r"   r#   )r   Zarraysr   r   r   r   )r   r   &test_numpy_arrays_use_different_memory  s    r   c                 sX  ddd} t jd}t j|d j|dks8t|}j|dksNtj|d j|dkslt~tj  tj	dkstj|dkst~| j	 t jd}j|d tj	dkstj|dkstt
t jt jd W d Q R X ~| j	 d	d
  t fddtdD }tj dkrTt|dk sTtd S )Nrh   c             S   sF   x0t |D ]$}t| dkrd S tj  td q
W t| dksBtd S )Nr   g?)r}   r   gccollectr   r   )	containerretriesr   r   r   r   assert_empty_after_gc_collect/  s    z>test_weak_array_key_map.<locals>.assert_empty_after_gc_collect*   rJ   rK   r5   r6   c          
   S   sN   t jd}tt | j| W d Q R X | j|| | j||ksFtt|S )Nr   )r   r   r	   KeyErrorr   r   r   r   )mr   rJ   r   r   r   get_set_get_collectV  s    

z4test_weak_array_key_map.<locals>.get_set_get_collectc                s   g | ]} |qS r   r   )r~   r   )r   r   r   r   r   ^  s    z+test_weak_array_key_map.<locals>.<listcomp>i  CPython)rh   )r   r   r   r   r   r   r   r   r   _datar	   r   r}   r   python_implementation)r   rJ   rK   r6   Z
unique_idsr   )r   r   r   test_weak_array_key_map,  s6    





r   c           	   C   s*   t  } ttj tj|  W d Q R X d S )N)r   r	   picklePicklingErrordumps)r   r   r   r   #test_weak_array_key_map_no_picklingg  s    r   c                s   t | jdtjddd}|j fdd dd   }td	d
fdd|gD }td	d
fdd|gD }tjj||  fddtd	d
fddtdD }tjj|d | d S )Nzarr.datrj   rm   )r+   c                 s@   t  } tj| j dtjdd}W d Q R X tjdtj|ddS )Nr   )accessr3   rj   )r+   rI   r3   )rj   )openrq   filenoZACCESS_READr   rF   rm   )fdmm)testfiler   r   _read_arrayt  s    
"z%test_direct_mmap.<locals>._read_arrayc             S   s   | d S )Nr)   r   )r7   r   r   r   r   y  s    ztest_direct_mmap.<locals>.funcr)   )r   c             3   s   | ]}t  |V  qd S )N)r   )r~   r7   )r   r   r   r     s    z#test_direct_mmap.<locals>.<genexpr>c             3   s   | ]}t  |V  qd S )N)r   )r~   r7   )r   r   r   r     s    c                  s     S )Nr   r   )r  r   r   worker  s    z test_direct_mmap.<locals>.workerc             3   s   | ]}t   V  qd S )N)r   )r~   r   )r  r   r   r     s    r5   r   )	strr<   r   r@   Ztofiler   r"   r#   r}   )rG   rJ   r   refr   r   )r  r   r  r  r   test_direct_mmapm  s    
 r
  )Ur   rq   r[   r   r   r   	itertoolstimer   rY   r   Zjoblib.test.commonr   r   r   r   r   r   Zjoblib.testingr	   r
   r   r   r   Zjoblib.backportsr   Zjoblib.parallelr   r   Zjoblib.poolr   Zjoblib.executorr   ZTestExecutorZjoblib._memmapping_reducerr   r   r   r   r   r   Z_memmapping_reducerr   r   r   r!   r(   r*   rR   rg   rl   ru   Zget_memmapping_executorr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   listproductr}   r   r  r   r   r   r   r   r
  r   r   r   r   <module>   s   
M
-.3$!0'G 2	"'592
*;