U
    Nvf"                     @   s  d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlmZ d dl	Z	d dl
Z
d dlZd dlZd dlmZmZ d dlmZ d dlmZ d dlmZmZmZ d dlmZ d dlmZmZ d d	lmZ d d
lmZ d dl m!Z! d dl m"Z" d dl m#Z# d dl m$Z$ d dl m%Z% d dl m&Z& d dl m'Z( dd Z)dd Z*dd Z+dd Z,dd Z-eedd Z.eeejdkprddd d!d" Z/eed#d$ Z0ed%d& Z1eeed'eej2gd(d)gd*d+d, Z3eeed'eej2gd(d)gd*d-d. Z4eeed/d(d)gd0d1 Z5eeed/d(d)gd2d3 Z6eeed/d(d)gd4d5 Z7eeed/d(d)gd6d7 Z8eed8d9 Z9eed:d; Z:eeed/d(d)gd<d= Z;eeed/d(d)gd>d? Z<eeed/d(d)gd@dA Z=eeed'eej2gd(d)gd*dBdC Z>eeed/ej?d(ej@jAdDd dEd)gdFdG ZBeeed'eej2gd(d)gd*dHdI ZCeeeed'eej2gd(d)gd*dJdK ZDeeeed'eej2gd(d)gd*dLdM ZEeeed'eej2gd(d)gd*dNdO ZFdPdQ ZGeeed'eej2gd(d)gd*dRdS ZHdTdU ZIeeedVeJeKeej2geLdWdXdY eKd(d)geMeNeLdWD d*dZd[ ZOd\d] ZPd^d_ ZQeeejdkd`d edadbdcgddde ZRedfdg ZSdhdi ZTeedjdk ZUdS )l    N)sleep)
with_numpynp)with_multiprocessing)with_dev_shm)raisesparametrizeskipif)make_memmap)Paralleldelayed)MemmappingPool)_TestingMemmappingExecutor)has_shareable_memory)ArrayMemmapForwardReducer)_strided_from_memmap)_get_temp_dir)_WeakArrayKeyMap)_get_backing_memmapc                   C   s   t jddd d S )Ni,  T)timeoutexit)faulthandlerZdump_traceback_later r   r   ?/tmp/pip-unpacked-wheel-dylwa62s/joblib/test/test_memmapping.pysetup_module!   s    r   c                   C   s   t   d S N)r   Zcancel_dump_traceback_laterr   r   r   r   teardown_module%   s    r   c                 C   s   t | d k	st| S r   )r   AssertionErrorarrayr   r   r   check_memmap_and_send_back)   s    r    c                 C   s    | \}}}t j|| | dS )zDummy helper function to be executed in subprocesses

    Check that the provided array has the expected values in the provided
    range.

    N)r   testingassert_array_equalargsdatapositionexpectedr   r   r   check_array.   s    
r(   c                 C   sD   | \}}}|| |kst ||  d9  < tj|| d|  dS )zDummy helper function to be executed in subprocesses


    Check that the input array has the right values in the provided range
    and perform an inplace modification to double the values in the range by
    two.

       N)r   r   r!   r"   r#   r   r   r   inplace_double9   s    	
r*   c                    s  t jj}| dj}t j|t jddd}dt j|jd |j	d |dd< |
  t j|t jd	d
ddd}t d|j|dd< |ddddddf }t |}|j}td| jdd  fdd}||}	t|	stt|	t jst||	| ||}
t|
st||
| ||}t|t jr2tt|s@t||| ||}t|t jrdtt|srt||| |d }t|rt||}t|rtt|t jrt|||d  t |}t|rt||}t|t jstt|rt||| dS )z9Check that it is possible to reduce a memmap backed array	test.mmapi  w+dtypeshapemodeg      r   r.   N)         r+Fr4   )r.   r/   r0   orderoffset<      r)   cTc                    s    | \}}|| S r   r   xZconsr$   Zreducerr   r   reconstruct_array_or_memmapi   s    zEtest_memmap_based_array_reducing.<locals>.reconstruct_array_or_memmapr2   )r   r!   r"   joinstrpathmemmapfloat64aranger/   r.   flushreshapeasarrayTr   r   r   
isinstancendarray)tmpdirr"   filenamebufferabr<   dr@   a_reconstructedb_reconstructedc_reconstructedd_reconstructeda3Za3_reconstructedZb3Zb3_reconstructedr   r?   r    test_memmap_based_array_reducingH   sV    "  





rW   win32r   z2PermissionError only easily triggerable on Windows)reasonc                 C   s   |  dj}dj|d}tjtjd|gtjtjd}|  |	 \}}|j
dksVt|dksbtd|}|| ks|td S )	Nr+   a  if 1:
    import os
    import numpy as np
    import time
    from joblib.externals.loky.backend import resource_tracker
    resource_tracker.VERBOSE = 1

    # Start the resource tracker
    resource_tracker.ensure_running()
    time.sleep(1)

    # Create a file containing numpy data
    memmap = np.memmap(r"{filename}", dtype=np.float64, shape=10, mode='w+')
    memmap[:] = np.arange(10).astype(np.int8).data
    memmap.flush()
    assert os.path.exists(r"{filename}")
    del memmap

    # Create a np.memmap backed by this file
    memmap = np.memmap(r"{filename}", dtype=np.float64, shape=10, mode='w+')
    resource_tracker.register(r"{filename}", "file")

    # Ask the resource_tracker to delete the file backing the np.memmap , this
    # should raise PermissionError that the resource_tracker will log.
    resource_tracker.maybe_unlink(r"{filename}", "file")

    # Wait for the resource_tracker to process the maybe_unlink before cleaning
    # up the memmap
    time.sleep(2)
    )rM   -cstderrstdoutr       z'tried to unlink {}, got PermissionError)rA   rB   format
subprocessPopensys
executablePIPEwaitcommunicate
returncoder   decode)rL   rM   cmdpouterrmsgr   r   r   2test_resource_tracker_retries_when_permissionerror   s    
rn   c                    sX  t jj}| dj}t j|t jddd}t d|j	|d d < |dd }|d d ddf }|d d d d d d df }|d	d
d }t
d | jdd  fdd}||}	t|	stt|	t jst||	| ||}
t|
st||
| ||}t|s
t||| ||}t|s*t||| ||}t|sJt||| d S )Nr+   )d      rp   r2   r,   r-   i r   
   r3   r:   r2   r4   r<   Tc                    s    | \}}|| S r   r   r=   r?   r   r   r@      s    zNtest_high_dimension_memmap_array_reducing.<locals>.reconstruct_array_or_memmap)r   r!   r"   rA   rB   rC   rD   rE   rG   r/   r   r   r   rJ   )rL   r"   rM   rO   rP   r<   rQ   er@   rR   rS   rT   rU   Ze_reconstructedr   r?   r   )test_high_dimension_memmap_array_reducing   s8    



rs   c                 C   s   |  dj}dtj }tjd }tj|d|| d}t|dd|d|d d d	d
	}t|tjs^t|j	|ksltt|dd|d|d fd|d	d
	}t
|j	|kstd S )Nr+   r3   r:   r,   )r0   r/   uint8rCF)r.   r0   r8   r7   r/   stridesZtotal_buffer_lenZunlink_on_gc_collectr)   )r)   )rA   rB   mmapALLOCATIONGRANULARITYr   rC   r   rJ   r   r8   r   )rL   fnamesizer8   Z
memmap_objZmemmap_backed_objr   r   r   test__strided_from_memmap   s4    

         r|   factorymultiprocessingloky)idsc              	      s  t jj}|dj}| dd|d}zh|dj}t j|t jddd  	d	 |
t fd
dt jd D  | dt  j  t j|t jddd|
tfddtjd D  t|g kst| dt  j  |dt j  t j|t jddddtt* |
tfddtjd D  W 5 Q R X tttf* |
tfddtjd D  W 5 Q R X W 5 |  ~X dS )z@Check that subprocess can access and update shared memory memmappoolrq   r)   
max_nbytestemp_folderr+   r2   r3   r,   r-         ?c                    s.   g | ]&}t  jd  D ]} ||fdfqqS r:   r   ranger/   .0ij)rO   r   r   
<listcomp>  s    z)test_pool_with_memmap.<locals>.<listcomp>r   )r3   r2   r<   c                    s.   g | ]&}t  jd  D ]} ||fdfqqS )r:          @r   r   rP   r   r   r   #  s    rq   ru      )r.   r/   r0   r8   c                    s   g | ]} |d fqS )g      @r   r   r   r<   r   r   r   4  s     c                    s   g | ]} |d fqS )r   r   r   r   r   r   r   9  s     N)r   r!   r"   mkdirrB   	terminaterA   rC   float32fillmapr*   r   r/   onesoslistdirr   r   r(   RuntimeError
ValueErrorr}   rL   r"   pool_temp_folderrj   rM   r   )rO   rP   r<   r   test_pool_with_memmap
  s6    

.2r   c                    s   t jj}|dj}| dd|d}z|dj}t j|t jddd  	d	 t 
 tt jrhttstt|t fd
dt jd D  | dt  j  |dt  j  t|g kstW 5 |  ~X dS )z?Check that subprocess can access and update shared memory arrayr   rq   r)   r   r+   r   r,   r-   r   c                    s.   g | ]&}t  jd  D ]}||fdfqqS r   r   r   rO   Za_viewr   r   r   V  s    z4test_pool_with_memmap_array_view.<locals>.<listcomp>r   N)r   r!   r"   r   rB   r   rA   rC   r   r   rH   rJ   r   r   r   r*   r   r/   r   r   r   r   r   r   r    test_pool_with_memmap_array_view@  s$    

r   backendc                 C   sb   dj | d}tjtjd|gtjtjd}|  | \}}|jdks^t	|
 d |
  d S )N  if 1:
        import numpy as np
        from joblib import Parallel, delayed


        data = np.random.rand(int(2e6)).reshape((int(1e6), 2))

        # Build a complex cyclic reference that is likely to delay garbage
        # collection of the memmapped array in the worker processes.
        first_list = current_list = [data]
        for i in range(10):
            current_list = [current_list]
        first_list.append(current_list)

        if __name__ == "__main__":
            results = Parallel(n_jobs=2, backend="{b}")(
                delayed(len)(current_list) for i in range(10))
            assert results == [1] * 10
    r   rZ   r[   r   

)r_   r`   ra   rb   rc   rd   re   rf   rg   r   rh   r   ri   rj   rk   rl   r   r   r   -test_permission_error_windows_reference_cycleg  s    	r   c                 C   s   dj | d}tdD ]}tj }tjt|d< tj	t
jd|gtjtj|d}|  | \}}|jdksvt||dkstt
jd d d	krd
|kstqd S )Nak  if 1:
        import os
        import time

        import numpy as np

        from joblib import Parallel, delayed
        from testutils import return_slice_of_data

        data = np.ones(int(2e6))

        if __name__ == '__main__':
            # warm-up call to launch the workers and start the resource_tracker
            _ = Parallel(n_jobs=2, verbose=5, backend='{b}')(
                delayed(id)(i) for i in range(20))

            time.sleep(0.5)

            slice_of_data = Parallel(n_jobs=2, verbose=5, backend='{b}')(
                delayed(return_slice_of_data)(data, 0, 20) for _ in range(10))
    r   r2   
PYTHONPATHrZ   r\   r]   envr   r^   )r2      r   )r2   r   r:      resource_tracker)r_   r   r   environcopypathdirname__file__r`   ra   rb   rc   rd   re   rf   rg   r   version_info)r   ri   _r   rj   rk   rl   r   r   r   3test_permission_error_windows_memmap_sent_to_parent  s"    

 r   c                    sz   t td td| dd fddtdD \}td| dd fddtdD \}tj|tj|ksvtd S )	N      Y@r)   rq   n_jobsr   r   c                 3   s   | ]}t t d V  qdS rM   Nr   getattrr   r   r   r   r   	<genexpr>  s    z6test_parallel_isolated_temp_folders.<locals>.<genexpr>r:   c                 3   s   | ]}t t d V  qdS r   r   r   r   r   r   r     s    	r   rE   intr   r   r   r   r   r   )r   
filename_1
filename_2r   r   r   #test_parallel_isolated_temp_folders  s    

r   c              	      s   t td td| dd>}| fddtdD \}| fddtdD \}W 5 Q R X tj|tj|ks|td S )	Nr   r)   rq   r   c                 3   s   | ]}t t d V  qdS r   r   r   r   r   r   r     s    z9test_managed_backend_reuse_temp_folder.<locals>.<genexpr>r:   c                 3   s   | ]}t t d V  qdS r   r   r   r   r   r   r     s    r   )r   rj   r   r   r   r   r   &test_managed_backend_reuse_temp_folder  s    
r   c                  C   s   t td} t }t }dd }tj|| |fd}tj|| |fd}|  |  |  |  t|dksvt	t|dkst	||kst	d S )Nr   c              	      sZ   t ddddB}tdD ]2}| fddtdD \}|tj| qW 5 Q R X d S )Nr   r)   rq   )r   r   r   c                 3   s   | ]}t t d V  qdS r   r   r   r   r   r   r     s    z]test_memmapping_temp_folder_thread_safety.<locals>.concurrent_get_filename.<locals>.<genexpr>r:   )r   r   addr   r   r   )r   Z	temp_dirsrj   r   rM   r   r   r   concurrent_get_filename  s    
zJtest_memmapping_temp_folder_thread_safety.<locals>.concurrent_get_filename)targetr$   r:   )
r   rE   r   set	threadingThreadstartrA   lenr   )r   Ztemp_dirs_thread_1Ztemp_dirs_thread_2r   t1t2r   r   r   )test_memmapping_temp_folder_thread_safety  s&      r   c                  C   s   d} dddg}|D ]n\}}}t jtjd| j||dgt jt jd}|  | \}}|j|kslt	|
 d|kst	|
 qd S )	Na>  if 1:
        import os
        import numpy as np
        from joblib import Parallel, delayed
        from joblib.externals.loky.backend import resource_tracker
        from concurrent.futures import ThreadPoolExecutor, wait

        resource_tracker.VERBOSE = 0

        array = np.arange(int(1e2))

        temp_dirs_thread_1 = set()
        temp_dirs_thread_2 = set()


        def raise_error(array):
            raise ValueError


        def parallel_get_filename(array, temp_dirs):
            with Parallel(backend="loky", n_jobs=2, max_nbytes=10) as p:
                for i in range(10):
                    [filename] = p(
                        delayed(getattr)(array, "filename") for _ in range(1)
                    )
                    temp_dirs.add(os.path.dirname(filename))


        def parallel_raise(array, temp_dirs):
            with Parallel(backend="loky", n_jobs=2, max_nbytes=10) as p:
                for i in range(10):
                    [filename] = p(
                        delayed(raise_error)(array) for _ in range(1)
                    )
                    temp_dirs.add(os.path.dirname(filename))


        executor = ThreadPoolExecutor(max_workers=2)

        # both function calls will use the same loky executor, but with a
        # different Parallel object.
        future_1 = executor.submit({f1}, array, temp_dirs_thread_1)
        future_2 = executor.submit({f2}, array, temp_dirs_thread_2)

        # Wait for both threads to terminate their backend
        wait([future_1, future_2])

        future_1.result()
        future_2.result()
    )parallel_get_filenamer   r   )r   parallel_raiser:   )r   r   r:   rZ   )f1f2r[   r   )r`   ra   rb   rc   r_   rd   re   rf   rg   r   rh   )ri   Zfunctions_and_returncodesr   r   rg   rj   rk   rl   r   r   r   ?test_multithreaded_parallel_termination_resource_tracker_silent  s    3 r   c                 C   s   dj | d}tj }tjt|d< tjt	j
d|gtjtj|d}|  | \}}|jdksjt||dksvtt	jd d d	krd
|kstd S )Na  if 1:
        import os
        import time

        import numpy as np

        from joblib import Parallel, delayed
        from testutils import return_slice_of_data

        data = np.ones(100)

        if __name__ == '__main__':
            for i in range(5):
                slice_of_data = Parallel(
                    n_jobs=2, max_nbytes=1, backend='{b}')(
                        delayed(return_slice_of_data)(data, 0, 20)
                        for _ in range(10)
                    )
    r   r   rZ   r   r   r^   r2   r   r   )r_   r   r   r   r   r   r   r`   ra   rb   rc   rd   re   rf   rg   r   r   )r   ri   r   rj   rk   rl   r   r   r   'test_many_parallel_calls_on_same_objectL  s"    	

r   c                    sH   t td td| dd fddtdD \}t|d ksDtd S )Ng     @@r)   ro   r   c                 3   s   | ]}t t V  qd S r   )r   r    r   r%   r   r   r     s    z8test_memmap_returned_as_regular_array.<locals>.<genexpr>r:   )r   r   r   r   r   r   r   )r   resultr   r   r   %test_memmap_returned_as_regular_array|  s
    
r   c                 C   s   | dkrt jdrtd dj| d}tjt jd|gtj	tj	d}|
  | \}}| }| }|jdkst|d	 | d
|kst|d S )Nr   winzVThe temporary folder cannot be deleted on Windows in the presence of a reference cycler   r   rZ   r[   r   r   Zresource_tracker)rb   platform
startswithpytestxfailr_   r`   ra   rc   rd   re   rf   rh   rg   r   r   r   r   r   2test_resource_tracker_silent_when_reference_cycles  s     r   c              
      s  t |jg kst| dd|jdd}z"t |jg ks>tt j|j	rPttjdtjd	jd
ksnt|tfddtjd D  t |jg ksttjdtjd	  jdkst|t fddt jd D  t j|j	stt |j	}t|dksttjdgd dd	}|t|g}|d rHtW 5 |  tdD ]$}td t j|j	s\ qq\td
|j	~X dS )z0Check that large arrays are not copied in memoryr2   (   r)   )r   r   verboserq   皙?z#temporary folder {} was not deletedr3   r1   r   c                    s   g | ]} |d fqS r   r   r   )smallr   r   r     s     z9test_memmapping_pool_for_large_arrays.<locals>.<listcomp>r   ro      c                    s   g | ]} |d fqS r   r   r   larger   r   r     s     r:   abcobjectN)r   r   rB   r   r   r   r   r   exists_temp_folderr_   r   r   r   nbytesr   r(   r/   rD   isdirr   r   r   )r}   rL   rj   r   Zdumped_filenamesZobjectsresultsr   )r   r   r   %test_memmapping_pool_for_large_arrays  s6    $$

r   z,https://github.com/joblib/joblib/issues/1086)Zmarksc                 C   s   dj | d}tj }tjt|d< tjt	j
d|gtjtj|d}|  | \}}| |  }}|dd }|jdkst|p||dksttj|rtd S )	Na  if 1:
        import os
        from pathlib import Path
        from time import sleep

        import numpy as np
        from joblib import Parallel, delayed
        from testutils import print_filename_and_raise

        data = np.random.rand(1000)

        def get_temp_folder(parallel_obj, backend):
            if "{b}" == "loky":
                return Path(parallel_obj._backend._workers._temp_folder)
            else:
                return Path(parallel_obj._backend._pool._temp_folder)


        if __name__ == "__main__":
            try:
                with Parallel(n_jobs=2, backend="{b}", max_nbytes=100) as p:
                    temp_folder = get_temp_folder(p, "{b}")
                    p(delayed(print_filename_and_raise)(data)
                              for i in range(1))
            except ValueError as e:
                # the temporary folder should be deleted by the end of this
                # call but apparently on some file systems, this takes
                # some time to be visible.
                #
                # We attempt to write into the temporary folder to test for
                # its existence and we wait for a maximum of 10 seconds.
                for i in range(100):
                    try:
                        with open(temp_folder / "some_file.txt", "w") as f:
                            f.write("some content")
                    except FileNotFoundError:
                        # temp_folder has been deleted, all is fine
                        break

                    # ... else, wait a bit and try again
                    sleep(.1)
                else:
                    raise AssertionError(
                        str(temp_folder) + " was not deleted"
                    ) from e
    r   r   rZ   r   
r    )r_   r   r   r   r   r   r   r`   ra   rb   rc   rd   re   rf   rh   splitrg   r   r   )r   ri   r   rj   rk   rl   rM   r   r   r   &test_child_raises_parent_exits_cleanly  s     -.
 r   c                    s   | dd|j d}znt|j g ks&ttjdtjd  jdksDt|	t
 fddt jd	 D  t|j g ks|tW 5 |  ~X dS )
z2Check that large arrays memmapping can be disabledr2   Nr   ro   r1   r   c                    s   g | ]} |d fqS r   r   r   r   r   r   r   ^  s     zBtest_memmapping_pool_for_large_arrays_disabled.<locals>.<listcomp>r   )rB   r   r   r   r   r   r   rD   r   r   r(   r   r/   )r}   rL   rj   r   r   r   .test_memmapping_pool_for_large_arrays_disabledN  s    $r   c                 C   s,  t j}ztdt _| ddd}z|j}d}||s:ttj	|sJtt
jdt
jd}|jdksht|t|gd  tt|d	kstt
jdt
jdd
 }|jdkst|t|gd  tt|d
kstW 5 |  ~X tdD ] }tj	|s
 qtd qtdW 5 |t _X dS )z1Check that memmapping uses /dev/shm when possibleg    ~Ar2   rq   r   z"/dev/shm/joblib_memmapping_folder_ro   r1   r   r:   r)   r   z(temporary folder of pool was not deletedN)jmrSYSTEM_SHARED_MEM_FS_MIN_SIZEr   r   r   r   r   r   r   r   r   r   rD   r   r   idr   r   r   r   )r}   	orig_sizerj   r   Zfolder_prefixrO   rP   r   r   r   r   'test_memmapping_on_large_enough_dev_shmi  s2    

r   c                 C   sd   t j}zPtdt _| ddd}z|j}|dr4tW 5 |  ~X tj	|rTtW 5 |t _X d S )Ng@6Dr2   rq   r   z/dev/shm)
r   r   r   r   r   r   r   r   r   r   )r}   r   rj   r   r   r   r   $test_memmapping_on_too_small_dev_shm  s    
r   c                 C   sb   t jj}| dd|jd}z8|jt jdd}| }t|r>t	||t d W 5 |  ~X dS )z:Check that large arrays are not copied in memory in returnr2   rq   r   )  r$   r   N)
r   r!   r"   rB   r   apply_asyncr   getr   r   )r}   rL   r"   rj   resr   r   r   r   /test_memmapping_pool_for_large_arrays_in_return  s    	r   c                 C   s   t | st| | S )z4Multiplication function to be executed by subprocess)r   r   )rO   Zn_timesr   r   r   _worker_multiply  s    r  c                 C   s   t jj}| dd|jd}zdt jt dddddddd	ddf }|jt	|dfd

 }t|rlt||d|  W 5 |  ~X dS )zCheck that memmaps with a bad buffer are returned as regular arrays

    Unary operations and ufuncs on memmap instances return a new memmap
    instance with an in-memory buffer (probably a numpy bug).
    r2   rq   r   ip  )r   r)   r2   r6   )r7   Nr:   r   )r   r!   r"   rB   r   rH   rE   rG   r   r  r   r   r   )r}   rL   r"   rj   rO   rP   r   r   r   6test_workaround_against_bad_memmap_with_copied_buffers  s    
r  c                 C   s   | S r   r   )argr   r   r   identity  s    r  zfactory,retry_nor2   c                 C   s   g | ]\}}d  ||qS )z{}, {})r_   )r   r>   yr   r   r   r     s     r   c           	      C   s   | dj}dtj }tjd }t|d|d|d}| d|jd}|jt|fd	 }t|t	j
sdt|j|ksrtt	j|| |  d S )
Nr+   r3   r:   r,   rt   )r0   r/   r.   r8   r)   r   r   )rA   rB   rx   ry   r
   r   r  r   rJ   r   rC   r   r8   r!   r"   r   )	r}   Zretry_norL   rz   r{   r8   objrj   r   r   r   r    test_pool_memmap_with_big_offset  s    


r  c                 C   sn   d}t || j\}}|dks t|| djks4tt |d d\}}tjdr\|dks\t||sjtd S )Ntest.tmpdirFr  r   )r   rB   r   rA   rb   r   r   endswith)rL   pool_folder_namepool_folder
shared_memr   r   r   test_pool_get_temp_dir  s    r  c                 C   sd   d}ddl }t|jjdr*||jjd t|dd\}}tjdrR|dksRt	|
|s`t	dS )z`Check that _get_temp_dir works when os.statvfs is not defined

    Regression test for #902
    r	  r   Nstatvfsr  r   F)joblib._memmapping_reducerhasattr_memmapping_reducerr   delattrr   rb   r   r   r   r
  )rL   Zmonkeypatchr  Zjoblibr  r  r   r   r   !test_pool_get_temp_dir_no_statvfs$  s    r  z1This test fails with a PermissionError on Windows	mmap_moder5   r,   c                    sb   dd  dd t dD }t| ddd fd	d
t|D }t|D ]\}}tj|| qFd S )Nc                 S   s   || d d < | S r   r   )arrvaluer   r   r   func:  s    z4test_numpy_arrays_use_different_memory.<locals>.funcc                 S   s   g | ]}t jd ddqS ))rq   rq   rD   r1   )r   zerosr   r   r   r   r   >  s     z:test_numpy_arrays_use_different_memory.<locals>.<listcomp>rq   r   r)   )r  r   r   c                 3   s    | ]\}}t  ||V  qd S r   r   )r   r   r  r  r   r   r   @  s    z9test_numpy_arrays_use_different_memory.<locals>.<genexpr>)r   r   	enumerater   r!   r"   )r  Zarraysr   r   r  r   r  r   &test_numpy_arrays_use_different_memory5  s    r  c               	      sp  ddd} t d}t |d |dks8t|}|dksNt|d |dkslt~t  tj	dkst|dkst~| j	 t d}|d tj	dkst|dkstt
t t d W 5 Q R X ~| j	 d	d
  t fddtdD }t dkrlttjddrVdnd}t||k sltd S )Nro   c                 S   sD   t |D ]&}t| dkr d S t  td qt| dks@td S )Nr   r   )r   r   gccollectr   r   )	containerretriesr   r   r   r   assert_empty_after_gc_collectJ  s    
z>test_weak_array_key_map.<locals>.assert_empty_after_gc_collect*   rO   rP   r:   r<   c              	   S   sN   t d}tt | | W 5 Q R X | || | ||ksFtt|S )Nr#  )r   r   r   KeyErrorr   r   r   r   )mr   rO   r   r   r   get_set_get_collectq  s    

z4test_weak_array_key_map.<locals>.get_set_get_collectc                    s   g | ]} |qS r   r   r   r&  r%  r   r   r   y  s     z+test_weak_array_key_map.<locals>.<listcomp>r   CPythonZnogilFi  )ro   )r   r   r   r   r   r   r  r  r   _datar   r$  r   r   python_implementationr   rb   flags)r"  rO   rP   r<   Z
unique_idsZmax_len_unique_idsr   r'  r   test_weak_array_key_mapG  s8    





r,  c               	   C   s*   t  } ttj t|  W 5 Q R X d S r   )r   r   picklePicklingErrordumps)r%  r   r   r   #test_weak_array_key_map_no_pickling  s    r0  c                    s   t | dtjddd}| fdd dd   }td	d
fdd|fD }td	d
fdd|fD }tj||  fddtd	d
fddtdD }tj|d | d S )Nzarr.datrq   rt   r1   c               	      s@   t  } tj|  dtjdd}W 5 Q R X tjdtj|ddS )Nr   )accessr8   r   )r.   rN   r8   )openrx   filenoZACCESS_READr   rK   rt   )fdmm)testfiler   r   _read_array  s    
"z%test_direct_mmap.<locals>._read_arrayc                 S   s   | d S )Nr)   r   )r>   r   r   r   r    s    ztest_direct_mmap.<locals>.funcr)   )r   c                 3   s   | ]}t  |V  qd S r   r  r   r>   r  r   r   r     s     z#test_direct_mmap.<locals>.<genexpr>c                 3   s   | ]}t  |V  qd S r   r  r8  r  r   r   r     s     c                      s     S r   r   r   )r7  r   r   worker  s    z test_direct_mmap.<locals>.workerc                 3   s   | ]}t   V  qd S r   r  r   )r9  r   r   r     s     r:   r   )	strrA   r   rE   Ztofiler   r!   r"   r   )rL   rO   r  refr   r   )r7  r  r6  r9  r   test_direct_mmap  s    
 r<  )Vr   rx   rb   r   r  r-  	itertoolstimer   r`   r   r   r   Zjoblib.test.commonr   r   r   r   Zjoblib.testingr   r   r	   Zjoblib.backportsr
   Zjoblib.parallelr   r   Zjoblib.poolr   Zjoblib.executorr   ZTestExecutorr  r   r   r   r   r   r   r  r   r   r   r    r(   r*   rW   rn   rs   r|   Zget_memmapping_executorr   r   r   r   r   r   r   r   r   r   r   r   parammarkr   r   r   r   r   r   r  r  r  listproductr   r   r:  r  r  r  r  r,  r0  r<  r   r   r   r   <module>   sT  L,-
2# /&F-44J1 
 
<