U
    Nvf                     @   s   d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlZddlZddlZddlmZ ddlmZ ddlmZmZ ddlmZmZ ddlmZ ddlmZmZ dd	lmZmZ dd
lmZ ddlmZmZ ddlmZm Z  ddl!m"Z"m#Z# ddl!m$Z$ ddl%m&Z&m'Z'm(Z( ddl)m*Z* dddZ+dd Z,dd Z-dd Z.dd Z/e&dddgdd  Z0d!d" Z1d#d$ Z2d%d& Z3d'd( Z4d)d* Z5d+d, Z6d-d. Z7d/d0 Z8g fd1d2Z9d3d4 Z:e"e&d5dd6gd7d8 Z;e"d9d: Z<d;d< Z=d=d> Z>d?d@ Z?dAdB Z@e&dCdDgdEd6fg dFdfgdGdH ZAdIdJ ZBdKdL ZCdMdN ZDdOdP ZEdQdR ZFdSdT ZGdUdV ZHdWdX ZIdYdZ ZJd[d\ ZKd]d^ ZLd_d` ZMdadb ZNdcdd ZOdedfdgdhdiZPeQeReRdjdkdlZSdmdn ZTdodp ZUddqdrZVdsdt ZWdudv ZXdwdx ZYdydz ZZd{d| Z[d}d~ Z\dd Z]dddZ^e$dd Z_e$dd Z`dd ZaG dd deZbG dd deZce&dded ee gdd Zfdd Zgdd Zhdd Zidd Zjdd Zkdd Zldd Zmdd Zndd ZodddZpejq&ddddd6dddidgdd Zrdd Zsdd ZtG dd dZudS )z
Test the memory module.
    N)Memory)expires_after)MemorizedFuncNotMemorizedFunc)MemorizedResultNotMemorizedResult)_FUNCTION_HASHES)register_store_backend_STORE_BACKENDS)_build_func_identifier_store_backend_factory)JobLibCollisionWarning)Paralleldelayed)StoreBackendBaseFileSystemStoreBackend)
with_numpynp)with_multiprocessing)parametrizeraiseswarns)hash   c                 C   s   | d | S )z3 A module-level function for testing purposes.
        xyr   r   ;/tmp/pip-unpacked-wheel-dylwa62s/joblib/test/test_memory.pyf*   s    r    c                 C   sZ   t |dd}|| } tdD ]6}tdD ](}| ||ks>tt||d ks*tq*qdS )z Given a function and an accumulator (a list that grows every
        time the function is called), check that the function can be
        decorated by memory to be a lazy identity.
    r   locationverbose   r   r   N)r   cacherangeAssertionErrorlen)funcaccumulatorr"   memoryi_r   r   r   check_identity_lazy2   s    
r.   c              	   C   s@   | j  \}tj|jd}t|d}|d W 5 Q R X d S )N
output.pklwgarbage)store_backend	get_itemsospathjoinopenwrite)r+   Zsingle_cache_itemZoutput_filenamer    r   r   r   corrupt_single_cache_itemA   s    r9   c                    s"   g   fdd}| | d|  S )Nc                    s     |  d S Nappend)itemZrecordedr   r   append_to_recordO   s    z6monkeypatch_cached_func_warn.<locals>.append_to_recordwarn)setattr)r)   Zmonkeypatch_fixturer?   r   r>   r   monkeypatch_cached_func_warnH   s    rB   c                    s   t    fdd}t| | j dD ]}dD ]T}t| jd||d}tj| jdd ||}|d	 |jd
d t }|d	}q,t |d	 kst	|
|d	|kst	t |d	 ks$t	q$d|_t| jdd}||d	 dS )z, Simple test of memory lazy evaluation.
    c                    s     d | S Nr   r;   argr*   r   r   r    _   s    
z"test_memory_integration.<locals>.f)FT)rN
   )r"   r#   	mmap_modecompressT)ignore_errorsr   F)r@   __main__r   r!   N)listr.   strpathr   shutilrmtreer%   clearr(   r'   eval
__module__)tmpdirr    rJ   rI   r+   gcurrent_accumulatoroutr   rF   r   test_memory_integrationW   s*     

rX   call_before_reducingTFc           
         s  dD ]}d}d |}ttt||dd t d }d|_|ddksPt|jj	|ks`tt
| jd	d
}|| tt| d dkstt| d d	 }d|kst| d | }|d	krt|dgkstt|d g kst|r8 d tt|d dksttdd fdddD }	n@tdd fdddD }	tt|d dkspt d tt|d dkstqtt|d dkst d tt|d dkstqd S )N)r   r   z0
        def f(x):
            return x
        z<ipython-input-{}-000000000000>exec)filenamemoder    rL   r   r   r!   joblibzipython-inputr$   r   Zn_jobsc                 3   s   | ]}t  |V  qd S r:   r   .0r,   Zcached_fr   r   	<genexpr>   s     zHtest_parallel_call_cached_function_defined_in_jupyter.<locals>.<genexpr>)r   r   c                 3   s   | ]}t  |V  qd S r:   r_   r`   rb   r   r   rc      s        )formatrZ   compiletextwrapdedentlocalsrS   r'   __code__co_filenamer   rN   r%   r(   r4   listdirr   )
rT   rY   Z
session_noZipython_cell_sourceZipython_cell_idZ	aliased_fr+   Zf_cache_relative_directoryZf_cache_directoryresr   rb   r   5test_parallel_call_cached_function_defined_in_jupyter   sD    





	rn   c                     s^   t    fdd} tddd}|| }tdD ](}t }|d t |d ks0tq0dS )z, Test memory with location=None: no memoize c                    s     d | S rC   r;   rD   rF   r   r   ff   s    
ztest_no_memory.<locals>.ffNr   r!   rd   r   )rM   r   r%   r&   r(   r'   )ro   r+   Zggr-   rV   r   rF   r   test_no_memory   s    
rp   c                    sR   t   d
 fdd	}t| | j t| jdd}||}|ddd	dksNtdS )z4 Test memory with a function with keyword arguments.Nr   c                    s     d | S rC   r;   Zarg1Zarg2rF   r   r   rU      s    
ztest_memory_kwarg.<locals>.gr   r!      r   rq   )Nr   )rM   r.   rN   r   r%   r'   )rT   rU   r+   r   rF   r   test_memory_kwarg   s    
rs   c                    s,   t    fddtfdd | j dS )z+ Test memory with a function with a lambda.c                    s     d | S z4 A helper function to define l as a lambda.
        r   r;   r   rF   r   r   helper	  s    
z"test_memory_lambda.<locals>.helperc                    s    | S r:   r   ru   )rv   r   r   <lambda>      z$test_memory_lambda.<locals>.<lambda>N)rM   r.   rN   )rT   r   )r*   rv   r   test_memory_lambda  s    ry   c              	   C   s   t | jdd}|jdd }|}|jdd }|}tt}|d |d W 5 Q R X t|dksftdt|d jks|tdS )	z> Check that name collisions with functions will raise warningsr   r!   c                 S   s   | S )z0 A first function called name_collision
        r   ru   r   r   r   name_collision  s    z2test_memory_name_collision.<locals>.name_collisionc                 S   s   | S )z1 A second function called name_collision
        r   ru   r   r   r   rz     s    r   Z	collisionN)	r   rN   r%   r   r   r(   r'   strmessage)rT   r+   rz   abwarninfor   r   r   test_memory_name_collision  s    


r   c              	   C   s   t | jdd}|dd }|dd }tt6}|ddksDt|ddksTt|ddksdtW 5 Q R X t|dks~td S )	Nr   r!   c                 S   s   | S r:   r   ru   r   r   r   rw   1  rx   z7test_memory_warning_lambda_collisions.<locals>.<lambda>c                 S   s   | d S rC   r   ru   r   r   r   rw   2  rx   r   r   rd   )r   rN   r%   r   r   r'   r(   )rT   r+   r}   r~   r   r   r   r   %test_memory_warning_lambda_collisions.  s    
r   c              	   C   s   t | jdd}td}||}td}||}tt}|d |d |d W 5 Q R X t|dksntdt|d j	
 kstd S )Nr   r!   lambda x: xzlambda x: x+1r   r   zcannot detect)r   rN   rR   r%   r   r   r(   r'   r{   r|   lower)rT   r+   Za1Zb1r   r   r   r   'test_memory_warning_collision_detection>  s    


r   c                    s8   t    fdd}ddl}||d}t| | j dS )z$ Test memory with functools.partial.c                    s     d |S rt   r;   r   rF   r   r   r)   T  s    
z!test_memory_partial.<locals>.funcr   Nr   )rM   	functoolspartialr.   rN   )rT   r)   r   functionr   rF   r   test_memory_partialP  s
    r   c                 C   s4   t | jdd}td}||}|ddks0tdS )zF Smoke test memory with a function with a function defined in an eval.r   r!   r   r   N)r   rN   rR   r%   r'   )rT   r+   mmmr   r   r   test_memory_eval`  s    
r   c                 C   s   t | }| d |S )zx A function with a side effect in its arguments.

        Return the length of its argument and append one element.
    N)r(   r<   )r   Zlen_xr   r   r   count_and_appendj  s    
r   c                 C   s8   t | jdd}|t}| dks&t| dks4tdS )zx Check that if a function has a side effect in its arguments, it
        should use the hash of changing arguments.
    r   r!   r   N)r   rN   r%   r   r'   )rT   r+   r)   r   r   r   test_argument_changet  s    
r   rI   rG   c           	         s   t   d	 fdd	}t| j|dd}||}tjd}tdD ]F}|d}tdD ].}t	|||ksrt
t |d ksXt
qXqBdS )
z/ Test memory with a function with numpy arrays.Nc                    s     d | S rC   r;   rD   rF   r   r   n  s    
ztest_memory_numpy.<locals>.nr   r"   rI   r#   r$   )rH   rH   r   )N)rM   r   rN   r%   r   randomZRandomStater&   Zrandom_sampleallr'   r(   )	rT   rI   r   r+   Zcached_nZrndr,   r}   r-   r   rF   r   test_memory_numpy  s    

r   c           
      C   s   t | jddd}| dd }td}||}||}t|tjsJt|jdksXtt|tjsht|jdksvt~~t	
  t| t||}||}t|dkstd}	|	|d kstt|tjst|jdkstd	S )
z8Check that mmap_mode is respected even at the first callrG   r   r   c                 S   s   | d S Nr   r   r}   r   r   r   twice  s    z0test_memory_numpy_check_mmap_mode.<locals>.twicer$   r   Exception while loading resultsN)r   rN   r%   r   Zones
isinstanceZmemmapr'   r\   gcZcollectr9   rB   r(   )
rT   monkeypatchr+   r   r}   r~   crecorded_warningsdexception_msgr   r   r   !test_memory_numpy_check_mmap_mode  s*    


r   c              
      sf   t | jdd}G dd dt |jd
 fdd	}|  tdD ] }t  |d W 5 Q R X q@d	S )z1 Smoketest the exception handling of Memory.
    r   r!   c                   @   s   e Zd ZdS )z*test_memory_exception.<locals>.MyExceptionN__name__rS   __qualname__r   r   r   r   MyException  s   r   c                    s   | r d S r:   r   )excr   r   r   h  s    z test_memory_exception.<locals>.hr$   r   N)r   )r   rN   	Exceptionr%   r&   r   )rT   r+   r   r-   r   r   r   test_memory_exception  s    
r   c                    s   t | jdd}t  |jdgdd fdd	}|jdgks@t|ddd t dks\t|ddd t dksxt|dd	d t dkstd
S )z# Test the ignore feature of memory r   r!   r   ignorer   c                    s     d d S rC   r;   r   rF   r   r   z  s    ztest_memory_ignore.<locals>.zr   r   N)r   r   rN   rM   r%   r   r'   r(   )rT   r+   r   r   rF   r   test_memory_ignore  s    r   c                    s   t | jdd}t  dd }|jdgd|d fdd		}|jdgksLt|ddd
 t dksht|ddd
 t dkst|ddd
 t dkstdS )z; Test the ignore feature of memory on a decorated function r   r!   c                    s   t   fdd}|S )Nc                     s
    | |S r:   r   )argskwargsr    r   r   wrapped  s    z?test_memory_ignore_decorated.<locals>.decorate.<locals>.wrapped)r   wraps)r    r   r   r   r   decorate  s    z.test_memory_ignore_decorated.<locals>.decorater   r   r   c                    s     d d S rC   r;   r   rF   r   r   r     s    z'test_memory_ignore_decorated.<locals>.zr   r   N)r   r   )rT   r+   r   r   r   rF   r   test_memory_ignore_decorated  s    r   c                 C   sT   t | jdd}|jdd }|ddks,t|dddks>t|dddksPtd	S )
z_Non-regression test against 0.12.0 changes.

    https://github.com/joblib/joblib/pull/751
    r   r!   c                 S   s   | d S rC   r   r   r   r   r   plus_one
  s    z,test_memory_args_as_kwargs.<locals>.plus_oner   r   r   r$   N)r   rN   r%   r'   )rT   r+   r   r   r   r   test_memory_args_as_kwargs  s    
r   zignore, verbose, mmap_moder   d   rH   c                 C   sT   t | jdd}|j|||ddd }|j|ks4t|j|ksBt|j|ksPtdS )z7Check cache may be called with kwargs before decoratingr   r!   )r   r#   rI   c                 S   s   d S r:   r   ru   r   r   r   r     s    z"test_partial_decoration.<locals>.zN)r   rN   r%   r   r'   Z_verboserI   )rT   r   r#   rI   r+   r   r   r   r   test_partial_decoration  s    
r   c           	      C   s,  t | jdd}td}|d | jd	| j}|t}tt}t	j
|jj|}||ksbtt	j
|srt|jt	j
|jjkstt  | rtt	j
t	j
|dst| st|d\}}t	j
|jj||}|d}t	j
|stt	t	j
|d ||dks(td S )
Nr   r!   .r    r]   zfunc_code.pyr   r/   )r]   )r   rN   r   splitr<   r6   r%   r    r   r4   r5   r2   r"   r'   existsdirnamer   rQ   _check_previous_func_code_get_output_identifiersremove)	rT   r+   r5   rU   func_idr"   args_id
output_dirr}   r   r   r   test_func_dir&  s(    


r   c           
      C   s   t | jdd}|t}|d}tt|}|d\}}tj	
|jj||}tj	|sbt||j||gkszttt|}|jj|jjkstt d dd}tt| |t}tt|}	|	d d S )Nr   r!   r   )r   rN   r%   r    pickleloadsdumpsr   r4   r5   r6   r2   r"   r   r'   Z	load_item)
rT   r+   rU   outputr   r   r   r   Zmemory2Zgpr   r   r   test_persistenceF  s    

r   c                 C   s   t t| jt| jddtfD ]\}|d}|r6tt|tsDt|ddksTt|d}|sftt|tstt|	  q d S )Nr   r!   r      )
r   r    rN   r   r%   Zcheck_call_in_cacher'   r   boolrQ   )rT   r)   resultr   r   r   test_check_call_in_cache]  s    


r   c              
   C   s   t tt| jttt| jddttd dtfttttfD ]l\}}|ddksZt	|
d}t||srt	| dkst	|  tt |  W 5 Q R X |  qBd S )Nr   r!   r"   r   r   )zipr   r    rN   r   r   r%   r   r   r'   call_and_shelver   getrQ   r   KeyError)rT   r)   ZResultr   r   r   r   test_call_and_shelvej  s,     

r   c              	   C   sz   t | jddt}|d}t|ts,ttt	}|j
|jksFtW 5 Q R X t|dks`tdt|d jksvtd S )Nr   r!   r   r   z1The 'argument_hash' attribute has been deprecated)r   rN   r%   r    r   r   r   r'   r   DeprecationWarningargument_hashr   r(   r{   r|   )rT   r)   r   r0   r   r   r   "test_call_and_shelve_argument_hash  s    

r   c           
      C   s  |  d}|d t|jj}td | dks<t	|t|jjkrXt
d t| jdd}|t}|d\}}tj |jj||d}|ddkst	t|j}td	 |d}	t|	tst	t|j|kst	td	 |	 dkst	t|j|kst	d
S )z6Check call_and_shelve only load stored data if needed.Ztest_access      ?z>filesystem does not support fine-grained access time attributer   r!   r   r/   r   r   N)r6   r8   r4   statrN   st_atimetimesleepreadr'   pytestskipr   r%   r    r   r5   r2   r"   r   r   r   r   )
rT   Ztest_access_time_fileZtest_access_timer+   r)   r   r   result_pathZfirst_access_timer   r   r   r   .test_call_and_shelve_lazily_load_stored_result  s0    




  


r   c              
   C   s   t t| jttfD ]z}| dj}|d}t|d}t|| W 5 Q R X t|d}t	|}W 5 Q R X |
 |
 kstt| qd S )Nzpickling_test.datr   wbrb)r   r    rN   r   r6   r   r7   r   dumploadr   r'   r4   r   )rT   r)   r[   r   fpresult2r   r   r   test_memorized_pickling  s    
r   c                 C   s
  t t| j}|d}t t| j}|d}| | ks@tt|t|ksTttt}t| t|d t t| jdt d}|d}|  t t| jdd}|d}|  t t| jdt d}|d}|  t t| jdd}|d}|  d S )Nr      )r#   	timestamp)r#   r   )	r   r    rN   r   r   r'   reprr   r   )rT   r)   r   Zfunc2r   r   r   r   test_memorized_repr  s*    





r   c              	   C   sX  | dj}tj|d}d}t|d}|| W 5 Q R X || dd l}t	|jdd}|
|j}	|	d |	d |	d t|d}|d	|  W 5 Q R X |	d |	d t| t | d
}t|d}|| W 5 Q R X |	d |	d tjd tjd dd l}|
|j}	|	d |	d |  \}
}|
dksTtd S )NZ
tmp_importztmp_joblib_.pyz$def f(x):
    print(x)
    return x
r0   r   r!   r   r   z

z-def f(x):
    print("x=%s" % x)
    return x
z
Reloading
tmp_joblib_z1
2
Reloading
x=1
)mkdirrN   r4   r5   r6   r7   r8   Zsyspath_prependr   r   r%   r    rO   rP   sysstdoutmodulespop
readouterrr'   )ZcapsysrT   r   Zdir_namer[   contentZmodule_filetmpr+   r    rW   errr   r   r   test_memory_file_modification  s>    


r   c                 C   s   d S r:   r   r}   r~   r   r   r   _function_to_cache  s    r   c                 C   s   | | S r:   r   r   r   r   r   _sum  s    r   c                 C   s   | | S r:   r   r   r   r   r   _product  s    r   c              	   C   s   t jt_t| jdd}|t}|dddks2t|dddksDttt2 t	jt_|dddksht|dddksztW 5 Q R X d S )Nr   r!   r   r   r$   )
r   rj   r   r   rN   r%   r'   r   r   r   )rT   r+   r    r   r   r   *test_memory_in_memory_function_code_change"  s    

r   c                  C   s   t d d} |   d S Nr   r   rQ   )r+   r   r   r   $test_clear_memory_with_none_location2  s    
r   kw1kw2r   r   c                C   s   | |||fS r:   r   )r}   r~   r   r   r   r   r   func_with_kwonly_args7  s    r  )r}   r~   returnc                 C   s   | | S r:   r   r   r   r   r   func_with_signature;  s    r  c              	   C   s   t | jdd}|t}|dddddks.ttt}|ddddd	 W 5 Q R X |d
 |ddddd tt}|ddddd	 W 5 Q R X |d
 |jtdgd}|ddddddkst|ddddddkstd S )Nr   r!   r   r   r$   )r   )r   r   r$   r   rd   )r   z?Keyword-only parameter 'kw1' was passed as positional parameterr  r   r   )r   r   r$   rd   Zignored)r   rN   r%   r  r'   r   
ValueErrormatch)rT   r+   func_cachedexcinfor   r   r   !test_memory_func_with_kwonly_args?  s    




r	  c                 C   s.   t | jdd}|t}|dddks*td S )Nr   r!   r   g       @g      @)r   rN   r%   r  r'   )rT   r+   r  r   r   r   test_memory_func_with_signature[  s    
r
  c                    st   t | jdd}| dd tt|}|D ]}| q.t fdd|D } fdd|D }||fS )Nr   r!   c                 S   s   dS )N  aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaar   rD   r   r   r   get_1000_bytese  s    z(_setup_toy_cache.<locals>.get_1000_bytesc                    s   g | ]}  |d  qS )r   )r   )ra   rE   )r  r   r   
<listcomp>n  s   z$_setup_toy_cache.<locals>.<listcomp>c                    s    g | ]}t jjj |qS r   )r4   r5   r6   r2   r"   )ra   r   r   r  r   r   r  q  s
    )r   rN   r%   rM   r&   r   )rT   Z
num_inputsr+   inputsrE   Zhash_dirnamesZfull_hashdirsr   r  r   _setup_toy_cacheb  s    


r  c                    s   t | \}}}|j }dd |D }t|t|ks:tdd   fdd|D }dd |D }||ksntdd |D }dd |D }	d	d |D }
|
|	kstd S )
Nc                 S   s   g | ]
}|j qS r   )r5   ra   cir   r   r   r  z  s     z#test__get_items.<locals>.<listcomp>c                    s*    fddt  D }tdd |D S )Nc                    s   g | ]}t j |qS r   r4   r5   r6   ra   fn	directoryr   r   r  ~  s   z;test__get_items.<locals>.get_files_size.<locals>.<listcomp>c                 s   s   | ]}t j|V  qd S r:   )r4   r5   getsize)ra   r   r   r   r   rc     s     z:test__get_items.<locals>.get_files_size.<locals>.<genexpr>)r4   rl   sum)r  Z
full_pathsr   r  r   get_files_size}  s    
z'test__get_items.<locals>.get_files_sizec                    s   g | ]} |qS r   r   ra   Zhash_dirr  r   r   r    s   c                 S   s   g | ]
}|j qS r   )sizer  r   r   r   r    s     c                 S   s   g | ]}t j|d qS )r/   r  r  r   r   r   r    s   c                 S   s    g | ]}t j tj|qS r   )datetimefromtimestampr4   r5   getatimer  r   r   r   r    s   c                 S   s   g | ]
}|j qS r   Zlast_accessr  r   r   r   r    s     )r  r2   r3   setr'   )rT   r+   Zexpected_hash_dirsr-   itemsZ	hash_dirsZexpected_hash_cache_sizesZhash_cache_sizesZoutput_filenamesZexpected_last_accessesZlast_accessesr   r  r   test__get_itemsw  s$    

r$  c                 C   s   t | \}}}|j }|jd}t|}tt|t|sDtt||d ksXt|jd}t|t|ksxt|jd}|g kstd}	|j|	}
t|
stt||jd}t|	|}t
dd |D td	d |D kstd S )
NZ2Kr   i   1M  ip  c                 s   s   | ]}|j V  qd S r:   r!  r  r   r   r   rc     s     z,test__get_items_to_delete.<locals>.<genexpr>c                 s   s   | ]}|j V  qd S r:   r!  r  r   r   r   rc     s     )r  r2   r3   Z_get_items_to_deleter(   r"  issubsetr'   sorted
differencemaxmin)rT   r+   Zexpected_hash_cachedirsr-   r#  Zitems_to_deleteZ	nb_hashesZitems_to_delete_2048bZitems_to_delete_emptybytes_limit_too_smallZitems_to_delete_500bZitems_to_delete_6000bZsurviving_itemsr   r   r   test__get_items_to_delete  s(    
r-  c                 C   s   t | \}}}|j }|  |j }t|t|ks>t|jdd |j }t|t|ksht|jdd |j }tt|t|stt|dkstd}|j|d |j }|g kstd S )Nr%  )bytes_limitZ3Kr   r&  	r  r2   r3   reduce_sizer(  r'   r"  r'  r(   )rT   r+   r-   ref_cache_itemscache_itemsr,  r   r   r   #test_memory_reduce_size_bytes_limit  s     




r3  c                 C   s   t | \}}}|j }|  |j }t|t|ks>t|jdd |j }t|t|ksht|jdd |j }tt|t|stt|dkst|jdd |j }|g kstd S )NrH   )Zitems_limitr   r   r/  )rT   r+   r-   r1  r2  r   r   r   #test_memory_reduce_size_items_limit  s    




r4  c                 C   s  dd l }dd l}t| \}}}|j }|  |j }t|t|ksNt|j|jddd |j }t|t|kst|	d |d |d |j|jddd |j }t
t
|t
|rtt|dkst|j|jddd |j }|g kstd S )	Nr   r   )days)Z	age_limitr   secondsr   )r   r  r  r2   r3   r0  r(  r'   	timedeltar   r"  r'  r(   )rT   r   r  r+   r-   Z	put_cacher1  r2  r   r   r   !test_memory_reduce_size_age_limit  s(    





r:  c                 C   s@   t | \}}}|  t|jjg ks,t|jddr<td S )Nrd   )
stacklevel)r  rQ   r4   rl   r2   r"   r'   r   )rT   r+   r-   rU   r   r   r   test_memory_clear	  s    r<  c                  C   s   dgd } | S )Nr  i  r   )complex_objr   r   r   fast_func_with_complex_output  s    
r>  c                 C   s"   dd t tdD }| r|S dS )Nc                 S   s   i | ]}t ||qS r   )r{   r`   r   r   r   
<dictcomp>  s      z=fast_func_with_conditional_complex_output.<locals>.<dictcomp>g     j@zsimple output)r&   int)Zcomplex_outputr=  r   r   r   )fast_func_with_conditional_complex_output  s    rA  c                    sb   t | jd}|t tdd fddtdD  | \}}d}||ksRt||ks^td S )Nr   r   r^   c                 3   s   | ]}t   V  qd S r:   r_   r`   r  r   r   rc   %  s     zMtest_cached_function_race_condition_when_persisting_output.<locals>.<genexpr>r$   r   )r   rN   r%   r>  r   r&   r   r'   rT   Zcapfdr+   r   stderrr   r   rB  r   :test_cached_function_race_condition_when_persisting_output  s    
 rE  c                    sb   t | jd}|t tdd fddtdD  | \}}d}||ksRt||ks^td S )Nr   r   r^   c                 3   s*   | ]"}t  |d  dkrdndV  qdS )r   r   TFNr_   r`   rB  r   r   rc   <  s   zOtest_cached_function_race_condition_when_persisting_output_2.<locals>.<genexpr>r$   r   )r   rN   r%   rA  r   r&   r   r'   rC  r   rB  r   <test_cached_function_race_condition_when_persisting_output_21  s    
rF  c              
   C   s   t | jd}dd }||}d}||\}}||ks:tt| t||}||\}	}
t|dkshtd}||d ks|t|	|kst|
|kstt| ||}z|  tdW n8 t	k
r } zd	}|t
|jkstW 5 d }~X Y nX d S )
Nr   c                 S   s   t d | t   fS )Ng{Gz?r   r   rD   r   r   r   r)   M  s    
zItest_memory_recomputes_after_an_error_while_loading_results.<locals>.funcrE   r   r   r   z<It normally not possible to load a corrupted MemorizedResultzis corrupted)r   rN   r%   r'   r9   rB   r(   r   r   r   r{   r   )rT   r   r+   r)   Zcached_funcZ	input_argrE   r   r   Zrecomputed_argZrecomputed_timestampr   	referenceer|   r   r   r   ;test_memory_recomputes_after_an_error_while_loading_resultsI  s0    


rJ  c                   @   s   e Zd ZdZdS )IncompleteStoreBackendzAThis backend cannot be instantiated and should raise a TypeError.N)r   rS   r   __doc__r   r   r   r   rK  x  s   rK  c                   @   sP   e Zd ZdZdd Zdd Zdd Zdd	 Zd
d Zdd Z	dd Z
dd ZdS )DummyStoreBackendz(A dummy store backend that does nothing.c                 O   s   dS )zOpen an item on store.Nr   )selfr   r   r   r   r   
_open_item  s    zDummyStoreBackend._open_itemc                 C   s   dS )z!Check if an item location exists.Nr   rN  r"   r   r   r   _item_exists  s    zDummyStoreBackend._item_existsc                 C   s   dS )z&Move an item from src to dst in store.Nr   )rN  srcdstr   r   r   
_move_item  s    zDummyStoreBackend._move_itemc                 C   s   dS )zCreate location on store.Nr   rP  r   r   r   create_location  s    z!DummyStoreBackend.create_locationc                 C   s   dS )z&Check if an object exists in the storeFr   rN  objr   r   r   r     s    zDummyStoreBackend.existsc                 C   s   dS )zClear object on storeNr   rV  r   r   r   clear_location  s    z DummyStoreBackend.clear_locationc                 C   s   g S )z3Returns the whole list of items available in cache.r   )rN  r   r   r   r3     s    zDummyStoreBackend.get_itemsc                 O   s   dS )zConfigure the storeNr   )rN  r"   r   r   r   r   r   	configure  s    zDummyStoreBackend.configureN)r   rS   r   rL  rO  rQ  rT  rU  r   rX  r3   rY  r   r   r   r   rM  }  s   rM  invalid_prefixc              	   C   s,   t t}t| d  W 5 Q R X |d d S )Nz&Store backend name should be a string*r   r  r	   r  )rZ  r  r   r   r   (test_register_invalid_store_backends_key  s    
r\  c               	   C   s,   t t} tdd  W 5 Q R X | d d S )Nfsz.Store backend should inherit StoreBackendBase*r[  r  r   r   r   +test_register_invalid_store_backends_object  s    
r_  c               	   C   s.   t t} tddd W 5 Q R X | d d S )Nz/tmp/joblibunknown)r"   backendzUnknown location*)r   	TypeErrorr   r  r^  r   r   r   !test_memory_default_store_backend  s    
rc  c               	   C   sR   G dd d} |  }t t}td|d W 5 Q R X d}|t|d jksNtd S )Nc                   @   s   e Zd ZdS )zHtest_warning_on_unknown_location_type.<locals>.NonSupportedLocationClassNr   r   r   r   r   NonSupportedLocationClass  s   rd  localr   zbInstantiating a backend using a NonSupportedLocationClass as a location is not supported by joblibr   )r   UserWarningr   r{   r|   r'   )rd  Zunsupported_locationr   Zexpected_mesager   r   r   %test_warning_on_unknown_location_type  s    
rg  c               	   C   sN   d} t | t | tft ks"ttt}t| d W 5 Q R X |d d S )NZisbZfake_locationznCan't instantiate abstract class IncompleteStoreBackend (without an implementation for|with) abstract methods*)	r	   rK  r
   r#  r'   r   rb  r   r  )backend_namer  r   r   r   )test_instanciate_incomplete_store_backend  s    

ri  c                  C   s>   d} t | t | tft ks"tt| d}t|ts:td S )NZdsbZdummy_location)r	   rM  r
   r#  r'   r   r   )rh  backend_objr   r   r   test_dummy_store_backend  s
    

rk  c                  C   s&   t d} td| }|jdks"td S )NZsome_folderre  )pathlibPathr   r"   r'   )r5   rj  r   r   r   0test_instanciate_store_backend_with_pathlib_path  s    

rn  c                 C   sj   d}t  }|jd kstt| t||jd dks8t|| j t||j| jdks^tt| d S )Nz-FileSystemStoreBackend(location="{location}")r   )r   r"   r'   r   r{   re   rY  rN   )rT   Zrepr_patternra  r   r   r   "test_filesystem_store_backend_repr  s    ro  c                 C   s   dd }t | jdd}||}d}t||j||jjdksBt|dd}d}t||j|jj|j	|j
d	ksvtt|d
j|jjdkstd S )Nc                 S   s   | | S r:   r   r   r   r   r   my_func  s    z)test_memory_objects_repr.<locals>.my_funcr   r!   z/MemorizedFunc(func={func}, location={location}))r)   r"   *   zJMemorizedResult(location="{location}", func="{func}", args_id="{args_id}"))r"   r)   r   zMemory(location={location})r   )r   rN   r%   r{   re   r2   r"   r'   r   r   r   )rT   rp  r+   Zmemorized_funcZmemorized_func_reprmemorized_resultZmemorized_result_reprr   r   r   test_memory_objects_repr  s$    



rs  c                 C   s   t | jd}|jdd }|d}t|}t|}|jj|jjksLt	|j
|j
ks\t	|j|jkslt	t|t|kst	d S )Nr   c                 S   s   | d S r   r   ru   r   r   r   rU     s    z'test_memorized_result_pickle.<locals>.grd   )r   rN   r%   r   r   r   r   r2   r"   r'   r)   r   r{   )rT   r+   rU   rr  Zmemorized_result_pickleZmemorized_result_loadsr   r   r   test_memorized_result_pickle  s    



rt  c                 C   sh   |d krg }t | }t |}t| t| ks8t| D ]"}||krNq@|| || ks@tq@d S r:   )varsr"  keysr'   )leftrightignored_attrsZ	left_varsZ
right_varsattrr   r   r   compare+  s    r{  memory_kwargsr$   r   )rJ   r#   r   Z	parameterZunused)rI   r#   Zbackend_optionsc                 C   s  t f d| ji|}tt|}t|j|j t||tdddgd t|t|ks^t	|
t}tt|}t|j|j t||tdddgd t|t|kst	|d}tt|}t|j|j t||tdddgd t|t|kst	d S )Nr"   r2   r   Z_func_code_id)ry  r   )r   rN   r   r   r   r{  r2   r"  r   r'   r%   r    r   )rT   r|  r+   Zmemory_reloadedr  Zfunc_cached_reloadedrr  Zmemorized_result_reloadedr   r   r   test_memory_pickle_dump_load8  s.    

r}  c                 C   s   | tj d}t| jdd}|jdd }||}d|jksBt|  t| jdd}|jdd }||}d|jks|t|  d S )	Nr$      r!   c                 S   s   | d S r   r   ru   r   r   r   r    b  s    ztest_info_log.<locals>.fZQueryingr   c                 S   s   | d S r   r   ru   r   r   r   r    l  s    )	Z	set_levelloggingINFOr   rN   r%   textr'   rQ   )rT   Zcaplogr   r+   r    r-   r   r   r   test_info_log\  s    

r  c              	   C   sH   ddl m} |dkrtdtjtdd t| jdd}W 5 Q R X d S )	Nr   )__version__z1.5z6Bytes limit is deprecated and should be removed by 1.4r.  r  Z1K)r"   r.  )r]   r  r   r   r   r   rN   )rT   r  r-   r   r   r   test_deprecated_bytes_limitu  s    r  c                   @   s\   e Zd ZdZe dd ZdddZdd Zej	
d	d
dgdd Zdd Zdd ZdS )TestCacheValidationCallbackz.Tests on parameter `cache_validation_callback`c                 c   s   t |d}|V  |  d S r   r   )rN  Ztmp_pathZmemr   r   r   r+     s    
z"TestCacheValidationCallback.memoryNc                 C   s"   d|d< |d k	rt | |d S )NTrunr   rG  )rN  r   r   delayr   r   r   foo  s    
zTestCacheValidationCallback.fooc              	   C   s.   d}t jt|d |jdd W 5 Q R X dS )z2Test invalid values for `cache_validation_callbackz9cache_validation_callback needs to be callable. Got True.r  T)cache_validation_callbackN)r   r   r  r%   )rN  r+   r  r   r   r   &test_invalid_cache_validation_callback  s    zBTestCacheValidationCallback.test_invalid_cache_validation_callbackconsider_cache_validTFc                    sr   |j | j fdddgd}ddiddi }}|d|dks@t|d|dksRt|d s^t|d  ksntd	S )
zTest expiry of old resultsc                    s    S r:   r   )r-   r  r   r   rw     rx   zUTestCacheValidationCallback.test_constant_cache_validation_callback.<locals>.<lambda>r   r  r   r  Fr   rd   Nr%   r  r'   )rN  r+   r  r    d1d2r   r  r   'test_constant_cache_validation_callback  s     
zCTestCacheValidationCallback.test_constant_cache_validation_callbackc                 C   s   dd }|j | j|dgd}ddiddi }}|d|dd	d
ksDt|d|dd	d
ksZt|d sft|d srtddiddi }}|d|dd	d
kst|d|dd	d
kst|d st|d rtdS )z*Test cache validity based on run duration.c                 S   s   | d }|dkrdS d S )Ndurationg?Tr   )metadatar  r   r   r   r    s    z^TestCacheValidationCallback.test_memory_only_cache_long_run.<locals>.cache_validation_callbackr   r  r  Fr   r   )r  rd   g?Nr  )rN  r+   r  r    r  r  r   r   r   test_memory_only_cache_long_run  s      z;TestCacheValidationCallback.test_memory_only_cache_long_runc                 C   s   |j | jtdddgd}ddiddiddi  }}}|d|dksHt|d|dksZttd	 |d|dksvt|d st|d rt|d std
S )z!Test expiry of old cached resultsg333333?r7  r   r  r  Fr   rd   r   N)r%   r  r   r'   r   r   )rN  r+   r    r  r  Zd3r   r   r   test_memory_expires_after  s     
z5TestCacheValidationCallback.test_memory_expires_after)N)r   rS   r   rL  r   Zfixturer+   r  r  markr   r  r  r  r   r   r   r   r    s   


r  )r   )rH   )T)N)vrL  r   r   r  rO   r4   os.pathrl  r   r   r   r  rg   r   Zjoblib.memoryr   r   r   r   r   r   r   r	   r
   r   r   r   Zjoblib.parallelr   r   Zjoblib._store_backendsr   r   Zjoblib.test.commonr   r   r   Zjoblib.testingr   r   r   Zjoblib.hashingr   r    r.   r9   rB   rX   rn   rp   rs   ry   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r@  floatr  r	  r
  r  r$  r-  r3  r4  r:  r<  r>  rA  rE  rF  rJ  rK  rM  dictrM   r\  r_  rc  rg  ri  rk  rn  ro  rs  rt  r{  r  r}  r  r  r  r   r   r   r   <module>   s   
+
b


'
 $ 8
"



/$


 
