U
    Nvf                  
   @   s>  d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlZddlZddlmZ ddlZddlmZ zddlZW n ek
r   dZY nX ddlZddlmZmZmZmZ ddlmZmZ ddlmZmZmZ ddl m!Z!m"Z" dd	l#m$Z$ dd
l%m&Z& ddl%m'Z' ddl%m(Z( ddl%m)Z) ddl*m+Z+m,Z,m-Z-m.Z.m/Z/ g Z0dZ1e02e1 e3Z4e02e4 e5dZ6e02e6 e7dZ8e02e8 e9dZ:e02e: e;dZ<e02e< e=dZ>e02e> dZ?e02e? g Z@e02e@ i ZAe02eA eBZCe02eC dd ZDG dd dZEG dd deFZGe02eD e02eE e02eG eE ZHe02eH eG ZIe02eI edddgede0dd ZJdd ZKedddeL gd d! ZMeedd"d#dd$d%gd&d' ZNed(d) ZOed*d+ ZPed,d- ZQed.d/ ZRed0d1 ZSeed2dddgd3d4 ZTeeedd#d"gd5d6 ZUed7d8 ZVdd9d:ZWed;d< ZXed=d> ZYed?d@ ZZedAdBdCgdDdE Z[edFdGdHdI\dJfdKdL\dJfgdMdN Z]edOd%dPgdQdR Z^eeddd$dSgedTe+dUdV Z_dWdX Z`dYdZ Zaed[d\eafd]e`fgd^d_ Zbed`dadbdcdddedfdggdhdi Zcedjdk Zdedldm Zeedndo Zfedpdq Zgedrds Zhedtdudvdw\eiddxdx jdy gdzd{gd|ed}dd$d~gdd Zkeddddddi gdd Zledddddddgdd Zmedddi gdd Znedk	rVG dd dejoZpdd ZqG dd dZredd Zsdd Ztedd Zuedd Zvedd Zwedd Zxdd Zyedddi gdd Zzdd Z{G dd de-Z|G dd de-Z}dd Z~edd Zedd ZejgZejejkr$e2ej eededd ZdS )z@Test the numpy pickler as a replacement of the standard pickler.    N)closing)Path)np
with_numpywith_lz4without_lz4)with_memory_profilermemory_used)parametrizeraiseswarns)numpy_pickleregister_compressor)data)_IO_BUFFER_SIZE)_detect_compressor)#_is_numpy_array_byte_order_mismatch)_ensure_native_byte_order)_COMPRESSORS_LZ4_PREFIXCompressorWrapperLZ4_NOT_INSTALLED_ERRORBinaryZlibFile    c                 c   s
   | V  d S Nr   )xr   r   A/tmp/pip-unpacked-wheel-dylwa62s/joblib/test/test_numpy_pickle.py	_functionK   s    r   c                   @   s   e Zd Zdd ZdS )_classc                 C   s   d S r   r   selfr   r   r   _methodP   s    z_class._methodN__name__
__module____qualname__r"   r   r   r   r   r   O   s   r   c                   @   s   e Zd Zdd ZdS )	_newclassc                 C   s   d S r   r   r    r   r   r   r"   U   s    z_newclass._methodNr#   r   r   r   r   r'   T   s   r'   compressmemberc                 C   sD   |  dj}tj|||d t|}|t|kr@||ks@td S )Ntest.pklr(   )joinstrpathr   dumploadcopydeepcopyAssertionError)tmpdirr(   r)   filenameZ_memberr   r   r   test_standard_typese   s
    
r5   c                	   C   s&   t t tdt  W 5 Q R X d S )Nfoo)r   
ValueErrorr   r.   dictr   r   r   r   test_value_errorr   s    
r9   wrong_compress
   c              	   C   s<   d | }tt}tjdd| d W 5 Q R X || d S )Nz%Non valid compress level given: "{0}"dummyr6   r+   )formatr   r7   r   r.   match)r:   Zexception_msgexcinfor   r   r   test_compress_level_errorx   s    
rA   FT   zlibc                 C   s  |  dj}tjd}|d}t|f|jf||f|||gfD ]\}}tj	|||d}t
|dksjt|d |kszttj|d stt|}|D ]}	t|	tjstqtjt|t| qBtj|d ddtjd	}tj	|||d}t
|dkstt|}t|tjk	rBt|d
rBt|t|sBttj|| t }tj	|||d}t
|dksxtt|}
t|
t|sttj|
j|j tj|
j|j tj|
j|j d S )Nr*   r   r<      r+   r   mmapw+   )modeshapedtypeZ__array_prepare__)r,   r-   r   randomRandomStaterandom_sample	enumerateTr   r.   lenr2   ospathexistsr/   
isinstancendarraytestingassert_array_equalarraymemmapfloat64typehasattrComplexTestObjectarray_float	array_int	array_obj)r3   r(   r4   rndaindexobj	filenamesZobj_item
obj_loadedr   r   r   test_numpy_persistence   s:    
(


ri   c                 C   sL   t jtd t jd}| dj}tj||dd t|}t j	
|| d S )Nd   rK   r*   Tr+   )r   onesr   uint8r,   r-   r   r.   r/   rW   rX   )r3   	big_arrayr4   Zarr_reloadedr   r   r   2test_numpy_persistence_bufferred_array_compression   s
    
ro   c                 C   s  t jd}|d}| dj}t|| tj|dd}t	|t j
sLt| dj}t }t|| tj|dd}t	|t|stt	|jt j
st|jjjrtt	|jt j
st|jjjrtt	|jt j
rtt j|j|j t j|j|j t j|j|j tj|dd}|jjjs0td|jdd< |jjjsNtd|jdd< tj|dd}t j|j|j t j|j|j tj|d	d |jjjst|jjdkst|jjjst|jjdkstd S )
Nr   r<   z	test1.pklr	mmap_modez	test2.pklr+g      $@rG   )r   rL   rM   rN   r,   r-   r   r.   r/   rU   rZ   r2   r^   r\   r_   flagsZ	writeabler`   ra   rW   rX   rI   )r3   rb   rc   r4   bre   rh   obj_reloadedr   r   r   test_memmap_persistence   sT    





rw   c                 C   s~   t jd}|d}t jddgtd}||f}| dj}t	|| tj
|dd\}}t|t jsjtt|t jrztd S )	Nr   r<   r   ru   rk   r*   rp   rq   )r   rL   rM   rN   rY   objectr,   r-   r   r.   r/   rU   rZ   r2   )r3   rb   rc   ru   Z	constructr4   Za_cloneZb_cloner   r   r   $test_memmap_persistence_mixed_dtypes   s    
ry   c                 C   s`   t jd}|d}t j|d}| dj}t	|| tj
|dd}t|t jjs\td S )Nr   r<   g      ?r*   rp   rq   )r   rL   rM   rN   maZmasked_greaterr,   r-   r   r.   r/   rU   Zmasked_arrayr2   )r3   rb   rc   r4   ru   r   r   r   test_masked_array_persistence  s    
r{   c              	   C   s   t jd}|d}| dj}tj||dd tt	}tj
|dd W 5 Q R X dd	d
 |D }dd
 |D }t|dkst|t|d d| dkstd S )Nr   r<   r*   r   r+   rs   rq   
c                 S   s   g | ]}t |qS r   )str.0wr   r   r   
<listcomp>  s     z3test_compress_mmap_mode_warning.<locals>.<listcomp>c                 S   s   g | ]
}|j qS r   )messager~   r   r   r   r     s     z6mmap_mode "r+" is not compatible with compressed file z. "r+" flag will be ignored.)r   rL   rM   rN   r,   r-   r   r.   r   UserWarningr/   rQ   r2   r}   )r3   rb   rc   Zthis_filenamewarninfoZ	debug_msgr   r   r   test_compress_mmap_mode_warning  s    



r   
cache_sizec              	   C   s   |  dj}tjd}|d}td tjdd}t	j
|||d W 5 Q R X |d k	r`dnd}t||kstt|D ]*}|jtkstt|jd	|ksxtqxd S )
Nr*   r   rD   alwaysTrecord)r   r   zzPlease do not set 'cache_size' in joblib.dump, this parameter has no effect and will be removed. You used 'cache_size={0}')r,   r-   r   rL   rM   rN   warningssimplefiltercatch_warningsr   r.   rQ   r2   categoryDeprecationWarningr}   r   r>   )r3   r   r4   rb   rc   r   expected_nb_warningsr   r   r   r   test_cache_size_warning%  s    

r   c                 C   s   |  dj}td}tjdtd tjd}||fD ]n}|jd }|ttj	dd }t
tj|||d}td	 }	||	kstt
tj|}d
t }
|||
 k s6tq6d S )Nr*   r<   r<   rj   g    .ArJ   rK   r     r+   g0@    )r,   r-   r   rl   intrm   nbytesr}   rL   randintr	   r   r.   r   r2   r/   )r3   r(   r4   Zsmall_arrayrn   re   sizeZobj_filenameZmem_usedZwrite_buf_sizeZread_buf_sizer   r   r   test_memory_usage:  s     

  r   c              
   C   s  t jdt ddt jdt ddt jdt ddt jdt ddt jdddd	d
gddt jdt jd dg}| dj}tj	||dd}t
|dkstt|}t||D ]H\}}t|t jrt|}|j|jkstt j|| q||kstqd S )N   <i8rk   >i8<f8>f8r   abcrE   rc   ru   O      C'est l'été !ztemp.pkl.gzr+   )r   arangerK   rY   rm   tobytesr,   r-   r   r.   rQ   r2   r/   ziprU   rV   r   rW   assert_equal)r3   expected_listfnameZdumped_filenamesresult_listresultexpectedr   r   r   $test_compressed_pickle_dump_and_loadU  s$    
r   c              
   C   s  t d| }t|d}ddd}|dd}||d}||krPzptjdd.}td	 tjd
ddd t	j
| |d}	W 5 Q R X tj| }
d|
ksd|
krdnd}t d|
r|dk	rdnd}|| }t||kstdd |D }dd |D }|D ] }t|jd| kstq|D ]0}t | }t d| dt|js(tq(t|	|D ]P\}}t|tjrt|}|j|jksttj|| n||ksdtqdW n tk
rL } zt|dkrt|tstd}|t|ks<tn>| dr:t j!d r:t|ts&tt"t|ks<tn W 5 d}~X Y nX nZzt	
|  tdW nB tk
r } z"d|}|t|j!kstW 5 d}~X Y nX dS )zHelper function to test joblib pickle content.

    Note: currently only pickles containing an iterable are supported
    by this function.
    z.+py(\d)(\d).+r   rE   rB   )rE   rB   rH   Tr   r   ignoreZnumpyz"The compiler package is deprecated)moduler   rq   z_0.9z_0.8.4r   z_0.1.+.pkl$Nc                 S   s   g | ]}t |jtr|qS r   )
issubclassr   r   r~   r   r   r   r     s      z!_check_pickle.<locals>.<listcomp>c                 S   s   g | ]}t |jtr|qS r   )r   r   r   r~   r   r   r   r     s      zkThe file '{0}' has been generated with a joblib version less than 0.10. Please regenerate this pickle file.zmemmapped.+z.+segmentation faultzPYou may be trying to read with python 3 a joblib pickle generated with python 2..lz4z>Numpy pickle loading should have raised a ValueError exceptionz unsupported pickle protocol: {0})#rer?   r   groupgetr   r   r   filterwarningsr   r/   rR   rS   basenamesearchrQ   r2   r}   r   r>   escaper   rU   r   rV   r   rK   rW   r   	Exceptionr7   endswithr   argsr   )r4   r   rr   Zversion_matchZpy_version_used_for_writingZ%py_version_to_default_pickle_protocolZpickle_reading_protocolZpickle_writing_protocolr   r   filename_baseZ expected_nb_deprecation_warningsZexpected_nb_user_warningsr   Zdeprecation_warningsZuser_warningsr   Zescaped_filenamer   r   excr   er   r   r   _check_picklem  s    
 

 




r   c               	      s   t jdt ddt jdt ddt jdddddgd	dt jd
t jd t jdddgt dddg} tj	tj
tjd td k	r d7   fddtD }|D ]}t||  qd S )Nr   r   rk   r   r   r   rE   r   r   r   r   r   ).pkl.gzz.gzip.bz2lz4).xz.lzmac                    s0   g | ]( t  fd dD rtj qS )c                 3   s   | ]}  |V  qd S r   )r   )r   extfnr   r   	<genexpr>  s     zGtest_joblib_pickle_across_python_versions.<locals>.<listcomp>.<genexpr>)anyrR   rS   r,   )r   Zpickle_extensionstest_data_dirr   r   r     s   z=test_joblib_pickle_across_python_versions.<locals>.<listcomp>)r   r   rK   rY   rm   r   matrixrR   rS   dirnameabspathr   __file__lzmalistdirr   r   Zpickle_filenamesr   r   r   r   )test_joblib_pickle_across_python_versions  s     r   c               	      s   t jdt ddt jdt ddt jdddddgd	dt jd
t jd t jdddgt dddg} tj	tj
tj  fddt D }|D ]}t|| dd qd S )Nr   r   rk   r   r   r   rE   r   r   r   r   r   c                    s$   g | ]}| d rtj |qS )r   )r   rR   rS   r,   )r   r   r   r   r   r     s    
zGtest_joblib_pickle_across_python_versions_with_mmap.<locals>.<listcomp>rp   rq   )r   r   rK   rY   rm   r   r   rR   rS   r   r   r   r   r   r   r   r   r   r   3test_joblib_pickle_across_python_versions_with_mmap  s    

r   c                  C   sf  t jddgddgdt jdt ddt jdt ddg} | D ]j}tjd	kr\t|rhtnt|shtt|}|jj	r|jj	
 D ]}|d
 jdk qq@|jjdks@tq@t jddgddgdt jdt ddt jdt ddg}|D ]t}tjdkrt|rtnt|stt|}|jj	rP|jj	
 D ]}|d
 jdk q8q|jjdkstqd S )N)r   g       @)rB   g      @) r   )r   r   rk   rB   r   r   bigr   =)r   r   )r   r   r   r   little)r   rY   r   rK   sys	byteorderr   r2   r   fieldsvalues)Z	be_arraysrY   Z	convertedfZ	le_arraysr   r   r   .test_numpy_array_byte_order_mismatch_detection  s<    



r   compress_tuple)rC   rB   gziprB   c              	   C   sJ   |  dj}tjd||d t|d}t||d ks<tW 5 Q R X d S )Nr*   r=   r+   rbr   r,   r-   r   r.   openr   r2   )r3   r   r4   r   r   r   r   test_compress_tuple_argument  s    r   zcompress_tuple,message))rC   rB   extraz9Compress argument tuple should contain exactly 2 elements)wrongrB   z(Non valid compression method given: "{}"r   )rC   r   z$Non valid compress level given: "{}"c              	   C   s>   |  dj}tt}tjd||d W 5 Q R X || d S )Nr*   r=   r+   )r,   r-   r   r7   r   r.   r?   )r3   r   r   r4   r@   r   r   r   &test_compress_tuple_argument_exception*  s    
r   compress_stringr   c              	   C   sF   |  dj}tjd||d t|d}t||ks8tW 5 Q R X d S )Nr*   r=   r+   r   r   )r3   r   r4   r   r   r   r   test_compress_string_argument9  s    r      cmethodc           	   
   C   s  |  dj}tjdddtddddg d	i d
df}|dkrPtd krPtd n|dkrltj	d
 rltd |d | }|D ]}t
j||||fd t|d}t||kstW 5 Q R X t
|}t|t|stt|tjrtj|| q|||ks|tq|d S )Nr*   )rj   rj   Zf8r   r<   r   ru   )rc   rE   r   r   g      ?r   xzzlzma is support not availabler   zlz4 is not installed..r+   r   )r,   r-   r   rl   ranger   pytestskipr   r   r   r.   r   r   r2   r/   rU   r\   rV   rW   rX   )	r3   r(   r   r4   ZobjectsZdump_filenamere   r   rv   r   r   r   test_joblib_compression_formatsD  s0         

r   c              	   C   sH   t t| d}| }W 5 Q R X t|d}|| W 5 Q R X dS )zDecompress a gzip file.r   wbN)r   r   GzipFilereadr   writeZsource_filenameZtarget_filenamefobufr   r   r   _gzip_file_decompressd  s    r   c              	   C   sH   t | d}t| }W 5 Q R X t |d}|| W 5 Q R X dS )zDecompress a zlib file.r   r   N)r   rC   
decompressr   r   r   r   r   r   _zlib_file_decompressm  s    r   zextension,decompress.zr   c                 C   sH   d}|  dj}|| }t|| ||| t|}||ksDtd S )Nza string to persistr*   )r,   r-   r   r.   r/   r2   )r3   	extensionr   re   Zfilename_rawZfilename_compressedrv   r   r   r   'test_load_externally_decompressed_filesv  s    

r  zextension,cmethod)r   rC   )r   r   )r   bz2)r   r   )r   r   )r   not-compressed)r   r  c              	   C   s   |dkrt d krtd | dj}d}|| }t|| t|d}t||ksZt	W 5 Q R X t
|}t|t|st	||kst	d S )Nr   zlzma is missingr*   zobject to dumpr   )r   r   r   r,   r-   r   r.   r   r   r2   r/   rU   r\   )r3   r  r   r4   re   Z
dump_fnamer   rv   r   r   r   %test_compression_using_file_extension  s    

r  c           	      C   s   t jddg}tjtjg}td k	r0|tjg7 }| dj	}|D ]}|D ]}||d}t
|| W 5 Q R X ||d}t
|}W 5 Q R X t|d}t
|}W 5 Q R X t|t jrt j|| t j|| qH||kst||ksHtqHq@d S )Nr   	some datar*   r   r   )r   rL   r  BZ2Filer   r   r   LZMAFiler,   r-   r   r.   r/   r   rU   rV   rW   rX   r2   )	r3   objsZfobjsr4   re   Zfobjr   rv   Zobj_reloaded_2r   r   r   test_file_handle_persistence  s$    r
  c                  C   sd   t jddg} | D ]J}t }t|| t|}t|t jrRt j	
|| q||kstqd S )Nr   r  )r   rL   ioBytesIOr   r.   r/   rU   rV   rW   rX   r2   )r	  re   r   rv   r   r   r   test_in_memory_persistence  s    
r  c              	   C   sp   t jd}| dj}t|d}t|| W 5 Q R X t|d}tj|dd}W 5 Q R X t j	|| d S )Nr   r*   r   r   rs   rq   )
r   rL   r,   r-   r   r   r.   r/   rW   rX   )r3   re   r4   r   rv   r   r   r   !test_file_handle_persistence_mmap  s    r  c              
   C   s   t jd}| dj}t|d}tj||dd W 5 Q R X tt	|dX}t
t}tj|dd W 5 Q R X t|d	kstt|d
 jd|dd kstW 5 Q R X d S )Nr   r*   r   r   r+   r   rs   rq   r   r   zP"%(fileobj)r" is not a raw file, mmap_mode "%(mmap_mode)s" flag will be ignored.)fileobjrr   )r   rL   r,   r-   r   r   r.   r   r   r   r   r   r/   rQ   r2   r}   r   )r3   re   r4   r   r   r   r   r   ,test_file_handle_persistence_compressed_mmap  s    
r  c               	   C   st   t jd} t }t| | tt}tj|dd W 5 Q R X t	|dksRt
t|d jdddi kspt
d S )Nr   rs   rq   r   r   zuIn memory persistence is not compatible with mmap_mode "%(mmap_mode)s" flag passed. mmap_mode option will be ignored.rr   )r   rL   r  r  r   r.   r   r   r/   rQ   r2   r}   r   )re   r   r   r   r   r   +test_file_handle_persistence_in_memory_mmap  s    
r  r   s   a little data as bytes.i'  z{}r   zlatin-1za little data as bytes.za large data as bytes.)idscompress_level	   c                 C   sD  |  dj}t|d}t|d|dl}| s4t|| | | ksRttt	j
 |  W 5 Q R X tt	j
 |  W 5 Q R X W 5 Q R X |jsttt |  W 5 Q R X W 5 Q R X t|d}t|}| st| st| | kst| |ksttt	j
 |  W 5 Q R X | sJt|d | dksftW 5 Q R X |js|tW 5 Q R X t|d|d}| st|| W 5 Q R X t|d&}| |kst| stW 5 Q R X t|d|d}| s
t|| |  t|d}| |ks8t|  d S )Nr*   r   compresslevelr   r   )r,   r-   r   r   writabler2   r   filenor   r  UnsupportedOperation_check_can_read_check_can_seekclosedr7   _check_not_closedreadableseekabler   _check_can_writeseektellclose)r3   r   r  r4   r   Zfzr   r   r   test_binary_zlibfile  sX    






r$  	bad_value   rc   c              	   C   sL   |  dj}tt}t|d|d W 5 Q R X td|}|| d S )Nr*   r   r  zS'compresslevel' must be an integer between 1 and 9. You provided 'compresslevel={}')	r,   r-   r   r7   r   r   r   r>   r?   )r3   r%  r4   r@   patternr   r   r   +test_binary_zlibfile_bad_compression_levels<  s    
r(  bad_moder   rp   r   rE   c              	   C   s8   |  dj}tt}t|| W 5 Q R X |d d S )Nr*   zInvalid mode)r,   r-   r   r7   r   r?   )r3   r)  r4   r@   r   r   r   "test_binary_zlibfile_invalid_modesF  s    
r*  bad_filec              	   C   s,   t t}t| d W 5 Q R X |d d S )Nr   z1filename must be a str or bytes object, or a file)r   	TypeErrorr   r?   )r+  r@   r   r   r   *test_binary_zlibfile_invalid_filename_typeN  s    
r-  c                   @   s   e Zd Zdd ZdS )SubArrayc                 C   s   t t| ffS r   )_load_sub_arrayr   Zasarrayr    r   r   r   
__reduce__[  s    zSubArray.__reduce__N)r$   r%   r&   r0  r   r   r   r   r.  Y  s   r.  c                 C   s   t | j}| |d d < |S r   )r.  rJ   )Zarrdr   r   r   r/  ^  s    
r/  c                   @   s   e Zd ZdZdd ZdS )r^   z7A complex object containing numpy arrays as attributes.c                 C   s:   t jddd| _t jddd| _t jdddgdd| _d S )	Nrj   r[   rk   Zint32rc   r<   g      4@rx   )r   r   r_   rl   r`   rY   ra   r    r   r   r   __init__f  s    zComplexTestObject.__init__N)r$   r%   r&   __doc__r2  r   r   r   r   r^   c  s   r^   c                 C   sJ   |  dj}td}t|| t|}t|ts8ttj	
|| d S )Nr*   )r<   )r,   r-   r.  r   r.   r/   rU   r2   r   rW   rX   )r3   r4   rc   cr   r   r   test_numpy_subclassl  s    
r5  c                 C   sX   |  dj}d}t|t| t||ks2tt|| tt||ksTtd S )Nr*   {   )r,   r-   r   r.   r   r/   r2   )r3   r4   valuer   r   r   test_pathlibv  s    r8  c                 C   s   |  dj}tddgddggdd  tjdddd d d dd d f fD ]@}|jjr^t|jjrjtt	
|| t	|}tj|| qNd S )	Nr*   r   rE   rB   rH   )r<   2      F)order)r,   r-   r   Zasfortranarrayrl   rt   c_contiguousr2   f_contiguousr   r.   r/   rW   rX   )r3   r4   rY   array_reloadedr   r   r   "test_non_contiguous_array_pickling  s    "
r@  c                 C   sD   |  dj}td}tj||tjd t|}tj	
|| d S )Nr*   r<   protocol)r,   r-   r   zerosr   r.   pickleHIGHEST_PROTOCOLr/   rW   rX   )r3   r4   
test_arrayr?  r   r   r   test_pickle_highest_protocol  s
    

rG  c            
   
   C   s   t d} d}ttjtj}|| |d t|}| \}}|	d}t
| | W 5 Q R X |	d}t
|}W 5 Q R X t j||  t }	t
| |	 ||	  |	d}t
|}W 5 Q R X t j||  W 5 Q R X d S )Nr<   )	localhosti90  r   r   r   )r   r   socketAF_INETSOCK_STREAMbindlistencreate_connectionacceptmakefiler   r.   r/   rW   rX   r  r  sendgetvalue)
rF  Z_ADDRZlistenerclientserverclient_addrZsfcfr?  Zbytes_to_sendr   r   r   test_pickle_in_socket  s$    


rW  c                 C   s   |  dj}tj}tj|ddtj|ddg}t|| tj	|dd}t
|d tjs\t|d j|ksnttj|| d S )N	test.mmaprm   rk   rp   rq   r   )r,   r-   rF   ZALLOCATIONGRANULARITYr   rC  rl   r   r.   r/   rU   rZ   r2   offsetrW   rX   )r3   r   r   re   Zmemmapsr   r   r    test_load_memmap_with_big_offset  s    rZ  c                    sn   d}dG dd dt j G  fdddt}t||  t| j ksNtt| jks`tt| d S )N	test-nameztest-prefixc                   @   s   e Zd ZdS )z:test_register_compressor.<locals>.BinaryCompressorTestFileNr$   r%   r&   r   r   r   r   BinaryCompressorTestFile  s   r]  c                       s   e Zd Z fddZdS )z=test_register_compressor.<locals>.BinaryCompressorTestWrapperc                    s   t j|  d d S )Nre   prefixr   r2  r    r]  Zcompressor_prefixr   r   r2    s    zFtest_register_compressor.<locals>.BinaryCompressorTestWrapper.__init__Nr$   r%   r&   r2  r   ra  r   r   BinaryCompressorTestWrapper  s   rc  )	r  BufferedIOBaser   r   r   fileobj_factoryr2   r_  pop)r3   compressor_namerc  r   ra  r   test_register_compressor  s    rh  invalid_namec              	   C   s,   t t}t| d  W 5 Q R X |d d S )Nz"Compressor name should be a string)r   r7   r   r?   )ri  r@   r   r   r   %test_register_compressor_invalid_name  s    
rj  c               	      sP   G dd d G  fdddt } tt}td|   W 5 Q R X |d d S )Nc                   @   s   e Zd ZdS )zCtest_register_compressor_invalid_fileobj.<locals>.InvalidFileObjectNr\  r   r   r   r   InvalidFileObject  s   rk  c                       s   e Zd Z fddZdS )zJtest_register_compressor_invalid_fileobj.<locals>.InvalidFileObjectWrapperc                    s   t j|  dd d S Ns   prefixr^  r`  r    rk  r   r   r2    s    zStest_register_compressor_invalid_fileobj.<locals>.InvalidFileObjectWrapper.__init__Nrb  r   rm  r   r   InvalidFileObjectWrapper  s   rn  invalidzQCompressor 'fileobj_factory' attribute should implement the file object interface)r   r   r7   r   r?   )rn  r@   r   rm  r   (test_register_compressor_invalid_fileobj  s
    
rp  c                   @   s   e Zd Zdd ZdS )AnotherZlibCompressorWrapperc                 C   s   t j| tdd d S rl  )r   r2  r   r    r   r   r   r2    s    z%AnotherZlibCompressorWrapper.__init__Nrb  r   r   r   r   rq    s   rq  c                   @   s   e Zd Zdd ZdS ) StandardLibGzipCompressorWrapperc                 C   s   t j| tjdd d S rl  )r   r2  r   r   r    r   r   r   r2  
  s    z)StandardLibGzipCompressorWrapper.__init__Nrb  r   r   r   r   rr    s   rr  c               	   C   s~   d} t | t  tt}t | t  W 5 Q R X |d|  t | t dd | tks\tt|  j	t
jksptt|  d S )Nr[  z#Compressor '{}' already registered.T)force)r   rq  r   r7   rr  r?   r>   r   r2   re  r   r   rf  )rg  r@   r   r   r   +test_register_compressor_already_registered  s    
rt  c              	   C   s   dd l }d}|tkstt| j|jjks.t| dj}d}tj	|||d t
|d}|tttksptW 5 Q R X t||kstt	||d  t
|d}|tttkstW 5 Q R X t||kstd S )Nr   r   r*   	test datar+   r   r   )Z	lz4.framer   r2   re  frameZLZ4FrameFiler,   r-   r   r.   r   r   rQ   r   r/   )r3   r   Z
compressorr   r   r   r   r   r   test_lz4_compression&  s      rw  c              	   C   st   |  dj}d}t}tt}tj||dd W 5 Q R X || tt}t||d  W 5 Q R X || d S )Nz
test.nolz4ru  r   r+   r   )r,   r-   r   r   r7   r   r.   r?   )r3   r   r   msgr@   r   r   r    test_lz4_compression_without_lz4=  s    


ry  rB  c                 C   s4  |  dj}tjd}tj|||d tj|dd}t|tj	sFt
tj|| |jjtj dksjt
|jjsvt
tjdtjdtjdtjdg}|  dj}tj|||d tj|dd}t|D ]N\}}t|tj	st
tj|| | |jjtj dkst
|jjst
qtjdtjdtjd	tjdtjd
tjdtjdtjdtjdtjdtjdtjdtjdtjdtjdtjdtjdtjdd	}|  dj}tj|||d tj|dd}	|	 D ]T\}
}t|tj	st
tj||
 | |jjtj dkst
|jjst
qd S )NrX  rE   rA  rp   rq   r   z
test1.mmaprk   rB   r                     )	Za0Za1Za2a3Za4Za5Za6Za7Za8z
test2.mmap)r,   r-   r   rL   Zrandnr   r.   r/   rU   rZ   r2   rW   rX   ctypesr   ZNUMPY_ARRAY_ALIGNMENT_BYTESrt   ZalignedrO   r   rm   items)r3   rB  r   rc   rZ   Z
array_listZ
l_reloadedidxZ
array_dictZ
d_reloadedkeyr   r   r   test_memmap_alignment_paddingQ  sT    
 

 
r  )N)r3  r0   rR   rL   r   r  r   r   r   rC   r  rD  rI  
contextlibr   rF   pathlibr   r   ImportErrorr   Zjoblib.test.commonr   r   r   r   r   r	   Zjoblib.testingr
   r   r   Zjoblibr   r   Zjoblib.testr   Zjoblib.numpy_pickle_utilsr   r   r   r   Zjoblib.compressorr   r   r   r   r   ZtypelistZ_noneappendr\   _typeboolZ_boolr   _intfloatZ_floatcomplexZ_complexr}   _string_tupleZ_list_dictrQ   Z_builtinr   r   rx   r'   Z	_instanceZ_objectr5   r9   r8   rA   ri   ro   rw   ry   r{   r   r   r   r   r   r   r   r   r   r>   r   r   r   r   r   r  r  r
  r  r  r  r  r   encoder$  r(  r*  r-  rV   r.  r/  r^   r5  r8  r@  rG  rW  rZ  rh  rj  rp  rq  rr  rt  rw  ry  DEFAULT_PROTOCOLZ	protocolsrE  r  r   r   r   r   <module>   s  
	
















	2
	
1




S
 

(




		







4
	


	
		






