U
    >vfF                     @   s2  d Z ddlmZ ddlZddlZddlZddlmZ ddlZddlZddlZddl	Z	ddl
mZ ddlZddlZddlZddlZddlmZ ddlZddlZddlZddlmZmZ ddlmZ dd	lmZ ddlm  mZ  ddl!Z"dd
l!m#Z#m$Z$m%Z% ddl&m'Z( ddl)m  m*Z+ ddl,m-Z-m.Z. ej/dddd Z0dd Z1e	j23e	j24e5ddZ6ee	j23e6ddZ7ej/e7ddd Z8ej9:ddde;de<de=deddddge<d>dde<d>ddddd e?d j@d!d"d#e?d j@d!d$d#e?d j@d!d"d#dddddf gd%d& ZAd'd( ZBd)d* ZCd+d, ZDej9:d-ejEeCd.d/ejEe"jFd0d/ejEeje"jFejGd1d2d/ejEeje"jFd3d1d4d/ejEeje"jFd5d1d6d/gej9:d7e"jFeCgd8d9 ZHd:d; ZId<d= ZJej/d>d? ZKG d@dA dAZLG dBdC dCZMej9:dDdEgdFe#dGdHdIgfdJe"jNOdGdHdIgdKdd"ggfgdLdM ZPdNdO ZQej9:dPdQdRdSgdTdU ZRe SdVdWdX ZTG dYdZ dZejUZVd[d\ ZWd]d^ ZXd_d` ZYdadb ZZej9:dcdddedcedfdgdgdhgdidj Z[dkdl Z\dmdn Z]e j^dodp Z_ej9:dqej`ejGgdrds Zadtdu ZbdS )va<  
manage legacy pickle tests

How to add pickle tests:

1. Install pandas version intended to output the pickle.

2. Execute "generate_legacy_storage_files.py" to create the pickle.
$ python generate_legacy_storage_files.py <output_dir> pickle

3. Move the created pickle to "data/legacy_pickle/<version>" directory.
    )arrayN)partial)Path)catch_warnings)get_lzma_fileis_platform_little_endian)import_optional_dependency)flatten_buffer)IndexSeriesperiod_range)DayMonthEndmodule)Zscopec               
   C   s0   ddl m}  t  |  W  5 Q R  S Q R X d S )Nr   create_pickle_data)Z-pandas.tests.io.generate_legacy_storage_filesr   r   r    r   ?/tmp/pip-unpacked-wheel-vdrwu74i/pandas/tests/io/test_pickle.pycurrent_pickle_data8   s    r   c                 C   s   t |trt||  d S |dr2t| | nP|dkrb|tjkrT| tjks`tq| |kstn t	td| dtj
}|| | d S )NZsp_	timestampZassert_Z_equal)
isinstancer
   tmassert_index_equal
startswithZassert_equalpdZNaTAssertionErrorgetattrZassert_almost_equal)resultexpectedtypZ
comparatorr   r   r   compare_elementD   s    


r    datalegacy_pickle*z*.pickle)paramsc                 C   s
   || j S N)param)requestdatapathr   r   r   r"   Y   s    s   123s   123456I         B)r,   r+      )r,      C)orderFc                 C   s   t | }t| d}||ks"tt| ttfr>|| kstnBt|tr|jdksVt|jdksdt|j	snt|j
|jfkstd S )NAr*   r-   )r	   
memoryviewtobytesr   r   bytes	bytearrayndimformat
contiguousshapenbytes)r!   r   r   r   r   r   test_flatten_bufferc   s    

r=   c           	      C   s  t  std t| }| D ]\}}| D ]\}}|| | }|dkr2|dkr2t|| |jj	|jj	ks~t
|jj	jrt
t|dk|dk |jj	}|td tdkst
|tjdd }t|tjst
|tjdddkst
|tjdd	 }t|tjst
|tjddd
kst
q4|dkr|dkrt|| t|j	tsdt
|j	t ksvt
|jdkst
t|d|d q4|dkr|dkrt|| q4|dkr|dkrt|| q4t||| q4q"d S )Nz"known failure on non-little endianZseriestsr   r*   r+   )hours)daysr?   )nanoseconds)r@   rA   indexZperiodM)Zdt_tzcatframe)Zdt_mixed_tzsZ
cat_onecolZcat_and_float)r   pytestskipr   read_pickleitemsr   assert_series_equalrB   freqr   	normalizer   Z	Timedeltar   r   r   Zfreqstrshiftassert_frame_equalr    )	r"   r!   r   dvdtr   r   rK   resr   r   r   test_pickles   s<    

rR   c              	   C   s*   t |d}tj| |dd W 5 Q R X d S Nwbprotocolopenpickledumpobjpathfhr   r   r   python_pickler   s    r`   c              
   C   s6   t | d"}|d t|W  5 Q R  S Q R X d S )Nrbr   )rY   seekrZ   load)r^   r_   r   r   r   python_unpickler   s    
rd   pickle_writerpython)idZpandas_proto_defaultrV   Zpandas_proto_highestr/   Zpandas_proto_4   Zpandas_proto_5writerc                 C   s   | }|  D ]\}}|  D ]\}}t }||| t|}	t|	|| t|}	t|	|| t|dd}
||| |
d W 5 Q R X t|dd}
t|
}	|
d W 5 Q R X t|	|| W 5 Q R X qqd S )NrT   moder   ra   )	rI   r   ensure_cleanr   rH   r    rd   rY   rb   )r   re   ri   r!   r   rO   rP   r   r^   r   handler   r   r   test_round_trip_current   s     




rn   c                  C   s(   t  } t | jtj}t | | d S r%   )r   makeDataFrameround_trip_pathlib	to_pickler   rH   rN   dfr   r   r   r   test_pickle_path_pathlib   s    rt   c                  C   s(   t  } t | jtj}t | | d S r%   )r   ro   Zround_trip_localpathrq   r   rH   rN   rr   r   r   r   test_pickle_path_localpath   s    ru   c                   C   s   dt   dS )N__z	__.pickle)uuidZuuid4r   r   r   r   get_random_path   s    rx   c                   @   sX   e Zd ZejZdd Zdd Zej	
ddddd	gd
d Zdd Zdd Zdd ZdS )TestCompressionc           	   
   C   s`  |d krt || d S |dkr.t|d}n|dkrDt|d}n|dkrtj|dtjd}|	|t
j| W 5 Q R X n|dkrt|d@}tj|dd&}||t
j|}||| W 5 Q R X W 5 Q R X nD|d	krt |d}n.|d
krtd|d}nd| }t||dkr\t|d$}| |	|  W 5 Q R X W 5 Q R X d S )Ngzipwbz2zipcompressiontarra   rj   xzZzstdZ	zstandardrT   zUnrecognized compression type: )r}   r   )shutilcopyfilerz   rY   r|   BZ2FilezipfileZipFileZIP_DEFLATEDwriteosr^   basenametarfile
gettarinfoaddfiler   r   
ValueErrorread)	selfZsrc_path	dest_pathr   fr_   r   tarinfomsgr   r   r   compress_file   s2     "


zTestCompression.compress_filec                 C   s   |}|d }|d }t |}t |t}t  }|j||d t j||d*}	t|d}
|
|	  W 5 Q R X W 5 Q R X tj	|d d}t 
|| W 5 Q R X W 5 Q R X d S )N.compressed.rawr~   rT   )r   rl   ro   rq   decompress_filerY   r   r   r   rH   rN   )r   r   rx   basepath1path2p1p2rs   r   r_   df2r   r   r   test_write_explicit  s    "z#TestCompression.test_write_explicitr    NonebadZ7zc              
   C   sJ   t jtdd2 t|}t }|j||d W 5 Q R X W 5 Q R X d S )NzUnrecognized compression type)matchr~   )rF   Zraisesr   r   rl   ro   rq   )r   r   rx   r^   rs   r   r   r   test_write_explicit_bad/  s    z'TestCompression.test_write_explicit_badc                 C   s   |}|| }|d }| j | }t|}t|p}t }	|	| tj||d*}
t|d}|	|

  W 5 Q R X W 5 Q R X tj|d d}t|	| W 5 Q R X W 5 Q R X d S )Nr   r~   rT   )_extension_to_compressiongetlowerr   rl   ro   rq   r   rY   r   r   r   rH   rN   )r   compression_extrx   r   r   r   r   r   r   rs   r   r_   r   r   r   r   test_write_infer6  s    
"z TestCompression.test_write_inferc           
   
   C   s   |}|d }|d }t |\}t |F}t  }|j|d d | j|||d tj||d}	t ||	 W 5 Q R X W 5 Q R X d S )Nr   r   r~   )r   rl   ro   rq   r   r   rH   rN   )
r   r   rx   r   r   r   r   r   rs   r   r   r   r   test_read_explicitL  s    z"TestCompression.test_read_explicitc              
   C   s   |}|d }|| }| j | }t|X}t|B}t }	|	j|d d | j|||d t	|}
t
|	|
 W 5 Q R X W 5 Q R X d S )Nr   r~   )r   r   r   r   rl   ro   rq   r   r   rH   rN   )r   r   rx   r   r   r   r   r   r   rs   r   r   r   r   test_read_infer^  s    
zTestCompression.test_read_inferN)__name__
__module____qualname__icomZextension_to_compressionr   r   r   rF   markparametrizer   r   r   r   r   r   r   r   ry      s   
ry   c                   @   s*   e Zd Zejdddddgdd ZdS )	TestProtocolrW   rU   r   r*   r+   c              	   C   sF   t |2}t  }|j||d t|}t || W 5 Q R X d S )NrV   r   rl   ro   rq   r   rH   rN   )r   rW   rx   r^   rs   r   r   r   r   	test_readx  s
    
zTestProtocol.test_readN)r   r   r   rF   r   r   r   r   r   r   r   r   w  s   r   pickle_fileexcolsztest_py27.pklabcztest_mi_py27.pklr3   c                 C   s*   | ddd|}t |}t|j| d S )Nior!   rZ   )r   rH   r   r   columns)r(   r   r   r^   rs   r   r   r   test_unicode_decode_error  s    
r   c               
   C   sl   t  Z} t  }t| d}|| W 5 Q R X t| d}t|}W 5 Q R X t || W 5 Q R X d S )NrT   ra   )r   rl   ro   rY   rq   r   rH   rN   )r^   rs   r_   r   r   r   r   test_pickle_buffer_roundtrip  s    
r   mockurlzhttp://url.comzftp://test.comzhttp://gzip.comc              	      sp   dd }G dd d t  H fdd}t  }|| | d| t|}t || W 5 Q R X d S )Nc              	   S   s*   t |d}tj| |dd W 5 Q R X d S rS   rX   r\   r   r   r   r`     s    z3test_pickle_generalurl_read.<locals>.python_picklerc                   @   s:   e Zd ZddddZdd Zdd Zd	d
 Zdd ZdS )z5test_pickle_generalurl_read.<locals>.MockReadResponseNreturnc                 S   s.   t |d| _d|kr ddi| _n
ddi| _d S )Nra   rz   zContent-Encodingr   )rY   fileheaders)r   r^   r   r   r   __init__  s    z>test_pickle_generalurl_read.<locals>.MockReadResponse.__init__c                 S   s   | S r%   r   r   r   r   r   	__enter__  s    z?test_pickle_generalurl_read.<locals>.MockReadResponse.__enter__c                 W   s   |    d S r%   )close)r   argsr   r   r   __exit__  s    z>test_pickle_generalurl_read.<locals>.MockReadResponse.__exit__c                 S   s
   | j  S r%   )r   r   r   r   r   r   r     s    z:test_pickle_generalurl_read.<locals>.MockReadResponse.readc                 S   s
   | j  S r%   )r   r   r   r   r   r   r     s    z;test_pickle_generalurl_read.<locals>.MockReadResponse.close)r   r   r   r   r   r   r   r   r   r   r   r   MockReadResponse  s
   r   c                     s    S r%   r   )r   kwargsr   r^   r   r   mock_urlopen_read  s    z6test_pickle_generalurl_read.<locals>.mock_urlopen_readzurllib.request.urlopen)r   rl   ro   setattrr   rH   rN   )Zmonkeypatchr   r`   r   rs   r   r   r   r   test_pickle_generalurl_read  s    


r   Zfsspecc               	   C   sD   t  2 d} t  }||  t| }t || W 5 Q R X d S )Nzmemory://afiler   )r   rs   r   r   r   r   test_pickle_fsspec_roundtrip  s    


r   c                   @   s   e Zd ZddddZdS )MyTzNr   c                 C   s   d S r%   r   r   r   r   r   r     s    zMyTz.__init__)r   r   r   r   r   r   r   r   r     s   r   c                  C   sD   t tdt f} t| }t|d | d  t|d ts@td S )N)Zdtyper   r*   )r   objectr   r   round_trip_picklerJ   r   r   )r   r   r   r   r   test_read_pickle_with_subclass  s    
r   c              	   C   s   t  }t   }|j|| d t| }W 5 Q R X t }|j|| d |d |	 |ksn| dksnt
tj|| d}|d t || dS )ze
    Read/write from binary file-objects w/wo compression.

    GH 26237, GH 29054, and GH 29570
    r~   r   )rz   r}   r   N)r   ro   rl   rq   r   
read_bytesr   BytesIOrb   getvaluer   r   rH   rN   )r   rs   r^   	referencebufferZread_dfr   r   r   %test_pickle_binary_object_compression  s    


r   c                 C   s8   | }|}dd }|| ||j  || ||j  d S )Nc                 S   s   t | }t | | d S r%   )r   r   rN   )rE   	unpickledr   r   r   _test_roundtrip  s    
zDtest_pickle_dataframe_with_multilevel_index.<locals>._test_roundtrip)T)Z/multiindex_year_month_day_dataframe_random_dataZ multiindex_dataframe_random_dataZymdrE   r   r   r   r   +test_pickle_dataframe_with_multilevel_index  s    
r   c                  C   sB   t dddd} ttjt| | }t|}|jj	dks>t
d S )Nz1/1/2011z1/1/2012rC   )rK   )r   r   nprandomZrandnlenr   r   rB   rK   r   )Zprngr>   Znew_tsr   r   r   "test_pickle_timeseries_periodindex  s    
r   namei	  g     H@i     )r*   r+   c                 C   s$   t t j| d}|j| ks td S )N)r   )r   r   ZmakeTimeSeriesr   r   )r   r   r   r   r   test_pickle_preserve_name  s    r   c                 C   s   t | }t ||  d S r%   r   r   rJ   )Zdatetime_seriesZunp_tsr   r   r   test_pickle_datetimes%  s    
r   c                 C   s   t | }t ||  d S r%   r   )Zstring_seriesZ
unp_seriesr   r   r   test_pickle_strings*  s    
r   c                  C   sf   t tddjdg } t| }|jjd jdks:t	|jjd j
dksPt	t|dg |  d S )Nabccategoryr   r*   )r*   T)r   listZastypeZilocr   r   Z_mgrblocksr8   r   r;   rJ   )ZserrQ   r   r   r    test_pickle_preserves_block_ndim/  s
    
r   rW   c                 C   s@   t td}tt|j| |dtt j|d}t|| d S )Ni )rW   r   r~   )	r   	DataFrameranger   rp   r   rq   rH   rN   )rW   r   rs   r   r   r   r   %test_pickle_big_dataframe_compression<  s    r   c               	   C   sN   t jtdd} t| d}t|}W 5 Q R X tjg g d}t	
|| d S )Nz1.2.4zempty_frame_v1_2_4-GH#42345.pklra   )rB   r   )r   r^   joinlegacy_dirnamerY   rZ   rc   r   r   r   rN   )r^   fdrs   r   r   r   r   #test_pickle_frame_v124_unpickle_130G  s
    r   )c__doc__r   r|   datetime	functoolsr   globrz   r   r   pathlibr   rZ   r   r   rw   warningsr   r   Znumpyr   rF   Zpandas.compatr   r   Zpandas.compat._optionalr   Zpandas.compat.compressorsr	   Zpandas.util._test_decoratorsutilZ_test_decoratorstdZpandasr   r
   r   r   Zpandas._testingZ_testingr   Zpandas.io.commoncommonr   Zpandas.tseries.offsetsr   r   Zfixturer   r    r^   r   dirname__file__r   filesr"   r   r   r7   r4   PickleBuffercastZarangeZreshaper=   rR   r`   rd   r&   rq   HIGHEST_PROTOCOLrn   rt   ru   rx   ry   r   Z
MultiIndexZfrom_arraysr   r   r   Z
skip_if_nor   tzinfor   r   r   r   r   r   r   r   Zskip_array_manager_invalid_testr   DEFAULT_PROTOCOLr   r   r   r   r   r   <module>   s   



	$
.
~


 
%
		 



