U
    >vfF                     @   s  d dl mZ d dlmZ d dlmZ d dlZd dlZd dlm	Z	m
Z
 d dlmZ d dlmZ ejdd Zd	d
 Zdd Zdd Zdd Zdd Zdd Zejdddgdd Zdd Zdd Zdd Zd d! Zejdddgd"d# Zejdd d$d%d&gd'd( Z ejdddd)gd*d+ Z!d,d- Z"ejd.dd)gd/d0 Z#ejd1d2d3gd4d5 Z$d6d7 Z%d8d9 Z&d:d; Z'ejd<d=d>d?gd@dA Z(dBdC Z)ejdDdEdFgdGdH Z*dIdJ Z+dKdL Z,dMdN Z-dOdP Z.dS )Q    )StringIO)Path)IteratorN)	DataFrame	read_json)
JsonReaderc                  C   s(   t dddgdddgd} | jdd	d
S )N                  ABTrecordslinesorient)r   to_json)df r   G/tmp/pip-unpacked-wheel-vdrwu74i/pandas/tests/io/json/test_readlines.pylines_json_df   s    r   c                  C   s8   t ddd} tddgddggddgd}t| | d S )	Nz!{"a": 1, "b": 2}
{"b":2, "a" :1}
Tr   r   r	   abcolumnsr   r   tmassert_frame_equal)resultexpectedr   r   r   test_read_jsonl   s    r$   c                 C   sB   t | ddddd|d}tddd	gd
ddgd}t|| d S )Niojsondataline_delimited.jsonTr   enginer   r
   r   r	   r   r   r   r   r   )datapathr*   r"   r#   r   r   r   test_read_jsonl_engine_pyarrow   s    r-   c                 C   s   |dkr$d}| j tjj|td tddgddgdfgd	d
dgd}|jddd}t||d}tdddgdddggd	d
dgd}t	
|| d S )NpyarrowzEPyarrow only supports a file path as an input and line delimited jsonreasonraisesr   r	   z
2020-03-05z2020-04-08T09:58:49+00:00ZhectorZaccountsdatenamer   Tr   r   r*   )node
add_markerpytestmarkxfail
ValueErrorr   r   r   r    r!   )requestr*   r0   r   Z	json_liner"   r#   r   r   r   test_read_datetime(   s    r<   c                  C   s|   d} t | } t| dd}tddgddggddgd	}t|| d} t| dd}tddgddggddgd	}t|| d S )
Nu5   {"a": "foo”", "b": "bar"}
{"a": "foo", "b": "bar"}
Tr   u   foo”barfoor   r   r   )r   r   r   r    r!   )r&   r"   r#   r   r   r   test_read_jsonl_unicode_chars<   s    r?   c                  C   s   t ddgddggddgd} | jddd}d	}||ks:tt d
dgddggddgd} | jddd}d}||kstttt|dd|  t ddgddggddgd} | jddd}d}||ksttt|dd|  d S )Nr   r	   r   r   r   r   Tr   r   z{"a":1,"b":2}
{"a":1,"b":2}
zfoo}r=   zfoo"z/{"a":"foo}","b":"bar"}
{"a":"foo\"","b":"bar"}
r   zfoo\za\z4{"a\\":"foo\\","b":"bar"}
{"a\\":"foo\"","b":"bar"}
)r   r   AssertionErrorr    r!   r   )r   r"   r#   r   r   r   test_to_jsonlN   s    rB   c                  C   sD   t ddgddggddgd} | jdddd	}d}||ks@td S )
Nr   r	   r   r   r   r   Tr@   
)r   r   countrA   )r   Zactual_new_lines_countZexpected_new_lines_countr   r   r   test_to_jsonl_count_new_linesc   s    rE   	chunksizer   g      ?c              	   C   sn   |dkr$d}| j tjj|td tt|dd}tt|d||d}t	|}W 5 Q R X t
|| d S )Nr.   mPyarrow only supports a file path as an input and line delimited jsonand doesn't support chunksize parameter.r/   Tr   r   rF   r*   )r5   r6   r7   r8   r9   r:   r   r   pdconcatr    r!   )r;   r   rF   r*   r0   	unchunkedreaderchunkedr   r   r   test_readjson_chunksk   s       rN   c              
   C   sB   d}t jt|d& tt| dd|d}W 5 Q R X W 5 Q R X d S )Nz*chunksize can only be passed if lines=TruematchFr	   rH   r7   r1   r:   r   r   )r   r*   msg_r   r   r   &test_readjson_chunksize_requires_lines   s       rT   c              	   C   s   |dkr"d}| j tjj|d tddd}t|jddd	}t	|dd
|d}t|jddd	}t	|dd
d|d}t
|}W 5 Q R X t|| d S )Nr.   rG   )r0   r   r	   r   Tr   r   Series)r   typr*   )r   rV   rF   r*   )r5   r6   r7   r8   r9   rI   rU   r   r   r   rJ   r    Zassert_series_equal)r;   r*   r0   sZstriorK   rL   rM   r   r   r   test_readjson_chunks_series   s$        rX   c              	   C   st   |dkr$d}| j tjj|td tt|dd|d}t|}W 5 Q R X |d j	dks^t
|d	 j	d
kspt
d S )Nr.   rG   r/   Tr	   rH   r   r	   r	   r   )r   r	   )r5   r6   r7   r8   r9   r:   r   r   listshaperA   )r;   r   r*   r0   rL   chunksr   r   r   test_readjson_each_chunk   s       r]   c              
   C   s   |dkr$d}| j tjj|td tdp}tdddgdd	d
gd}|j	|ddd t
|dd|d}t|}W 5 Q R X t
|d|d}t|| W 5 Q R X d S )Nr.   rG   r/   	test.jsonr   r	   r
   r   r   r   r   Tr   r   rH   r)   )r5   r6   r7   r8   r9   r:   r    ensure_cleanr   r   r   rI   rJ   r!   )r;   r*   r0   pathr   rL   rM   rK   r   r   r   test_readjson_chunks_from_file   s    ra   c                 C   s   t d}tdddgdddgd}|j|d	d
d t|d dd	d	d	d	dd d d	| d d d}| |  W 5 Q R X |jjjst	d|  W 5 Q R X d S )Nr^   r   r	   r
   r   r   r   r   Tr   r   frameF)r   rV   ZdtypeZconvert_axesZconvert_datesZkeep_default_datesZprecise_floatZ	date_unitencodingr   rF   compressionnrowsz%didn't close stream with chunksize = )
r    r_   r   r   r   readZhandleshandleclosedrA   )rF   r`   r   rL   r   r   r   test_readjson_chunks_closes   s2    ri   g@r>   c              
   C   sB   d}t jt|d& tt| d||d}W 5 Q R X W 5 Q R X d S )Nz"'chunksize' must be an integer >=1rO   TrH   rQ   )r   rF   r*   rR   rS   r   r   r   test_readjson_invalid_chunksize   s       rk   r	   c              	   C   sh   d}t dddgdddgd}t|d	| d
}| d k	rN| t|}W 5 Q R X tj||d|  d d S )NzF

    {"A":1,"B":4}



    {"A":2,"B":5}







    {"A":3,"B":6}
    r   r	   r
   r   r   r   r   Tr   rF   zchunksize: )obj)r   r   rI   rJ   r    r!   )rF   jorigtestr   r   r   )test_readjson_chunks_multiple_empty_lines   s    rq   c              
   C   s   |dkr$d}| j tjj|td td`}|ddd  t	|dd	d
}|
d W 5 Q R X t||d}tddgi}t|| W 5 Q R X d S )Nr.   rG   r/   r^   zlocale.getpreferredencodingc                 S   s   dS )Ncp949r   )do_setlocaler   r   r   <lambda>      z'test_readjson_unicode.<locals>.<lambda>wzutf-8)rc   u6   {"£©µÀÆÖÞßéöÿ":["АБВГДабвгд가"]}r4   u   £©µÀÆÖÞßéöÿu   АБВГДабвгд가)r5   r6   r7   r8   r9   r:   r    r_   setattropenwriter   r   r!   )r;   Zmonkeypatchr*   r0   r`   fr"   r#   r   r   r   test_readjson_unicode	  s    r{   re   c                 C   sJ   d}t |d| d}tddddgdd	d
dgdjd |  }t|| d S )N[{"a": 1, "b": 2}
        {"a": 3, "b": 4}
        {"a": 5, "b": 6}
        {"a": 7, "b": 8}T)r   re   r   r
   r      r	   r   r      r+   )r   r   ilocr    r!   )re   r*   jsonlr"   r#   r   r   r   test_readjson_nrows  s    (r   znrows,chunksizerY   )r   r	   c           	   	   C   s   |dkr$d}| j tjj|td d}t|d|||d}t|}W 5 Q R X t	ddd	d
gddddgdj
d | }t|| d S )Nr.   rG   r/   r|   T)r   re   rF   r*   r   r
   r   r}   r	   r   r   r~   r+   )r5   r6   r7   r8   r9   r:   r   rI   rJ   r   r   r    r!   )	r;   re   rF   r*   r0   r   rL   rM   r#   r   r   r   test_readjson_nrows_chunks)  s         (r   c              	   C   s6   d}d}t jt|d t|dd| d W 5 Q R X d S )Nr|   z&nrows can only be passed if lines=TruerO   Fr	   )r   re   r*   )r7   r1   r:   r   )r*   r   rR   r   r   r   "test_readjson_nrows_requires_linesA  s    r   c           
   	   C   s   |dkr$d}| j tjj|td tddggddgdgd	td
dggddgdgd	tddggddgdgd	g}|dddd}t| }t	|dd|d(}t
|D ]\}}	t|	||  qW 5 Q R X d S )Nr.   rG   r/   r   r	   r   r   r   )r   indexr
   r   r   r   r%   r&   r'   r(   TrH   )r5   r6   r7   r8   r9   r:   r   r   as_urir   	enumerater    r!   )
r;   r,   r*   r0   Zdf_list_expectedZos_pathZfile_urlZ
url_readerr   Zchuckr   r   r   "test_readjson_lines_chunks_fileurlM  s    r   c                  C   sN   dd } G dd d}|| }t tt|ddddks<t|jd	ksJtd S )
Nz\{"a": 1, "b": 2}
        {"a": 3, "b": 4}
        {"a": 5, "b": 6}
        {"a": 7, "b": 8}
i  c                   @   s0   e Zd ZddddZdd ZedddZdS )	z/test_chunksize_is_incremental.<locals>.MyReaderN)returnc                 S   s   d| _ t|| _d S )Nr   )
read_countr   stringio)selfcontentsr   r   r   __init__o  s    z8test_chunksize_is_incremental.<locals>.MyReader.__init__c                 W   s   |  j d7  _ | jj| S Nr   )r   r   rf   )r   argsr   r   r   rf   s  s    z4test_chunksize_is_incremental.<locals>.MyReader.readc                 S   s   |  j d7  _ t| jS r   )r   iterr   )r   r   r   r   __iter__w  s    z8test_chunksize_is_incremental.<locals>.MyReader.__iter__)__name__
__module____qualname__r   rf   r   r   r   r   r   r   MyReadern  s   r   Td   rl   r   
   )lenrZ   r   rA   r   )r   r   rL   r   r   r   test_chunksize_is_incrementald  s    r   orient_splitr   tablec              	   C   sF   t ddgddgd}d}tjt|d |jd| d W 5 Q R X d S )	Nr   r	   r   r   col1col2Omode='a' \(append\) is only supported whenlines is True and orient is 'records'rO   )moder   r   r7   r1   r:   r   )r   r   rR   r   r   r   test_to_json_append_orient  s
    r   c               	   C   sH   t ddgddgd} d}tjt|d | jddd	d
 W 5 Q R X d S )Nr   r	   r   r   r   r   rO   Fr   r   r   r   r   )r   rR   r   r   r   test_to_json_append_lines  s
    r   mode_rxc              	   C   sP   t ddgddgd}d|  d}tjt|d |j| d	d
d W 5 Q R X d S )Nr   r	   r   r   r   zmode=z@ is not a valid option.Only 'w' and 'a' are currently supported.rO   Fr   r   r   )r   r   rR   r   r   r   test_to_json_append_mode  s
    
r   c               	   C   s   t ddgddgd} t ddgdd	gd}t ddddgdddd	gd}td
@}| j|ddd |j|dddd t|dd}t|| W 5 Q R X d S )Nr   r	   r   r   r   r
   r   cdr^   Tr   r   r   r   r   r    r_   r   r   r!   )df1df2r#   r`   r"   r   r   r   -test_to_json_append_output_consistent_columns  s    r   c               	   C   s   t ddgddgd} t ddgdd	gd
}t ddd d gddddgd d dd	gd}tdB}| j|dddd |j|dddd t|dd}t|| W 5 Q R X d S )Nr   r	   r   r   r   erz   !#r   col3)r   r   r   r^   Tr   r   r   r   )r   df3r#   r`   r"   r   r   r   /test_to_json_append_output_inconsistent_columns  s    


r   c                  C   s(  t ddgddgd} t ddgdd	gd}t d
dgddgd}t dddgi}t ddddd d d d gdddd	d
dd d gd d d d ddd d gd d d d d d ddgdddi}tdf}| j|dddd |j|dddd |j|dddd |j|dddd t|dd}t|| W 5 Q R X d S )Nr   r	   r   r   r   r
   r   r   r   r   rz   r   r   r   col4TF)r   r   r   r   floatr^   r   r   r   r   Zastyper    r_   r   r   r!   r   r   r   Zdf4r#   r`   r"   r   r   r   ,test_to_json_append_output_different_columns  s(    r   c                  C   s(  t ddgddgd} t ddgdd	gd}t d
dgddgd}t dddgi}t ddd d d d d d gd d d
ddd	ddgd d ddd d d d gd d d d ddddgdddi}tdf}|j|dddd |j|dddd |j|dddd | j|dddd t|dd}t|| W 5 Q R X d S )Nr   r	   r   r   r   r
   r   r   r   r   rz   r   r   r   r   TF)r   r   r   r   r   r^   r   r   r   r   r   r   r   r   6test_to_json_append_output_different_columns_reordered  s(    r   )/r%   r   pathlibr   typingr   r7   ZpandasrI   r   r   Zpandas._testingZ_testingr    Zpandas.io.json._jsonr   Zfixturer   r$   r-   r<   r?   rB   rE   r8   ZparametrizerN   rT   rX   r]   ra   ri   rk   rq   r{   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   <module>   sX   


	







