U
    Kvfl                     @   s  d dl mZmZ d dlmZ d dlmZ d dlZd dl	m
Z d dlZd dl	mZ d dlZd dlZd dlmZ dd Zd	d
 Zdd Zdd ZG dd dZG dd dZG dd dZdd ZG dd dZdd Zdd Zdd Zdd  ZG d!d" d"ZG d#d$ d$eZG d%d& d&eZ G d'd( d(eZ!G d)d* d*eZ"G d+d, d,eZ#G d-d. d.eZ$G d/d0 d0eZ%G d1d2 d2eZ&G d3d4 d4e"Z'dS )5    )assert_series_equalassert_frame_equal)StringIO)dedentN)assert_equal)rosc                  C   sF   t d} t| }|d |jd d df< |d dk|jd d df< |S )Na  res,qual
2.00,=
4.20,=
4.62,=
5.00,ND
5.00,ND
5.50,ND
5.57,=
5.66,=
5.75,ND
5.86,=
6.65,=
6.78,=
6.79,=
7.50,=
7.50,=
7.50,=
8.63,=
8.71,=
8.99,=
9.50,ND
9.50,ND
9.85,=
10.82,=
11.00,ND
11.25,=
11.25,=
12.20,=
14.92,=
16.77,=
17.81,=
19.16,=
19.19,=
19.64,=
20.18,=
22.97,=
resconcqualZNDcensored)r   pandasread_csvloc)Zraw_csvdf r   I/tmp/pip-unpacked-wheel-2v6byqio/statsmodels/imputation/tests/test_ros.pyload_basic_data   s    
r   c               )   C   s  t dddddddddddddddddddddd	d
dddd	d
ddddddddddddddddddddddddddddddddddddddddddddddddddd
dddddddddddddddddddddddddddddd ddd!d
dddd"d
dddd#ddddd#ddddd$ddddd%dd
ddd&ddddd'ddddd(ddddd)ddddd*ddddd+dd ddd,dd-dg#} | S ).NT      @   )r   r	   det_limit_indexrank         @      @         #@         &@   F       @r   @{Gz@HzG@p=
ף@q=
ףp@皙@Q@)\(@      @      (\B!@   Qk!@	   {G!@
   33333#@p=
ף%@     &@ffffff(@ףp=
-@Q0@(\1@)\(3@q=
ף03@p=
ף3@Gz.4@Q6@   r   	DataFramer   r   r   r   load_intermediate_data    sL    &rA   c               +   C   s>  t ddddddddddddd	dd
ddd	ddddddddddddddddddddddd	dddddddddddddddddd ddd	dd!dd"dd#ddd$dd%d	d&ddd'dd(d	d)d	dd*dd+dd,ddd-dd.dd/d	dd0dd1dd2ddd3dd4dd5ddd6dd7dd8ddd9dd7dd:d;dd<dd7dd=d>dd?dd@ddAdBddCddDddEdFddGddHddIdJddKddLddMdddNddOddPd	ddQddRddSdddTddRddUd	ddVddWddXdddYddZdd[ddd\dd]dd^ddd_dd`ddad;ddbddcdddd>ddeddfddgdBddhddiddjdFddkddlddmdJddnddoddpdqdg#} | S )rNgCB!Tr   r   g$F?)ZZprelimr   r	   r   Zplot_posr   gXHg6iqx?r   gJ@r   g$F?gI@r   r   g$F?gXr   r   gs?gW'>ٿg__?gz0zʿr   r   s?g{Fr   r   g6iqx?r    g?r!   g)YUZ?gL{br"   <8 E?gMQnr#   g!Rkf?g#ܿr$   gGGt)?g!6|Rf׿r%   g j`?g@>(xҿr&   gL_?g?	$R˿r'   gr8-?g@Dr(   g+$?g?V$kmg{?r)   g.+NƷ?gϋ~K3?r*   gȎE?r+   g+(?r,   g[ T?r-   gj?r.   go4%?r/   g?r0   g?r1   gU	?g?糼k?r2   gyڽN?g?r3   g~?g&.Ef?gyy?gXu?r4   gPuPu?gU?r5   gKK?gK4?r6   g""""""?g?r7   g?gM?r8   g<<?g۸?r9   g]ʥ\ʥ?g%͢?r:   gW|W|?g1_B?r;   g.R.R?gj2
?r<   gr)r)?r=   r>   r@   r   r   r   load_advanced_dataJ   sd                                                                         IrD   c                  C   s   t ddddddddddddddddd	ddd
dd
ddddddddddddddddddtjdtjtjtjtjdtjdg} | S )Nr                 @      ?r   lower_dl
ncen_equal
nobs_belownuncen_aboveprob_exceedanceupper_dlK?r         @r         "@      $@܄?r         5@]R0?r         8@AA?)r   r?   numpyinfnan)cohnr   r   r   load_basic_cohn   sL                  r\   c                   @   s$   e Zd Zdd Zdd Zdd ZdS )Test__ros_sortc              '   C   sJ  t  | _tddddddddddddddddddddddd	ddd
ddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd ddd!dg#d"d#g | _| jjd d$ | _d S )%NTr   )r   r	   r   r   r   r   Fr   r    r!   r"   r#   r$   r%   r&   r'   r(   r+   r-   r/   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r	   r   )r   r   r   r?   expected_baselineZilocexpected_with_warningselfr   r   r   setup_method   sR                     zTest__ros_sort.setup_methodc                 C   s    t | jdd}t|| j d S )Nr	   r   )r   	_ros_sortr   r   r_   rb   resultr   r   r   test_baseline   s    zTest__ros_sort.test_baselinec                 C   sB   | j  }|d  }d|j|df< t|dd}t|| j d S )Nr	   Tr   )r   copyZidxmaxr   r   rd   r   r`   )rb   r   Zmax_rowrf   r   r   r   test_censored_greater_than_max   s
    
z-Test__ros_sort.test_censored_greater_than_maxN)__name__
__module____qualname__rc   rg   ri   r   r   r   r   r]      s   r]   c                   @   s$   e Zd Zdd Zdd Zdd ZdS )Test_cohn_numbersc                 C   s   t  | _ddddddg| _tdddd	d
ddddddddddd
ddddddd
dddddddddddddd
dddtjdtjtjtjtjdtjdg| j | _d S )NrI   rN   rL   rK   rJ   rM   r   rE   rF   rG   r   rH   rO   r   rP   r   rQ   rR   rS   r   rT   rU   r   rV   rW   )	r   r   Z
final_colsr   r?   rX   rY   rZ   r_   ra   r   r   r   rc      sX                   zTest_cohn_numbers.setup_methodc                 C   s"   t j| jddd}t|| j d S )Nr	   r   ZobservationsZ
censorship)r   cohn_numbersr   r   r_   re   r   r   r   rg      s    zTest_cohn_numbers.test_baselinec                 C   s4   | j  }d|d< tj|ddd}|jdks0td S )NFr
   r	   rn   )r   r)   )r   rh   r   ro   shapeAssertionError)rb   Z_dfrf   r   r   r   test_no_NDs   s    
zTest_cohn_numbers.test_no_NDsN)rj   rk   rl   rc   rg   rr   r   r   r   r   rm      s   rm   c                   @   s,   e Zd Zdd Zdd Zdd Zdd Zd	S )
Test__detection_limit_indexc                 C   s   t  | _ttd| _d S )N)r   r*   )r\   r[   r   r?   rX   empty
empty_cohnra   r   r   r   rc      s    z(Test__detection_limit_index.setup_methodc                 C   s   t td | jd d S Nr   )r   r   _detection_limit_indexru   ra   r   r   r   
test_empty   s    z&Test__detection_limit_index.test_emptyc                 C   s@   t td| jd t td| jd t td| jd d S )N      @r   rP   r         (@r   )r   r   rw   r[   ra   r   r   r   test_populated   s    z*Test__detection_limit_index.test_populatedc              	   C   s(   t t td| j W 5 Q R X d S rv   )pytestZraises
IndexErrorr   rw   r[   ra   r   r   r   test_out_of_bounds  s    z.Test__detection_limit_index.test_out_of_boundsN)rj   rk   rl   rc   rx   r{   r~   r   r   r   r   rs      s   rs   c                  C   sz   t dgd tdtd ttdd} t| dd}t jdddddd	dddd
dd	gdd}t|t	|t	 d S )Nr      ZAABCCCDEZDCBA)dl_idxparamsvaluesr   r   r   r   r   r   )name)
r   r?   listranger   Z_ros_group_rankZSeriesr   Zastypeint)r   rf   expectedr   r   r   test__ros_group_rank  s    
&r   c                   @   s4   e Zd Zdd Zdd Zdd Zdd Zd	d
 ZdS )Test__ros_plot_posc                 C   s   t  | _d S )N)r\   r[   ra   r   r   r   rc     s    zTest__ros_plot_pos.setup_methodc                 C   s*   dddd}t |d| j}t|d d S )NFr   r   r   r   r   r   rC   r   Z_ros_plot_posr[   r   rb   rowrf   r   r   r   test_uncensored_1  s    z$Test__ros_plot_pos.test_uncensored_1c                 C   s*   dddd}t |d| j}t|d d S )NFr   r   r   r   g}?r   r   r   r   r   test_uncensored_2  s    z$Test__ros_plot_pos.test_uncensored_2c                 C   s*   dddd}t |d| j}t|d d S )NTr   r   r   r   g__?r   r   r   r   r   test_censored_1!  s    z"Test__ros_plot_pos.test_censored_1c                 C   s*   dddd}t |d| j}t|d d S )NTr   r   r   r   rB   r   r   r   r   r   test_censored_2&  s    z"Test__ros_plot_pos.test_censored_2N)rj   rk   rl   rc   r   r   r   r   r   r   r   r   r     s
   r   c                  C   s4   t ddddg} tddddg}t| | d S )	Nr   r   r   r   gT]?g`>?gO:`?gyj?)r   Z_norm_plot_posrX   arraynptassert_array_almost_equal)rf   r   r   r   r   test__norm_plot_pos,  s    r   c               %   C   sz   t  } t }t| d|}tdddddddddd	d
dddddddddddddddddddddd d!d"g#}t|| d S )#Nr   g@,U?gUqx?g^c@?g?g_?g?gaˀqx?gBUZ?geL$E?gKnf?g*?gh	?gޮ_?gqo-?g> ?g{X={?gK3?g m(?g(?g1^?g?g}N?g"}?gCy?g&(Nu?gC"
K?g`C#""?g p?g!R<?gB4_ʥ?gb W|?g!/R?gr)?)rA   r\   r   Zplotting_positionsrX   r   r   r   )r   r[   resultsr   r   r   r   test_plotting_positions2  sR                                	r   c               %   C   s   t dddddddddd	d
ddddddddddddddddddddddddg#} t }t|d d!t jt j}|d" j}t	||  d S #NgC@gَ@g7!/@g@gKڿx@gQP@r   r    r!   r"   r#   r$   r%   r&   r'   r(   r+   r-   r/   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r	   r   final)
rX   r   rD   r   Z_imputelogexpr   r   r   r   r   rf   r   r   r   test__imputeC  sR                              
r   c               %   C   s   t dddddddddd	d
ddddddddddddddddddddddddg#} t }t|d d!t jt j}|d" j}t	||  d S r   )
rX   r   r   r   Z_do_rosr   r   r   r   r   r   r   r   r   test__do_rosU  sR                              
r   c                   @   s$   e Zd Zdd Zdd Zdd ZdS )CheckROSMixinc                 C   s6   t j| j| j| jd}tjt|t| j| j	d d S Nr@   )decimal)
r   
impute_rosrescolcencolr   r   r   sortedexpected_finalr   re   r   r   r   test_ros_dfi  s    zCheckROSMixin.test_ros_dfc                 C   s@   t j| j| j | j| j d d}tjt|t| j| j	d d S r   )
r   r   r   r   r   r   r   r   r   r   re   r   r   r   test_ros_arraysq  s     zCheckROSMixin.test_ros_arraysc                 C   sH   ddddg}t | j| j| j}tt|| dt| j| d d S )NrL   rK   rJ   rM   r   )	r   ro   r   r   r   r   nproundexpected_cohn)rb   colsr[   r   r   r   	test_cohny  s      zCheckROSMixin.test_cohnN)rj   rk   rl   r   r   r   r   r   r   r   r   h  s   r   c                   @   s  e Zd ZdZdZedddddddddddddd	d
dddgZeddddddddddddddddddgZdZ	dZ
ee	ee
eiZedddddddddddddd	d
dddgZeeddejgeddejgeddejgedddgdZd S )!Test_ROS_HelselAppendixBz
    Appendix B dataset from "Estimation of Descriptive Statists for
    Multiply Censored Water Quality Data", Water Resources Research,
    Vol 24, No 12, pp 1997 - 2004. December 1988.
    r   rG   rR   rF         @rQ   rz         .@      4@      ;@     @@      I@TFobscengGz?g333333?g(\?gRQ?g)\(?gQ@g      @g=
ףp=@g333333@rP   g%?g1ZGU?rE   rL   rK   rJ   rM   Nrj   rk   rl   __doc__r   rX   r   r   r   r   r   r   r?   r   r   rZ   r   r   r   r   r   r     s                                                  r   c                   @   s6  e Zd ZdZdZeddddddddddddddddd	d	d
d
ddddgZeddddddddddddddddddddddddgZdZ	dZ
ee	ee
eiZeddddddddddddddddd	dd
d
ddddgZeeddddejgedd d!d"ejgeddd#d$ejgedd%d&d'dgd(Zd)S )*Test_ROS_HelselArsenica!  
    Oahu arsenic data from Nondetects and Data Analysis by
    Dennis R. Helsel (John Wiley, 2005)

    Plotting positions are fudged since relative to source data since
    modeled data is what matters and (source data plot positions are
    not uniformly spaced, which seems weird)
    r   g	@gffffff@r   g333333?      ?rG   ?gffffff?g333333?      ?FTr   r   gQ?g=
ףp=?gffffff?Q?g(\?g=
ףp=?gq=
ףp?gffffff?g\(\?gRQ?g(\?g=
ףp=?gQ?rP   rE   r   rz   g      6@      @       @g      ?g=m?g&S?r   Nr   r   r   r   r   r     s                                                                 r   c                4   @   s   e Zd ZdZeedZdZdZe	j
eddZeddd	d
ddddddddddddddddddddddd d!d"d#d$d%d&d'd(d(d)d*d+d+d,d-d.d/d0d1d2d3d4d5g2Ze	ed6d7d8ejged+d9d:ejged+d+d;ejged<d=d=d>gd?Zd@S )ATest_ROS_RNADAdatar   a          res cen
        0.090  True
        0.090  True
        0.090  True
        0.101 False
        0.136 False
        0.340 False
        0.457 False
        0.514 False
        0.629 False
        0.638 False
        0.774 False
        0.788 False
        0.900  True
        0.900  True
        0.900  True
        1.000  True
        1.000  True
        1.000  True
        1.000  True
        1.000  True
        1.000 False
        1.000  True
        1.000  True
        1.000  True
        1.000  True
        1.000  True
        1.000  True
        1.000  True
        1.000  True
        1.000  True
        1.000  True
        1.000  True
        1.000  True
        1.100 False
        2.000 False
        2.000 False
        2.404 False
        2.860 False
        3.000 False
        3.000 False
        3.705 False
        4.000 False
        5.000 False
        5.960 False
        6.000 False
        7.214 False
       16.000 False
       17.716 False
       25.000 False
       51.000 Falser   r   z\s+)sepg6j?g}%?gM"?gB`"۹?g rh?g(\?gsh|??g r?gT㥛 ?g"~j?gS㥛?gK7?gJ	c?gπz3*?gN?g&V?g7F?gV?gqC/7?gɹEGϹ?rG   gzc?gfD̙+?gcT?g-?g",(?gG4L?gL6S?g?ΰ(W~?gR*?gZKfd?gJE?g?r   gZd;@gzG@rF   gp=
ף@r   r   gףp=
@rP   gB`"@g      0@gK1@g      9@g     I@rQ   rE   g      2@r   g      @@      1@gzG?g
ףp=
?r   r   N)rj   rk   rl   r   r   r   Z
datastringr   r   r   r   r   rX   r   r   r?   rZ   r   r   r   r   r   r     s~   4                                        r   c                   @   s   e Zd ZdZejd dZejjedZ	dge Z
dZdZeee	ee
iZedd	d
dddddddddddddddddgZeeg eg eg eg dZdS )Test_NoOp_ZeroNDr   r      )sizeFr   r   RQ?Q?r   Q?r   Gz?ffffff?Q?ffffff?ףp=
?)\(?(\?Q@Q@HzG@Q@ףp=
@\(\@zG@"@r   N)rj   rk   rl   r   rX   randomseedNZ	lognormalr   r   r   r   r   r?   r   r   r   r   r   r   r   r   r     sH   
                 r   c                   @   s   e Zd ZdZeddddddddddddddd	d
ddgZeddddddddddddddddddgZdZdZ	e
eee	eiZeddddddddddddddd	d
ddgZe
edejgedejgedejgeddgdZdS )Test_ROS_OneNDr   rG   rR   rF   r   rQ   rz   r   r   r   r   r   TFr	   r   gQ?r   g~!<8?rE   r   Nrj   rk   rl   r   rX   r   r   r   r   r   r   r?   r   r   rZ   r   r   r   r   r   r   2  s                                                  r   c                   @   s(  e Zd ZdZeddddddddddddddd	d
ddgZeddddddddddddddddddgZdZdZ	e
eee	eiZedddddddddddddddd
ddgZe
eddddddddejg	eddddddddejg	eddddddddejg	edgd dg dZdS ) Test_HalfDLs_80pctNDsr   rG   rR   rF   r   rQ   rz   r   r   r   r   r   TFvaluer
   r   r   r   ry   g      @rP   r(   rE   r   g      *@g      ,@gKqU?r,   r   Nr   r   r   r   r   r   M  s                                                  r   c                
   @   s   e Zd ZdZeddddgZeddddgZdZdZ	e
eee	eiZed	d	d
dgZe
eddejgeddejgeddejgedddgdZdS )Test_HaflDLs_OneUncensoredr   rG   rz   r   TFr   r
   r   rP   rE   r   rF   g      ?r   Nr   r   r   r   r   r   h  s   r   c                   @   sp   e Zd Zedddddddddddddddd	d
dddgZeddddddddddddddddddddgZdS )Test_ROS_MaxCen_GT_MaxUncenrG   rR   rF   r   rQ   rz   r   r   r   r   r   <   F   TFN)rj   rk   rl   rX   r   r   r   r   r   r   r   r   y  sX                                     r   c                   @   s^   e Zd Zejd dZddddddd	d
ddddddddddddddgZdge ddg ZdS )Test_ROS_OnlyDL_GT_MaxUncenr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   rR   FTN)	rj   rk   rl   rX   r   r   r   r   r   r   r   r   r   r     s4                      r   )(Zstatsmodels.compat.pandasr   r   ior   textwrapr   rX   r   Znumpy.testingZtestingr   r   r   r|   Zstatsmodels.imputationr   r   rA   rD   r\   r]   rm   rs   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   <module>   s@   *M'# 'N