U
    Kvfw`                  >   @   s  d Z ddlZddlZddlmZmZmZ ddlZ	ddl
Z
ddlZddlmZmZmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm  mZ dd	lmZmZ dd
lm Z m!Z!m"Z" ddl#m$Z$m%Z% ddl&m'Z'm(Z(m)Z) G dd deeZ*G dd deeZ+G dd deeZ,G dd deeZ-ej./ej.0e1Z2ej.3e2ddZ4e	5e4Z6ej.3e2ddZ4e	5e4Z7e7dddddg 8 Z9G dd dZ:G d d! d!e:Z;G d"d# d#e:Z<G d$d% d%e:Z=G d&d' d'e:Z>G d(d) d)Z?G d*d+ d+Z@G d,d- d-ZAeBd.d/d0d1d2d3d4d5d6d7d8d8d9d:d;gZCeBddddddddddd<d=d>d?d@dAdBdCdDdEdFdGdHdIdJdKdLdMdNdOdPdQdRdSdTdUdVdWdXdYdZd[d\d]d^d_d`dadbdcdddedfdgdhdidjdkdldmg<jDdndodpdqZEG drds dsZFG dtdu duZGG dvdw dwZHG dxdy dye:ZIG dzd{ d{e:ZJG d|d} d}eJZKdS )~z,
unit test for GAM

Author: Josef Perktold

    N)assert_allcloseassert_equalassert_)PoissonLogitProbit)GLM)family)TheilGLS)PenalizedMixin)BSplinesCyclicCubicSplines)GLMGamGLMGamResultsGLMGamResultsWrapper)matrix_sqrttransf_constraints   )results_plsresults_mpg_bsresults_mpg_bs_poissonc                   @   s   e Zd ZdS )PoissonPenalizedN__name__
__module____qualname__ r   r   H/tmp/pip-unpacked-wheel-2v6byqio/statsmodels/gam/tests/test_penalized.pyr   #   s   r   c                   @   s   e Zd ZdS )LogitPenalizedNr   r   r   r   r   r   '   s   r   c                   @   s   e Zd ZdS )ProbitPenalizedNr   r   r   r   r   r   +   s   r   c                   @   s   e Zd ZdS )GLMPenalizedNr   r   r   r   r   r    /   s   r    resultszmotorcycle.csvz	autos.csvcity_mpgfueldriveweighthpc                   @   s8   e Zd Zedd Zdd Zdd Zejj	dd Z
d	S )
CheckGAMMixinc                 C   sh   t ttd dgd}t|jd}t|}|j|}|j	|j
d |}t|}|||fS )Ntimes   dfr   )r   npasarraydata_mcycleZ
atleast_2dZbasisZmeanr   dotTZpenalty_matricesr   )clsZcc_hconstraintsZtransfexogpenalty_matrixrestrictionr   r   r   _init?   s    zCheckGAMMixin._initc                 C   s   | j }| j}t|j|jdd tt| |j| j ddd t|j	|j	| j dd tt|j
|jt| j ddd d S )Nh㈵>rtolư>g&.>)r9   atol:0yE>)res1res2r   paramsr,   r-   
cov_paramsZVpcovp_corrfactscaleZbsesesqrtselfr=   r>   r   r   r   test_paramsL   s     
   zCheckGAMMixin.test_paramsc                 C   s$   | j }| j}t|j|j| jd d S )Nr8   )r=   r>   r   fittedvaluesfitted_valuesrtol_fittedrE   r   r   r   test_fittedZ   s
    
zCheckGAMMixin.test_fittedc                 C   s   | j j d S N)r=   ZllnullrF   r   r   r   test_null_smoke`   s    zCheckGAMMixin.test_null_smokeN)r   r   r   classmethodr6   rG   rK   pytestmarkZsmokerN   r   r   r   r   r'   =   s   
r'   c                   @   s,   e Zd ZdZedd Zdd Zdd ZdS )	TestTheilPLS5z
data-priorc           	      C   sh   |   \}}}td }t|||d}d}d}d| |  | _}|j|| jd| _tj| _	d| _
d| _d S )	Naccel)Zr_matrix<mǳO?g<-.@r   
pen_weightcov_typeHz>g]j?)r6   r.   r
   pwfitrW   r=   r   pls5r>   rJ   rA   )	r1   r3   r4   r5   endogZmodp	s_scale_rZsigma_erY   r   r   r   setup_classi   s    zTestTheilPLS5.setup_classc                 C   sF   | j }| j}|j}|jj|dd}tt| |j	| j
 dd d S )NZsandwichrU   -C6?r8   )r=   r>   Zpenalization_factormodelrZ   r   r,   r-   r@   VerA   rF   r=   r>   rY   r   r   r   test_cov_robusty   s    
 zTestTheilPLS5.test_cov_robustc                 C   s   t d d S )Nzllnull not available)rP   skiprM   r   r   r   rN      s    zTestTheilPLS5.test_null_smokeN)r   r   r   rW   rO   r^   rc   rN   r   r   r   r   rR   e   s
   
rR   c                   @   s$   e Zd ZdZedd Zdd ZdS )TestGLMPenalizedPLS5	nonrobustc                 C   s   |   \}}}td }tj|d}t||t |d}d}d| d  | _|_|j	| j
ddd	d
dd| _tj| _d| _d| _d S )NrS   )r5   )r	   penalrT   r      bfgsd   r   FZx2)rW   methodmaxiterZdispZtrimrB   r7   g#n
?)r6   r.   smpenZL2ConstraintsPenaltyr    r	   ZGaussianrY   rV   rZ   rW   r=   r   r[   r>   rJ   rA   )r1   r3   r4   r5   r\   Zpenmodr]   r   r   r   r^      s       z TestGLMPenalizedPLS5.setup_classc                 C   sH   | j }| j}|jj}|jj|dd}tt| |j	| j
 dd d S )NZHC0rU   r_   r8   )r=   r>   r`   rV   rZ   r   r,   r-   r@   ra   rA   rb   r   r   r   _test_cov_robust   s    
 z%TestGLMPenalizedPLS5._test_cov_robustN)r   r   r   rW   rO   r^   ro   r   r   r   r   re      s   
re   c                   @   s   e Zd ZdZedd ZdS )TestGAM5Pirlsrf   c                 C   s^   d}t d j}t d }t|dgdd}t||d| d d	}| | _tj| _d
| _	d| _
d S )NO?r(   rS   r)   centerr+   r2   r   rh   smootheralphag-q=r.   valuesr   r   rZ   r=   r   r[   r>   rJ   rA   r1   s_scalexr\   ccgam_ccr   r   r   r^      s    

zTestGAM5Pirls.setup_classN)r   r   r   rW   rO   r^   r   r   r   r   rp      s   rp   c                   @   s$   e Zd ZdZedd Zdd ZdS )TestGAM5Bfgsrf   c                 C   sb   d}t d j}t d }t|dgdd}t||d| d d	}|jd
d| _tj| _d| _	d| _
d S )Nrq   r(   rS   r)   rr   rs   r   rh   rt   ri   rk   r7   rw   ry   r   r   r   r^      s    
zTestGAM5Bfgs.setup_classc                 C   s\   | j }| j}|d |jjjdd }t||jdd dd t||jdd | j	d d S Nrh      vIh%<=r8   )
r=   r>   predictr`   ru   r{   r   rH   rI   rJ   rF   r=   r>   	predictedr   r   r   test_predict   s    zTestGAM5Bfgs.test_predictN)r   r   r   rW   rO   r^   r   r   r   r   r   r~      s   
r~   c                   @   s    e Zd Zedd Zdd ZdS )TestGAM6Pirlsc                 C   s@   d}t td jdgd}ttd |d| d d}| | _d S )	Nrq   r(   r)   r*   rS   r   rh   rt   r   r.   rx   r   rZ   r=   r1   rz   r|   r}   r   r   r   r^      s    

zTestGAM6Pirls.setup_classc                 C   s   | j }| }d| _tdddddddd	d
ddddddg}t|jd d || jd t|jd d || jd |d |j	j
jdd }t||dd | jd d S )NrX   Oę@ z߰!	@@灢ld@Di/@t;+@篙(T-@	g@-@纰6<.@eBW-@c)+@x7\\}&@|x#@ Z @   r8   rh   r   )r=   get_predictionrJ   r,   arrayr   rH   predicted_meanr   r`   ru   r{   )rF   r=   predpls6_fittedvaluesr   r   r   r   rK      s<              zTestGAM6Pirls.test_fittedNr   r   r   rO   r^   rK   r   r   r   r   r      s   
r   c                   @   s    e Zd Zedd Zdd ZdS )TestGAM6Bfgsc                 C   sD   d}t td jdgd}ttd |d| d d}|jd	d
| _d S )Nrq   r(   r)   r*   rS   r   rh   rt   ri   r   r   r   r   r   r   r^      s    

zTestGAM6Bfgs.setup_classc                 C   st   | j }| }d| _tdddddddd	d
ddddddg}t|jd d || jd t|jd d || jd d S )Nr7   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r8   r=   r   rJ   r,   r   r   rH   r   rF   r=   r   r   r   r   r   rK     s4              zTestGAM6Bfgs.test_fittedNr   r   r   r   r   r      s   
r   c                   @   s    e Zd Zedd Zdd ZdS )TestGAM6Bfgs0c                 C   s<   d}t td jdgd}ttd |dd}|jdd	| _d S )
Nrq   r(   r)   r*   rS   r   rt   ri   r   r   r   r   r   r   r^     s    
zTestGAM6Bfgs0.setup_classc                 C   st   | j }| }d| _tdddddddd	d
ddddddg}t|jd d || jd t|jd d || jd d S )Nr7   gGmg@g+8M@g4vqr @gd8i@gϲO!@g5I9h/@g%,vs0@g׎\0@gH4V1@gdX31@g)-0@g:{+@gO'@gM_,C%@r   r8   r   r   r   r   r   rK     s4              zTestGAM6Bfgs0.test_fittedNr   r   r   r   r   r     s   
r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   gΜ`eտgV_ӿg3}]8ɿg)H#gI7g_1Gr?g Gl?gĩpk?g%ٿ?g*di?gk5Eg0LgfIUg;\<MPgIv?g"KXgSeg\ȲBg|dig<g߿g܋ֿg	bGֿgjh9JտgA6>MԿgp\i ӿg"Αȿg3$ƿgx!ĿghiNgqHĻg[	>տg:eԿg{nӿgxRkҿgEmѿg'Iȿg5'Cǿg<|ƿg>W¿g͸?Ϝg1%ڿglQ6ڿgdڿg}4ڿg1D+ٿgh#ԿgWPh\ӿg%Cҿg2XοgA˿
   r)   F)orderc                   @   s(   e Zd Zedd Zdd Zdd ZdS )TestGAM6ExogBfgsc                 C   s\   d}t d jd }tt d jdgdd}tt d t||d| d	 d
}|jdd| _d S )Nrq   r(   r   r)   rr   rs   rS   r   rh   rt   ri   r   	r.   shaper   rx   r   r,   ZonesrZ   r=   r1   rz   Znobsr|   r}   r   r   r   r^   M  s     
zTestGAM6ExogBfgs.setup_classc                 C   sL   | j }| }d| _t|jd d t| jd t|jd d t| jd d S Nr7   r   r8   r=   r   rJ   r   rH   r   r   rF   r=   r   r   r   r   rK   W  s    zTestGAM6ExogBfgs.test_fittedc                 C   s$   | j jj}t|d d tdd d S Nr   r   r8   r=   r`   r3   r   	pls6_exogrF   r3   r   r   r   	test_exoga  s    
zTestGAM6ExogBfgs.test_exogNr   r   r   rO   r^   rK   r   r   r   r   r   r   K  s   
	
r   c                   @   s(   e Zd Zedd Zdd Zdd ZdS )TestGAM6ExogPirlsc                 C   s`   d}t d jd }tt d jdgdd}tt d t|df|d| d	 d
}|jdd| _d S )Nrq   r(   r   r)   rr   rs   rS   r   rh   rt   Zpirlsr   r   r   r   r   r   r^   i  s     
zTestGAM6ExogPirls.setup_classc                 C   sL   | j }| }d| _t|jd d t| jd t|jd d t| jd d S r   r   r   r   r   r   rK   s  s    zTestGAM6ExogPirls.test_fittedc                 C   s$   | j jj}t|d d tdd d S r   r   r   r   r   r   r   }  s    
zTestGAM6ExogPirls.test_exogNr   r   r   r   r   r   g  s   
	
r   c                   @   s(   e Zd Zedd Zdd Zdd ZdS )
TestGAMMPGc                 C   s   t ddg}t ddg}tddg j}tjdtd}t|d	d
gdd}ttd ||d| | d  d}|	 | _
ttd ||d| | d  d}|j	dd| _d S )Ngj=^Y@gv3#?gܓ>gɱ ?r%   r&   fuel + drivedatar)      rr   rs   r"   r   rh   r3   ru   rv   newtonr   )r,   r   df_autosrx   patsydmatrixr   r   tolistrZ   res1ares1b)r1   sprz   x_spliner3   r|   r}   r   r   r   r^     s    
zTestGAMMPG.setup_classc                 C   sL   t jtdd}t|}|j}| j| jfD ]}|j	j
}t||dd q,d S )Nr!   zautos_exog.csvg+=)r;   )ospathjoincur_dirpdread_csvrx   r   r   r`   r3   r   )rF   	file_pathZdf_exogZ	res2_exogr=   r3   r   r   r   r     s    
zTestGAMMPG.test_exogc                 C   s   t jtdd}tj|dd}|jd |_|d j}|d j}| j| j	fD ]J}|
 }d| _t|j|d	d
 t|j|d	d
 d}t|j|| d	d
 qJd S )Nr!   zautos_predict.csvz	Row.names)Z	index_colr   rZ   Zse_fitr7   g|=r8   )r   r   r   r   r   r   indexrx   r   r   r   rJ   r   rH   r   Zse_mean)rF   r   Zdf_predZres2_fittedvaluesZres2_se_meanr=   r   Z	corr_factr   r   r   rK     s     

zTestGAMMPG.test_fittedN)r   r   r   rO   r^   r   rK   r   r   r   r   r     s   
	r   c                   @   sD   e Zd Zedd Zedd Zdd Zdd Zd	d
 Zdd Z	dS )TestGAMMPGBSc                 C   s   t ddg}t ddg | _}tddg j}t tjdtd| _t	|d	d
gddgddgddd}d| | d }t
td | j|| d}|jdd| _|jddd| _| jj| _tj| _d| _d| _ddg| _d S )Ng??g@{LǕz@j{ZxZ>E?r%   r&   r   r      r      rr   Tr+   ZdegreeZvariable_namesr2   Zinclude_interceptr   rh   r"   r   use_tr   rk   r   r<   g B޾AgO&#@)r,   r   rz   r   rx   r-   r   r   r3   r   r   r   rZ   r   r   _resultsr=   r   Zmpg_bsr>   rJ   rA   rv   r1   r   rz   r   bsalpha0gam_bsr   r   r   r^     s(    
zTestGAMMPGBS.setup_classc                 C   s   d S rL   r   r1   r   r   r   r6     s    zTestGAMMPGBS._initc                 C   s:   | j }| j}t|j|jdd | }t||jdd d S )Nr:   r8   )r=   r>   r   edfedf_allget_hat_matrix_diaghatrF   r=   r>   r   r   r   r   test_edf  s
    zTestGAMMPGBS.test_edfc                 C   s>   | j }| j}|jjj}|d j}t||jj|jj	 dd d S Nr   r:   r8   
r=   r>   r`   ru   	smoothersZcov_der2r   Zsmooth0SZS_scalerF   r=   r>   r   Zpen_matrix0r   r   r   test_smooth  s    

zTestGAMMPGBS.test_smoothc                 C   sf   | j }| j}|| jdd |jjjdd }t||jdd dd t||j	dd | j
d d S r   )r=   r>   r   r3   r`   ru   r{   r   rH   rI   rJ   r   r   r   r   r     s    $zTestGAMMPGBS.test_predictc                 C   sR  | j j}t|j| j t| j jddd | d }t|ddgdd t|j| j t|jjd |j	 }t|d d d df d t|d dd d f d t| j jddd t
jd	 |jd
dd\}}t|ddgdd t|j| j t|jjd |j	 }t|d d d df d t|d dd d f d t| j jddd d S )NgJp@r   r8   r   gK}v@g5<`@gMbP?r   i r   r)   )Zk_foldsZk_gridg    cAg{9@r7   )r=   r`   r   rv   r   rB   select_penweightrg   Z	start_idxr4   r,   randomseedZselect_penweight_kfold)rF   rn   Z	alpha_aicZpmZalpha_cv_r   r   r   test_crossval  s(    

zTestGAMMPGBS.test_crossvalN)
r   r   r   rO   r^   r6   r   r   r   r   r   r   r   r   r     s   

	r   c                   @   sL   e Zd Zedd Zedd Zdd Zdd Zd	d
 Zdd Z	dd Z
dS )TestGAMMPGBSPoissonc                 C   s   t ddg}t ddg | _}tddg j}tjdtd| _t|d	d
gddgddgddd}d| | d }t	td | j|t
 |d}| jjj|jj }||jd d < |jdd| _|jddd| _| jj| _tj| _d| _d| _d S )N<,l@H=<`Ar   r   r%   r&   r   r   r   r   r   rr   Tr   r   rh   r"   )r3   ru   r	   rv   Fr   r   r   r<   )r,   r   rz   r   rx   r   r   r3   r   r   r	   r   Zdesign_infocolumn_namesru   Z	col_names
exog_namesrZ   r   r   r   r=   r   mpg_bs_poissonr>   rJ   rA   )r1   r   rz   r   r   r   r   xnamesr   r   r   r^     s,     
zTestGAMMPGBSPoisson.setup_classc                 C   s   d S rL   r   r   r   r   r   r6   0  s    zTestGAMMPGBSPoisson._initc                 C   sp   | j }| j}t|j|jdd | }t||jdd t|j|jdd t|j|jdd t|j	|j
dd d S )Nr:   r8   r<   )r=   r>   r   r   r   r   r   ZaicZdevianceZdf_residZresidual_dfr   r   r   r   r   4  s    zTestGAMMPGBSPoisson.test_edfc                 C   s>   | j }| j}|jjj}|d j}t||jj|jj	 dd d S r   r   r   r   r   r   r   >  s    

zTestGAMMPGBSPoisson.test_smoothc                 C   s   | j }| j}|tjdd |jjjdd }t||j	dd dd t||j
dd | jd t|jjjdd }|jtjdd |dd}t||jdd | jd t|jjddg t|jjddg d S )Nrh   r   r   r8   Zlinear)whichr   )r=   r>   r   r   ilocr`   ru   r{   r   rH   rI   rJ   r   Z	DataFrameZlinear_predictorsr   r   rx   )rF   r=   r>   r   ZxpZlinpredr   r   r   r   G  s(    z TestGAMMPGBSPoisson.test_predictc                 C   sr   | j }| j}|jdddddgdd}t|jd d |jdd	 t|jd d |jd
d	 t|j	d d |j
 d S )NTr#   r$   r%   r&   )Zskip_singleZcombine_termsZscalarrh   rX   r8   r:   )r=   r>   Zwald_test_termsr   Z	statisticZpTerms_chi_sqZpvaluesZ	pTerms_pvr   Zdf_constraintsZ	pTerms_df)rF   r=   r>   Zwttr   r   r   	test_wald^  s     zTestGAMMPGBSPoisson.test_waldc                 C   s.   | j }|jj}|j }t|d |dd d S )Nr   g-C6
?r8   )r=   r`   rv   r   r   )rF   r=   Z
alpha_mgcvZres_sr   r   r   test_select_alphaj  s    
z%TestGAMMPGBSPoisson.test_select_alphaN)r   r   r   rO   r^   r6   r   r   r   r   r   r   r   r   r   r     s   


	r   c                   @   s    e Zd Zedd Zdd ZdS )TestGAMMPGBSPoissonFormulac                 C   s   t ddg}t ddg | _}tjdtd| _tddg j}t|d	d
gddgddgddd}d| | d }t	j
dt|t |d}|jdd| _|jddd| _| jj| _tj| _d| _d| _d S )Nr   r   r   r   r   r   r%   r&   r   r   r   rr   Tr   r   rh   zcity_mpg ~ fuel + drive)ru   r	   rv   Fr   r   r   r<   )r,   r   rz   r   r   r   r3   rx   r   r   Zfrom_formular	   r   rZ   r   r   r   r=   r   r   r>   rJ   rA   r   r   r   r   r^   t  s*     
z&TestGAMMPGBSPoissonFormula.setup_classc                 C   s   | j }ddddddddd	d
ddddddddddddddg}t|jj| t|jjj|d d  t|jjdd jj	ddg t|j
jj	| tt|j
tj tt|t tt|jt d S )NZ	Interceptzfuel[T.gas]zdrive[T.fwd]zdrive[T.rwd]Z	weight_s0Z	weight_s1Z	weight_s2Z	weight_s3Z	weight_s4Z	weight_s5Z	weight_s6Z	weight_s7Z	weight_s8Z	weight_s9Z
weight_s10Zhp_s0Zhp_s1Zhp_s2Zhp_s3Zhp_s4Zhp_s5Zhp_s6Zhp_s7Zhp_s8r   rh   r   )r   r   r`   r   Zdesign_info_linearr   rH   r   r   rx   r?   r   
isinstancer   ZSeriesr   r   r   )rF   r   r   r   r   r   test_names_wrapper  s@                   

z-TestGAMMPGBSPoissonFormula.test_names_wrapperN)r   r   r   rO   r^   r   r   r   r   r   r   q  s   
r   )L__doc__r   Znumpyr,   Znumpy.testingr   r   r   Zpandasr   rP   r   Z#statsmodels.discrete.discrete_modelr   r   r   Z+statsmodels.genmod.generalized_linear_modelr   Zstatsmodels.genmod.familiesr	   Z(statsmodels.sandbox.regression.penalizedr
   Zstatsmodels.base._penalizedr   Zstatsmodels.base._penaltiesbaseZ
_penaltiesrm   Zstatsmodels.gam.smooth_basisr   r   Z*statsmodels.gam.generalized_additive_modelr   r   r   Zstatsmodels.tools.linalgr   r   r!   r   r   r   r   r   r   r    r   dirnameabspath__file__r   r   r   r   r.   Z	df_autos_Zdropnar   r'   rR   re   rp   r~   r   r   r   r   r   Zreshaper   r   r   r   r   r   r   r   r   r   r   <module>   s   

( "                                                       3Za