U
    Kvf                     @   s  d Z ddlmZmZmZ ddlZddlmZm	Z	m
Z
 ddlZddlZddlmZ ddlmZmZmZmZ ej jZejddd	d
e_dd Zdd Zdd Zdd Zdd Zej !dddgej !dddgej !ddddgdd Z"ej !dddgej !dddgej !dddgd d! Z#ej !dddgej !dddgd"d# Z$ej !dddgej !dddgd$d% Z%ej !dddgej !dddgej !dddgd&d' Z&ej !dddgej !dddgd(d) Z'ej !dddgej !dddgd*d+ Z(d,d- Z)ej !d.ddgd/d0 Z*ej !dd1d2d3d4d5gd6d7 Z+d8d9 Z,ej !ddd:d;gd<d= Z-ej !ddd>gd?d@ Z.ej !ddAdBgdCdD Z/dS )Ez<
Tests for news results

Author: Chad Fulton
License: BSD-3
    )FUTURE_STACKassert_frame_equalassert_series_equalN)assert_assert_allcloseassert_equal)datasets)dynamic_factorsarimax
structuralvarmaxZ1959Q12009Q3Qstartendfreqc                 C   s8   dD ].}t | |}t|j| t|j | qd S )N)total_impactsupdate_impactsrevision_impactspost_impacted_forecastsprev_impacted_forecasts)getattrr   indexequalsr   columnstolist)newsimpact_datesimpacted_variablesattrval r"   N/tmp/pip-unpacked-wheel-2v6byqio/statsmodels/tsa/statespace/tests/test_news.pycheck_impact_indices   s    
r$   c                    s   | j jd kr^tt| jd d tt| jd d tt| jd d tt| jd d n| jjj | jjj	t
trg| jdd}| } fdd|d D |d< fdd|d D |d< t| j|| jj t| j| d S )	Nrevision dater   revised variableTZdropc                    s   g | ]}  |qS r"   Zget_loc.0datedatesr"   r#   
<listcomp>;   s    z*check_revision_indices.<locals>.<listcomp>c                    s   g | ]}  |qS r"   r   r*   nameendog_namesr"   r#   r.   =   s   )news_resultsr   r   lenrevisions_ilocrevisions_ixpreviousmodel_indexr3   
isinstancestrto_framereset_indexcopyr   r   astypedtypes)r   revisions_index
desired_ixdesired_ilocr"   r-   r3   r#   check_revision_indices(   s*    






rF   c                    s   | j jd kr^tt| jd d tt| jd d tt| jd d tt| jd d n| jjj | jjj	t
trg| jdd}| } fdd|d D |d< fdd|d D |d< t| j|| jj t| j| d S )	Nupdate dater   updated variableTr'   c                    s   g | ]}  |qS r"   r(   r)   r,   r"   r#   r.   X   s    z(check_update_indices.<locals>.<listcomp>c                    s   g | ]}  |qS r"   r/   r0   r2   r"   r#   r.   Z   s   )r4   r   r   r5   updates_iloc
updates_ixupdatedr9   r:   r3   r;   r<   r=   r>   r?   r   r   r@   rA   )r   updates_indexrC   rD   r"   rE   r#   check_update_indicesF   s*    






rM   c                 C   s   t |r|}ntjjg g gddgd}| jjj}t|tr@|g}t	| j
j| t	| jj| t	| jj| t	| jj| tj||g}t	| jj| d S )NrG   rH   names)r5   pd
MultiIndexfrom_productr8   r9   r3   r;   r<   r   r   r   r   update_forecastsupdate_realizedweightsr   )r   rL   r   Z
news_indexr3   Zweights_columnsr"   r"   r#   check_news_indicesc   s     

rV   c                 C   s  d| _ t| || t| | t| | t| || |rHt| j|dd ntt	| j
  |rrt| j|dd n$t| jjd k tt	| j
  | jtd| jtd }t| j|dd t| j|	dd t| j|
dd t| j|dd t| j|dd t| j|dd t| j|dd t| jj dddg t| jjjdd	g t| jj| t| jj d
dg t| jjjddg t| jj| jj t| jd | jdd t| jd
 | jdd | j}d
ddddg}t|j | ddddg}t|jj| |r| jd   !ddgd }t|| jdd | jd
   !ddgd
 }t|| jdd | jd   !ddgd }t|| jdd t|d "ddg| jdd | jd "ddgj#dd"d}t|| jdd | j$}dddg}t|j | ddd
dddg}t|jj| |r| j$d   !ddddgd }t|| jdd t| j$d "ddg| jdd | j$d "ddgj#dd"d}t|| jdd | j%}dddddg}t|j | ddg}t|jj| t|j&d d df | jj'f t(dd t|j&d d df | jtdj'f t(dd t|j&d d df | jtdj'f t(dd t|j&d d df | jj'f t(dd t|j&d d df | jj'f t(dd d S )Ng|۽g-q=)Zatolr   revisedobserved (prev)detailed impacts computedr%   r&   Zobservedzforecast (prev)rG   rH   r   weightimpactzimpact datezimpacted variable         )Zaxis      estimate (prev)impact of revisionsimpact of newstotal impactestimate (new)))	tolerancer$   rF   rM   rV   r   r   r   npallisnullr   r4   r@   floatZfillnar   r   r   rS   rT   r   rU   r   data_revisionsr   r   r   rO   r   Zdata_updatesdetails_by_impactZdrop_duplicatesr>   Zunstacksumdetails_by_updateimpactslocstackr   )r   	revisionsupdatesr   r   rB   rL   r   r   r   r   rS   rT   news_desiredrU   r   rl   desiredactualrn   ro   r"   r"   r#   
check_newsx   s&   




   

  
 
 rw   rr   TFrs   revisions_details_startc                    sx  t d  }d }|r:|jd d  }|jd d  }n(|jd d  }|jd d  }d}| rpd|jd< t|}|ddg}|j|dd	||d
}	tj	dd	dd}
dg}| r|rtj
j|jdd  dggddgd}|jd dtddd  }nr| rZtj
j|jdd  dggddgd}tjd|jd dtd  f dd}ntj
jg g gddgd}d }|rtj
jtj	dddddggddgd}td|jd d|jd   d|jd  d|jd   d|jd  d|jd   ggj}ntj
jg g gddgd}d }t| |rPtj|jd dtd   dd}n0tj|jd |jd dtd  f dd}tj|jd dtd |jd  f dd}|rd|jd  g |jd g fddtt D }ttjddtd f j}n@tjg tjd tjg tjdtjg tjd}ttd}t|	| ||
||||||| || d S )Ninfl2009Q2r   rK                 ?      ?2010Q1)r   r   comparison_typerx   r   r   r%   r&   rN   ry   r_   r\   r   r^   r   periodsr   rG   rH   g      ?g      ?c                    s   g | ]}|  |  qS r"   r"   r*   irS   rT   r"   r#   r.   o  s   z/test_sarimax_time_invariant.<locals>.<listcomp>Zdtype)r   r_   )dtar?   rp   ilocr
   SARIMAXsmoothr   rP   period_rangerQ   Zfrom_arraysr   rg   arangeZreshaper_arrayTprintranger5   	DataFrameSeriesfloat64zerosrw   )rr   rs   rx   endogr   endog1endog2modresr   r   r   rB   r   rL   r   r   r   rt   rU   r"   r   r#   test_sarimax_time_invariant  s    


"         
"      r   whichexogtrendc                 C   sj  t d  }d }|r:|jd d  }|jd d  }n(|jd d  }|jd d  }d}| rpd|jd< d }d }d}	|dkrt|}t|}n|d	krd
}	tj|||	d}
|
dddg}|j	||dd|d}t|}|ddg}|j	|dd|d}dddddddddddddg}|D ]>}t
||}t
||}t|tjrXt|| n
t|| q&d S )Nrz   r{   r   rK   r|   r}   nr   r   tr   r   r~   r   2008Q1r   r   r   r   r   r   r   r   r   r   r   rU   rS   rT   r   r   r6   r7   rI   rJ   )r   r?   rp   r   rg   	ones_liker
   r   r   r   r   r;   rP   r   r   r   )rr   rs   r   r   r   r   r   exog1exog2r   mod1res1news1mod2res2news2attrsr    wxr"   r"   r#   test_sarimax_time_varying  sZ    




      

r   c                 C   sJ  t d  }d }|r:|jd d  }|jd d  }n(|jd d  }|jd d  }d}| rpd|jd< t|}t|}tj|d|d}|d	d
dg}	|	j	||dd|d}
t|d}|d	d
g}|j	|dd|d}dddddddddddddg}|D ]>}t
|
|}t
||}t|tjr8t|| n
t|| qd S )Nrz   r{   r   rK   r|   r}   Zllevel)r   r~   皙?r   r   r   r   r   r   r   rU   rS   rT   r   r   r6   r7   rI   rJ   )r   r?   rp   r   rg   r   r   ZUnobservedComponentsr   r   r   r;   rP   r   r   r   )rr   rs   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r   r   r"   r"   r#   'test_unobserved_components_time_varying  sN    	



      

r   c                    sr  t ddg  }t|d  d |d< |jdd  }d }|rf|jd d  }|jd d  }n(|jd d  }|jd d  }d}| rd|jd	< tj|d
d}tj	d }|
|}|j|dd|d}	tjdddd}
ddg}td}td}|d }|| }|| }| rx|rxtjj|jd	d  ddggddgd}|jd j}tj|| || || || f j}nz| rtjj|jd	d  ddggddgd}|jd	 j}tj|| || || || f j}ntjjg g gddgd}d }|rZ|jd	 j}tj|| || || || f j}|jd j}tj|| || || || f j}nl|jd	 j}tj||jd  || || || f j}|jd	 j}tj||jd  || || || f j}|jd	 j}tj||jd  || || || f j}|r4tjjtjddddddggddgd}|| }ntjjg g gddgd}d }|r
||jd j  |jd j fddtt D }tjj|
|gddgd}tjtd||d }||jd d df< ||jd d df< |j|jd d d!f< |j|jd d df< n@tjg tjd" tjg tjd"tjg tjd"}ttd#}t|	| ||
||||||| || d S )$Nrealgdpunemp  r\   r{   r   rK   r|   r}   r   r   r~   皙?r   ?r   r   皙?r   r   r   r   )r]   r]   r]   Z
transitionr%   r&   rN   ry   r   rG   rH   c                    s   g | ]}|  |  qS r"   r"   r   r   r"   r#   r.   W  s   z.test_varmax_time_invariant.<locals>.<listcomp>zimpact dateszimpacted variables)r]      )r   r   Z2009Q4r   )r   r   )r   r?   rg   logdiffr   rp   r   VARMAXr   r   r   rP   r   r   ZeyerQ   rR   r   valuesZc_r   r   r5   r   r   r   rw   )rr   rs   r   r   r   r   r   paramsr   r   r   r   ZZT0ZT1ZT2ZT3rB   tmpr   r   Zrev_impacted_forecastsr   rL   r   rt   r   rU   r"   r   r#   test_varmax_time_invariant  s    





&&  
   
 
      r   c                 C   s  t ddg  }t|d  d |d< |jdd  }d }|rf|jd d  }|jd d  }n(|jd d  }|jd d  }d}| rd|jd	< d }d }d
}	|dkrt|d }t|d }tjd }
tjd }n"|dkrd}	tjd }
tjd }t	j
|||	d}||
}|j||dd|d}t	j
|d
d}||}|j|dd|d}ddddddddddd d!d"g}|D ]>}t||}t||}t|tjrt|| n
t|| q~d S )#Nr   r   r   r\   r{   r   rK   r|   r}   r   r   )	r~   r   r   r   r|   r|   r   r   r   r   r   r   )	r|   r|   r~   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   rU   rS   rT   r   r   r6   r7   rI   rJ   )r   r?   rg   r   r   r   rp   r   r   r   r   r   r   r   r;   rP   r   r   r   )rr   rs   r   r   r   r   r   r   r   r   params1params2r   r   r   r   r   r   r   r    r   r   r"   r"   r#   test_varmax_time_varyingp  sf    








      

r   c                 C   s  t ddg  }t|d  d |d< |jdd  }d }|rf|jd d  }|jd d  }n(|jd d  }|jd d  }d}| rd|jd	< t|d }t|d }tjd
 }tjd }	t	j
||ddd}
|
|}|j||dd|d}t	j
|ddd}||	}|j|dd|d}dddddddddddddg}|D ]>}t||}t||}t|tjr|t|| n
t|| qJd S )Nr   r   r   r\   r{   r   rK   r|   r}   )r   r   r|   r|   333333?r   r~   r   )r   r   r   r   r~   r   r]   )r   	k_factorsfactor_orderr   r   )r   r   r   r   r   r   r   rU   rS   rT   r   r   r6   r7   rI   rJ   )r   r?   rg   r   r   r   rp   r   r   r	   ZDynamicFactorr   r   r   r;   rP   r   r   r   )rr   rs   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r   r   r"   r"   r#    test_dynamic_factor_time_varying  s\    	


 


      

r   c                 C   sp  t d  }|r6|jd d  }|jd d  }n$|jd d  }|jd d  }| rhd|jd< t|}|ddg}t|}|ddg}|j|dd	}	|j|d
d	}
|j|dd	}|j|d
d	}dddddddddddddg}|D ]x}t|	|}t|
|}t||}t||}t	|t
jrLt|| t|| t|| qt|| t|| t|| qd S )Nrz   r{   r   r|   r}   r~   r   rK   r   r8   r   r   r   r   rU   rS   rT   r   r   r6   r7   rI   rJ   )r   r?   rp   r   r
   r   r   r   r   r;   rP   r   r   r   )rr   rs   r   r   r   r   r   r   r   Znews_updated_dataZnews_previous_dataZnews_updated_resultsZnews_previous_resultsr   r    r   r   yzr"   r"   r#   test_defaults  sN    


      







r   c               	   C   s   t d  } tj| jd< d}t| }|ddg}tj	t
|d ||  W 5 Q R X tj	t
|d || W 5 Q R X |j| dd}t|jd	 |j| d
d}t|jd	 |j|d
d}t|jd	 |j|d
d}t|jd	 d S )Nrz   r}   z8Could not automatically determine the type of comparisonr~   r   matchr8   r   r   rK   )r   r?   rg   nanr   r
   r   r   pytestraises
ValueErrorr   r   r   )r   msgr   r   r   r"   r"   r#   test_comparison_types  s"    
r   use_periodsc                 C   s  t d  }| rtj}ndd }|jdd}t|jd d }|ddg}|	|}|d	d
dd}t
|jj| |j	|dd
d}|dd
dd}t
|jj| |j	|ddd}|dddd}t
|jj| |j	||jd
 d
d}|d	d
dd}t
|jj| |j	||jd dd}|dddd}t
|jj| |j	|dd
d}|dd
dd}t
|jj| |j	|ddd}|dddd}t
|jj| |j	||jd
 d
d}|d	d
dd}t
|jj| |j	||jd dd}|dddd}t
|jj| |j	|ddd}|dddd}t
|jj| |j	|dd	d}|dd	dd}t
|jj| |j	|d|jd d}|dddd}t
|jj| |jd|jd d}t
|jj|j |j	|d|jd
 d}|dd	dd}t
|jj| |jd|jd
 d}t
|jj|j |j	||jd dd}|dddd}t
|jj| |j|jd dd}t
|jj|j |j	||jd d	d}|dd	dd}t
|jj| |j|jd d	d}t
|jj|j t|}t||jd
  |dddd}|jd |d dfD ]T}	|j	|d|	d}t
|jj| |	dkr"|jd|	d}t
|jj|j q"|dddd}|jd |d dfD ]T}
|j	||
dd}t
|jj| |	dkr|j|
dd}t
|jj|j qd S )Nrz   c                  _   s   t j| |jddS )Nr   r   )rP   r   to_timestamp)argskwargsr"   r"   r#   index_rangeA  s    z)test_start_end_dates.<locals>.index_ranger   r   r}   r~   r   r{   r\   r   Z2009Q1r   r   r]   r   r   )r   r   r   r   r   r   r^   r   )r   r?   rP   r   r   r
   r   r   r   r   r   r   r   r   nobspredictr5   r   )r   r   r   r   r   r   ru   	predictedZ
total_nobsr   r   r"   r"   r#   test_start_end_dates;  s    


r   r   range2int64numpylistc           
      C   sl  t d  }t|}| dkrDt||_|jd d }t|d }| dkrtj|d dd|_|jd d }tj|d d dd}n| dkrtt	||_|jd d }tt	|d }nV| d	kr|j
}|d d }t|d }n,| d
kr| }|d d }t|d }t|}|ddg}||}|dd }t|jj| |j||jd dd}|dd }t|jj| |j||jd dd}|dd }t|jj| |j||jd dd}|dd }t|jj| |j||jd dd}|dd }t|jj| |j||jd |jd d}|dd }t|jj| | dkr|j|jd |jd d}	t|jj|	j |j||j|jd}|dd }t|jj| | dkr|j|j|jd}	t|jj|	j |j||j|jd d}|dd }t|jj| | dkrh|j|j|jd d}	t|jj|	j d S )Nrz   r   r}   r]   r   )stopstep)r   r   r   r   r~   r   r   r\   r   r   ir   )r   r   ry   )r   r?   r5   rP   Z
RangeIndexr   r   ZIndexrg   r   r   r   r
   r   r   r   r   r   r   r   r   )
r   r   r   Z
endog_initZindex_plus2r   r   r   ru   r   r"   r"   r#   test_start_end_int  sr    





r   c               	   C   s\   t d  } t| jd d }|ddg}d}tjt|d |	| j
 W 5 Q R X d S )Nrz   r}   r~   r   z?The index associated with the updated results is not a supersetr   )r   r?   r
   r   r   r   r   r   r   r   r   )r   r   r   r   r"   r"   r#   test_invalid  s    r   i   c                 C   s  t tddddg  jdd  d }t j|jd< | }dd	d
ddd}| D ]\}}|j|  |7  < qTt	j
|dd}ddddd}t jt t|  ddddddddddg
f }||}	|	|}
|
j|	dd| d}|j}|jddg}| D ]\}}t|j|df |j|  t|j|df |j|  tt |j|df d t|j|df |j|  t|j|df |j|  t|j|df | qd}t|j|d f d t|j|d!f |d"  d#}t|j|d f d t|j|d!f |d$  d%}t|j|d f |d  t|j|d!f |d |d&   d'}t|j|d f d t|j|d!f d d(}t|j|d f d t|j|d!f d t|jd) d t|jd* |jd+  d,D ]N}t|jjd-|fd.f |jd-|f  t|jjd-|fd/f |jd-|f  qd}t|jjd-|fd.f |jd0|f ||   t|jjd-|fd/f |jd0|f ||   t|jd+ |jddgd1 d!  d S )2Nr   realconsrealinvcpir\   d   r}   r   r   r~   333333ӿr   r{   r   r   r   )r{   r   )r{   r   r   r   r   r   r   皙?ffffff?333333?r   r   r   r   r   r8   r}   r   rf   rx   r]   r^   rW   rX   rY   Trevisionr   r   r   r   rZ   r[   r   r   r   r   r   r   )r{   r   r   r   r   )r{   r   r   r   )r{   r   r   r   rc   rd   rb   r   r   r   r   re   ra   r{   level)rg   r   r   r   r   r   r?   itemsrp   r   r   r   diagr   r   flattenr   applyr   rk   revision_details_by_updater>   r   r   r   ro   groupbyrm   )rx   r   	y_revisedrr   keyr   r   ar_coeffr   r   res_revisedr   rk   revision_detailsr1   r"   r"   r#   test_detailed_revisions  s    
 



r     c                 C   s  t tddddg  jdd  d }t j|jd< | }dd	d
ddd}| D ]\}}|j|  |7  < qTt	j
|dd}ddddd}t jt t|  ddddddddddg
f }||}	|	|}
|
j|	dd| d}|j}|jddg}| D ]X\}}t|j|df |j|  t|j|df |j|  tt |j|df d qd}dddd g}t |j||f  stt|j|d! d"f |d |d#   t|j|d$ d"f |d%  t|j|d& d"f d t|j|d' d"f |d(  d)D ]N}t|jjd*|fd+f |jd*|f  t|jjd*|fd,f |jd*|f  qd}t|jjd*|fd+f |jd-|f ||   t|jjd*|fd,f |jd-|f ||   t|jd. |jddgd/ d"  d S )0Nr   r   r   r   r\   r   r   r   r~   r   r   r   r   r   r   r   r   r   r   r   r8   r}   r   r]   r^   rW   rX   rY   F)r   all prior revisionsr   r  rZ   r   r[   r   )r   r   r   )r   r   r  r   re   ra   r{   rb   r  rg   r   r   r   r   r   r?   r  rp   r   r   r   r  r   r   r	  r   r
  r   rk   r  r>   r   r   r   rh   ri   AssertionErrorro   r  rm   )rx   r   r  rr   r  r   r   r  r   r   r  r   rk   r  colsr1   r"   r"   r#   test_grouped_revisionse  s    
 


r  r}      c                 C   s  t tddddg  jdd  d }t j|jd< | }dd	d
ddd}| D ]\}}|j|  |7  < qTt	j
|dd}ddddd}t jt t|  ddddddddddg
f }||}	|	|}
|
j|	dd| d}|j}|jddg}| D ]d\}}t|j|df |j|  t|j|df |j|  |d dk}tt |j|df | qd}dddd g}t |j||f  stt|j|d! d"f |d |d#   t|j|d$ d"f d d%}t|j|d f d t|j|d"f |d&  d'}t|j|d f d t|j|d"f |d(  d)D ]N}t|jjd|fd*f |jd|f  t|jjd|fd+f |jd|f  q8d}t|jjd|fd*f |jd,|f ||   t|jjd|fd+f |jd,|f ||   t|jd- |jddgd. d"  d S )/Nr   r   r   r   r\   r   r   r   r~   r   r   r   r   r   r   r   r   r   r   r   r8   r}   r   r]   r^   rW   rX   r   rY   )r{   r  r   r  rZ   r  r[   r   r  r  r   r  r   r  re   ra   r{   rb   r  r  )rx   r   r  rr   r  r   r   r  r   r   r  r   rk   r  Zexpected_details_computedr  r1   r"   r"   r#   test_mixed_revisions  s    
 


r  )0__doc__Zstatsmodels.compat.pandasr   r   r   r   rg   Znumpy.testingr   r   r   ZpandasrP   r   Zstatsmodelsr   Zstatsmodels.tsa.statespacer	   r
   r   r   Z	macrodataZload_pandasdatar   r   r   r$   rF   rM   rV   rw   markZparametrizer   r   r   r   r   r   r   r   r   r   r   r  r  r  r"   r"   r"   r#   <module>   sl   
 '_:4{A:-
k 
H

i
W