U
    OvfNB                    @   sj  d Z ddlZddlZddlZddlZddlZddlmZ ddlm	Z	 ddl
mZ ddlZddlZddlmZmZmZ ddlmZmZ ddlmZ dd	lmZmZmZmZmZmZ dd
lm Z  ddl!m"Z" ddl#m$Z$ ddl%m&Z&m'Z'm(Z(m)Z)m*Z* ddl+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4 ddl5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;m<Z<m=Z=m>Z>m?Z?m@Z@mAZAmBZBmCZCmDZD ddlEmFZFmGZGmHZH ddlImJZJ ddlKmLZL ddlMmNZN ddlOmPZP ddlQmRZR ddlSmTZT ddlUmVZVmWZW ddlXmYZYmZZZ ddl[m\Z\ ddl]m^Z^m_Z_ ddl`maZambZbmcZcmdZd ddlemfZf G dd deZgG dd  d egZhG d!d" d"eZiG d#d$ d$eiZjG d%d& d&Zkeld'ZmeemZneoddd(d(d)d)d*d*d+d+g
Zpeod(d(d(d)d)d)d*d*d*d*g
Zqeerd,Zsd-d. Ztd/d0 Zud1d2 Zvd3d4 Zwejxyd5d6d7gezd8d9d:Z{d;d< Z|d=d> Z}d?d@ Z~ejxdAdBdC ZdDdE ZdFdG ZdHdI ZdJdK ZdLdM ZdNdO ZdPdQ ZdRdS ZdTdU ZdVdW ZdXdY ZdZd[ Zd\d] Zd^d_ Zd`da Zdbdc Zddde ZejxdAdfdg Zdhdi Zdjdk Zdldm Zdndo Zdpdq Zdrds Zdtdu Zdvdw Zdxdy Zdzd{ Zd|d} Zd~d Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd ZG dd de Zdd Zdd Zdd Zdd Zdd Zdd ZejxdAdd Zdd Zdd ZG dd deZejxydejdgdd Zejxydejdgdd Zdd ZejxdáejxydejddgddƄ Zejxdáejxydejddgejxydd7d6gejxydd6d7gddʄ Zdd̄ Zejxydd6ed)dddfd7eedќd*dddfd6eedќddddfgddք Zdd؄ Zddڄ Zdd܄ Zddބ ZdS )zTest the validation module    N)partial)StringIO)sleep)
coo_matrix
csr_matrixissparse)BaseEstimatorclone)KMeans)load_diabetesload_digits	load_irismake_classificationmake_multilabel_classificationmake_regression)RandomForestClassifier)FitFailedWarning)SimpleImputer)LogisticRegressionPassiveAggressiveClassifierRidgeRidgeClassifierSGDClassifier)	accuracy_scorecheck_scoringconfusion_matrixexplained_variance_scoremake_scorermean_squared_errorprecision_recall_fscore_supportprecision_scorer2_score)GridSearchCV
GroupKFoldGroupShuffleSplitKFoldLeaveOneGroupOutLeaveOneOutLeavePGroupsOutShuffleSplitStratifiedKFoldcross_val_predictcross_val_scorecross_validatelearning_curvepermutation_test_scorevalidation_curve)_check_is_permutation_fit_and_score_score)OneTimeSplitter)FailingClassifier)OneVsRestClassifier)KNeighborsClassifier)MLPRegressor)Pipeline)LabelEncoderscale)SVC	LinearSVC)shuffle)CheckingClassifierMockDataFrame)assert_allcloseassert_almost_equalassert_array_almost_equalassert_array_equal)_num_samplesc                   @   s<   e Zd ZdZdd ZdddZdd Zdd	d
Zdd ZdS )MockImprovingEstimatorz+Dummy classifier to test the learning curvec                 C   s   || _ d| _d | _d S Nr   )n_max_train_sizestrain_sizesX_subset)selfrH    rL   Q/tmp/pip-unpacked-wheel-qu3nn_q2/sklearn/model_selection/tests/test_validation.py__init__Z   s    zMockImprovingEstimator.__init__Nc                 C   s   || _ |jd | _| S rG   rJ   shaperI   rK   rJ   Zy_subsetrL   rL   rM   fit_   s    zMockImprovingEstimator.fitc                 C   s   t d S NNotImplementedErrorrK   XrL   rL   rM   predictd   s    zMockImprovingEstimator.predictc                 C   s2   |  |rdt| j| j  S t| j| j S d S )Ng       @)_is_training_datafloatrI   rH   rK   rW   YrL   rL   rM   scoreg   s    
zMockImprovingEstimator.scorec                 C   s
   || j kS rS   rJ   rV   rL   rL   rM   rY   n   s    z(MockImprovingEstimator._is_training_data)N)NN	__name__
__module____qualname____doc__rN   rR   rX   r]   rY   rL   rL   rL   rM   rF   W   s   

rF   c                       s4   e Zd ZdZd	 fdd	Zdd Zd
ddZ  ZS )!MockIncrementalImprovingEstimatorz*Dummy classifier that provides partial_fitNc                    s   t  | d | _|| _d S rS   )superrN   xexpected_fit_params)rK   rH   rg   	__class__rL   rM   rN   u   s    z*MockIncrementalImprovingEstimator.__init__c                 C   s
   | j |kS rS   rf   rV   rL   rL   rM   rY   z   s    z3MockIncrementalImprovingEstimator._is_training_datac              	   K   s   |  j |jd 7  _ |d | _| jrt| jt| }|rNtdt| d| D ]F\}}|| jkrVt|t|krVtd| dt| dt| dqVd S )Nr   zExpected fit parameter(s) z
 not seen.zFit parameter z has length z; expected .)	rI   rP   rf   rg   setAssertionErrorlistitemsrE   )rK   rW   yparamsmissingkeyvaluerL   rL   rM   partial_fit}   s"    
z-MockIncrementalImprovingEstimator.partial_fit)N)N)r`   ra   rb   rc   rN   rY   ru   __classcell__rL   rL   rh   rM   rd   r   s   rd   c                   @   s<   e Zd ZdZdddZdd Zdd Zdd
dZdd Zd	S )MockEstimatorWithParameterz-Dummy classifier to test the validation curve      ?c                 C   s   d | _ || _d S rS   )rJ   param)rK   ry   rL   rL   rM   rN      s    z#MockEstimatorWithParameter.__init__c                 C   s   || _ |jd | _| S rG   rO   rQ   rL   rL   rM   rR      s    zMockEstimatorWithParameter.fitc                 C   s   t d S rS   rT   rV   rL   rL   rM   rX      s    z"MockEstimatorWithParameter.predictNc                 C   s   |  |r| jS d| j S )N   )rY   ry   )rK   rW   rp   rL   rL   rM   r]      s    z MockEstimatorWithParameter.scorec                 C   s
   || j kS rS   r^   rV   rL   rL   rM   rY      s    z,MockEstimatorWithParameter._is_training_data)rx   )NNr_   rL   rL   rL   rM   rw      s   

rw   c                       s(   e Zd ZdZ fddZdd Z  ZS )%MockEstimatorWithSingleFitCallAllowedz<Dummy classifier that disallows repeated calls of fit methodc                    s&   t | drtdd| _t ||S )Nfit_called_zfit is called the second timeT)hasattrrm   r|   re   rR   rQ   rh   rL   rM   rR      s    z)MockEstimatorWithSingleFitCallAllowed.fitc                 C   s   t d S rS   rT   rV   rL   rL   rM   rX      s    z-MockEstimatorWithSingleFitCallAllowed.predict)r`   ra   rb   rc   rR   rX   rv   rL   rL   rh   rM   r{      s   r{   c                	   @   sH   e Zd ZdZdddZdddZd	d
 Zdd ZdddZdddZ	dS )MockClassifierz-Dummy classifier to test the cross-validationr   Fc                 C   s   || _ || _d S rS   aallow_nd)rK   r   r   rL   rL   rM   rN      s    zMockClassifier.__init__Nc                 C   sT  || _ || _|	| _|
dk	r"|
|  | jr8|t|d}|jdkrP| jsPtd|dk	r|jd |jd kst	d
|jd |jd |dk	r|jd tttkst	d
|jd ttt|dk	rd}|jd |jd kst	|
|jd |jd |dk	rPd	}|jtjksPt	|
|jd |jd
 tjd tjd
 | S )zThe dummy arguments are to test that this fit function can
        accept non-array arguments through cross-validation, such as:
            - int
            - str (this is actually array-like)
            - object
            - function
        N   zX cannot be dr   zKMockClassifier extra fit_param sample_weight.shape[0] is {0}, should be {1}zIMockClassifier extra fit_param class_prior.shape[0] is {0}, should be {1}zRMockClassifier extra fit_param sparse_sample_weight.shape[0] is {0}, should be {1}zUMockClassifier extra fit_param sparse_param.shape is ({0}, {1}), should be ({2}, {3})rz   )	dummy_int	dummy_str	dummy_objr   reshapelenndim
ValueErrorrP   rm   formatnpuniquerp   P_sparse)rK   rW   r\   sample_weightclass_priorsparse_sample_weightsparse_paramr   r   r   callbackfmtrL   rL   rM   rR      sP      
 
zMockClassifier.fitc                 C   s&   | j r|t|d}|d d df S )Nr   r   )r   r   r   rK   TrL   rL   rM   rX      s    zMockClassifier.predictc                 C   s   |S rS   rL   r   rL   rL   rM   predict_proba   s    zMockClassifier.predict_probac                 C   s   ddt | j  S )N      ?rz   )r   absr   r[   rL   rL   rM   r]      s    zMockClassifier.scorec                 C   s   | j | jdS )Nr   r   )rK   deeprL   rL   rM   
get_params  s    zMockClassifier.get_params)r   F)	NNNNNNNNN)NN)F)
r`   ra   rb   rc   rN   rR   rX   r   r]   r   rL   rL   rL   rM   r~      s    
         
?
r~   )
      rz   r   r         c               	   C   s|  t  } tddD ]}|| _t| tt}t|| tt t	ttd d d g}t| t
|}t|| t
| t| t
t}t|| t
t t| t
|}t|| t
| qdd }t|d} t| t t dd}t|d	} t| tt dd}tt t| ttd
d W 5 Q R X td d d d tjf }t dd} t| |t}t dd} tt t| |tdd W 5 Q R X d S )Nir   r   c                 S   s
   t | tS rS   
isinstancern   rj   rL   rL   rM   <lambda>(      z&test_cross_val_score.<locals>.<lambda>check_Xr   cvcheck_yZsklearnscoringT)r   Fraiseerror_score)r~   ranger   r,   rW   y2rD   r]   r   column_stackX_sparser?   tolistpytestraisesr   newaxis)clfr   scoresmultioutput_y
list_checkX_3drL   rL   rM   test_cross_val_score  s2    



r   c                  C   s@   t dd\} }tdd}t|dddgid}t|| |d	d
 d S )NTZ
return_X_yauto)gammaCrz   r   )Z
param_gridr   )n_jobs)r   r<   r"   r-   )rW   rp   r   ZgridrL   rL   rM   test_cross_validate_many_jobs<  s    
r   c               	   C   s  t dd\} }t }d}tjt|d" t|| |ttttfd W 5 Q R X tjt|d t|| |ttfd W 5 Q R X tjt|d d t|| |dd W 5 Q R X tjt|d d t|| |d	d W 5 Q R X tjt|d t|| |ttggd W 5 Q R X tjtd
d t|| |t	 d W 5 Q R X tt
}dtj d}tjt|d t|| ||d W 5 Q R X tjt|d t|| |d|id W 5 Q R X d S )Nr   random_statez.*must be unique strings.*matchr   zEmpty list.*rL   zDuplicate.*)f1_micror   zAn empty dict[Scoring failed. The score on this train-test partition for these parameters will be set to z. Details: 
Zfoo)r   r~   r   r   r   r-   r   r    r   dictr   r   nanwarnsUserWarning)rW   rp   	estimatorerror_message_regexpZmulticlass_scorerwarning_messagerL   rL   rM   )test_cross_validate_invalid_scoring_paramF  s6     "r   c                  C   sf   t dd\} }tdt fdt fg}t|| |dd}|d }t|tsLttdd |D sbtd S )	NTr   imputer
classifier)return_estimatorr   c                 s   s   | ]}t |tV  qd S rS   )r   r9   ).0r   rL   rL   rM   	<genexpr>  s     z7test_cross_validate_nested_estimator.<locals>.<genexpr>)	r   r9   r   r~   r-   r   rn   rm   all)rW   rp   ZpipelineresultsZ
estimatorsrL   rL   rM   $test_cross_validate_nested_estimatorz  s    r   
use_sparseFT)r   c              	   C   s  t  }tddd\}}tdd}tddd\}}tddd}| rPt|}t|}|||f|||ffD ]6\}}	}
t|
dd}t|
d	d}g }g }g }g }g }|||	D ]\}}t|
	|| |	| }
|
||
|| |	|  |
||
|| |	|  |
||
|| |	|  |
||
|| |	|  |
|
 qt|}t|}t|}t|}t|}|||||f}t|
||	|| t|
||	|| qdd S )
N   r   	n_samplesr   r   linearkernelr   neg_mean_squared_errorr   r2)r%   r   r   r   r<   r   r   splitr	   rR   appendr   array"check_cross_validate_single_metric!check_cross_validate_multi_metric)r   r   ZX_regZy_regregZX_clfZy_clfr   rW   rp   estZ
mse_scorerZ	r2_scorertrain_mse_scorestest_mse_scorestrain_r2_scorestest_r2_scoresfitted_estimatorstraintestr   rL   rL   rM   test_cross_validate  sH    
 




r   c              	   C   sz  |\}}}}}	dD ]\}
}|
rBt | ||dd|d}t|d | nt | ||dd|d}t|tsdtt||ksttt|d | |
rt | ||dgd|d}t|d	 |d nt | ||dgd|d}t|tstt||kstt|d
 | qt | ||dd|d}t|d D ]b\}}|j }t	|r6|
 }|	| j }t	|rV|
 }t|| t|j|	| j qd S )N))Tr   )Fr   r   Tr   return_train_scorer   train_scoreF
test_scorer   train_r2test_r2)r   r   r   r   )r-   rC   r   r   rm   r   	enumerateZcoef_copyr   toarrayrB   Z
intercept_)r   rW   rp   r   r   r   r   r   r   r   r   Zdict_lenZmse_scores_dictZr2_scores_dictkr   Zest_coefZfitted_est_coefrL   rL   rM   r     s                   



r   c              
   C   s  |\}}}}}	dd }
dt tdd|
f}ddddh}|d	d
h}dD ]\}|D ]P}|rt| |||d|d}t|d	 | t|d
 | nt| |||d|d}t|tstt|	 |r|n|kstt|d | t|d | t
|d tjkstt
|d tjkstt
|d tjks.tt
|d tjksFtt|d dks^tt|d dk svtt|d dkstt|d dk sRtqRqHd S )Nc                 S   s"   |  |}t||t|| dS )Nr   r   )rX   r!   r   )r   rW   rp   y_predrL   rL   rM   custom_scorer  s    

z8check_cross_validate_multi_metric.<locals>.custom_scorerr   r   r   Ztest_neg_mean_squared_errorZfit_timeZ
score_timer   Ztrain_neg_mean_squared_error)TFTr   Fr   r   )r   r!   unionr-   rC   r   r   rm   rl   keystyper   Zndarrayr   )r   rW   rp   r   r   r   r   r   r   r   r   Zall_scoringZkeys_sans_trainZkeys_with_trainr   r   
cv_resultsrL   rL   rM   r     s~    


           

 r   c               
   C   s   t dddd\} }tdd}t tdt t g}d}|D ]X}tjt|d t	|| ||d	 W 5 Q R X tjt|d t
|| ||d	 W 5 Q R X q:d S )
N   r   r   )r   	n_classesr   r   r   z*The 'groups' parameter should not be None.r   )r   rW   rp   r   )r   r<   r&   r(   r#   r$   r   r   r   r,   r+   )rW   rp   r   Z	group_cvserror_messager   rL   rL   rM   #test_cross_val_score_predict_groupsP  s    
r  z(ignore: Using or importing the ABCs fromc                     s   t t fg} z"ddlm}m} | ||f W n tk
r@   Y nX | D ]N\  tt }} fdd}fdd}t||d}t	|||dd qFd S )	Nr   	DataFrameSeriesc                    s
   t |  S rS   r   rj   InputFeatureTyperL   rM   r   t  r   z-test_cross_val_score_pandas.<locals>.<lambda>c                    s
   t |  S rS   r
  rj   
TargetTyperL   rM   r   u  r   r   r   r   r   )
r@   pandasr  r	  r   ImportErrorrW   r   r?   r,   typesr  r	  X_dfy_sercheck_dfcheck_seriesr   rL   r  r  rM   test_cross_val_score_pandasf  s    
r  c                  C   s   t dd} t }|j|j }}td}t| |||d}td}g }|||D ]J\}}tjt	|t
d}	tjt	|t
d}
d|	|< d|
|< |||f qNt| |||d}t|| d S )Nr   r  r   r   dtyperz   )r<   r   datatargetr%   r,   r   r   zerosr   boolr   rD   )svmirisrW   rp   ZkfoldZscores_indicesZcv_masksr   r   Z
mask_trainZ	mask_testZscores_masksrL   rL   rM   test_cross_val_score_maskz  s    
r"  c               	   C   s   t dd} t }|j|j }}t||j}t| ||}t dd} t| ||}t|| t dd d} t| ||}t|| t dd} t	
t t| || W 5 Q R X t	
t t| | | W 5 Q R X d S )NZprecomputedr  r   c                 S   s   t | |jS rS   )r   dotr   )rf   rp   rL   rL   rM   r     r   z2test_cross_val_score_precomputed.<locals>.<lambda>)r<   r   r  r  r   r#  r   r,   rC   r   r   r   r   )r   r!  rW   rp   Zlinear_kernelZscore_precomputedZscore_linearZscore_callablerL   rL   rM    test_cross_val_score_precomputed  s     




r$  c               	      s   t  } tjd }ttt}ttdgtdgtdgffdd}tt	d}d dt
  fdd	}t|t|d
| || |d}t| tt|d d S )Nr   rz   )r   rz   rP   r   *   Z42c                    s.   | j  kst| jkst| jks*td S rS   )r   rm   r   r   )r   Z	DUMMY_INTZ	DUMMY_OBJZ	DUMMY_STRrL   rM   assert_fit_params  s    z:test_cross_val_score_fit_params.<locals>.assert_fit_paramsr   )r   r   r   r   r   r   r   r   
fit_params)r~   rW   rP   r   r   r   rp   r   r   eyeobjectonesfullr,   )r   r   r  ZW_sparser   r(  r*  rL   r'  rM   test_cross_val_score_fit_params  s,    
" 	
r/  c               	      sl   t  } g   fdd}tjdd  t|}t| tt|dd}W 5 Q R X t|dddg t dksht	d S )Nc                    s     | |f dS )Nr   )r   )Zy_testZ	y_predictZ_score_func_argsrL   rM   
score_func  s    z3test_cross_val_score_score_func.<locals>.score_funcTrecordr   )r   r   r   )
r~   warningscatch_warningsr   r,   rW   rp   rD   r   rm   )r   r1  r   r]   rL   r0  rM   test_cross_val_score_score_func  s    r6  c               	   C   s4   G dd d} t t t|  t W 5 Q R X d S )Nc                   @   s   e Zd ZdS )z4test_cross_val_score_errors.<locals>.BrokenEstimatorN)r`   ra   rb   rL   rL   rL   rM   BrokenEstimator  s   r7  )r   r   	TypeErrorr,   rW   )r7  rL   rL   rM   test_cross_val_score_errors  s    r9  c                  C   s   t  } tdd}t|| j| j}t|dddddgd t|| j| jdd}t|dddddgd t|| j| jdd}t|dddddgd d S )	Nr   r  
ףp=
?r   r   accuracyr   Zf1_weighted)r   r<   r,   r  r  rC   )r!  r   r   Z	zo_scoresZ	f1_scoresrL   rL   rM   3test_cross_val_score_with_score_func_classification  s    
r<  c            	      C   s   t ddddd\} }t }t|| |}t|ddddd	gd
 t|| |dd}t|ddddd	gd
 t|| |dd}tdddddg}t||d
 tt}t|| ||d}t|ddddd	gd
 d S )Nr   r  r   r   )r   
n_featuresn_informativer   gGz?r:  gGz?gq=
ףp?r   r   r   r   g(\؇gzGIgGz&qg\(qg)\G)r   r   r,   rC   r   r   r   r   )	rW   rp   r   r   Z	r2_scoresZneg_mse_scoresZexpected_neg_mser   Z	ev_scoresrL   rL   rM   /test_cross_val_score_with_score_func_regression  s    r?  c               
   C   s  t  } | j}t|}| j}tdd}td}t|||d|dd\}}}|dksRtt|dd	 t|||d|dt	
|jd
d\}	}
}|	|kst||ksttdd}td}t|||d|dt	
|jd
d\}	}
}|	|kst||kstdd }t|}t|||d||d
d\}}
}t|dd t|dd t	t	t|d}t|||d|dd\}}}|dk snt|dks|td S )Nr   r  r   r   r;  )n_permutationsr   r   g?        rz   r   )r@  r   r   groupsr   c                 S   s"   | |k  | |k   | jd  S rG   )sumrP   )Zy_truer   rL   rL   rM   custom_score;  s    z,test_permutation_score.<locals>.custom_scored   )r@  r   r   r   g(\?g{Gz?r   rx   皙?)r   r  r   r  r<   r*   r/   rm   rB   r   r-  sizer   modaranger   )r!  rW   r   rp   r   r   r]   r   ZpvalueZscore_group_Zpvalue_groupZ
svm_sparseZ	cv_sparserD  scorerrL   rL   rM   test_permutation_score  s    
     



           rL  c                  C   st   t jdt jddd} t j| dd d f< t ddg| jd d }tdtd	t jd
fdt	 fg}t
|| | d S N   r  r   r   r   r   rz   r   mean)ZstrategyZmissing_valuesr   )r   rI  float64r   r   repeatrP   r9   r   r~   r/   rW   rp   prL   rL   rM   &test_permutation_test_score_allow_nansP  s    rT  c               	   C   s   t ddd} t dgd dgd  }tdd}d}tjt|d	 t|| | W 5 Q R X d
}tjt	|d	  t|| |dt 
did W 5 Q R X t|| |dt 
did d S )NrE  r   r   r   rz   TZexpected_sample_weight#Expected sample_weight to be passedr   /sample_weight.shape == \(1,\), expected \(8,\)!r   r)  )r   rI  r   r   r?   r   r   rm   r/   r   r-  rW   rp   r   err_msgrL   rL   rM   &test_permutation_test_score_fit_params^  s    
$rZ  c                  C   st   t jdt jddd} t j| dd d f< t ddg| jd d }tdtd	t jd
fdt	 fg}t
|| | d S rM  )r   rI  rP  r   r   rQ  rP   r9   r   r~   r,   rR  rL   rL   rM   test_cross_val_score_allow_nansm  s    r[  c            	      C   s*  t ddgddgddgddgddgddgddgddgddgddgg
} t ddgddgddgddgddgddgddgddgddgddgg
}tdd	}ttd
d}ttdd}ttdd}t|| ||d}t|| ||d}t|| ||d}t|dddddg t|dddddg t|dddddg d S )Nr   r   r   r   rz   r   )Zn_neighborsmicro)ZaveragemacroZsamplesr   rx         ?gUUUUUU?      ?)r   r   r7   r   r    r,   rB   )	rW   rp   r   Zscoring_microZscoring_macroZscoring_samplesZscore_microZscore_macroZscore_samplesrL   rL   rM   test_cross_val_score_multilabel{  s4    >
rb  c               	   C   s  t dd\} }t }t }t|}|| |D ].\}}|| | ||  || | ||< q0t|| ||d}t	|| t|| |}t
|t
|kstt }t|| ||d}t
|t
|kst|  }||t|k9 }t|}t|||}t	t
|t
| ttdd| }t
|t
|ks*tG dd d}	tt t|| ||	 d W 5 Q R X tdd\} }d}
tjt|
d	" ttd
d| |dtdd W 5 Q R X d S )NTr   r   r   )Zn_initc                   @   s   e Zd ZdddZdS )z%test_cross_val_predict.<locals>.BadCVNc              	   s   s:   t dD ],}tddddgtddddd	gfV  qd S )
Nr   r   rz   r   r   r            )r   r   r   )rK   rW   rp   rB  irL   rL   rM   r     s    z+test_cross_val_predict.<locals>.BadCV.split)NN)r`   ra   rb   r   rL   rL   rL   rM   BadCV  s   rg  zNumber of classes in training fold \(2\) does not match total number of classes \(3\). Results may not be appropriate for your use case.r   	liblinearsolverr   r   methodr   )r   r%   r   r   Z
zeros_liker   rR   rX   r+   rC   r   rm   r'   r   Zmedianr   r
   r   r   r   r   r   RuntimeWarningr   )rW   rp   r   r   Zpreds2r   r   predsZXsprg  r   rL   rL   rM   test_cross_val_predict  sF    

ro  c               	   C   sD  t dddd\} }ttdd| |dd}|jd	ks6ttd
d\} }ttdd| |dd}|jdksht| d d } |d d }d}tjt|d tt	 | |dt
dd W 5 Q R X td
d\} }tddd}t|| |dd}|jdkstt|}| | ||  } }d}tjt|d t|| |t
dddd W 5 Q R X d S )Nr   2   r   r  r   r   rh  ri  decision_functionrl  )rp  Tr      r   rE  zOnly 1 class/es in training fold, but 2 in overall dataset. This is not supported for decision_function with imbalanced folds. To fix this, use a cross-validation technique resulting in properly stratified foldsr   rk  r   Zovo)r   Zdecision_function_shape)i  -   zOutput shape \(599L?, 21L?\) of decision_function does not match number of classes \(7\) in fold. Irregular decision_function .*r   n_splitsr   rl  )r   r+   r   rP   rm   r   r   r   r   r   r%   r   r<   r   Zargsort)rW   rp   rn  r  r   indr   rL   rL   rM   .test_cross_val_predict_decision_function_shape  sL              
r{  c                  C   sl   t dddd\} }ttdd| |dd}|jd	ks6ttd
d\} }ttdd| |dd}|jdkshtd S )Nr   rp  r   rq  rh  ri  r   rs  rp  r   Tr   rt  r   r+   r   rP   rm   r   rW   rp   rn  rL   rL   rM   *test_cross_val_predict_predict_proba_shape  s           r  c                  C   sl   t dddd\} }ttdd| |dd}|jd	ks6ttd
d\} }ttdd| |dd}|jdkshtd S )Nr   rp  r   rq  rh  ri  predict_log_probars  r|  Tr   rt  r}  r~  rL   rL   rM   .test_cross_val_predict_predict_log_proba_shape  s           r  c            
      C   sb  t  } | j| j }}t|}t||d d d g}tddd}t|||}|jdksZt	t|||}|jdkstt	t|||}t
|jd t|||}t
|jd dd }t|d	}t|| | }t|d
}t||| }ttdd| | dd}ttdd|| dd}|d d d d tjf }dd }	t|	d	}t|||}t
|jd d S )Nr   Fr   )Zfit_interceptr   )ru  )ru  r   c                 S   s
   t | tS rS   r   rj   rL   rL   rM   r   =  r   z4test_cross_val_predict_input_types.<locals>.<lambda>r   r   rh  ri  rr  rs  c                 S   s
   | j dkS )Nr   )r   rj   rL   rL   rM   r   T  r   )r   r  r  r   r   r   r   r+   rP   rm   rD   r?   r   r   r   )
r!  rW   rp   r   r   r   predictionsr   r   Zcheck_3drL   rL   rM   "test_cross_val_predict_input_types%  sF    


r  c                     s   t t fg} z"ddlm}m} | ||f W n tk
r@   Y nX | D ]N\  tt }} fdd}fdd}t||d}t	|||dd qFd S )	Nr   r  c                    s
   t |  S rS   r
  rj   r  rL   rM   r   h  r   z/test_cross_val_predict_pandas.<locals>.<lambda>c                    s
   t |  S rS   r
  rj   r  rL   rM   r   i  r   r  r   r   )
r@   r  r  r	  r   r  rW   r   r?   r+   r  rL   r  rM   test_cross_val_predict_pandasZ  s    
r  c                  C   s   t ddddddd\} }d|d< tddd}tdd}t|| |\}}t|| ||d	d
}||d  d dksttt||d  d d df dkstt||d  d d ddf dkstt||d  dkstt	|j
ddt|jdd d S )NrE  r   r   rz   )r   r=  n_redundantr>  n_clusters_per_classr   rh  r   rj  rw  r   ry  Zaxis   )decimal)r   r   r*   rn   r   r+   rm   r   r   rC   rC  r-  rP   )rW   rp   r   r   r   r   Z
yhat_probarL   rL   rM   !test_cross_val_predict_unbalancedn  s$    
	
&*r  c                  C   sf   t  } tjd}|dd}t| |d ddd}t|d d df | t| |d ddd}t|| d S )	Nr&  rE  r   r   rX   )rp   r   rl  r   r   )r~   r   randomRandomStateZrandr+   rA   )Zmock_classifierrngrW   Zy_hatZy_hat_probarL   rL   rM   test_cross_val_predict_y_none  s        r  c                  C   sX   t  } | j| j }}t }dtt|jd i}t||||dd}t	|t
d d S )Nr   r   r   )r*  r   )r   r  r  r~   r   r   r+  rP   r,   rD   r-  )r!  rW   rp   r   r*  r   rL   rL   rM   &test_cross_val_score_sparse_fit_params  s    r  c                  C   s  d} d}t | ddddddd\}}t| |d |  }dD ]}tjdd	6}t|||t|d
tddd|dd\}}}	}
}W 5 Q R X t|dkrt	d|d j
 |jdkst|	jdkst|
jdkst|jdkstt|tddd t|jddtddd t|	jddtddd |
jdks:t|jdksJttjdd	2}t|||t|| dtddd|d\}}}W 5 Q R X t|dkrt	d|d j
 t|| t||	 q:d S )Nr   r   rz   r   r   r   r=  r>  r  r  r  r   FTTr2  rw  皙?r   r   )r   rI   r>   Zreturn_timesUnexpected warning: %r)r   r   r  r  ffffff?rP  rx  r   )r   rI   r>   )r   rF   r4  r5  r.   r%   r   linspacer   RuntimeErrormessagerP   rm   rD   rC   rO  r  r4   )r   rx  rW   rp   r   shuffle_trainwrI   train_scorestest_scoresZ	fit_timesZscore_timesZtrain_sizes2Ztrain_scores2Ztest_scores2rL   rL   rM   test_learning_curve  sl    
	



r  c               
   C   s   t dddddddd\} }td}t|| d dtdd	d
d\}}}t|tddd
 t|jddtdd	d
 t|jddtdd	d
 d S )Nr   rz   r   r   r  r  r   r  r   r   )rp   r   rI   r  r  )r   rF   r.   r   r  rD   rC   rO  rW   rJ  r   rI   r  r  rL   rL   rM    test_learning_curve_unsupervised  s(    
	    r  c               	   C   sz   t dddddddd\} }td}tj}t t_zt|| |ddd\}}}W 5 tj }tj  |t_X d	|ksvtd S )
Nr   rz   r   r   r  r  r   )r   verbosez[learning_curve])	r   rF   sysstdoutr   getvaluecloser.   rm   )rW   rp   r   Z
old_stdoutoutrI   r  r  rL   rL   rM   test_learning_curve_verbose  s0    
	    

r  c               	   C   sL   t dddddddd\} }td}tt t|| |dd W 5 Q R X d S )Nr   rz   r   r  T)exploit_incremental_learningr   rF   r   r   r   r.   rW   rp   r   rL   rL   rM   5test_learning_curve_incremental_learning_not_possible  s    

r  c                  C   s   t dddddddd\} }td}dD ]v}t|| |dd	td
dd|d\}}}t|tddd t|jddtddd t|jddtd
dd q&d S )Nr   rz   r   r   r  r  r  r   Tr  r   r   )r   r  rI   r>   r  r  r   rd   r.   r   r  rD   rC   rO  )rW   rp   r   r  rI   r  r  rL   rL   rM   (test_learning_curve_incremental_learning  s.    
		r  c                  C   s   t dddddddd\} }td}t|| d ddtd	d
dd\}}}t|tddd t|jddtdd
d t|jddtd	d
d d S )Nr   rz   r   r   r  r  r   Tr  r   r   )rp   r   r  rI   r  r  r  r  rL   rL   rM   5test_learning_curve_incremental_learning_unsupervised.  s*    
	r  c            
   	   C   s   t dddddddd\} }tddd}tdd d	d
}t|| ||ddd\}}}t|| |d|d	d\}}}	t|| t|jdd|jdd t|jdd|	jdd d S )Nr   rz   r   r   r  rF  r   r   Fmax_itertolr>   r   T)rI   r   r  )r   rI   r  r  )r   r   r  r   r.   rD   rC   rO  )
rW   rp   rI   r   train_sizes_inctrain_scores_inctest_scores_inctrain_sizes_batchtrain_scores_batchtest_scores_batchrL   rL   rM   <test_learning_curve_batch_and_incremental_learning_are_equalF  sH    
		

 

 
r  c               	   C   s  t dddddddd\} }td}tt t|| |dddgd W 5 Q R X tt t|| |dd	d
gd W 5 Q R X tt t|| |dddgd W 5 Q R X tt t|| |dddgd W 5 Q R X tt t|| |dddgd W 5 Q R X d S )Nr   rz   r   r   r  r  r   r   rI   rA  r   r  g?   r  r  rL   rL   rM   0test_learning_curve_n_sample_range_out_of_boundsm  s(    
	    r  c                  C   st   t dddddddd\} }td}d}tjt|d( t|| |dtdd	dd
\}}}W 5 Q R X t|ddg d S )Nr   rz   r   r   r  zzRemoved duplicate entries from 'train_sizes'. Number of ticks will be less than the size of 'train_sizes': 2 instead of 3.r   gQ?r   r  )	r   rF   r   r   rm  r.   r   r  rD   )rW   rp   r   r   rI   rJ  rL   rL   rM   1test_learning_curve_remove_duplicate_sample_sizes  s*    
	    r  c               
   C   s   t dddddddd\} }td}tdd}t|| ||td	d
dd\}}}t|tddd t|jddtdd
d t|jddtd	d
d d S )Nr   rz   r   r   r  r  r   rw  r  r   r   r  r  r  )	r   rF   r%   r.   r   r  rD   rC   rO  )rW   rp   r   r   rI   r  r  rL   rL   rM   (test_learning_curve_with_boolean_indices  s*    
	
    r  c                  C   s  t ddgddgddgddgd	d
gddgddgddgddgddgddgd	d
gddgddgddgg} t dddddddddddddddg}t dddddddddddddddg}tdd dd}tdd}t|| ||dt ddd|ddd	\}}}t|jddt dddg t|jddt dddg t	t
( t|| ||dt ddd|d d! W 5 Q R X t|| ||dt ddd|dddd"
\}}	}
t|	jdd|jdd t|
jdd|jdd d S )#Nrz   r   r   r   r   rc  rd  re     r                       r  	   r   Fr  rw  g333333?r   T)r   r   rI   rB  r>   r   r  r`  gIq?ra  r   )r   r   rI   rB  r   )r   r   rI   rB  r>   r   r  )r   r   r   r#   r.   r  rC   rO  r   r   r   )rW   rp   rB  r   r   r  r  r  r  r  r  rL   rL   rM    test_learning_curve_with_shuffle  s    ((

 
 
 

 
r  c               
   C   s   t ddd} t dgd dgd  }tdd}d}tjt|d	 t|| |d
d W 5 Q R X d}tjt	|d	" t|| |d
dt 
did W 5 Q R X t|| |d
dt 
did d S )NrE  r   r   r   rz   TrU  rV  r   r   r   z/sample_weight.shape == \(1,\), expected \(2,\)!r   )r   r*  )r   rI  r   r   r?   r   r   rm   r.   r   r-  rX  rL   rL   rM   test_learning_curve_fit_params  s,    
        r  c                  C   s   t dddddddd\} }tddg}d}tjt|d	& t|| |d
dtddddd W 5 Q R X d}tjt|d	2 t|| |d
dtdddddtd
id W 5 Q R X t|| |d
dtdddddtdid d S )Nr   rz   r   r   r  r  r   z9Expected fit parameter\(s\) \['sample_weight'\] not seen.r   r   Tr  r   r   r   )r   r  rI   r   z2Fit parameter sample_weight has length 3; expected)r   r  rI   r   r*  )	r   rd   r   r   rm   r.   r   r  r-  )rW   rp   r   rY  rL   rL   rM   3test_learning_curve_incremental_learning_fit_params  sV    
	
r  c               	   C   s   t dddddddd\} }tddd}tjdd }tt | |d|dd	\}}W 5 Q R X t|dkrxtd
|d j	 t
|jdd| t
|jddd|  d S )Nr   rz   r   r  r   Tr2  ry   
param_nameparam_ranger   r  r  )r   r   r  r4  r5  r0   rw   r   r  r  rC   rO  )rW   rp   r  r  r  r  rL   rL   rM   test_validation_curve?  s.    
	r  c               	   C   sF   t dddddddd\} }tddd}tt | |d|dd\}}d S )Nr   rz   r   r  r   ry   r  )r   r   r  r0   r{   )rW   rp   r  rJ  rL   rL   rM   %test_validation_curve_clone_estimatorZ  s$    

r  c               
   C   s   d} d}t ddd\}}ttddd||dddd	d	gt|| d
d}ttt|dd d f d  ttddd||dddd	d	gt|ddd}ttt|dd d f d  ttddd||dddd	d	gt|dd}tt	|t	| d S )NrE  r   r   r   r   r   r   r  rF  r  r  )r   r   rz   r   r   Trx  r>   rw  )
r   r0   r<   r4   rC   r   Zvsplithstackr%   r   )r   rx  rW   rp   Zscores1Zscores2Zscores3rL   rL   rM   +test_validation_curve_cv_splits_consistencyp  s<    


"


"


r  c                  C   s   t ddd} t dgd dgd  }tdd}d}tjt|d	  t|| |d
dddgdd W 5 Q R X d}tjt	|d	, t|| |d
dddgddt 
did W 5 Q R X t|| |d
dddgddt 
did d S )NrE  r   r   r   rz   TrU  rV  r   Z	foo_paramr   r   r   )r  r  r   rW  r   )r  r  r   r*  )r   rI  r   r   r?   r   r   rm   r0   r   r-  rX  rL   rL   rM    test_validation_curve_fit_params  sB    
		r  c                  C   sv   t jd} t d}| | t|ds.ttt |ddrDtd|d< t|drZttt |dfdrrtd S )Nr   rE     )	r   r  r  rI  r>   r1   rm   deleter  )r  rS  rL   rL   rM   test_check_is_permutation  s    

r  c                  C   sj   t dddddd\} }t| }t|}ttdd}t|| |dd	}t|||dd	}| }t|| d S )
Nr   rz   FT)r  n_labelsZallow_unlabeledZreturn_indicatorr   r   r  r   r   )r   r   r6   r<   r+   r   rC   )rW   rp   r   Zy_sparseZclassifrn  Zpreds_sparserL   rL   rM   (test_cross_val_predict_sparse_prediction  s    
r  c           
   	   C   s   t ddd}|jdkr6|dkr(t|fn
t|df}n|j}t|}|||D ]6\}}t| || || } t	| ||| ||< qR||d |d |
dfD ]}	tt| ||	||d| qd	S )
z@Helper for tests of cross_val_predict with binary classificationr   Fr  rz   rr  r   strrk  N)r%   r   r   rP   r   r  r   r	   rR   getattrastyperA   r+   )
r   rW   rp   rl  r   	exp_shapeexpected_predictionsr   r   tgrL   rL   rM   check_cross_val_predict_binary  s    
 
 r  c              	   C   s   t ddd}ttjj}||dd}tjt|tt|f|| tjd}tj|dd\}}	|	||	D ]P\}
}t
| ||
 |	|
 } t| ||| }t|	|
 }||t||< qj||d	 |d
 |dfD ]}tt| ||||d| qdS )zDHelper for tests of cross_val_predict with multiclass classificationr   Fr  r   rr  r  r   r  TZreturn_inverserz   r   r  rk  N)r%   r   finforP  minr.  r   rl   r   r   r	   rR   r  ix_r  rA   r+   )r   rW   rp   rl  r   	float_mindefault_valuesr  rJ  y_encr   r   
fold_predsZ
i_cols_fitr  rL   rL   rM   "check_cross_val_predict_multiclass  s,       r  c                    s  t ddd}ttjj}||dd} jd }g }t|D ]`}	tt dd|	f }
|
dkrt|d	krtt|f}nt||
f}|	tj
||| tjd
 q< fddt jd D }tj|dd}|||D ]\}}t| || || } t| ||| }t|D ]^}	t|| dd|	f }||	 jdkrN||	 ||	 |< nt||}||	 ||	 |< qqԈ  d  d  dfD ]T}t| ||||d}t|t|ksttt|D ]}t|| ||  qqdS )zCheck the output of cross_val_predict for 2D targets using
    Estimators which provide a predictions as a list with one
    element per class.
    r   Fr  r   r  rz   Nr   rr  r  c                    s8   g | ]0}t j d d |f ddd d d t jf qS )NTr  rz   )r   r   r   )r   rf  rp   rL   rM   
<listcomp>0  s   z6check_cross_val_predict_multilabel.<locals>.<listcomp>r  r  rk  )r%   r   r  rP  r  rP   r   r   rl   r   r.  Zconcatenater   r	   rR   r  r   r   r  r  r+   rm   rA   )r   rW   rp   rl  r   r  r  	n_targetsZexpected_predsZi_colZn_classes_in_labelr  Z
y_enc_colsr  r   r   r  Z	fold_colsidxr  Zcv_predict_outputrf  rL   r  rM   "check_cross_val_predict_multilabel  sD    

r  c                 C   s,   t ddd\}}dD ]}t| ||| qd S )Nr   r   )r  r   rr  r   r  )r   r  )r   rW   rp   rl  rL   rL   rM   *check_cross_val_predict_with_method_binaryI  s    r  c                 C   sB   t  }|j|j }}t||dd\}}dD ]}t| ||| q*d S )Nr   r   r  )r   r  r  r>   r  )r   r!  rW   rp   rl  rL   rL   rM   .check_cross_val_predict_with_method_multiclassQ  s
    r  c                   C   s    t tdd ttdd d S )Nrh  ri  )r  r   r  rL   rL   rL   rM   "test_cross_val_predict_with_methodY  s    r  c                  C   sN   t  } | j| j }}t||dd\}}dD ]}tddd}t|||| q*d S )Nr   r   r  Zlog_lossr   )Zlossr   )r   r  r  r>   r   r  )r!  rW   rp   rl  r   rL   rL   rM   &test_cross_val_predict_method_checking`  s    r  c                  C   s`   t  } | j| j }}t||dd\}}ttddddddgid	d
}dD ]}t|||| qHd S )Nr   r   r&  rh  r  r   r  rz   r   r   r  )r   r  r  r>   r"   r   r  )r!  rW   rp   r   rl  rL   rL   rM   /test_gridsearchcv_cross_val_predict_with_methodk  s    
 
 r  c                  C   sL   d} d}t | d|ddd\}}ttddd	}d
D ]}t||||d q2d S )NrE  r   r   r   r&  r   r  r  r=  r   rh  r   )rj  r   )r   rr  rs  )r   r6   r   r  )Zn_sampr  rW   rp   r   rl  rL   rL   rM   1test_cross_val_predict_with_method_multilabel_ovrv  s        
r  c                   @   s   e Zd Zdd ZdS )RFWithDecisionFunctionc                 C   s2   |  |}d}t|ts t|dd |D }|S )Nz?This helper should only be used on multioutput-multiclass tasksc                 S   s.   g | ]&}|j d  dkr&|dddf n|qS )rz   r   Nr   r%  )r   rS  rL   rL   rM   r    s     z<RFWithDecisionFunction.decision_function.<locals>.<listcomp>)r   r   rn   rm   )rK   rW   ZprobsmsgrL   rL   rM   rr    s
    
z(RFWithDecisionFunction.decision_functionN)r`   ra   rb   rr  rL   rL   rL   rM   r    s   r  c               
   C   s   d} t dd| ddd\}}|d d df  |d d df 7  < d	D ]>}tddd
}t   td t||||d W 5 Q R X qBd S )Nr   rE  r   r   r&  r  r   rz   r   r  rr  Zn_estimatorsr   ignorers  )r   r  r4  r5  simplefilterr  )r  rW   rp   rl  r   rL   rL   rM   0test_cross_val_predict_with_method_multilabel_rf  s        
$

r  c                  C   s   t jd} | jdddd}t ddddddddddddddg}tdd}d	D ]0}t  td
 t	|||| W 5 Q R X qPd S )Nr   rz   )r  r   rG  r   r   rh  ri  r  r  )
r   r  r  normalr   r   r4  r5  r  r  )r  rW   rp   r   rl  rL   rL   rM   -test_cross_val_predict_with_method_rare_class  s    &


r  c               
   C   s   t jd} | jdddd}t ddgddgddgddgddgg}dD ]>}tddd}t   td	 t	||||d
 W 5 Q R X qHd S )Nr   rz   )r   r   r  r   )r   r  r   r  r  rs  )
r   r  r  r  r   r  r4  r5  r  r  )r  rW   rp   rl  r   rL   rL   rM   ;test_cross_val_predict_with_method_multilabel_rf_rare_class  s    (

r   c                 C   s   t t||g}t||}|| |D ]|\}}	|| | ||  || |	 }
|dkrlt t|	|f}nt t|	|ft |jj	}|
|d d |j
f< |||	< q(|S )Nr   )r   r  r   r  r   rR   r.  r  r  r  Zclasses_)rW   rp   r   classesr   rl  r  funcr   r   Zexpected_predictions_Zexp_pred_testrL   rL   rM   get_expected_predictions  s    

 
r  c                  C   s  t ddd} t dd tdD }d}tdd}td	d}t }d
ddg}|D ]}tdd}t|| |||d}	t	| |||||}
t
|
|	 t|| |||d}	t	| |||||}
t
|
|	 tt tdddd}t|| |||d}	||}t	| |||||}
t
|
|	 qVd S )NrN  rE  r   c                 S   s   g | ]}|d  qS )r   rL   )r   rf   rL   rL   rM   r    s     z7test_cross_val_predict_class_subset.<locals>.<listcomp>r   r   rw  r   rr  r   r  rh  ri  rk  r   r   )r   rI  r   r   r   r%   r:   r   r+   r  rC   r>   rQ  Zfit_transform)rW   rp   r  Zkfold3Zkfold4lemethodsrl  r   r  r  rL   rL   rM   #test_cross_val_predict_class_subset  sR    



     
     

     r  c                     s   t  } | j| j }}t }tjddd}|d |  tj	|j
tjdtj	|j
ddtjd zJt||| fddd tt t|||fddd W 5 Q R X W 5 d	\ td
D ]8}zt|j
 W  qW q tk
r   td Y qX qX d S )NwbF)moder  s   Hello world!!!!!r  rL   r)rP   r  r  )NNr   r   c                    s    S rS   rL   r   rW   rp   )r]   rL   rM   r     r   z#test_score_memmap.<locals>.<lambda>r   c                    s    S rS   rL   r
  )r   rL   rM   r   	  r   )r   r  r  r~   tempfileNamedTemporaryFilewriter  r   ZmemmapnamerP  r   osunlinkOSErrorr   r,   r   r   r   )r!  rW   rp   r   tfrJ  rL   )r]   r   rM   test_score_memmap  s&    
&
r  c                     s   t t fg} z"ddlm}m} | ||f W n tk
r@   Y nX | D ]^\ t }|j|j }} || }} fdd}fdd}	t	||	d}
t
|
|| qFd S )Nr   r  c                    s
   t |  S rS   r
  rj   r  rL   rM   r   %  r   z4test_permutation_test_score_pandas.<locals>.<lambda>c                    s
   t |  S rS   r
  rj   r  rL   rM   r   &  r   r  )r@   r  r  r	  r   r  r   r  r  r?   r/   )r  r  r	  r!  rW   rp   r  r  r  r  r   rL   r  rM   "test_permutation_test_score_pandas  s    
r  c               	   C   s   t t j} tdd}| |d t d d dd d g	}ddi}tjtdd t|| W 5 Q R X d}tjt|d t	| |d	d
d W 5 Q R X | 
 dkstd S )Nrz   r   r   r   r   z%Failing classifier failed as requiredr   zIThe 'error_score' parameter of cross_validate must be .*. Got .* instead.r   zunvalid-stringr   r   rA  )r5   FAILING_PARAMETERr   rI  r   r   r   r   r2   r,   r]   rm   )failing_clfrW   fit_and_score_argsfit_and_score_kwargsZerror_message_cross_validaterL   rL   rM   test_fit_and_score_failing+  s    
r  c                  C   sx   t ddd\} }tddd}tt | \}}|| |t ||dg}dddd d	d
}t||}|d |d ksttd S )Nr   r   r   r   r   rE  r  )r  r  T)
parametersr*  Zreturn_parametersr  )r   r<   nextr)   r   r   r2   rm   )rW   rp   r   r   r   r  r  resultrL   rL   rM   test_fit_and_score_workingB  s    
r  c                   @   s*   e Zd ZdddZd	ddZd
ddZdS )DataDependentFailingClassifierNc                 C   s
   || _ d S rS   max_x_value)rK   r!  rL   rL   rM   rN   R  s    z'DataDependentFailingClassifier.__init__c                 C   s&   || j k }|r"td| dd S )NzClassifier fit failed with z values too high)r!  rC  r   )rK   rW   rp   Znum_values_too_highrL   rL   rM   rR   U  s
    
z"DataDependentFailingClassifier.fitc                 C   s   dS )NrA  rL   r[   rL   rL   rM   r]   \  s    z$DataDependentFailingClassifier.score)N)N)NN)r`   ra   rb   rN   rR   r]   rL   rL   rL   rM   r  Q  s   

r  r   c              	   C   s   t dd}tdd}td}|||g}d| d}d}tjd	|d
  d| tjd}tjt	|d t
|| W 5 Q R X d S )Nre  r   rz   r   r  r   r  z8ValueError: Classifier fit failed with 1 values too highzh2 fits failed.+total of 3.+The score on these train-test partitions for these parameters will be set to r   z.+flagsr   )r  r   rI  r-  recompileDOTALLr   r   r   r-   )r   r  rW   rp   cross_validate_argscross_validate_kwargsindividual_fit_error_messager   rL   rL   rM   -test_cross_validate_some_failing_fits_warning`  s    



	r*  c              	   C   sv   t t j}tdd}td}|||g}d| d}d}tjd| tjd}tj	t
|d	 t|| W 5 Q R X d S )
Nrz   r   r  rd  r  z1ValueError: Failing classifier failed as requiredz4All the 7 fits failed.+your model is misconfigured.+r"  r   )r5   r  r   rI  r-  r$  r%  r&  r   r   r   r-   )r   r  rW   rp   r'  r(  r)  r  rL   rL   rM   *test_cross_validate_all_failing_fits_error|  s    



r+  c                 C   s   t |d S rS   )r   )r   rW   rp   	error_msgrL   rL   rM   _failing_scorer  s    r-  zignore:lbfgs failed to converger   c              	   C   s   t dd\}}tdd||}d}tt|d}| dkrhtjt|d t|||d	|| d
 W 5 Q R X nBd|  }tj	t
|d$ t|||d	|| d
}t||  W 5 Q R X d S )NTr   r   r  "This scorer is supposed to fail!!!r,  r   r   r   )r   r   r   r   )r   r   rR   r   r-  r   r   r   r,   r   r   rA   )r   rW   rp   r   r,  failing_scorerwarning_msgr   rL   rL   rM   #test_cross_val_score_failing_scorer  s4              r3  r   with_multimetricc              
   C   s  t dd\}}tdd||}d}tt|d}|rJtt}|||d}	n|}	| dkrtjt	|d	 t
|||d
|	|| d W 5 Q R X nd|  }
tjt|
d	b t
|||d
|	|| d}|D ]@}d|krd|kr|| D ]}t|tstqqt|| |  qW 5 Q R X d S )NTr   r   r.  r/  r0  )Zscore_1Zscore_2Zscore_3r   r   r   )r   r   r   r   r   r3   Z_score_2)r   r   rR   r   r-  r   r   r   r   r   r-   r   r   r   rZ   rm   rA   )r   r   r4  rW   rp   r   r,  r1  Znon_failing_scorerr   r2  r   rs   rf  rL   rL   rM   "test_cross_validate_failing_scorer  sP    	r5  c                 C   s   dS )Ng;pΈ^@rL   )rf  jr   rL   rL   rM   three_params_scorer  s    r7  z:train_score, scorer, verbose, split_prg, cdt_prg, expected)rz   r   r   rz   zS\[CV\] END .................................................... total time=   0.\ds)Zsc1Zsc2ze\[CV 2/3\] END  sc1: \(train=3.421, test=3.421\) sc2: \(train=3.421, test=3.421\) total time=   0.\dsr   zW\[CV 2/3; 1/1\] END ....... sc1: \(test=3.421\) sc2: \(test=3.421\) total time=   0.\dsc              	   C   s   t ddd\}}tddd}	tt |\}
}|	||||
||d d g	}|||d}t|| |  \}}|d}t|dkrt	||d	 st
nt	||d st
d S )
Nr   r   r   r   r   )r   Zsplit_progressZcandidate_progress
r   rz   )r   r<   r  r)   r   r2   Z
readouterrr   r$  r   rm   )Zcapsysr   rK  r  Z	split_prgZcdt_prgexpectedrW   rp   r   r   r   r  r  r  rJ  ZoutlinesrL   rL   rM   test_fit_and_score_verbosity  s    #

r;  c               	   C   sF   d} dd }d d d |g}t jt| d t|dtji W 5 Q R X d S )Nz&scoring must return a number, got Nonec                 S   s   d S rS   rL   )r   ZX_testrL   rL   rM   two_params_scorer3	  s    z%test_score.<locals>.two_params_scorerr   r   )r   r   r   r3   r   r   )r  r<  r  rL   rL   rM   r   0	  s
    r   c                  C   sp   dd } t dddd\}}tddd}||| t|||d	| d
}ddddg}|D ]}d||ksTtqTd S )Nc                 S   s2   |  |}t||}|d |d |d |d dS )N)r   r   r8  )rz   r   )rz   rz   )tnfpfntp)rX   r   )r   rW   rp   r   cmrL   rL   rM   r   <	  s    

zPtest_callable_multimetric_confusion_matrix_cross_validate.<locals>.custom_scorer(   r   r&  )r   r=  r   r   )Zdualr   r   )r   r   r=  r>  r?  r@  ztest_{})r   r=   rR   r-   r   rm   )r   rW   rp   r   r  Zscore_namesr  rL   rL   rM   9test_callable_multimetric_confusion_matrix_cross_validate;	  s    rC  c                  C   s&   t dd\} }tt | |ddd dS )z^Check that regressors with partial_fit is supported.

    Non-regression test for #22981.
    r&  r   Tr   )r  r   N)r   r.   r8   )rW   rp   rL   rL   rM   *test_learning_curve_partial_fit_regressorsK	  s    rD  c                 C   s  t dd\}}t|}t }tdd| d}t||||ddd}d|ksJtt||||ddd}d|ksjt|d d	 }|d d
 }t||jkstt||jksttdd |D d tdd |D d t	|
||D ](\}\}	}
t|| |	 t|| |
 qdS )z<Check the behaviour of `return_indices` in `cross_validate`.Tr   r   )rx  r>   r   r   F)r   r   Zreturn_indicesindicesr   r   c                 S   s   g | ]
}|j qS rL   r  r   rE  rL   rL   rM   r  g	  s     z6test_cross_validate_return_indices.<locals>.<listcomp>rE  c                 S   s   g | ]
}|j qS rL   r  rF  rL   rL   rM   r  h	  s     rp  N)r   r;   r   r%   r-   rm   r   rx  rD   r   r   )Zglobal_random_seedrW   rp   r   r   r  Ztrain_indicesZtest_indicesZ	split_idxZexpected_train_idxZexpected_test_idxrL   rL   rM   "test_cross_validate_return_indicesV	  s"    rG  )rc   r  r$  r  r  r4  	functoolsr   ior   timer   Znumpyr   r   Zscipy.sparser   r   r   Zsklearn.baser   r	   Zsklearn.clusterr
   Zsklearn.datasetsr   r   r   r   r   r   Zsklearn.ensembler   Zsklearn.exceptionsr   Zsklearn.imputer   Zsklearn.linear_modelr   r   r   r   r   Zsklearn.metricsr   r   r   r   r   r   r   r    r!   Zsklearn.model_selectionr"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   Z#sklearn.model_selection._validationr1   r2   r3   Z$sklearn.model_selection.tests.commonr4   Z)sklearn.model_selection.tests.test_searchr5   Zsklearn.multiclassr6   Zsklearn.neighborsr7   Zsklearn.neural_networkr8   Zsklearn.pipeliner9   Zsklearn.preprocessingr:   r;   Zsklearn.svmr<   r=   Zsklearn.utilsr>   Zsklearn.utils._mockingr?   r@   Zsklearn.utils._testingrA   rB   rC   rD   Zsklearn.utils.validationrE   rF   rd   rw   r{   r~   r-  rW   r   r   rp   r   r+  r   r   r   r   r   markZparametrizer  r   r   r   r  filterwarningsr  r"  r$  r/  r6  r9  r<  r?  rL  rT  rZ  r[  rb  ro  r{  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r  r  r  r  r  r  r  r   r*  r+  r-  r3  r5  r7  r;  r   rC  rD  rG  rL   rL   rL   rM   <module>   s@   ,DW
*
47BI

#B845

	='N0.&4(





=

 
