U
    OvfA                    @   s  d dl Z d dlZd dlZd dlZd dlmZ d dlmZ d dl	m
Z
mZ d dlmZ d dlmZ d dlmZ d dlmZmZmZmZmZmZmZmZmZmZmZmZmZ d d	l m!Z!m"Z" d d
l#m$Z$ d dl%m&Z& d dl'm(Z( d dl)m*Z*m+Z+m,Z,m-Z- d dl.m/Z/ d dl0m1Z1m2Z2m3Z3 eeegZ4dddZ5dd Z6dd Z7dd Z8dd Z9ej:;dddgdd Z<dd  Z=d!d" Z>d#d$ Z?d%d& Z@d'd( ZAd)d* ZBd+d, ZCd-d. ZDd/d0 ZEd1d2 ZFd3d4 ZGej:;d5eHd d6d d7gd d6d7gfeHd d6d d7gdfd8d9d8d:gd8d9d:gfd8d9d8d:gdfgd;d< ZIej:;d5eHd d7d d7gd d6d7gfeHd8d=d8d=gd8d9d=gfgd>d? ZJej:;d5eHd d6d7d7gdfd8d9d:d:gdfd d6d7d7gd d6d7gfd8d9d:d:gd8d9d:gfgd@dA ZKej:;dBdCdDdEgdFdG ZLdHdI ZMej:;dJdKeHd d6d7d7gd d7d gfdKeHd8d9d:d:gd8d8d9gfdLeHd d7d d7gdfdMeHd8d9d:d:gd8d:d9gfdNeHd d6d7d7gd d6gfdNeHd8d9d:d:gd8d9gfdOeHd d6d7d7gd d6d7dPgfdOeHd8d9d:d:gd8d9d:d=gfdQeHd8d9d:dRgd8d9d:gfdQeHd8d9d:d=gd8d9d:gfdQeHd d6d7dPgd d6d7gfgej:;dSdTdUgdVdW ZNej:;dXdYdZdTd[fd\dZdUd[fd]dTg d^fd_dTd`dafdbi fgdcdd ZOdedf ZPej:;dge4dhdi ZQej:;dge4djdk ZRej:;dge4dldm ZSej:;dddgdndo ZTdpdq ZUej:;dddgdrds ZVdtdu ZWdvdw ZXdxdy ZYdzd{ ZZd|d} Z[d~d Z\ej:;dd d d6gd d`d6gd gd gfd d d6gd dd`gd gd gfd d d6gd`dd6gd gd gfd d d6gdd`dgd gd gfd d6d gd d`d6gd`gd gfd d6d gd dd`gd`gd gfd d6d gd`dd6gd`gd gfd d6d gdd`dgd`gd gfd d6d6gd d`d6gdgd gfd d6d6gd dd`gd gd gfd d6d6gd`dd6gd gd gfd d6d6gdd`dgd gd gfd6d d gd d`d6gd6d6d`gd d6d6gfd6d d gd dd`gd6d6d`gd d6d6gfd6d d gd`dd6gd6d6d`gd d6d6gfd6d d gdd`dgd6d6d`gd d6d6gfd6d d6gd d`d6gd6d6d gd d`d`gfd6d d6gd dd`gd6d6d gd d`d`gfd6d d6gd`dd6gd6d6d gd d`d`gfd6d d6gdd`dgd6d6d gd d`d`gfgdd Z]ej:;dd6d gd`d`gd6gd gfd d6gd`d`gd6gd gfd d d6gdd`d`gd`gd gfd d6d gdd`d`gd`gd gfd d6d6gdd`d`gd gd gfd6d d gdd`d`gd6gd gfd6d d6gdd`d`gd6gd gfd6d6d gdd`d`gd6gd gfgdd Z^dd Z_ej:;dd dd`dd6gdd Z`ej:;dd d d d d d6gd d d d d6d6gd d d d6d6d6gd d d6d6d6d6gd d6d6d6d6d6ggdd Zaej:;dd d6gd d`d6gdfd d6d6gd d`gdfd d d gd d`d6gdfd6d6d6gd d`d6gdfdddgdddgdfgdd Zbdd Zcdd Zddd Zedd Zfdd Zgdd Zhdd ZidddZjej:;dedehegeefej:;deeifdd Zkdd Zlej:;ddej:;ddej:;demd6dd Zndd Zodd Zpdd Zqej:;dd6d d6gdd`d`gfd6d d6gdd`d`ggfd6d d6ggdd`d`gfgdd ZrddĄ ZsddƄ ZtddȄ Zuddʄ Zvdd̄ Zwdd΄ ZxddЄ Zydd҄ ZzddԄ Z{ej:;dddgddׄ Z|ddل Z}ddۄ Z~dd݄ Zdd߄ Zej:;dd d6d7dPgd6dfd d6d7dPgd7d`fd d6d7dPgdPdfgdd Zej:;deHddd6d6gd6d6feHdd6dd6gd6d`feHdd6dd6gd7d6feHddddgd6d6feHddddgd6d`feHddddgd7d6fgdd Zej:;deHd d6d6d7gdd d6d7dPgfeHd d6d6d6gd`d d6d7dPgfeHd6d6d6d6gd`d d6d7dPgfeHd8dRdRd8gdd8d9d=dRgfgej:;dddgdd Zdd Zej:;dd d6d7dPgd6dfd d6d7dPgd7d`fd d6d7dPgdPd6fgdd Zej:;dd d6d7dPgdfd d6d7dPgdfgdd Zej:;dd dd6d7gdddgdddgdddgdd`dggddfd d6d7dPgdddgdddgdddgdd`dggddfd:d:d8d9gdddgdddgdddgdd`dggd8d9d:d:gdfd:d:d8d9gdddgdddgdddgdd`dggd8d:d9gdfd d d6d7gdddgdddgdddgdd`dggd d6d7dPgdfd d d6d7gdddgdddgdddgdd`dggd d6dPgdfd d6gd`ddgdddggddfgdd  Zdd Zej:;deeeegej:;dddddgd	d
 Zdd ZdS (      N)stats)
csr_matrix)datasetssvm)make_multilabel_classification)UndefinedMetricWarning)LogisticRegression)accuracy_scoreaucaverage_precision_scorecoverage_error	dcg_score	det_curve%label_ranking_average_precision_scorelabel_ranking_loss
ndcg_scoreprecision_recall_curveroc_auc_score	roc_curvetop_k_accuracy_score)_dcg_sample_scores_ndcg_sample_scores)train_test_split)label_binarize)_sparse_random_matrix)assert_allcloseassert_almost_equalassert_array_almost_equalassert_array_equal)softmax)check_arraycheck_consistent_lengthcheck_random_stateFc                 C   s  | dkrt  } | j}| j}|r:||dk  ||dk   }}|j\}}t|}td}|| || ||  }}t	|d }tj
d}tj|||d| f }tjdddd}	|	|d| |d| ||d }
|r|
ddd	f }
|	||d }||d }|||
fS )
zMake some classification predictions on a toy dataset using a SVC

    If binary is True restrict to a binary classification problem instead of a
    multiclass classification problem
    N   %   r      ZlinearT)kernelZprobabilityrandom_state   )r   Z	load_irisdatatargetshapenparanger"   shuffleintrandomRandomStateZc_Zrandnr   ZSVCfitpredict_probaZpredict)ZdatasetbinaryXy	n_samples
n_featuresprngZhalfclfy_scorey_predy_true r?   F/tmp/pip-unpacked-wheel-qu3nn_q2/sklearn/metrics/tests/test_ranking.pymake_prediction7   s*    


*rA   c                 C   sd   t | d }|| |k }|| |k }|dd|dd }t |dk}|tt|t|  S )zKAlternative implementation to check for correctness of
    `roc_auc_score`.r(   r   )r,   uniquereshapesumfloatlen)r>   r<   	pos_labelposnegZdiff_matrixZ	n_correctr?   r?   r@   _aucg   s    rK   c           	      C   s   t | d }t | |k}t |ddd }|| }| | } d}tt|D ]P}| | |krPd}td|d D ]}| | |krr|d7 }qr||d  }||7 }qP|| S )a>  Alternative implementation to check for correctness of
    `average_precision_score`.

    Note that this implementation fails on some edge cases.
    For example, for constant predictions e.g. [0.5, 0.5, 0.5],
    y_true = [1, 0, 0] returns an average precision of 0.33...
    but y_true = [0, 0, 1] returns 1.0.
    r(   NrB   r         ?)r,   rC   rE   ZargsortrangerG   )	r>   r<   rH   Zn_posorderscoreiprecjr?   r?   r@   _average_precisionv   s    	

rS   c                 C   sd   t | |\}}}tt|}tt|}d}tdt|D ]$}||| || ||d    7 }q:|S )ao  A second alternative implementation of average precision that closely
    follows the Wikipedia article's definition (see References). This should
    give identical results as `average_precision_score` for all inputs.

    References
    ----------
    .. [1] `Wikipedia entry for the Average precision
       <https://en.wikipedia.org/wiki/Average_precision>`_
    r   r(   )r   listreversedrM   rG   )r>   r<   	precisionrecall	thresholdZaverage_precisionrP   r?   r?   r@   _average_precision_slow   s    
"rY   c                 C   s^   dd }|| ||\}}t ||}d}|}d||  ||  }	|| }
dd||	 |
|	    S )zcAlternative implementation to check for correctness of `roc_auc_score`
    with `max_fpr` set.
    c                 S   s   t | |\}}}|||k }t||}|||k }t||k}|d }	||	 || g}
||	 || g}t|t||
|}||fS )Nr(   )r   r,   appendargmaxZinterp)r>   	y_predictmax_fprfprtpr_new_fprnew_tprZidx_outZidx_inZx_interpZy_interpr?   r?   r@   _partial_roc   s    z,_partial_roc_auc_score.<locals>._partial_rocr         ?r(   )r
   )r>   r\   r]   rc   ra   rb   Zpartial_aucZfpr1Zfpr2Zmin_areaZmax_arear?   r?   r@   _partial_roc_auc_score   s    
re   dropTc           	      C   sz   t dd\}}}t||}t||| d\}}}t||}t||dd t|t|| |j|jksft|j|jksvtd S )NTr4   Zdrop_intermediater#   decimal)	rA   rK   r   r
   r   r   r   r+   AssertionError)	rf   r>   r`   r<   Zexpected_aucr^   r_   
thresholdsroc_aucr?   r?   r@   test_roc_curve   s    

rn   c                  C   s   t jd} t dgd dgd  }| jddd}t||dd\}}}|d dksXt|d	 dksht|j|jksxt|j|jkstd S )
Nr   2   r(      d   sizeTrh   rB   )r,   r0   r1   arrayrandintr   rk   r+   )r:   r>   r=   r^   r_   Zthrr?   r?   r@   test_roc_curve_end_points   s    rv   c            
      C   s   t dd\} }}t| |\}}}g }|D ]2}t||k| @ }t| }	|d| |	  q(t||dd |j|jkszt|j|jkstd S )NTrg   rL   r#   ri   )rA   r   r,   rE   rZ   r   r+   rk   )
r>   r`   r<   r^   r_   rl   Ztpr_correctttpr9   r?   r?   r@   test_roc_returns_consistency   s    
ry   c               	   C   s4   t dd\} }}tt t| | W 5 Q R X d S )NFrg   )rA   pytestraises
ValueErrorr   )r>   r`   r<   r?   r?   r@   test_roc_curve_multi   s    r}   c                  C   s`   t dd\} }}t| |d \}}}t||}t|ddd |j|jksLt|j|jks\td S )NTrg   rd   ?r#   ri   )rA   r   r
   r   r+   rk   )r>   r`   r<   r^   r_   rl   rm   r?   r?   r@   test_roc_curve_confidence   s    
r   c                  C   s  t dd\} }}t| j}t| |\}}}t||}t|ddd |j|jksTt|j|jksdtt| j}t| |\}}}t||}t|ddd |j|jkst|j|jkstt| |\}}}t||}t|ddd |j|jkst|j|jkstd S )NTrg   rd   r#   ri   g(\?)	rA   r,   onesr+   r   r
   r   rk   zeros)r>   predr<   Ztrivial_predr^   r_   rl   rm   r?   r?   r@   test_roc_curve_hard  s$    


r   c               
   C   s  ddddddddddg
} ddddddddddg
}d}t jt|d t| |\}}}W 5 Q R X t|tt|tj |j	|j	kst
|j	|j	kst
d}t jt|d  tdd | D |\}}}W 5 Q R X t|tt|tj |j	|j	kst
|j	|j	kst
d S )Nr(   r   INo negative samples in y_true, false positive value should be meaninglessmatchHNo positive samples in y_true, true positive value should be meaninglessc                 S   s   g | ]}d | qS )r(   r?   ).0xr?   r?   r@   
<listcomp>4  s     z,test_roc_curve_one_label.<locals>.<listcomp>)rz   warnsr   r   r   r,   fullrG   nanr+   rk   )r>   r=   expected_messager^   r_   rl   r?   r?   r@   test_roc_curve_one_label   s     $r   c               	   C   s  ddg} ddg}t | |\}}}t| |}t|dddg t|dddg t|d ddg} ddg}t | |\}}}t| |}t|dddg t|dddg t|d ddg} ddg}t | |\}}}t| |}t|ddg t|ddg t|d ddg} ddg}t | |\}}}t| |}t|dddg t|dddg t|d ddg} ddg}t | |\}}}t| |}t|ddg t|ddg t|d ddg} ddg}d}tjt|d	 t | |\}}}W 5 Q R X tt t| | W 5 Q R X t|dddg t|t	j
t	j
t	j
g ddg} ddg}d
}tjt|d	 t | |\}}}W 5 Q R X tt t| | W 5 Q R X t|t	j
t	j
t	j
g t|dddg t	ddgddgg} t	ddgddgg}tt t| |dd W 5 Q R X tt t| |dd W 5 Q R X tt| |ddd tt| |ddd t	ddgddgg} t	ddgddgg}tt t| |dd W 5 Q R X tt t| |dd W 5 Q R X tt| |ddd tt| |ddd t	ddgddgg} t	ddgddgg}tt| |ddd tt| |ddd tt| |ddd tt| |ddd t	ddgddgg} t	ddgddgg}tt| |ddd tt| |ddd tt| |ddd tt| |ddd d S )Nr   r(   rL           rd         ?      ?r   r   r   macroaverageweightedsamplesmicro)r   r   r   r   rz   r   r   r{   r|   r,   r   rt   )r>   r<   r_   r^   r`   rm   r   r?   r?   r@   test_roc_curve_toydata;  s    









r   c                  C   s   ddddddg} ddddddg}t | |d	d
\}}}t|tjdddg dddddddddddddg} dddddddddddddg}t | |d	d
\}}}t|tjdddddg d S )Nr   r(   r   皙?rd   333333?ffffff?rL   Trh   皙?皙?r~   )r   r   r,   inf)r>   r<   r_   r^   rl   r?   r?   r@    test_roc_curve_drop_intermediate  s    r   c                  C   st   dddddg} dddddg}t dd	}t| ||d
\}}}t |dk  dksVtt |dk  dksptd S )Nr   r(   r   r   333333?皙?rd   r      sample_weight)r,   repeatr   diffrE   rk   )r>   r<   r   r^   r_   r`   r?   r?   r@   !test_roc_curve_fpr_tpr_increasing  s    r   c                  C   s   ddg} ddg}t t| |d ddg} ddg}t t| |d dddg} dddg}t t| |d ddg} ddg}t t| |d dddg} dddg}t t| |d d S Nr   r(   rd   )r   r
   )r   r6   r?   r?   r@   test_auc  s    



r   c               	   C   s   t t tdddgddg W 5 Q R X t t tdgdg W 5 Q R X dddd	g} d
dddg}dt| }t jtt|d t| | W 5 Q R X d S )Nr   rd   rL   r   r   r#   r(   rp      r            z+x is neither increasing nor decreasing : {}r   )	rz   r{   r|   r
   formatr,   rt   reescape)r   r6   error_messager?   r?   r@   test_auc_errors  s    r   zy_true, labelsr(   r#   abcc              	   C   s^  t dddgdddgdddgdd	dgg}td
dd
gdddg}tdd
dgdddg}|| d }td
d
dgdddg}tddd
gdddg}|| d }td
dgdd	g}	tdd
gddg}
|	|
 d }|| | d }tt| ||dd| |||g}dddg}t j||d}tt| ||ddd| d}tjt|d t| ||dd d W 5 Q R X d S )Nr   r   r   r   ffffff?rd   333333?r   r   r(   r#   rp   ovolabelsmulti_classr   )weightsr   r   r   r   z6average=None is not implemented for multi_class='ovo'.r   )r,   rt   r   r   r   rz   r{   NotImplementedError)r>   r   y_scoresscore_01score_10Zaverage_score_01Zscore_02Zscore_20Zaverage_score_02Zscore_12Zscore_21Zaverage_score_12Zovo_unweighted_scoreZpair_scoresZ
prevalenceZovo_weighted_scorer   r?   r?   r@   #test_multiclass_ovo_roc_auc_toydata  sB    "

    r   dc                 C   s   t dddgdddgdddgdddgg}tdd	dd	gddddg}td	dd	dgddddg}|| d
 }tt| ||dd| tt| ||ddd| d S )Nr   r   r   r   r   g?g?r(   r   r#   r   r   r   r   r,   rt   r   r   )r>   r   r   r   r   Z	ovo_scorer?   r?   r@   *test_multiclass_ovo_roc_auc_toydata_binary  s(    "     r   c                 C   s   t dddgdddgdddgdddgg}tdd	d	d	g|d d d	f }td	dd	d	g|d d df }td	d	ddg|d d d
f }tt| |d|d d|||g || | d }tt| |d|d| |d |d  |d  }tt| |d|dd| d S )NrL   r   r   rd   r   r   r   r(   r   r#   ovr)r   r   r   g      @)r   r   r   r   r   )r>   r   r   Zout_0Zout_1Zout_2Zresult_unweightedZresult_weightedr?   r?   r@   #test_multiclass_ovr_roc_auc_toydata@  s4    "    r   zmulti_class, average)r   r   )r   r   )r   r   c                 C   s   t ddddg}ddddgddddgddddgdddd	gg}tt||| |d
d ddddgddddgddddgddddgg}t||| |d
dk stdt d }t||| |d
tdkstd S )Nrp   r(   r#   r   r   rL   r   g?r   r   r   r   )r   r   rd   )r,   rt   r   r   rk   r   rz   approx)r   r   r>   Z	y_perfectZy_imperfectZy_chancer?   r?   r@   0test_perfect_imperfect_chance_multiclass_roc_auck  s8    	







   r   c           	         s   |  t jjdddgd d}t fdd|D }t|dd	d
gd}t| | \}}}t||}t	||ddd}|t
|kstd S )N       @rL   rd     )rs   r'   c                    s"   g | ]}t jjd | d qS )r(   )nr9   r'   )r   Zmultinomialrvsr[   )r   Zy_pred_iseedr?   r@   r     s   z3test_micro_averaged_ovr_roc_auc.<locals>.<listcomp>r   r(   r#   )classesr   r   r   )r   Z	dirichletr   r,   asarrayr   r   Zravelr
   r   rz   r   rk   )	global_random_seedr=   r>   Zy_onehotr^   r_   r`   Zroc_auc_by_handZroc_auc_autor?   r   r@   test_micro_averaged_ovr_roc_auc  s    

r   zmsg, y_true, labelsz!Parameter 'labels' must be uniquezKNumber of classes in y_true not equal to the number of columns in 'y_score'z"Parameter 'labels' must be orderedzMNumber of given labels, 2, not equal to the number of columns in 'y_score', 3zMNumber of given labels, 4, not equal to the number of columns in 'y_score', 3rp   z2'y_true' contains labels not in parameter 'labels'er   r   r   c              	   C   sX   t dddgdddgdddgdd	dgg}tjt| d
 t||||d W 5 Q R X d S )Nr   r   r   r   r   rd   r   r   r   r   r   )r,   rt   rz   r{   r|   r   )msgr>   r   r   r   r?   r?   r@   *test_roc_auc_score_multiclass_labels_error  s
    I"r   zmsg, kwargszLaverage must be one of \('macro', 'weighted', None\) for multiclass problemsr   )r   r   zUaverage must be one of \('micro', 'macro', 'weighted', None\) for multiclass problemszksample_weight is not supported for multiclass one-vs-one ROC AUC, 'sample_weight' must be None in this case)r   r   z|Partial AUC computation not available in multiclass setting, 'max_fpr' must be set to `None`, received `max_fpr=0.5` insteadrd   )r   r]   z'multi_class must be in \('ovo', 'ovr'\)c              	   C   sX   t d}|dd}t|}|jdddd}tjt| d t||f| W 5 Q R X d S )N     rp   r   rr   r   )r"   randr   ru   rz   r{   r|   r   )r   kwargsr:   r<   Zy_probr>   r?   r?   r@   #test_roc_auc_score_multiclass_error  s    'r   c               
   C   st  t d} | d}tjddd}d}tjt|d t|| W 5 Q R X tjddd}tjt|d t|| W 5 Q R X tj	dddd}tjt|d t|| W 5 Q R X t
jdd	 t d} | d}tjddd}tjt|d t|| W 5 Q R X tjddd}tjt|d t|| W 5 Q R X tj	dddd}tjt|d t|| W 5 Q R X W 5 Q R X d S )
Nr   
   r/   ZdtypezROC AUC score is not definedr   rB   T)record)r"   r   r,   r   rz   r{   r|   r   r   r   warningscatch_warnings)r:   r=   r>   err_msgr?   r?   r@   test_auc_score_non_binary_class)  s0    

r   
curve_funcc              	   C   sN   t d}|jdddd}|d}d}tjt|d | || W 5 Q R X d S )Nr   r   rp   r   rr   z"multiclass format is not supportedr   )r"   ru   r   rz   r{   r|   )r   r:   r>   r=   r   r?   r?   r@   &test_binary_clf_curve_multiclass_errorI  s    
r   c              	   C   s   d}t jt|d" | tjddgddddg W 5 Q R X t jt|d" | tjddgtdddg W 5 Q R X d	}t jt|d" | tjd
dgddddg W 5 Q R X ddddg}| ddddg|}| ddddg|}t||D ]\}}tj|| qd S )Nzy_true takes value in {'a', 'b'} and pos_label is not specified: either make y_true take value in {0, 1} or {-1, 1} or pass pos_label explicitly.r   r   r   z<U1r   r   rL   zy_true takes value in {b'a', b'b'} and pos_label is not specified: either make y_true take value in {0, 1} or {-1, 1} or pass pos_label explicitly.   a   bz<S1r   gzG?r   r(   )	rz   r{   r|   r,   rt   objectzipZtestingr   )r   r   r=   Z	int_curveZfloat_curveZint_curve_partZfloat_curve_partr?   r?   r@   (test_binary_clf_curve_implicit_pos_labelS  s    &&&r   c                 C   s   dddddg}dddddg}dddddg}| |||d}| |d d	 |d d	 |d d	 d}t ||D ]\}}t|| qhd S )
Nr   r(   r   r   r   r   rd   r   rB   )r   r   )r   r>   r<   r   Zresult_1Zresult_2Zarr_1Zarr_2r?   r?   r@   (test_binary_clf_curve_zero_sample_weightw  s    &r   c           
      C   sj  t dd\}}}t|||  t|dd  |dd  | d\}}}|d dksPt|d |dd   ksltd|t|dk< | }t|||  t|| ddddg}ddd	d
g}	t||	| d\}}}| rt	|ddddg t	|ddddg t	|ddd
g n:t	|dddddg t	|dddddg t	|ddd	d
g |j
|j
ksPt|j
|j
d ksftd S )NTrg   r(   rh   r   rL   rB   r#   rp   r   rd   gQUU?r   )rA   _test_precision_recall_curver   rk   meanr,   wherecopyr   r   rs   )
rf   r>   r`   r<   r9   rrw   Zy_true_copyr   Zpredict_probasr?   r?   r@   test_precision_recall_curve  s*    $
r   c                 C   s   t | ||d\}}}t| |}t|dd t|t| | tt| ||dd |j|jks^t|j|jd ksrtt | t	||d\}}}|j|jkst|j|jd kstd S )Nrh   grh|?rp   r#   ri   r(   )
r   rY   r   r   r   rS   rs   rk   r,   
zeros_like)r>   r<   rf   r9   r   rl   Zprecision_recall_aucr?   r?   r@   r     s*    
     r   c              
   C   s  t jdd4 ddg}ddg}t||| d\}}}t||}t|dddg t|dddg t|d ddg}ddg}t||| d\}}}t||}t|dddg t|dddg t|d ddg}ddg}t||| d\}}}t||}t|ddg t|ddg t|d ddg}ddg}t||| d\}}}t||}t|dddg t|dddg t|d ddg}ddg}t||| d\}}}t||}t|ddg t|ddg t|d ddg}d	d
g}tjtdd t||| d\}}}W 5 Q R X tjtdd t||}W 5 Q R X t	|dddg t	|dddg t	|d ddg}d	d
g}t||| d\}}}tt||d t|dddg t|dddg t 
ddgddgg}t 
ddgddgg}tjtdd t	t||ddd W 5 Q R X tjtdd t	t||ddd W 5 Q R X t	t||ddd t	t||ddd t 
ddgddgg}t 
ddgddgg}tjtdd t	t||ddd W 5 Q R X tjtdd t	t||ddd W 5 Q R X t	t||ddd
 t	t||ddd t 
ddgddgg}t 
ddgddgg}tt||ddd tt||ddd tt||ddd tt||ddd t 
ddgddgg}t 
ddgddgg}tjtdd t	t||ddd W 5 Q R X t	t||ddd tjtdd t	t||ddd W 5 Q R X tjtdd t	t||ddd W 5 Q R X t 
ddgddgg}t 
ddgddgg}t	t||ddd t	t||ddd t	t||ddd t	t||ddd t 
ddgddgg}t 
ddgddgg}tt||ddd tt||ddd tt||ddd tt||ddd W 5 Q R X t jdd` t 
ddgddgg}t 
ddgddgg}tjtdd t	t||ddd W 5 Q R X W 5 Q R X d S )Nraise)allr   r(   rh   rd   rL   r   r   r   z!No positive class found in y_truer   r   r   r   r   r   ignore)r,   Zerrstater   r   r   r   rz   r   UserWarningr   rt   )rf   r>   r<   r9   r   r`   Zauc_prcr?   r?   r@   #test_precision_recall_curve_toydata  sD   










                     r   c                  C   s&  ddddddg} ddddddg}t | |d	d
\}}}t|dddg dddddddddddddg} dddddddddddddg}t | |d	d
\}}}t|ddddddg ddddg} ddddg}t | |d	d
\}}}t|ddg ddddg} ddddg}t | |d	d
\}}}t|ddddg dS )z9Check the behaviour of the `drop_intermediate` parameter.r   r(   r   r   rd   r   r   rL   Trh   r   r   r~   r   N)r   r   )r>   r<   rV   rW   rl   r?   r?   r@   -test_precision_recall_curve_drop_intermediate^  s@            r   c                  C   s<   t jdtd} d| d d d< t d}t| |dks8td S )Nrq   r   r(   r   r   )r,   r   r/   r   r   rk   r>   r<   r?   r?   r@   &test_average_precision_constant_values  s    
r   c               	   C   sL   t ddg} t ddg}d}tjt|d t| |dd W 5 Q R X d S )Nr   r(   z>pos_label=2 is not a valid label. It should be one of \[0, 1\]r   r#   rH   r,   rt   rz   r{   r|   r   r>   r=   r   r?   r?   r@   4test_average_precision_score_binary_pos_label_errors  s
    r   c               	   C   st   t ddgddgddgddgg} t ddgddgddgddgg}d}tjt|d t| |dd	 W 5 Q R X d S )
Nr(   r   r~   r   r   r   znParameter pos_label is fixed to 1 for multilabel-indicator y_true. Do not set pos_label or set pos_label to 1.r   r   r   r   r?   r?   r@   8test_average_precision_score_multilabel_pos_label_errors  s    ""r   c               
   C   s   t ddddddg} t dddgdddgddd	gdddgdddgdddgg}d
}tjt|d t| |dd W 5 Q R X d S )Nr   r(   r#   rd   r   r   r   r   r   zdParameter pos_label is fixed to 1 for multiclass y_true. Do not set pos_label or set pos_label to 1.r   rp   r   r   r   r?   r?   r@   8test_average_precision_score_multiclass_pos_label_errors  s    r  c                  C   s   t dd\} }}t| |}t| d| }t| d| }t| |d }||ksPt||ks\t||kshtt| |}t| d| }t| d| }	t| |d }
||kst||	kst||
kstd S )NTrg   rq   gư>r   )rA   r   rk   r   )r>   r`   r<   rm   Zroc_auc_scaled_upZroc_auc_scaled_downZroc_auc_shiftedZpr_aucZpr_auc_scaled_upZpr_auc_scaled_downZpr_auc_shiftedr?   r?   r@   test_score_scale_invariance  s    

r  z(y_true,y_score,expected_fpr,expected_fnrr   r   r   c                 C   s(   t | |\}}}t|| t|| d S Nr   r   r>   r<   Zexpected_fprZexpected_fnrr^   fnrr`   r?   r?   r@   test_det_curve_toydata  s    
r  c                 C   s(   t | |\}}}t|| t|| d S r  r  r  r?   r?   r@   test_det_curve_tie_handling  s    
r  c                
   C   s>   t tdddgdddgtddddddgddddddg d S r   )r   r   r?   r?   r?   r@   test_det_curve_sanity_check  s     r	  r<   c                 C   sN   t ddddddgtd| d\}}}t|dg t|dg t|| g d S )Nr   r(   r   r   )r   r,   r   r   )r<   r^   r  rX   r?   r?   r@   test_det_curve_constant_scores  s     
r
  r>   c                 C   s.   t | | d\}}}t|dg t|dg d S )Nr   r   r  )r>   r^   r  r`   r?   r?   r@   test_det_curve_perfect_scores  s    r  zy_true, y_pred, err_msgzinconsistent numbers of samplesz Only one class present in y_truecancer
not cancerr   r   r   zpos_label is not specifiedc              	   C   s(   t jt|d t| | W 5 Q R X d S )Nr   )rz   r{   r|   r   r   r?   r?   r@   test_det_curve_bad_input+  s    r  c            	      C   s   dgd dgd  } t ddddd	dd
dddg
}d| }t| |dd\}}}t| |dd\}}}|d tdksxt|d tdkstt||d d d  t||d d d  d S )Nr  rp   r  r   r   r   r   r   r   rd   r   r~   r(   r   r   rB   )r,   rt   r   rz   r   rk   r   )	r>   Zy_pred_pos_not_cancerZy_pred_pos_cancerZfpr_pos_cancerZfnr_pos_cancerZth_pos_cancerZfpr_pos_not_cancerZfnr_pos_not_cancerZth_pos_not_cancerr?   r?   r@   test_det_curve_pos_label?  s"    r  c                 C   sP  t | ddggddggd t | ddggddggd t | ddggddggd t | dddggdddggd t | dddggdddggd t | dddggdddggd t | dddggdddggd t | dddggdddggd t | dddggdddggd t | dddggdddggd t | dddggdddggd t | dddggdddggd t | dddggdddggd t | dddggdddggd t | dddggdddggd t | dddggdddggd t | dddggdddggd t | dddggdddggd t | dddggdddggd t | dddggdddggd t | dddggdddggd t | dddggdddggd t | dddggdddggd t | ddggddggd t | ddggddggd t | ddggddggd t | dddggdddggd t | dddggdddggd t | dddggdddggd t | dddggdddggd t | dddggdddggd t | dddggdddggd t | dddggdddggd t | dddggdddggd	 t | ddddggddddggd d S )
Nr   r(   r   r   rd   gUUUUUU?g?g?UUUUUU?)r   
lrap_scorer?   r?   r@   check_lrap_toyY  sj                                r  c                 C   s   t d}tddD ]}|jd|fd}t|}td|f}| ||dksPt| ||dksbttd|f}| ||dkst| ||dkstqt| dgdgdgdggdgdgdgdggd d S )Nr   r#   r   r(   rr   rL   rd   )	r"   rM   uniformr,   r   r   rk   r   r   )r  r'   n_labelsr<   Zy_score_tiesr>   r?   r?   r@   !check_zero_or_all_relevant_labels  s    
( r  c              	   C   s  t t | dddgdddg W 5 Q R X t t. | dddgdddgd	ddgd
ddgg W 5 Q R X t t. | dddgdddgd	ddgd
ddgg W 5 Q R X t t  | ddgddggddg W 5 Q R X t t" | ddgddggddgg W 5 Q R X t t$ | ddgddggdgdgg W 5 Q R X t t" | ddggddgddgg W 5 Q R X t t$ | dgdggddgddgg W 5 Q R X t t$ | ddgddggdgdgg W 5 Q R X d S )Nr   r(   r   r   r   r#   r   r   r   r   )rz   r{   r|   r  r?   r?   r@   check_lrap_error_raised  s*     2 $&(&(r  c              	   C   sz   t ddD ]j}td|f}t d|D ]L}t || D ]:}td|f}d|d||| f< t| ||||  q6q&q
d S )Nr#   r   r(   r   )rM   r,   r   r   r   )r  r  r<   
n_relevantrI   r>   r?   r?   r@   check_lrap_only_ties  s    r  c              	      s   t ddD ]}|t|d|fd  }td|f}d|d< d|d< t| ||d| d d  t d|D ]` t |  D ]Ntd|f}d|d  f< t| ||t fddt  D  q~qnq
d S )	Nr#   r   r(   )r   r   )r   rB   r   c                 3   s&   | ]}|d  | d     V  qdS )r(   Nr?   r   r   r  rI   r?   r@   	<genexpr>  s   z>check_lrap_without_tie_and_increasing_score.<locals>.<genexpr>)rM   r,   r-   rD   r   r   rE   )r  r  r<   r>   r?   r  r@   +check_lrap_without_tie_and_increasing_score  s     r  c                    s
  t | | t| } t|}| j\}}t|f}t|D ]}tj|| dd\}}|j}|| tj|d d	 }	|	 | | 
 d }
|
jdks|
j|krd||< q8d||< |
D ]4 t fdd|
D }||  |   7  < q||  |
j  < q8| S )	z8Simple implementation of label ranking average precisionT)Zreturn_inverser(   )Z	minlengthr   r   c                 3   s   | ]}|   kV  qd S r  r?   r  labelZrankr?   r@   r    s     z_my_lrap.<locals>.<genexpr>)r!   r    r+   r,   emptyrM   rC   rs   ZbincountZcumsumZnonzerorE   r   )r>   r<   r7   r  rO   rP   Zunique_rankZinv_rankZn_ranksZ	corr_rankZrelevantZn_ranked_abover?   r  r@   _my_lrap  s*    

r!  r   r   c           	      C   s   t dd|||d\}}t|jd |jd |d}t|drB| }t||}t||}t|| t|}|j	||fd}t||}t||}t|| d S )Nr(   F)r8   Zallow_unlabeledr'   	n_classesr7   r   )Zn_componentsr8   r'   toarrayrr   )
r   r   r+   hasattrr#  r   r!  r   r"   r  )	r  r"  r7   r'   r`   r>   r<   Z
score_lrapZscore_my_lrapr?   r?   r@   %check_alternative_lrap_implementation  s,    
	





r%  checkfuncc                 C   s   | | d S r  r?   )r&  r'  r?   r?   r@   test_label_ranking_avp5  s    r(  c                   C   s   t t d S r  )r  r   r?   r?   r?   r@   test_lrap_error_raisedC  s    r)  r7   )r(   r#   r   r   r"  )r#   r   r   r'   c                 C   s   t t|| | d S r  )r%  r   )r7   r"  r'   r?   r?   r@   $test_alternative_lrap_implementationG  s       r*  c                  C   s   t jddddgddddgddddggtd} t ddddgddddgddddgg}t dd	d
g}t d
d
dg}tt| ||dt || t |  d S )Nr(   r   r   r   r   r   r   rd   r   rL   r   r   )r,   rt   boolr   r   rE   )r>   r<   Zsamplewise_lrapsr   r?   r?   r@   &test_lrap_sample_weighting_zero_labelsP  s    ,   r,  c                   C   s$  t tddggddggd t tddggddggd t tddggddggd t tddggddggd t tdddggdddggd t tdddggdddggd t tdddggdddggd t tdddggdddggd t tdddggdddggd t tdddggdddggd t tdddggdddggd t tdddggdddggd t tdddggdddggd t tdddggdddggd t tdddggdddggd t tdddggdddggd t tdddggdddggd t tdddggdddggd t tdddggdddggd t tdddggdddggd t tdddggdddggd t tdddggdddggd t tdddggdddggd t tdddggdddggd t tdddggdddggd t tdddggdddggd t tdddggdddggd t tdddggdddggd t tdddgdddggdd	d
gdddggd t tdddgdddgdddggddd
gdddgdddggd t tdddgdddgdddggddd
gdddgdddggd d S )Nr   r(   r   r   r#   rd   rp   r         $@r   r   g@r   r   r?   r?   r?   r@   test_coverage_errore  s\                            (  r0  c                   C   st  t tddggddggd t tddggddggd t tddggddggd t tddggddggd t tdddggdddggd t tdddggdddggd t tdddggdddggd t tdddggdddggd t tdddggdddggd t tdddggdddggd t tdddggdddggd t tdddggdddggd d S )Nr   rd   r(   r#   r   rp   r/  r?   r?   r?   r@   test_coverage_tie_handling  s           r1  zy_true, y_scorec              	   C   s(   t jtdd t| | W 5 Q R X d S )Nz'Expected 2D array, got 1D array insteadr   )rz   r{   r|   r   r   r?   r?   r@   test_coverage_1d_error_message  s    r2  c                	   C   s  t tddggddggd t tddggddggd t tdddggdddggd t tdddggdddggd t tdddggdddggd t tdddggdddggd t tdddggdddggd t tdddggdddggd t tddggddggd t tddggddggd t tddggddggd t tddggddggd t tdddggdddggd t tdddggdddggd t tdddggdddggd t tdddggdddggd t tdddgdddggddd	gddd
ggd t tdddgdddgdddggddd	gddd
gdddggd t tdddgdddgdddggddd	gd
dd
gdddggd t tttdddgdddggddd	gd
dd
ggd d S )Nr   r(   r   r   rd   rL   r   r-  r.  rp   r   r#   )r   r   r   r,   rt   r?   r?   r?   r@   test_label_ranking_loss  sR              (   r3  c                	   C   s4  t t  tddgddggddg W 5 Q R X t t" tddgddggddgg W 5 Q R X t t$ tddgddggdgdgg W 5 Q R X t t" tddggddgddgg W 5 Q R X t t$ tdgdggddgddgg W 5 Q R X t t$ tddgddggdgdgg W 5 Q R X d S )Nr   r(   )rz   r{   r|   r   r?   r?   r?   r@   $test_ranking_appropriate_input_shape  s    $&(&(r4  c                   C   s   t tddggddggd t tddggddggd t tdddggdddggd t tdddggdddggd t tdddggdddggd t tdddggdddggd t tdddggdddggd t tdddggdddggd d S )Nr(   r   rd   r   )r   r   r?   r?   r?   r@   test_ranking_loss_ties_handling  s         r5  c                  C   sH   t ddd\} }| d }t|| tjdd\}}t|| d S Nr   r   )r'   r"  r(   )r#   rq   r   )r   _test_dcg_score_forr,   r0   r1   random_sampler`   r>   r<   r?   r?   r@   test_dcg_score
  s
    

r:  c                 C   s   t t | jd d }t| | }t| |}||k s>tt| | dd|k sXt|j| jd fksnt|j| jd fkst|tt 	| d d d d df | j
ddkstd S )Nr(   r#   r   kr   rB   Zaxis)r,   log2r-   r+   r   r   rk   rz   r   sortrE   )r>   r<   ZdiscountidealrO   r?   r?   r@   r7    s    

r7  c               	   C   sN  t t dg} t | j}t| |}t| |dd}dt t dd }|t|	 | 
  gkslt|t|| d d d d df  	 gkstd|dd	d f< t| |}t| |dd}|t|| d d d d df  	 gkst|t|d d 	 | dd	d f 
  |dd  	 | dd d	f 
   gksJtd S )
Nr   Tignore_tiesr(   r#   r   rB   r   rp   )r,   r   r-   r   r+   r   r>  rz   r   rE   r   rk   )r>   r<   ZdcgZdcg_ignore_tiesZ	discountsr?   r?   r@   test_dcg_ties  s"    
 .
.""rC  c               	   C   s>   t dd} t| | dddtt| | dddks:td S )N   )r#   r   rp   T)r<  rB  )r,   r-   rD   r   rz   r   rk   )r   r?   r?   r@   test_ndcg_ignore_ties_with_k1  s    rE  c               	   C   sf   t dddddgg} t dddd	d
gg}d}tjt|d t| |tdksXtW 5 Q R X d S )Ng{Gzg(\gGz޿g(\?gQ?gQ?gףp=
?r   gQ?gHzG?zndcg_score should not be used on negative y_true values. ndcg_score will raise a ValueError on negative y_true values starting from version 1.4.r   g&x@)r,   rt   rz   r   FutureWarningr   r   rk   )r>   r<   r   r?   r?   r@   test_ndcg_negative_ndarray_warn9  s    rG  c                  C   s   t ddd} | t jdjdd| jd }t| |}t| |dd	}|t	|ksZt
|t	d
kslt
|d7 }t| |t	d
kst
d S )NF   r   r   r   皙ɿr   rr   TrA  rL   r   )r,   r-   rD   r0   r1   r  r+   r   rz   r   rk   )r>   r<   ZndcgZndcg_no_tiesr?   r?   r@   test_ndcg_invariantD  s    
rJ  rB  c              
   C   s"  dt dd d  }t t dddd}|t jdjdd	|jd
 }t||| dt	
dt t dd ksxtt||| dt	
dt t dd kstt||| dt	
dt t dd kstt||d| dt	
dt t dd kstt||| dt	
dt t dd  ks:tt||| dt	
dt t dd  ksntdt d }dt t dd  }t||| dt	
|t d kstt||| dt	
t dkstt||| dt	
|ks tt||| dt	
dkstd S )Nrp   r   r   r   rB   )r   r(   r   rI  r   rr   rA  r#   r(   r   )Zlog_baserB  )r   r   	   rL   )r,   ZeyeZtiler-   r0   r1   r  r+   r   rz   r   r>  rk   r   log10r   r   r   r   rE   )rB  r>   r<   Zy_score_noisyZexpected_dcg_scorer?   r?   r@   test_ndcg_toy_examplesO  s|               
  
  
rM  c               	   C   s4   d} t jt| d tdggdgg W 5 Q R X dS )zhCheck that we raise an informative error message when trying to
    compute NDCG with a single document.zTComputing NDCG is only meaningful when there is more than 1 document. Got 1 instead.r   r(   N)rz   r{   r|   r   )r   r?   r?   r@   test_ndcg_error_single_documentv  s    rN  c                  C   sH   t ddd\} }| d }t|| tjdd\}}t|| d S r6  )r   _test_ndcg_score_forr,   r0   r1   r8  r9  r?   r?   r@   test_ndcg_score  s
    

rP  c                 C   s   t | | }t | |}||k s$t| dkjdd}||  tt|  ksXt|| tt| ksxt||  tt	| ||  t	| | |   kst|| tt| kst|j
| j
d fkst|j
| j
d fkstd S )Nr   r(   r=  )r   r   rk   rz   r   r,   r   rE   r   r   r+   )r>   r<   r@  rO   Zall_zeror?   r?   r@   rO    s    

$ 
 rO  c               	   C   sJ  t ddddg} t| | dddks(tt| | dddks>ttt t| | dds\tW 5 Q R X tt t| | ddstW 5 Q R X tt t| | ddstW 5 Q R X t ddddg}t| |dd}t| |}||kstt| |d	dd
ksttdd\} }}t dddD ]"}t	t| ||dt
| || q"d S )Nr   r(   )r]   gMbP?gg?r   g{Gz?r   rd   Trg   g-C6?r   )r,   rt   r   rk   rz   r{   r|   rA   Zlinspacer   re   )r>   r   Zroc_auc_with_max_fpr_oneZunconstrained_roc_aucr=   r`   r]   r?   r?   r@   test_partial_roc_auc_score  s(    

rQ  zy_true, k, true_scorec              	   C   sV   t ddddgddddgddddgddddgg}t| ||d}|t|ksRtd S )Nr   r   r   r   r;  r,   rt   r   rz   r   rk   )r>   r<  
true_scorer<   rO   r?   r?   r@   test_top_k_accuracy_score  s    	



rT  zy_score, k, true_scorerB   r   c                 C   s   ddddg}|   dkr(|  dkr(dnd}|dkrD| |ktjn|}t|| |d}t||}||  krzt|ksn t	d S )Nr   r(   rd   r;  )
minmaxZastyper,   Zint64r   r	   rz   r   rk   )r<   r<  rS  r>   rX   r=   rO   Z	score_accr?   r?   r@    test_top_k_accuracy_score_binary  s     
rW  zy_true, true_score, labelslabels_as_ndarrayc              	   C   sf   |rt |}t ddddgddddgddddgddddgg}t| |d|d}|t|ksbtdS )z,Test when labels and y_score are multiclass.r   r   r   r   r#   r<  r   N)r,   r   rt   r   rz   r   rk   )r>   rS  r   rX  r<   rO   r?   r?   r@   0test_top_k_accuracy_score_multiclass_with_labels  s    




	rZ  c                     s   t jddddd\ t dd\} }}}tdd| | t| |f||fD ]<\  fddtddD }tt	|dksTt
qTd S )	Nr   r   r   )r"  r7   Zn_informativer'   )r'   c                    s    g | ]}t  |d qS )r;  )r   r3   )r   r<  r5   r;   r6   r?   r@   r     s    z8test_top_k_accuracy_score_increasing.<locals>.<listcomp>r#   )r   Zmake_classificationr   r   r2   r   rM   r,   r   r   rk   )ZX_trainZX_testZy_trainZy_testZscoresr?   r[  r@   $test_top_k_accuracy_score_increasing  s       

r\  c              	   C   sR   t ddddgddddgddddgddddgg}t| ||dt|ksNtd S )Nr   r   r   r(   rp   r;  rR  )r>   r<  rS  r<   r?   r?   r@   test_top_k_accuracy_score_ties  s    




r]  z	y_true, kr   c              	   C   sn   t ddddgddddgddddgddddgg}d}tjt|d t| ||d}W 5 Q R X |dksjtd S )	Nr   r   r   r   zu'k' \(\d+\) greater than or equal to 'n_classes' \(\d+\) will result in a perfect score and is therefore meaningless.r   r;  r(   )r,   rt   rz   r   r   r   rk   )r>   r<  r<   r   rO   r?   r?   r@   !test_top_k_accuracy_score_warning$  s    



	r^  zy_true, y_score, labels, msgg=
ףp=?r   r   z9y type must be 'binary' or 'multiclass', got 'continuous'zZNumber of classes in 'y_true' \(4\) not equal to the number of classes in 'y_score' \(3\).z"Parameter 'labels' must be unique.z#Parameter 'labels' must be ordered.zSNumber of given labels \(4\) not equal to the number of classes in 'y_score' \(3\).z3'y_true' contains labels not in parameter 'labels'.z}`y_true` is binary while y_score is 2d with 3 classes. If `y_true` does not contain all the labels, `labels` must be providedc              	   C   s.   t jt|d t| |d|d W 5 Q R X d S )Nr   r#   rY  )rz   r{   r|   r   )r>   r<   r   r   r?   r?   r@   test_top_k_accuracy_score_error=  s    Sr_  c                  C   sR   t dddgdddgg} tdddgdddgg}t| |}|tdksNtd S )Nr(   r   rd   r~   r   r  )r   r,   rt   r   rz   r   rk   )r>   r<   resultr?   r?   r@   Otest_label_ranking_avg_precision_score_should_allow_csr_matrix_for_y_true_input  s    
ra  metricr   )FT)r   r(   )r   rL   )ZzeroZonec                 C   s   t jd}d|d  }}|j||dd}||}| |||d}t|tr\t |rtn@|\}}	}
t |	 rxtt |		 rtt |
	 rtdS )zCheck that the metric works with different types of `pos_label`.

    We can expect `pos_label` to be a bool, an integer, a float, a string.
    No error should be raised for those types.
    *   r   rB   T)rs   replacer   N)
r,   r0   r1   choicer   
isinstancerF   isnanrk   any)rb  r   r:   r7   rH   r>   Zy_probar`  Zmetric_1Zmetric_2rl   r?   r?   r@   #test_ranking_metric_pos_label_types  s    


ri  c                 C   sL   t j| }|jdddd}|d}t||\}}}t |d sHtdS )zCheck that thresholds do not exceed 1.0 when `y_score` is a probability
    estimate.

    Non-regression test for:
    https://github.com/scikit-learn/scikit-learn/issues/26193
    r   r#   r   rr   N)r,   r0   r1   ru   r   r   isinfrk   )r   r:   r>   r<   r`   rl   r?   r?   r@   (test_roc_curve_with_probablity_estimates  s
    
rk  )NF)r   r   r   )r   r   Znumpyr,   rz   Zscipyr   Zscipy.sparser   Zsklearnr   r   Zsklearn.datasetsr   Zsklearn.exceptionsr   Zsklearn.linear_modelr   Zsklearn.metricsr	   r
   r   r   r   r   r   r   r   r   r   r   r   Zsklearn.metrics._rankingr   r   Zsklearn.model_selectionr   Zsklearn.preprocessingr   Zsklearn.random_projectionr   Zsklearn.utils._testingr   r   r   r   Zsklearn.utils.extmathr   Zsklearn.utils.validationr    r!   r"   ZCURVE_FUNCSrA   rK   rS   rY   re   markZparametrizern   rv   ry   r}   r   r   r   r   r   r   r   r   rt   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r  r	  r
  r  r  r  r  r  r  r  r  r!  r%  r(  r)  rM   r*  r,  r0  r1  r2  r3  r4  r5  r:  r7  rC  rE  rG  rJ  rM  rN  rP  rO  rQ  rT  rW  rZ  r\  r]  r^  r_  ra  ri  rk  r?   r?   r?   r@   <module>   s  <
0
h	
2
	
"
"	




	G				#
 
	
#


 $"	""""""""






;'    
 	7
3
&

	









R	 
 