
    /iiJ                    0   d dl Z d dlZd dlZd dlZd dlmZ d dlmZ d dl	m
Z
 d dlmZ d dlmZ d dlmZmZmZmZmZmZmZmZmZmZmZmZmZmZ d dlmZm Z  d d	l!m"Z" d d
l#m$Z$ d dl%m&Z& d dl'm(Z(m)Z)m*Z*m+Z+m,Z, d dl-m.Z. d dl/m0Z0 d dl1m2Z2m3Z3m4Z4 eeeegZ5ddZ6d Z7d Z8d Z9d Z:d Z;ej<        =                    dddg          d             Z>d Z?d Z@d ZAd ZBd ZCd ZDd  ZEd! ZFd" ZGd# ZHd$ ZIej<        =                    d% ejJ        g d&          g d'f ejJ        g d&          dfg d(g d)fg d(dfg          d*             ZKej<        =                    d% ejJ        g d+          g d'f ejJ        g d,          g d-fg          d.             ZLej<        =                    d% ejJ        g d/          dfg d0dfg d/g d'fg d0g d)fg          d1             ZMej<        =                    d2g d3          d4             ZNd5 ZOej<        =                    d6d7 ejJ        g d/          g d8fd7 ejJ        g d0          g d9fd: ejJ        g d+          dfd; ejJ        g d0          g d<fd= ejJ        g d/          d d>gfd= ejJ        g d0          d?d@gfdA ejJ        g d/          g dBfdA ejJ        g d0          g dCfdD ejJ        g dE          g d)fdD ejJ        g dC          g d)fdD ejJ        g dB          g d'fg          ej<        =                    dFdGdHg          dI                         ZPej<        =                    dJdKdLdGdMfdNdLdHdMfdOdGg dPfdQdGdRdSfdTi fg          dU             ZQdV ZRej<        =                    dWe5          dX             ZSej<        =                    dWe5          dY             ZTej<        U                    dZ          ej<        =                    dWeeg          ej<        =                    d[d\d]g          d^                                     ZVej<        =                    dWe5          d_             ZWej<        =                    dddg          d`             ZXda ZYej<        =                    dddg          db             ZZdc Z[dd Z\de Z]df Z^dg Z_dh Z`ej<        =                    dig djg dkd gd gfg djg dld gd gfg djg dmd gd gfg djg dnd gd gfg dog dkg dpg dqfg dog dlg dpg dqfg dog dmg dpg dqfg dog dng dpg dqfg dqg dkdrgd gfg dqg dld gd gfg dqg dmd gd gfg dqg dnd gd gfg dsg dkg dtg dufg dsg dlg dtg dufg dsg dmg dtg dufg dsg dng dtg dufg dvg dkg dwg dxfg dvg dlg dwg dxfg dvg dmg dwg dxfg dvg dng dwg dxfg          dy             Zaej<        =                    g dzg dsg dkg d{g dudfg dsg dkg d|g dqdfg dsg dlg d{g dudfg dsg dlg d|g dqdfg dvg dkg dwg dxdfg dvg dkg dwg dxdfg dvg dlg dwg dxdfg dvg dlg dwg dxdfg          d}             Zbej<        =                    did>d gdRdRgd>d gd d>gfd d>gdRdRgd>d gd d>gfg djg d~dRd gd d>gfg dog d~dRd gd d>gfg dqg d~d gd gfg dsg d~g dwg dqfg dvg d~g dwg dkfg dwg d~g dwg dkfg          d             Zcd Zdej<        =                    dg d          d             Zeej<        =                    dg dg dg dg dg dg          d             Zfej<        =                    dd d>gg dkdfg dqd dRgdfg dg dkdfg dg dkdfg dg ddfg          d             Zgd Zhd Zid Zjd Zkd Zld Zmd Zn	 ddZoej<        =                    deiemelejf          ej<        =                    deenf          d                         Zpd Zqej<        =                    dd          ej<        =                    dd          ej<        =                    d erd>                    d                                     Zsd Ztd Zud Zvej<        =                    dg dvg d~fg dvg d~gfg dvgg d~fg          d             Zwd Zxej<        =                    de0          d             Zyd Zzd Z{d Z|d Z}d Z~d Zd Zd Zej<        =                    dddg          d             Zd Zd Zd Zd Zej<        =                    dg dBd>dfg dBddRfg dBddfg          dÄ             Zej<        =                    d ejJ        g dŢ          d>d>f ejJ        g dƢ          d>dRf ejJ        g dƢ          dd>f ejJ        g dǢ          d>d>f ejJ        g dȢ          d>dRf ejJ        g dȢ          dd>fg          dɄ             Zej<        =                    d ejJ        g dˢ          dg dBf ejJ        g du          dRg dBf ejJ        g d̢          dRg dBf ejJ        g d͢          dg d΢fg          ej<        =                    dddg          dЄ                         Zdф Zej<        =                    dg dBd>dfg dBddRfg dBdd>fg          d҄             Zej<        =                    dg dBdfg dBdfg          dՄ             Zej<        =                    dg dעg dآg d٢g dڢg dۢgddfg dBg dآg d٢g dڢg dۢgddfg dޢg dآg d٢g dڢg dۢgg d0dfg dޢg dآg d٢g dڢg dۢgg d<dfg dg dآg d٢g dڢg dۢgg dBdfg dg dآg d٢g dڢg dۢgg ddfd d>gg dg dgddfg          d             Zej<        =                    de0          d             Zej<        =                    deeeeg          ej<        =                    dg d          d                         Zd ZdS )    N)stats)datasets)make_multilabel_classification)UndefinedMetricWarning)LogisticRegression)accuracy_scoreaucaverage_precision_scoreconfusion_matrix_at_thresholdscoverage_error	dcg_score	det_curve%label_ranking_average_precision_scorelabel_ranking_loss
ndcg_scoreprecision_recall_curveroc_auc_score	roc_curvetop_k_accuracy_score)_dcg_sample_scores_ndcg_sample_scores)train_test_split)label_binarize)_sparse_random_matrix)_convert_containerassert_allcloseassert_almost_equalassert_array_almost_equalassert_array_equal)softmax)CSR_CONTAINERS)check_arraycheck_consistent_lengthcheck_random_stateFc                    | t          j                    } | j        }| j        }|r||dk              ||dk              }}|j        \  }}t          j        |          }t          d          }|                    |           ||         ||         }}t          |dz            }t
          j
                            d          }t
          j        ||                    |d|z            f         }t          d          }	|	                    |d|         |d|                                       ||d                   }
|r|
dddf         }
|	                    ||d                   }||d         }|||
fS )zMake some classification predictions on a toy dataset using an SVC

    If binary is True restrict to a binary classification problem instead of a
    multiclass classification problem
    N   %   r      random_state   )r   	load_irisdatatargetshapenparanger$   shuffleintrandomRandomStatec_randnr   fitpredict_probapredict)datasetbinaryXy	n_samples
n_featuresprnghalfclfy_scorey_predy_trues                h/var/www/html/bet.cuttalo.com/ml/venv/lib/python3.11/site-packages/sklearn/metrics/tests/test_ranking.pymake_predictionrI   :   ss    $&&AA "Qx1q51GIz
	)A
R
 
 CKKNNNQ41qAy1}D )


"
"C
a9cJ&67778A !
,
,
,Cggah%4%))77$%%AAG   !!!Q$-[[455""FtuuXF67""    c                 N   t          j        |           d         }|| |k             }|| |k             }|                    dd          |                    dd          z
  }t          j        |dk              }|t	          t          |          t          |          z            z  S )zKAlternative implementation to check for correctness of
    `roc_auc_score`.r+   r   )r0   uniquereshapesumfloatlen)rG   rE   	pos_labelposnegdiff_matrix	n_corrects          rH   _aucrW   j   s     	&!!!$I &I%
&C
&I%
&C++a$$s{{2q'9'99K{Q''IuSXXC01111rJ   c                    t          j        |           d         }t          j        | |k              }t          j        |          ddd         }||         }| |         } d}t	          t          |                    D ]D}| |         |k    r6d}t	          d|dz             D ]}| |         |k    r|dz  }||dz   z  }||z  }E||z  S )a>  Alternative implementation to check for correctness of
    `average_precision_score`.

    Note that this implementation fails on some edge cases.
    For example, for constant predictions e.g. [0.5, 0.5, 0.5],
    y_true = [1, 0, 0] returns an average precision of 0.33...
    but y_true = [0, 0, 1] returns 1.0.
    r+   NrL   r         ?)r0   rM   rO   argsortrangerQ   )	rG   rE   rR   n_posorderscoreiprecjs	            rH   _average_precisionrb   y   s     	&!!!$IF6Y&''EJw"%EenGE]FE3w<<   	 	!9	!! D1a!e__    !9	))CKDAGODTME5=rJ   c                 "   t          | |          \  }}}t          t          |                    }t          t          |                    }d}t          dt	          |                    D ]"}|||         ||         ||dz
           z
  z  z  }#|S )ao  A second alternative implementation of average precision that closely
    follows the Wikipedia article's definition (see References). This should
    give identical results as `average_precision_score` for all inputs.

    References
    ----------
    .. [1] `Wikipedia entry for the Average precision
       <https://en.wikipedia.org/wiki/Average_precision>`_
    r   r+   )r   listreversedr[   rQ   )rG   rE   	precisionrecall	thresholdaverage_precisionr_   s          rH   _average_precision_slowrj      s     $:&'#J#J IvyXi(())I(6""##F1c)nn%% H HYq\VAYA-FGGrJ   c                     d } || ||          \  }}t          ||          }d}|}d||z
  z  ||z   z  }	||z
  }
dd||	z
  |
|	z
  z  z   z  S )zcAlternative implementation to check for correctness of `roc_auc_score`
    with `max_fpr` set.
    c                 X   t          | |          \  }}}|||k             }t          j        ||          }|||k             }t          j        ||k              }|dz
  }	||	         ||         g}
||	         ||         g}t          j        |t          j        ||
|                    }||fS )Nr+   )r   r0   appendargmaxinterp)rG   	y_predictmax_fprfprtpr_new_fprnew_tpridx_outidx_inx_interpy_interps               rH   _partial_rocz,_partial_roc_auc_score.<locals>._partial_roc   s    	22S!cWn%)GW--cWn%)C'M**1KW.KW.)GRYw(%K%KLL!!rJ   r         ?r+   )r	   )rG   rp   rq   r{   ru   rv   partial_aucfpr1fpr2min_areamax_areas              rH   _partial_roc_auc_scorer      s    

" 
" 
" $|FIw??GWgw''K DDdTk"dTk2Hd{H!{X-(X2EFFGGrJ   c                 @   t           j                            |           }d}|                    ddd          }|                    d          }t          j        |          }||z
  }t          ||          \  }}}	}
}t          |          t          |          cxk    r;t          |	          cxk    r't          |
          cxk    rt          |          k    sn J t          |
|	z   |           t          ||z   |           t          ||z   |	z   |
z   |           dS )z.Smoke test for confusion_matrix_at_thresholds.d   r   r&   sizeN)	r0   r4   r5   randintuniformrO   r   rQ   r   )global_random_seedrB   r?   rG   rE   r\   n_negtnsfpsfnstps
thresholdss               rH   #test_confusion_matrix_at_thresholdsr      s"   
)

 2
3
3CI[[AC[((Fkksk##GF6NNEE%CFG%T%T"Cc3
s88s3xxJJJJ3s88JJJJs3xxJJJJ3z??JJJJJJC#Iu%%%C#Iu%%%C#IOc)955555rJ   dropTc                 B   t          d          \  }}}t          ||          }t          |||           \  }}}t          ||          }t	          ||d           t          |t          ||                     |j        |j        k    sJ |j        |j        k    sJ d S )NTr<   drop_intermediater&   decimal)rI   rW   r   r	   r   r   r   r/   )	r   rG   rt   rE   expected_aucrr   rs   r   roc_aucs	            rH   test_roc_curver      s     )555FAw((L$VWMMMCj#smmGg|Q????vw!?!?@@@9	!!!!9
(((((((rJ   c                  \   t           j                            d          } t          j        dgdz  dgdz  z             }|                     dd          }t          ||d          \  }}}|d         dk    sJ |d	         dk    sJ |j        |j        k    sJ |j        |j        k    sJ d S )
Nr   2   r+      r   r   Tr   rL   )r0   r4   r5   arrayr   r   r/   )rB   rG   rF   rr   rs   thrs         rH   test_roc_curve_end_pointsr      s     )


"
"CXqcBh!r)**F[[[%%FffEEEMCcq6Q;;;;r7a<<<<9	!!!!9	!!!!!!rJ   c                  d   t          d          \  } }}t          | |          \  }}}g }|D ]L}t          j        ||k    | z            }t          j        |           }	|                    d|z  |	z             Mt          ||d           |j        |j        k    sJ |j        |j        k    sJ d S )NTr   rY   r&   r   )rI   r   r0   rO   rm   r   r/   )
rG   rt   rE   rr   rs   r   tpr_correctttprA   s
             rH   test_roc_returns_consistencyr      s     )555FAw$VW55Cj K ) )VW\V+,,F6NN38a<(((( c;::::9	!!!!9
(((((((rJ   c                      t          d          \  } }}t          j        t                    5  t	          | |           d d d            d S # 1 swxY w Y   d S )NFr   )rI   pytestraises
ValueErrorr   )rG   rt   rE   s      rH   test_roc_curve_multir     s    (666FAw	z	"	" # #&'"""# # # # # # # # # # # # # # # # # #s   AAAc                      t          d          \  } }}t          | |dz
            \  }}}t          ||          }t          |dd           |j        |j        k    sJ |j        |j        k    sJ d S )NTr   r|   ?r&   r   )rI   r   r	   r   r/   )rG   rt   rE   rr   rs   r   r   s          rH   test_roc_curve_confidencer     s    (555FAw$VWs];;Cj#smmGgtQ77779	!!!!9
(((((((rJ   c                     t          d          \  } }}t          j        | j                  }t	          | |          \  }}}t          ||          }t          |dd           |j        |j        k    sJ |j        |j        k    sJ t          j        | j                  }t	          | |          \  }}}t          ||          }t          |dd           |j        |j        k    sJ |j        |j        k    sJ t	          | |          \  }}}t          ||          }t          |dd           |j        |j        k    sJ |j        |j        k    sJ d S )NTr   r|   r&   r   g(\?)rI   r0   onesr/   r   r	   r   zeros)rG   predrE   trivial_predrr   rs   r   r   s           rH   test_roc_curve_hardr     sp   +4888FD' 76<((L$V\::Cj#smmGgtQ77779	!!!!9
((((( 8FL))L$V\::Cj#smmGgtQ77779	!!!!9
((((( %VT22Cj#smmGgtQ77779	!!!!9
(((((((rJ   c                     g d} g d}d}t          j        t          |          5  t          | |          \  }}}d d d            n# 1 swxY w Y   t	          |t          j        t          |          t
          j                             |j	        |j	        k    sJ |j	        |j	        k    sJ d}t          j        t          |          5  t          d | D             |          \  }}}d d d            n# 1 swxY w Y   t	          |t          j        t          |          t
          j                             |j	        |j	        k    sJ |j	        |j	        k    sJ d S )N)
r+   r+   r+   r+   r+   r+   r+   r+   r+   r+   )
r   r+   r   r+   r   r+   r   r+   r   r+   INo negative samples in y_true, false positive value should be meaninglessmatchHNo positive samples in y_true, true positive value should be meaninglessc                     g | ]}d |z
  S )r+    ).0xs     rH   
<listcomp>z,test_roc_curve_one_label.<locals>.<listcomp>J  s    )@)@)@A!a%)@)@)@rJ   )
r   warnsr   r   r   r0   fullrQ   nanr/   )rG   rF   expected_messagerr   rs   r   s         rH   test_roc_curve_one_labelr   6  s   +++F+++F 	T  
,4D	E	E	E 9 9(88S*9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 sBGC
OORV<<===9	!!!!9
((((( 	S  
,4D	E	E	E J J()@)@)@)@)@&IIS*J J J J J J J J J J J J J J J sBGC
OORV<<===9	!!!!9
(((((((s#   AAAC88C<?C<c                  J   ddg} ddg}t          | |          \  }}}t          | |          }t          |g d           t          |g d           t          |d           ddg} ddg}t          | |          \  }}}t          | |          }t          |g d           t          |g d           t          |d           ddg} ddg}t          | |          \  }}}t          | |          }t          |ddg           t          |ddg           t          |d           ddg} ddg}t          | |          \  }}}t          | |          }t          |g d           t          |g d           t          |d           ddg} ddg}t          | |          \  }}}t          | |          }t          |ddg           t          |ddg           t          |d           ddg} dd	g}d
}t	          j        t          |          5  t          | |          \  }}}d d d            n# 1 swxY w Y   t          |g d           t          |t          j        t          j        t          j        g           d}t	          j        t          |          5  t          | |          }d d d            n# 1 swxY w Y   t          j
        |          sJ ddg} dd	g}d}t	          j        t          |          5  t          | |          \  }}}d d d            n# 1 swxY w Y   t          |t          j        t          j        t          j        g           t          |g d           d}t	          j        t          |          5  t          | |          }d d d            n# 1 swxY w Y   t          j
        |          sJ t          j        ddgddgg          } t          j        ddgddgg          }t	          j        t          |          5  t          | |d           d d d            n# 1 swxY w Y   t	          j        t          |          5  t          | |d           d d d            n# 1 swxY w Y   t          t          | |d          d           t          t          | |d          d           t          j        ddgddgg          } t          j        ddgddgg          }t	          j        t          |          5  t          | |d           d d d            n# 1 swxY w Y   t	          j        t          |          5  t          | |d           d d d            n# 1 swxY w Y   t          t          | |d          d           t          t          | |d          d           t          j        ddgddgg          } t          j        ddgddgg          }t          t          | |d          d           t          t          | |d          d           t          t          | |d          d           t          t          | |d          d           t          j        ddgddgg          } t          j        ddgddgg          }t          t          | |d          d           t          t          | |d          d           t          t          | |d          d           t          t          | |d          d           d S )Nr   r+   r   r   r+   r   r+   r+   rY           r|         ?      ?r   r   r   r|   rY   OOnly one class is present in y_true. ROC AUC score is not defined in that case.r   macroaverageweightedsamplesmicro)r   r   r   r   r   r   r   r0   r   mathisnanr   )rG   rE   rs   rr   rt   r   r   r	   s           rH   test_roc_curve_toydatar   Q  sd	   VF!fGFG,,KCaFG,,Gc999---c999---%%%VF!fGFG,,KCaFG,,Gc999---c999---%%%VF!fGFG,,KCaFG,,GcAq6***cAq6***%%%VF!fGFG,,KCaFG,,Gc999---c999---%%%VFCjGFG,,KCaFG,,GcAq6***cAq6***%%% VFTlG 	S  
,4D	E	E	E 1 100S!1 1 1 1 1 1 1 1 1 1 1 1 1 1 1c???333cBFBFBF#;<<<	5  
,4D	E	E	E - -FG,,- - - - - - - - - - - - - - -:c?? VFTlG 	T  
,4D	E	E	E 1 100S!1 1 1 1 1 1 1 1 1 1 1 1 1 1 1cBFBFBF#;<<<c???333	5  
,4D	E	E	E - -FG,,- - - - - - - - - - - - - - -:c?? X1v1v&''FhAA'((G	,4D	E	E	E 8 8fgw77778 8 8 8 8 8 8 8 8 8 8 8 8 8 8	,4D	E	E	E ; ;fgz::::; ; ; ; ; ; ; ; ; ; ; ; ; ; ;fgyIII3OOOfgwGGGMMMX1v1v&''FhAA'((G	,4D	E	E	E 8 8fgw77778 8 8 8 8 8 8 8 8 8 8 8 8 8 8	,4D	E	E	E ; ;fgz::::; ; ; ; ; ; ; ; ; ; ; ; ; ; ;fgyIII3OOOfgwGGGMMMX1v1v&''FhAA'((GfgwGGGKKKfgzJJJANNNfgyIII1MMMfgwGGGKKKX1v1v&''Fhc
S#J/00GfgwGGGMMMfgzJJJCPPPfgyIII3OOOfgwGGGMMMMMs   H''H+.H+J//J36J35LLLNN"%N"P--P14P1Q22Q69Q6T++T/2T/U00U47U4c            	          g d} g d}t          | |d          \  }}}t          |t          j        dddg           g d} g d	}t          | |d          \  }}}t          |t          j        dd
dddg           d S )Nr   r   r   r   r+   r+   r   皙?r|   333333?ffffff?rY   Tr   rY   r   r   r   r   r   r   r   r   r   r+   r+   r+   r+   r+   r+   r   皙?r   r   r   皙?r   r   r   r   r   r   rY   r   r   )r   r   r0   inf)rG   rE   rs   rr   r   s        rH    test_roc_curve_drop_intermediater     s    F,,,G$VWMMMCjj263S*ABBB 544FOOOG$VWMMMCjj263S#s*KLLLLLrJ   c                  ,   g d} g d}t          j        dd          }t          | ||          \  }}}t          j        |          dk                                     dk    sJ t          j        |          dk                                     dk    sJ d S )Nr   r   r+   r+   r+   )r   r   333333?皙?r|   r      sample_weightr   )r0   repeatr   diffrO   )rG   rE   r   rr   rs   rt   s         rH   !test_roc_curve_fpr_tpr_increasingr     s     __F'''GIc1%%MFG=IIIKCaGCLL1!!##q((((GCLL1!!##q((((((rJ   c                     ddg} ddg}t          t          | |          d           ddg} ddg}t          t          | |          d           g d} g d}t          t          | |          d           ddg} ddg}t          t          | |          d           g d} g d}t          t          | |          d           d S )Nr   r+   r|   r+   r   r   r   r   r|   r+   )r   r	   )r   r>   s     rH   test_aucr     s    	
AA	
AAc!Qii---	
AA	
AAc!Qii---		A		Ac!Qii---	
AA	
AAc!Qii+++AAc!Qii-----rJ   c                      t          j        t                    5  t          g dddg           d d d            n# 1 swxY w Y   t          j        t                    5  t          dgdg           d d d            n# 1 swxY w Y   g d} g d}d                    t          j        |                     }t          j        t          t          j        |                    5  t          | |           d d d            d S # 1 swxY w Y   d S )	Nr   r   r   r   )r&   r+   r      )r            z+x is neither increasing nor decreasing : {}r   )	r   r   r   r	   formatr0   r   reescape)r   r>   error_messages      rH   test_auc_errorsr     s   	z	"	" ) )OOOc3Z((() ) ) ) ) ) ) ) ) ) ) ) ) ) ) 
z	"	"  SEC5               	AAAHHRSUUM	z=)A)A	B	B	B  Aq			                 s1   ;??A>>BB%DD
Dzy_true, labels)r   r+   r   r&   r   r+   r&   )abr   c)r   r   r   c           	         t          j        g dg dg dg dg          }t          g dg d          }t          g dg d          }||z   d	z  }t          g d
g d          }t          g dg d          }||z   d	z  }t          ddgddg          }	t          ddgddg          }
|	|
z   d	z  }||z   |z   dz  }t          t          | ||d          |           |||g}g d}t          j        ||          }t          t          | ||dd          |           d}t          j        t          |          5  t          | ||dd            d d d            d S # 1 swxY w Y   d S )Nr   r   r   r   r   r   ffffff?r|   333333?r   r   r   r+   r   r+   )r   r   r   r   r+   r   )r   r   r|   r&   r+   r+   r   )r   r   r   r   )r   r  r   r+   r   r   r   r   r   r   ovolabelsmulti_class)r   r   r|   )weightsr   r  r	  r   z6average=None is not implemented for multi_class='ovo'.r   )r0   r   r   r   r   r   r   NotImplementedError)rG   r  y_scoresscore_01score_10average_score_01score_02score_20average_score_02score_12score_21average_score_12ovo_unweighted_scorepair_scores
prevalenceovo_weighted_scorer   s                    rH   #test_multiclass_ovo_roc_auc_toydatar    sQ    x	///+<+<+<mmmL H YYY(8(8(899HYYY88H 8+q0 YYY77HYYY(8(8(899H 8+q0 aVc3Z00HaVc3Z00H 8+q0 -/??BRRVWWfhv5III   $%57GHK###JKDDDHV
	
 	
 	
 		   MM	*-	@	@	@ X Xfhv5RVWWWWX X X X X X X X X X X X X X X X X Xs   ;EE!$E!)r   r&   r   r&   )r   dr   r  )r   r   r  c           	      ,   t          j        g dg dg dg dg          }t          g dg d          }t          g dg d          }||z   d	z  }t          t          | ||d
          |           t          t          | ||d
d          |           d S )N)r   r   r   )r   r   r   )皙?r   ?)r   r   r   )r+   r   r+   r   )r   r   r  r   )r   r+   r   r+   )r   r   r  r   r&   r  r  r   r  r0   r   r   r   )rG   r  r  r  r  	ovo_scores         rH   *test_multiclass_ovo_roc_auc_toydata_binaryr"  <  s     x	///+<+<+<oooN H \\\+@+@+@AAH\\\+@+@+@AAHH$)Ifhv5III9  
 HV
	
 	
 	
 		    rJ   )r   r+   r&   r&   )r   r   r   r   c           	         t          j        g dg dg dg dg          }t          g d|d d df                   }t          g d|d d df                   }t          g d	|d d d
f                   }t          t          | |d|d           |||g           ||z   |z   dz  }t          t          | |d|          |           |dz  |dz  z   |dz  z   }t          t          | |d|d          |           d S )NrY   r   r   )r   r|   r   )r   r   r   )r   r   r   r+   r   r   r   r   )r   r+   r   r   r+   r   r   r+   r+   r&   ovr)r	  r  r   g      @)r	  r  r   r|   r   r   )rG   r  r  out_0out_1out_2result_unweightedresult_weighteds           rH   #test_multiclass_ovr_roc_auc_toydatar-  `  s]    x	///???OOOL H
 ,,,A77E,,,A77E,,,A77EfhE&RVWWW	u   .#5fhE&III   dlUT\1ECK?OH%
	
 	
 	
 		    rJ   zmulti_class, average))r'  r   )r'  r   )r  r   c                 b   t          j        g d          }g dg dg dg dg}t          t          ||| |          d           g dg dg dg dg}t          ||| |          dk     sJ dt          j        d	          z  }t          ||| |          t          j        d
          k    sJ d S )N)r   r+   r&   r   )r   r   r   rY   )r   rY   r   r   )r   r   rY   r   )r   皙?r/  r  r	  r   rY   r   )r   r   r|   )r0   r   r   r   r   r   approx)r	  r   rG   	y_perfecty_imperfecty_chances         rH   0test_perfect_imperfect_chance_multiclass_roc_aucr5    s#    Xlll##F 	   	I fi['RRR   		K 	fk{GTTT
	 	 	 	
 bgfoo%Hk7  	s		     rJ   c                   	 | 	t           j                            g dd	          }t          j        	fd|D                       }t          |g d          }t          |                                |                                          \  }}}t          ||          }t          ||dd	          }|t          j        |          k    sJ d S )
N)       @rY   r|     )r   r*   c                 x    g | ]6}t           j                            d |                                          7S )r+   )nrA   r*   )r   multinomialrvsrn   )r   y_pred_iseeds     rH   r   z3test_micro_averaged_ovr_roc_auc.<locals>.<listcomp>  sM     	
 	
 	
 !!A!EELLNN	
 	
 	
rJ   r   )classesr'  r   r0  )r   	dirichletr<  r0   asarrayr   r   ravelr	   r   r   r1  )
r   rF   rG   y_onehotrr   rs   rt   roc_auc_by_handroc_auc_autor>  s
            @rH   test_micro_averaged_ovr_roc_aucrF    s    D
 _  t$ OOFZ	
 	
 	
 	
"	
 	
 	
 F fiii888HHNN,,fllnn==KCa#smmO UGTTTLfmL99999999rJ   zmsg, y_true, labelsz!Parameter 'labels' must be uniquer   r&   r   )r   r   r   zKNumber of classes in y_true not equal to the number of columns in 'y_score'z"Parameter 'labels' must be ordered)r   r   r   zMNumber of given labels, 2, not equal to the number of columns in 'y_score', 3r+   r   r   zMNumber of given labels, 4, not equal to the number of columns in 'y_score', 3)r   r+   r&   r   )r   r   r   r  z2'y_true' contains labels not in parameter 'labels')r   r   r   er	  r  r'  c                     t          j        g dg dg dg dg          }t          j        t          |           5  t          ||||           d d d            d S # 1 swxY w Y   d S )Nr   r   r   r  r   r  )r0   r   r   r   r   r   )msgrG   r  r	  r  s        rH   *test_roc_auc_score_multiclass_labels_errorrK    s    R x	///+<+<+<mmmL H 
z	-	-	- P Pfhv;OOOOP P P P P P P P P P P P P P P P P Ps   AA!$A!zmsg, kwargszLaverage must be one of \('macro', 'weighted', None\) for multiclass problemsr   )r   r	  zUaverage must be one of \('micro', 'macro', 'weighted', None\) for multiclass problemszksample_weight is not supported for multiclass one-vs-one ROC AUC, 'sample_weight' must be None in this case)r	  r   z|Partial AUC computation not available in multiclass setting, 'max_fpr' must be set to `None`, received `max_fpr=0.5` insteadr|   )r	  rq   z'multi_class must be in \('ovo', 'ovr'\)c                 "   t          d          }|                    dd          }t          |          }|                    ddd          }t	          j        t          |           5  t          ||fi | d d d            d S # 1 swxY w Y   d S )N     r   r   r   r   )r$   randr    r   r   r   r   r   )rJ  kwargsrB   rE   y_probrG   s         rH   #test_roc_auc_score_multiclass_errorrR    s    N S
!
!Chhr1ooGWF[[AB[''F	z	-	-	- 0 0ff/////0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0s   (BBBc                  l   t          d          } |                     d          }t          j        dd          }d}t	          j        t          |          5  t          ||           d d d            n# 1 swxY w Y   t          j        dd          }t	          j        t          |          5  t          ||           d d d            n# 1 swxY w Y   t          j	        ddd          }t	          j        t          |          5  t          ||           d d d            d S # 1 swxY w Y   d S )NrM  
   r3   dtyper   r   rL   )
r$   rO  r0   r   r   r   r   r   r   r   )rB   rF   rG   warn_messages       rH   test_auc_score_non_binary_classrX  I  s    S
!
!CXXb\\FXb&&&F	5  
,L	A	A	A & &ff%%%& & & & & & & & & & & & & & &WRu%%%F	,L	A	A	A & &ff%%%& & & & & & & & & & & & & & &WR5)))F	,L	A	A	A & &ff%%%& & & & & & & & & & & & & & & & & &s6   A55A9<A91CCCD))D-0D-
curve_funcc                    t          d          }|                    ddd          }|                    d          }d}t          j        t
          |          5   | ||           d d d            d S # 1 swxY w Y   d S )NrM  r   r   rT  r   z"multiclass format is not supportedr   )r$   r   rO  r   r   r   )rY  rB   rG   rF   rJ  s        rH   4test_confusion_matrix_at_thresholds_multiclass_errorr[  ^  s    
S
!
!C[[AB[''FXXb\\F
.C	z	-	-	- # #
66"""# # # # # # # # # # # # # # # # # #s   A44A8;A8c                    d}t          j        t          |          5   | t          j        ddgd          ddg           d d d            n# 1 swxY w Y   t          j        t          |          5   | t          j        ddgt
                    ddg           d d d            n# 1 swxY w Y   g d	} | g d
|          } | g d|          }t          ||          D ]%\  }}t          j                            ||           &d S )Nzy_true takes value in {'a', 'b'} and pos_label is not specified: either make y_true take value in {0, 1} or {-1, 1} or pass pos_label explicitly.r   r   r   z<U1rU  r   rY   )r   rY   r   gzG?)r   r+   r+   r   )r   rY   rY   r   )	r   r   r   r0   r   objectziptestingr   )rY  rJ  rF   	int_curvefloat_curveint_curve_partfloat_curve_parts          rH   6test_confusion_matrix_at_thresholds_implicit_pos_labelrd  h  s   
	  
z	-	-	- B B
28S#Je444sCjAAAB B B B B B B B B B B B B B B 
z	-	-	- C C
28S#Jf555SzBBBC C C C C C C C C C C C C C C
 #""F
<<<00I*1116::K,/	;,G,G E E((

"">3CDDDDE Es#   %AAA5*B++B/2B/z.ignore:Support for labels represented as byteslabels_typerd   r   c                     t          ddg|          }d}t          j        t          |          5   | |ddg           d d d            d S # 1 swxY w Y   d S )N   a   bz8Support for labels represented as bytes is not supportedr   r   rY   )r   r   r   	TypeError)rY  re  r  rJ  s       rH   <test_confusion_matrix_at_thresholds_implicit_bytes_pos_labelrj    s      tk::F
DC	y	,	,	, ' '
6C:&&&' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' 's   AAAc                     g d}g d}g d} | |||          } | |d d         |d d         |d d                   }t          ||          D ]\  }}t          ||           d S )Nr   )r   r   r   r   r|   )r+   r+   r+   r|   r   r   rL   )r^  r   )rY  rG   rE   r   result_1result_2arr_1arr_2s           rH   6test_confusion_matrix_at_thresholds_zero_sample_weightrp    s    __F'''G%%%Mz&'GGGHz&"+wss|=QTRTQTCUVVVHHh// & &uu%%%%& &rJ   c                    t          d          \  }}}t          |||            t          |dd          |dd          |           \  }}}|d         dk    sJ |d         |dd                                          k    sJ d|t	          j        |dk              <   |                                }t          |||            t          ||           g d}g d	}	t          ||	|           \  }}}| r7t          |g d
           t          |g d           t          |g d           n6t          |g d           t          |g d           t          |g d	           |j	        |j	        k    sJ |j	        |j	        dz   k    sJ d S )NTr   r+   r   r   rY   rL   r+   r   r   r+   )r+   r&   r   r   )r|   QUU?rY   rY   )rY   r|   r|   r   )r+   r&   r   )r|   rs  r|   rY   rY   )rY   r|   r|   r|   r   )
rI   _test_precision_recall_curver   meanr0   wherecopyr   r   r   )
r   rG   rt   rE   rA   rr   y_true_copyr  predict_probass
             rH   test_precision_recall_curver{    s   (555FAw $777 %VABBZPTUUUGAq!Q43;;;;Q46!"":??$$$$$$ %'F28FaK  !++--K $777{F+++\\F!\\N$V^tTTTGAq! )666777///000999%%%%;;;<<<444555<<<(((6QV6QVaZrJ   c                    t          | ||          \  }}}t          | |          }t          |dd           t          |t          | |                     t	          t          | |          |d           |j        |j        k    sJ |j        |j        dz   k    sJ t          | t          j        |          |          \  }}}|j        |j        k    sJ |j        |j        dz   k    sJ d S )Nr   g+?r   r&   r   r+   )	r   rj   r   r
   r   rb   r   r0   
zeros_like)rG   rE   r   rA   rx  r   precision_recall_aucs          rH   rt  rt    s"   -fgQUVVVAq*267CC2E1===5fgFF   67++-A1    6QV6Z_q(((((-g&&$  Aq* 6QV6Z_q(((((((rJ   c           	      (   t          j        d          5  ddg}ddg}t          |||           \  }}}t          ||          }t	          |g d           t	          |g d           t          |d           ddg}ddg}t          |||           \  }}}t          ||          }t	          |g d	           t	          |g d
           t          |d           ddg}ddg}t          |||           \  }}}t          ||          }t	          |ddg           t	          |ddg           t          |d           ddg}ddg}t          |||           \  }}}t          ||          }t	          |g d           t	          |g d           t          |d           ddg}ddg}t          |||           \  }}}t          ||          }t	          |ddg           t	          |ddg           t          |d           ddg}ddg}t          j        t          d          5  t          |||           \  }}}d d d            n# 1 swxY w Y   t          j        t          d          5  t          ||          }d d d            n# 1 swxY w Y   t          |g d           t          |g d           t          |d           ddg}ddg}t          |||           \  }}}t          t          ||          d           t	          |g d           t	          |g d           t          j
        ddgddgg          }t          j
        ddgddgg          }t          j        t          d          5  t          t          ||d          d           d d d            n# 1 swxY w Y   t          j        t          d          5  t          t          ||d          d           d d d            n# 1 swxY w Y   t          t          ||d          d           t          t          ||d          d           t          j
        ddgddgg          }t          j
        ddgddgg          }t          j        t          d          5  t          t          ||d          d           d d d            n# 1 swxY w Y   t          j        t          d          5  t          t          ||d          d           d d d            n# 1 swxY w Y   t          t          ||d          d           t          t          ||d          d           t          j
        ddgddgg          }t          j
        ddgddgg          }t          t          ||d          d           t          t          ||d          d           t          t          ||d          d           t          t          ||d          d           t          j
        ddgddgg          }t          j
        ddgddgg          }t          j        t          d          5  t          t          ||d          d           d d d            n# 1 swxY w Y   t          t          ||d          d           t          j        t          d          5  t          t          ||d          d           d d d            n# 1 swxY w Y   t          j        t          d          5  t          t          ||d          d           d d d            n# 1 swxY w Y   t          j
        ddgddgg          }t          j
        ddgddgg          }t          t          ||d          d           t          t          ||d          d           t          t          ||d          d           t          t          ||d          d           t          j
        ddgddgg          }t          j
        ddgddgg          }t          t          ||d          d           t          t          ||d          d           t          t          ||d          d           t          t          ||d          d           d d d            n# 1 swxY w Y   t          j        d          5  t          j
        ddgddgg          }t          j
        ddgddgg          }t          j        t          d          5  t          t          ||d          d           d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nraise)allr   r+   r   )r|   r+   r+   r  rY   )r|   r   rY   r$  r|   r   r   r   z!No positive class found in y_truer   r   )rY   rY   rY   )r+   r|   r   r   r   r   r   r   ignore)r0   errstater   r
   r   r   r   r   UserWarningr   r   )r   rG   rE   rA   rx  rt   auc_prcs          rH   #test_precision_recall_curve_toydatar    s   		!	!	! V
 V
Qa&(DQQQ1a)&'::!![[[111!!YYY///GS)))Qa&(DQQQ1a)&'::!!___555!!___555 	GS)))Qa&(DQQQ1a)&'::!!c1X...!!c1X...GS)))Qa&(DQQQ1a)&'::!![[[111!!YYY///GS)))Q*(DQQQ1a)&'::!!c1X...!!aX...GS)))Q,\+-PQQQ 	V 	V,VWPTUUUGAq!	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	V\+-PQQQ 	? 	?-fg>>G	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	?999%%%999%%%###Q,(DQQQ1a3FGDDcJJJ!!___555!!]]]333 Aq6Aq6*++(QFQF+,,\+-PQQQ 	 	'III3  	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 \+-PQQQ 	 	'LLLc  	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	#FGYGGG	
 	
 	
 	/QQQSVWWWAq6Aq6*++(QFQF+,,\+-PQQQ 	 	'III3  	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 \+-PQQQ 	 	'LLLc  	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	#FGYGGG	
 	
 	
 	/QQQSVWWWAq6Aq6*++(QFQF+,,#FGWEEEs	
 	
 	
 	#FGZHHH#	
 	
 	
 	#FGYGGG	
 	
 	
 	#FGWEEEs	
 	
 	
 Aq6Aq6*++(QFQF+,,\+-PQQQ 	 	'III3  	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	#FGZHHH#	
 	
 	
 \+-PQQQ 	 	'KKKS  	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 \+-PQQQ 	 	'III3  	 	 	 	 	 	 	 	 	 	 	 	 	 	 	
 Aq6Aq6*++(QFQF+,,/QQQSVWWW#FGZHHH#	
 	
 	
 	#FGYGGG	
 	
 	
 	/QQQSVWWWAq6Aq6*++(S#Jc
344#FGWEEEs	
 	
 	
 	#FGZHHH#	
 	
 	
 	#FGYGGG	
 	
 	
 	#FGWEEEs	
 	
 	
iV
 V
 V
 V
 V
 V
 V
 V
 V
 V
 V
 V
 V
 V
 V
p 
	"	"	"  Aq6Aq6*++(QFQF+,,\+-PQQQ 	 	'LLLa  	 	 	 	 	 	 	 	 	 	 	 	 	 	 		                 s  Hc$I;cI	cI	c-J
>c
J	cJ	C&c8!N%c%N)	)c,N)	-c!O8,c8O<	<c?O<	 Bc!R?3c?S	cS	c%!TcT	cT	Ec !ZcZ	cZ	>c!\ 4c \	c\	c&!]c]	c]	E+ccc2Af!e/#f/e3	3f6e3	7fffc                     g d} g d}t          | |d          \  }}}t          |g d           g d} g d}t          | |d          \  }}}t          |g d           g d	} g d
}t          | |d          \  }}}t          |ddg           g d} g d
}t          | |d          \  }}}t          |g d
           dS )z9Check the behaviour of the `drop_intermediate` parameter.r   r   Tr   )r   r   rY   r   r   )r   r   r   r   r   rY   r   r   r   r   )r   r   r   r   r   r   r+   r+   r+   r+   N)r   r   )rG   rE   rf   rg   r   s        rH   -test_precision_recall_curve_drop_intermediater  v  sH   F,,,G$:4% % %!Ivz J000 544FOOOG$:4% % %!Ivz J > > >??? \\F"""G$:4% % %!Ivz Jc
+++ \\F"""G$:4% % %!Ivz J 4 4 455555rJ   c                      t          j        dt                    } d| d d d<   t          j        d          }t	          | |          dk    sJ d S )Nr   rU  r+   r   r   )r0   r   r3   r   r
   rG   rE   s     rH   &test_average_precision_constant_valuesr    sW    
 Xc%%%FF33Q3KgcllG #6733t;;;;;;rJ   c                      t          j        ddg          } t          j        ddg          }d}t          j        t          |          5  t          | |d           d d d            d S # 1 swxY w Y   d S )Nr   r+   z>pos_label=2 is not a valid label. It should be one of \[0, 1\]r   r&   rR   r0   r   r   r   r   r
   rG   rF   err_msgs      rH   4test_average_precision_score_binary_pos_label_errorsr    s    Xq!fFXq!fFOG	z	1	1	1 = =!<<<<= = = = = = = = = = = = = = = = = =s   
A**A.1A.c                     t          j        ddgddgddgddgg          } t          j        ddgddgddgddgg          }d}t          j        t          |          5  t          | |d	           d d d            d S # 1 swxY w Y   d S )
Nr+   r   r   r   r   r   znParameter pos_label is fixed to 1 for multilabel-indicator y_true. Do not set pos_label or set pos_label to 1.r   r  r  r  s      rH   8test_average_precision_score_multilabel_pos_label_errorsr    s     X1v1v1v1v677FXSzC:SzC:FGGF	6  
z	1	1	1 = =!<<<<= = = = = = = = = = = = = = = = = =s   A>>BBc            	         t          j        g d          } t          j        g dg dg dg dg dg dg          }d}t          j        t          |          5  t          | |d	           d d d            d S # 1 swxY w Y   d S )
N)r   r+   r&   r   r+   r&   )r|   r   r   )r   r|   r   )r   r   r   )r   r   r|   zdParameter pos_label is fixed to 1 for multiclass y_true. Do not set pos_label or set pos_label to 1.r   r   r  r  r  s      rH   8test_average_precision_score_multiclass_pos_label_errorsr    s    X((())FXOOOOOOOOOOOO	
	 	F	6  
z	1	1	1 = =!<<<<= = = = = = = = = = = = = = = = = =s   A::A>A>c                     t          d          \  } }}t          | |          }t          | d|z            }t          | d|z            }t          | |dz
            }||k    sJ ||k    sJ ||k    sJ t          | |          }t          | d|z            }t          | d|z            }	t          | |dz
            }
||k    sJ ||	k    sJ ||
k    sJ d S )NTr   r   gư>rT  )rI   r   r
   )rG   rt   rE   r   roc_auc_scaled_uproc_auc_scaled_downroc_auc_shiftedpr_aucpr_auc_scaled_uppr_auc_scaled_downpr_auc_shifteds              rH   test_score_scale_invariancer    s    )555FAwFG,,G%fcGm<<'w??#FGbL99O''''')))))o%%%%$VW55F.vsW}EE0HH,VWr\BBN%%%%%'''''^######rJ   z(y_true,y_score,expected_fpr,expected_fnrr   r   )r   r   r|   )r|   r   r+   r   r|   r   r  )r|   r|   r   r   r   r   )r+   r+   r|   r   )r   r+   r+   r+   r  r  )r   r|   r|   c                 n    t          | |          \  }}}t          ||           t          ||           d S Nr   r   rG   rE   expected_fprexpected_fnrrr   fnrrt   s          rH   test_det_curve_toydatar    s?    6 FG,,KCaC&&&C&&&&&rJ   )rG   rE   r  r  r   )r+   r+   r|   r   )r+   r+   r   c                 r    t          | ||          \  }}}t          ||           t          ||           d S )Nr   r  )rG   rE   r  r  r   rr   r  rt   s           rH    test_det_curve_drop_intermediater    sE    & FG?PQQQKCaC&&&C&&&&&rJ   r   r|   r|   c                 n    t          | |          \  }}}t          ||           t          ||           d S r  r  r  s          rH   test_det_curve_tie_handlingr  $  s?     FG,,KCaC&&&C&&&&&rJ   c                  n    t          t          g dg d          t          g dg d                     d S )Nr   r   r   )r   r   r|   r|   r+   r+   )r   r   r   rJ   rH   test_det_curve_sanity_checkr  8  sN    )))[[[))$$$&<&<&<==    rJ   rE   )r   r   r|   r   r+   c                     t          g dt          j        d|                     \  }}}t          |ddg           t          |ddg           t          || t          j        g           d S )N)r   r+   r   r+   r   r+   r   r  r+   r   )r   r0   r   r   r   )rE   rr   r  rh   s       rH   test_det_curve_constant_scoresr  @  s{    #!!!271g+>+>  Ci C!Q   C!Q   I011111rJ   rG   )r   r   r   r   r   r+   r   )r   r   r   r+   r+   r+   )r   r   r+   r+   r+   r+   )r   r+   r+   r+   r+   r+   c                 t    t          | |           \  }}}t          |dg           t          |dg           d S )Nr  r   r  )rG   rr   r  rt   s       rH   test_det_curve_perfect_scoresr  K  sF     66:::KCaC!C!rJ   zy_true, y_pred, err_msgzinconsistent numbers of samplesr   r   r   z#Only one class is present in y_truer+   r+   r+   )cancerr  
not cancer)r   r   r   zpos_label is not specifiedc                     t          j        t          |          5  t          | |           d d d            d S # 1 swxY w Y   d S )Nr   )r   r   r   r   r  s      rH   test_det_curve_bad_inputr  \  s      
z	1	1	1 " "&&!!!" " " " " " " " " " " " " " " " " "   :>>c                     dgdz  dgdz  z   } t          j        g d          }d|z
  }t          | |d          \  }}}t          | |d          \  }}}|d         t          j        d	          k    sJ |d         t          j        d
          k    sJ t          ||d d d                    t          ||d d d                    d S )Nr  r   r  r   )
r   r   r   r   r   r   r|   r   r   r   r+   r  r   r   r   rL   )r0   r   r   r   r1  r   )	rG   y_pred_pos_not_cancery_pred_pos_cancerfpr_pos_cancerfnr_pos_cancerth_pos_cancerfpr_pos_not_cancerfnr_pos_not_cancerth_pos_not_cancers	            rH   test_det_curve_pos_labelr  p  s   Z!^|nq00FH%W%W%WXX114=5 5 51NNM
 AJA A A=*,= v}S111111Q6=#5#55555 N$6ttt$<===N$6ttt$<=====rJ   c                    t           | ddggddgg          d           t           | ddggddgg          d           t           | ddggddgg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g d	gg dg          d           t           | g d
gg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g d	gg dg          d           t           | g d
gg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g d	gg dg          d           t           | g d
gg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | ddggddgg          d           t           | ddggddgg          d           t           | ddggddgg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g d	gg dg          d           t           | g d
gg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           d S )Nr   r+   r   r   r|   r   r  r  r   r   gUUUUUU?r  g?r  g?r   r|   r   r  r|   r   r   r  )r|   r|   r|   UUUUUU?)r+   r+   r+   r   )r|   r|   r|   r|   )r   
lrap_scores    rH   check_lrap_toyr    s/   

QF8tTl^<<a@@@

QF8tTl^<<eDDD

QF8tTl^<<a@@@

III;1B1B1B0CDDaHHH

III;1B1B1B0CDDeLLL

III;1B1B1B0CDDaHHH

III;1B1B1B0CDDeLLL
III;!2!2!2 3446I   
III;!2!2!2 3446I   

III;1B1B1B0CDDeLLL

III;1B1B1B0CDDeLLL
III;!2!2!2 3446I   

III;1B1B1B0CDDaHHH

III;1B1B1B0CDDoVVV

III;1B1B1B0CDDaHHH

III;1B1B1B0CDDaHHH

III;1B1B1B0CDDeLLL

III;1B1B1B0CDDaHHH

III;1B1B1B0CDDoVVV

III;1B1B1B0CDDeLLL
III;!2!2!2 3446I   

III;1B1B1B0CDDaHHH

III;1B1B1B0CDDaHHH 

QF8sCj\::C@@@

QF8sCj\::C@@@

QF8sCj\::A>>>

III;1A1A1A0BCCSIII

III;1A1A1A0BCCSIII

III;1A1A1A0BCCQGGG

III;1A1A1A0BCCUKKK
III;!1!1!1 2335H   
III;!1!1!1 2335H   

III;1A1A1A0BCCQGGG

III;0ABBEJJJ

LLL>4H4H4H3IJJERRRRRrJ   c           	         t          d          }t          dd          D ]}|                    d|f          }t          j        |          }t          j        d|f          } | ||          dk    sJ  | ||          dk    sJ t          j        d|f          } | ||          dk    sJ  | ||          dk    sJ t           | dgdgdgdggdgdgdgdgg          d           d S )Nr   r&   r   r+   r   rY   r|   )r$   r[   r   r0   r}  r   r   r   )r  r*   n_labelsrE   y_score_tiesrG   s         rH   !check_zero_or_all_relevant_labelsr    sA   %a((L!QKK 7 7&&QM&::}W-- 1h-((z&'**c1111z&,//36666 !X''z&'**c1111z&,//366666 
QC!qcA3'3%#u)EFF    rJ   c                    t          j        t                    5   | g dg d           d d d            n# 1 swxY w Y   t          j        t                    5   | g dg dg dg dg           d d d            n# 1 swxY w Y   t          j        t                    5   | g dg dg dg dg           d d d            n# 1 swxY w Y   t          j        t                    5   | ddgddggddg           d d d            n# 1 swxY w Y   t          j        t                    5   | ddgddggddgg           d d d            n# 1 swxY w Y   t          j        t                    5   | ddgddggdgdgg           d d d            n# 1 swxY w Y   t          j        t                    5   | ddggddgddgg           d d d            n# 1 swxY w Y   t          j        t                    5   | dgdggddgddgg           d d d            n# 1 swxY w Y   t          j        t                    5   | ddgddggdgdgg           d d d            d S # 1 swxY w Y   d S )	Nr  )r   r   r   r   )r   r   r   )r   r   r   )r   r   r   r   r+   )r   r   r   r  s    rH   check_lrap_error_raisedr    s   	z	"	" 0 0
999...///0 0 0 0 0 0 0 0 0 0 0 0 0 0 0	z	"	" U U
999000///???STTTU U U U U U U U U U U U U U U	z	"	" 
 

OO//////R	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
z	"	" - -
QFQF#aV,,,- - - - - - - - - - - - - - -	z	"	" / /
QFQF#q!fX.../ / / / / / / / / / / / / / /	z	"	" 1 1
QFQF#qcA3Z0001 1 1 1 1 1 1 1 1 1 1 1 1 1 1	z	"	" / /
QF8q!fq!f-.../ / / / / / / / / / / / / / /	z	"	" 1 1
QC!:AA/0001 1 1 1 1 1 1 1 1 1 1 1 1 1 1	z	"	" 1 1
QFQF#qcA3Z0001 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1s   7;;A??BB#CCC+DDD0EEE6FF F=GG#&G#H&&H*-H*
I..I25I2c           	      &   t          dd          D ]}t          j        d|f          }t          d|          D ]V}t          ||z
            D ]A}t          j        d|f          }d|d|||z   f<   t	           | ||          ||z             BWd S )Nr&   rT  r+   r   )r[   r0   r   r   r   )r  r  rE   
n_relevantrS   rG   s         rH   check_lrap_only_tiesr    s     !RLL 	X 	X'1h-((  8,, 	X 	XJX
233 X X1h-0045q#j 0001#JJvw$?$?hAVWWWWX	X		X 	XrJ   c                 .   t          dd          D ] }|t          j        |                              d|f          dz   z
  }t          j        d|f          }d|d<   d|d<   t           | ||          d|z  dz   dz             t          d|          D ]zt          |z
            D ]et          j        d|f          }d|dz   f<   t           | ||          t          fdt                    D                                  f{d S )Nr&   rT  r+   )r   r   )r   rL   r   c              3   :   K   | ]}|d z   |z   d z   z  z  V  dS )r+   Nr   )r   rx  r  rS   s     rH   	<genexpr>z>check_lrap_without_tie_and_increasing_score.<locals>.<genexpr>  sM         QC!GaK:#=>     rJ   )r[   r0   r1   rN   r   r   rO   )r  r  rE   rG   r  rS   s       @@rH   +check_lrap_without_tie_and_increasing_scorer    sv    !RLL  bi11991h-HH1LM 1h-((tuJJvw77!h,:Ja9OPPP  8,, 	 	JX
233 	 	1h-0045q#j 0001#Jvw//     !&z!2!2       		 rJ   c                    t          | |           t          |           } t          |          }| j        \  }}t          j        |f          }t          |          D ]}t          j        ||         d          \  }}|j        }||z
  t          j        |dz             	                                }	|	         | |         
                                d         }
|
j        dk    s|
j        |k    rd||<   d||<   |
D ]7t          fd|
D                       }||xx         |         z  z  cc<   8||xx         |
j        z  cc<   |                                S )z8Simple implementation of label ranking average precisionT)return_inverser+   )	minlengthr   r   c              3   <   K   | ]}|                  k    V  d S r  r   )r   rx  labelranks     rH   r  z_my_lrap.<locals>.<genexpr><  s0       J JAaDK!7 J J J J J JrJ   )r#   r"   r/   r0   emptyr[   rM   r   bincountcumsumnonzerorO   ru  )rG   rE   r?   r  r^   r_   unique_rankinv_rankn_ranks	corr_rankrelevantn_ranked_abover  r  s               @@rH   _my_lrapr     s   FG,,,  F'""G ,IxHi\""E9 " " !#	'!*T J J JX"! K!<<<CCEE	!9$$&&q)=A(!:!:E!Ha 	5 	5E ! J J J J J J J JJJN !HHHe44HHHHaHM!::<<rJ   r   rN  c                    t          dd|||          \  }}t          |j        d         |j        d         |          }t          |d          r|                                }t          ||          }t          ||          }t          ||           t          |          }|	                    ||f          }t          ||          }t          ||          }t          ||           d S )Nr+   F)r@   allow_unlabeledr*   	n_classesr?   r   )n_componentsr@   r*   toarrayr   )
r   r   r/   hasattrr  r   r  r   r$   r   )	r  r  r?   r*   rt   rG   rE   
score_lrapscore_my_lraps	            rH   %check_alternative_lrap_implementationr  F  s    /!  IAv $\!_<?!  G w	"" $//##6vwGGJVW--M
M222 &l33L""I(>"??G6vwGGJVW--M
M22222rJ   checkfuncc                      | |           d S r  r   )r  r  s     rH   test_label_ranking_avpr  f  s     
E$KKKKKrJ   c                  .    t          t                     d S r  )r  r   r   rJ   rH   test_lrap_error_raisedr  t  s    ABBBBBrJ   r?   )r+   r&   r   rN  r  )r&   r   rT  r*   c                 4    t          t          || |           d S r  )r  r   )r?   r  r*   s      rH   $test_alternative_lrap_implementationr  x  s)     *-y)\    rJ   c                  t   t          j        g dg dg dgt                    } t          j        g dg dg dg          }t          j        g d          }t          j        g d	          }t          t	          | ||
          t          j        ||z            t          j        |          z             d S )Nr%  rr  r  rU  )r   r   r   r   )r   r   r   r   r   r   r   r   )r|   r   rY   )rY   rY   r   r   )r0   r   boolr   r   rO   )rG   rE   samplewise_lrapsr   s       rH   &test_lrap_sample_weighting_zero_labelsr     s     X|||\\\<<<@MMMFh			3335I5I5IJ G x 0 0 011H___--M-G=	
 	
 	
 	}//0026-3H3HH	    rJ   c            	         t          t          ddggddgg          d           t          t          ddggddgg          d           t          t          ddggddgg          d           t          t          ddggddgg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g d	gg dg          d           t          t          g d
gg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g d	gg dg          d           t          t          g d
gg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g d	gg dg          d           t          t          g d
gg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g d	g dgg dg dg          d           t          t          g d	g dg d
gg dg dg dg          d           t          t          g d	g dg d
gg dg dg dg          d           d S )Nr   r+   r   r   r&   r  r  r   r  r   r   r   r  r  r  r  r  r   g      $@r   r+   r   r7  r   rT  r  rG  g@r   r+   r   r   r   r   rJ   rH   test_coverage_errorr    st   Ax4,@@!DDDAx4,@@!DDDAx4,@@!DDDAx4,@@!DDD			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL 			999-/KLL  
 YY			999-yyy)))/T	
 	
 		   YY			999-yyy)))/T	
 	
 		    rJ   c                  f   t          t          ddggddgg          d           t          t          ddggddgg          d           t          t          ddggddgg          d           t          t          ddggddgg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g d	gg dg          d           t          t          g d
gg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           d S )Nr   r|   r+   r&   r  r  r   r  r   r   r   r  r  r  r  r   rJ   rH   test_coverage_tie_handlingr
    s   Ax3*>>BBBAx3*>>BBBAx3*>>BBBAx3*>>BBB			{5E5E5E4FGGKKK			{5E5E5E4FGGKKK			{5E5E5E4FGGKKK			{5E5E5E4FGGKKK			{5E5E5E4FGGKKK			{5E5E5E4FGGKKK			{5E5E5E4FGGKKK			{5E5E5E4FGGKKKKKrJ   zy_true, y_scorec                     t          j        t          d          5  t          | |           d d d            d S # 1 swxY w Y   d S )Nz'Expected 2D array, got 1D array insteadr   )r   r   r   r   r  s     rH   test_coverage_1d_error_messager    s     
z)S	T	T	T ( (vw'''( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( (r  c            	         t          t          ddggddgg          d           t          t          ddggddgg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g d	gg dg          d           t          t          g d
gg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          ddggddgg          d           t          t          ddggddgg          d           t          t          ddggddgg          d           t          t          ddggddgg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dg dgg dg dg          d           t          t          g dg dg d	gg dg dg dg          d           t          t          g dg dg d	gg dg dg dg          d           d S )Nr   r+   r   r   r   r  r  r|   r   r   rY   r  r  r  r  r  r  r  r  r  rG  r  r   r   r   rJ   rH   test_label_ranking_lossr    s   *QF8tTl^DDaHHH*QF8tTl^DDaHHH*III;9J9J9J8KLLaPPP*III;9J9J9J8KLLeTTT*III;9J9J9J8KLLaPPP*III;9J9J9J8KLLeTTT*III;9J9J9J8KLLeTTT*III;9J9J9J8KLLeTTT *QF8tTl^DDaHHH*QF8tTl^DDaHHH*QF8sCj\BBAFFF*QF8sCj\BBAFFF*III;9J9J9J8KLLaPPP*III;9J9J9J8KLLaPPP*III;9I9I9I8JKKQOOO*III;9I9I9I8JKKQOOO IIIyyy1OOOYYY3OPP  
 YY			999-yyy)))/T	
 	
 	"	   YY			999-yyy)))/T	
 	
 	"	    rJ   csr_containerc                     t          t           | t          j        g dg dg                    g dg dg          d           d S )Nr  r  r  r  r|   )r   r   r0   r   )r  s    rH   test_label_ranking_loss_sparser    sc    M"(IIIyyy#9::;;mmmYYY=W	
 	
 		    rJ   c                     t          j        t                    5  t          ddgddggddg           d d d            n# 1 swxY w Y   t          j        t                    5  t          ddgddggddgg           d d d            n# 1 swxY w Y   t          j        t                    5  t          ddgddggdgdgg           d d d            n# 1 swxY w Y   t          j        t                    5  t          ddggddgddgg           d d d            n# 1 swxY w Y   t          j        t                    5  t          dgdggddgddgg           d d d            n# 1 swxY w Y   t          j        t                    5  t          ddgddggdgdgg           d d d            d S # 1 swxY w Y   d S )Nr   r+   )r   r   r   r   r   rJ   rH   $test_ranking_appropriate_input_shaper  !  sE   	z	"	" 5 5QFQF+aV4445 5 5 5 5 5 5 5 5 5 5 5 5 5 5	z	"	" 7 7QFQF+q!fX6667 7 7 7 7 7 7 7 7 7 7 7 7 7 7	z	"	" 9 9QFQF+qcA3Z8889 9 9 9 9 9 9 9 9 9 9 9 9 9 9	z	"	" 7 7QF8q!fq!f%56667 7 7 7 7 7 7 7 7 7 7 7 7 7 7	z	"	" 9 9QC!:AA'78889 9 9 9 9 9 9 9 9 9 9 9 9 9 9	z	"	" 9 9QFQF+qcA3Z8889 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9si   ?AA#B		BB-CCC8DD"%D"E))E-0E-F55F9<F9c                  F   t          t          ddggddgg          d           t          t          ddggddgg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g d	gg dg          d           t          t          g d
gg dg          d           d S )Nr+   r   r|   r   r  r  r   r   r  r  r  r   rJ   rH   test_ranking_loss_ties_handlingr  1  sX   *QF8sCj\BBAFFF*QF8sCj\BBAFFF*III;9I9I9I8JKKUSSS*III;9I9I9I8JKKUSSS*III;9I9I9I8JKKQOOO*III;9I9I9I8JKKQOOO*III;9I9I9I8JKKQOOO*III;9I9I9I8JKKQOOOOOrJ   c                      t          dd          \  } }| dz   }t          ||           t          j                            d                              d          \  }}t          ||           d S Nr   rT  )r*   r  r+   )r&   r   rT  )r   _test_dcg_score_forr0   r4   r5   random_samplert   rG   rE   s      rH   test_dcg_scorer  =  sp    .ALLLIAvgkG(((i++A..<<\JJOFG(((((rJ   c                 :   t          j        t          j        | j        d                   dz             }t	          | |           }t	          | |          }||k                                    sJ t	          | | d          |k                                    sJ |j        | j        d         fk    sJ |j        | j        d         fk    sJ |t          j        t          j        |           d d d d df         |z  	                    d                    k    sJ d S )Nr+   r&   r   kr   rL   axis)
r0   log2r1   r/   r   r  r   r1  sortrO   )rG   rE   discountidealr^   s        rH   r  r  E  s   wrya11A566Hvv..Evw//EUN!!!!!vv333u<AACCCCC;6<?,,,,,;6<?,,,,,FM276??111ddd7#;h#F"K"KQR"K"S"STTTTTTTTrJ   c            	         t          j        t          j        d          g          } t          j        | j                  }t          | |          }t          | |d          }dt          j        t          j        dd                    z  }|t          j        |	                                | 
                                z  g          k    sJ |t          j        || d d d d df         z  	                                g          k    sJ d|dd	d f<   t          | |          }t          | |d          }|t          j        || d d d d df         z  	                                g          k    sJ |t          j        |d d         	                                | dd	d f         
                                z  |dd          	                                | dd d	f         
                                z  z   g          k    sJ d S )
Nr   Tignore_tiesr+   r&   r   rL   r   r   )r0   rA  r1   r   r/   r   r"  r   r1  rO   ru  )rG   rE   dcgdcg_ignore_ties	discountss        rH   test_dcg_tiesr,  P  s   Z1''Fhv|$$G
VW
-
-C(dKKKOBGBIaOO,,,I&-6;;==!@ ABBBBBBfmi&DDbD/.I-N-N-P-P,QRRRRRRGAqrrEN
VW
-
-C(dKKKOfmi&DDbD/.I-N-N-P-P,QRRRRRR&-bqbM&ABB-"4"4"6"66m!!F1bqb5M$6$6$8$889	
       rJ   c            	          t          j        d                              d          } t          | | dd          t	          j        t          | | dd                    k    sJ d S )N   )r&   r   r   T)r  r(  )r0   r1   rN   r   r   r1  )r   s    rH   test_ndcg_ignore_ties_with_kr/  d  sq    
	"f%%AaaT222fm1a1$///7 7      rJ   c                      t          j        g dg          } t          j        g dg          }d}t          j        t          |          5  t          | |           ddd           dS # 1 swxY w Y   dS )zDCheck `ndcg_score` exception when `y_true` contains negative values.)g{Gzg(\gGz޿g(\?gQ?)gQ?gףp=
?r   gQ?gHzG?z7ndcg_score should not be used on negative y_true valuesr   N)r0   r   r   r   r   r   )rG   rE   r   s      rH    test_ndcg_negative_ndarray_errorr1  k  s    X8889::Fh666788GP	z)9	:	:	: $ $67###$ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $s   A**A.1A.c                     t          j        d                              dd          } | t           j                            d                              dd| j                  z   }t          | |          }t          | |d	          }|t          j	        |          k    sJ |t          j	        d
          k    sJ |dz  }t          | |          t          j	        d
          k    sJ d S )NF   r   rT  r   皙ɿr   r   Tr'  rY   r8  )
r0   r1   rN   r4   r5   r   r/   r   r   r1  )rG   rE   ndcgndcg_no_tiess       rH   test_ndcg_invariantr7  t  s    Yr]]""1b))Fry,,Q//77c7UUUGfg&&Dfg4@@@L6=......6=%%%%%%tOGfg&&&-*<*<<<<<<<rJ   r(  c           
      v   dt          j        d          d d         z  }t          j        t          j        ddd          d          }|t           j                            d                              dd	|j        
          z   }t          |||           t          j
        dt          j        t          j        dd                    z            k    sJ t          |||           t          j
        dt          j        t          j        dd                    z            k    sJ t          |||           t          j
        dt          j        t          j        dd                    z            k    sJ t          ||d|           t          j
        dt          j        t          j        dd                    z            k    sJ t          |||           t          j
        dt          j        t          j        dd                    z                                            k    sJ t!          |||           t          j
        dt          j        t          j        dd                    z                                            k    sJ dt          j        d          z  }dt          j        t          j        dd                    z                                  }t          |||           t          j
        |t          j        d          z            k    sJ t          |||           t          j
        t          j        d                    k    sJ t!          |||           t          j
        |          k    sJ t          |||           t          j
        d          k    sJ d S )Nr   r   r   r   rL   )r   r+   r   r4  r   r   r'  r&   r+   rT  )log_baser(  )r   r   	   rY   )r0   eyetiler1   r4   r5   r   r/   r   r   r1  r"  r   log10r   ru  r   r   rO   )r(  rG   rE   y_score_noisyexpected_dcg_scores        rH   test_ndcg_toy_examplesr@    s   2A2Fgbi2r**F33Gbi33A66>>c ?   M [  	q2729Q??333	4	45 5 5 5 ;  	q2729Q??333	4	45 5 5 5 [  	q2729Q??333	4	45 5 5 5 "+  	q28BIaOO444	5	56 6 6 6 fg;???6=	
RWRYq!__%%	%++--D D     VW+>>>&-	
RWRYq!__%%	%++--C C      Fbgbi1oo666;;==[  	)BGAJJ6	7	78 8 8 8 [  	rwqzz	"	"# # # # VW+>>>&-C C     fg;???6=QTCUCUUUUUUUrJ   c                      d} t          j        t          |           5  t          dggdgg           ddd           dS # 1 swxY w Y   dS )zhCheck that we raise an informative error message when trying to
    compute NDCG with a single document.zTComputing NDCG is only meaningful when there is more than 1 document. Got 1 instead.r   r+   N)r   r   r   r   )r  s    rH   test_ndcg_error_single_documentrB    s    	  
z	1	1	1 ! !QC5A3%   ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! !s   A  AAc                      t          dd          \  } }| dz   }t          ||           t          j                            d                              d          \  }}t          ||           d S r  )r   _test_ndcg_score_forr0   r4   r5   r  r  s      rH   test_ndcg_scorerE    sp    .ALLLIAvgkG)))i++A..<<\JJOFG)))))rJ   c                 J   t          | |           }t          | |          }||k                                    sJ | dk                        d          }||          t          j        t	          j        |                                                     k    sJ ||         t          j        t	          j        |                                                    k    sJ ||          t          j        t          | |          |          t          | |           |          z            k    sJ ||         t          j        t	          j        |                                                    k    sJ |j	        | j	        d         fk    sJ |j	        | j	        d         fk    sJ d S )Nr   r+   r   )
r   r  r   r1  r0   r   rO   r   r   r/   )rG   rE   r%  r^   all_zeros        rH   rD  rD    s   //E00EUN!!!!!!  a ((H(v}RWxi__5F5F-G-GHHHHHH?fmBHX\\^^,D,DEEEEEE(v}67++XI6
VV
,
,hY
7	8        ?fmBHX\\^^,D,DEEEEEE;6<?,,,,,;6<?,,,,,,,rJ   c            	         t          j        g d          } t          | | d          dk    sJ t          | | d          dk    sJ t          j        t
                    5  t          | | d          sJ 	 d d d            n# 1 swxY w Y   t          j        t
                    5  t          | | d          sJ 	 d d d            n# 1 swxY w Y   t          j        t
                    5  t          | | d          sJ 	 d d d            n# 1 swxY w Y   t          j        g d          }t          | |d          }t          | |          }||k    sJ t          | |d	          d
k    sJ t          d          \  } }}t          j        ddd          D ]1}t          t          | ||          t          | ||                     2d S )Nr&  r+   )rq   gMbP?gg?r   )r   r   r   g{Gz?r   r|   Tr   g-C6?r   )
r0   r   r   r   r   r   rI   linspacer   r   )rG   r  roc_auc_with_max_fpr_oneunconstrained_roc_aucrF   rt   rq   s          rH   test_partial_roc_auc_scorerL    s   Xlll##F333q88887771<<<<	z	"	" ; ;VVT:::::::; ; ; ; ; ; ; ; ; ; ; ; ; ; ;	z	"	" : :VVS9999999: : : : : : : : : : : : : : :	z	"	" 8 8VVQ77777778 8 8 8 8 8 8 8 8 8 8 8 8 8 8 x+++,,H,VXqIII)&(;;#'<<<<<37773>>>>'t444FFA;tQ** 
 
&&':::"667;;	
 	
 	
 	

 
s6    BB	B&CCC,DDDzy_true, k, true_scorer   r&   r   r   c                     t          j        g dg dg dg dg          }t          | ||          }|t          j        |          k    sJ d S )Nr  r   r   r   r   r   r   r   r   r   r   r   r   r  r0   r   r   r   r1  )rG   r  
true_scorerE   r^   s        rH   test_top_k_accuracy_scorerS    su     h            		
 G !A666EFM*--------rJ   zy_score, k, true_score)rL   rL   r+   r+   )rL   r+   rL   r+   )r   r   r   r   )r   r   r   r   c                 V   g d}|                                  dk    r|                                 dk    rdnd}|dk    r#| |k                        t          j                  n|}t          || |          }t          ||          }||cxk    rt          j        |          k    sn J d S )Nr&  r   r+   r|   r  )	minmaxastyper0   int64r   r   r   r1  )rE   r  rR  rG   rh   rF   r^   	score_accs           rH    test_top_k_accuracy_score_binaryrZ    s     \\F{{}}))gkkmmq.@.@aI78Avvg	!))"(3336F A666Evv..II::::z!:!:::::::::rJ   zy_true, true_score, labels)r   r+   r+   r&   r  )r   rH  rH  r   )r   r   r  rH  labels_as_ndarrayc                     |rt          j        |          }t          j        g dg dg dg dg          }t          | |d|          }|t	          j        |          k    sJ dS )z,Test when labels and y_score are multiclass.r  rN  rO  rP  r&   r  r  N)r0   rA  r   r   r   r1  )rG   rR  r  r[  rE   r^   s         rH   0test_top_k_accuracy_score_multiclass_with_labelsr^    s      $F##h            		
 G !AfEEEEFM*--------rJ   c                     t          j        dddd          \  t          d          \  } }}}t          d                              | |           t          | |f||f          D ]O\  fdt          dd          D             }t          j        t          j	        |          dk              sJ Pd S )NrT  r8  r   )r  r?   n_informativer*   r)   c                 Z    g | ]'}t                                        |           (S )r  )r   r9   )r   r  r=   rD   r>   s     rH   r   z8test_top_k_accuracy_score_increasing.<locals>.<listcomp>8  sD     
 
 
CD C$5$5a$8$8A>>>
 
 
rJ   r&   )
r   make_classificationr   r   r8   r^  r[   r0   r  r   )X_trainX_testy_trainy_testscoresr=   rD   r>   s        @@@rH   $test_top_k_accuracy_score_increasingrh  ,  s    'BQ  DAq (811'M'M'M$GVWf
!
,
,
,CGGGWWf%'899 + +1
 
 
 
 
 
HMaQS
 
 
 vbgfoo)******+ +rJ   c                     t          j        g dg dg dg dg          }t          | ||          t          j        |          k    sJ d S )N)r   r   r   r   )r+   r   r   r   )r   r   r   r   r  r  rQ  )rG   r  rR  rE   s       rH   test_top_k_accuracy_score_tiesrj  ?  sh     hLLLLLLLL		
 G  1555z9R9RRRRRRRrJ   z	y_true, kr   c                     t          j        g dg dg dg dg          }d}t          j        t          |          5  t          | ||          }d d d            n# 1 swxY w Y   |dk    sJ d S )	Nr  )r   r   r   r   )r   r   r   r   )r   r   r   r   zu'k' \(\d+\) greater than or equal to 'n_classes' \(\d+\) will result in a perfect score and is therefore meaningless.r   r  r+   )r0   r   r   r   r   r   )rG   r  rE   r   r^   s        rH   !test_top_k_accuracy_score_warningrl  T  s     h            		
 G	6  
,4D	E	E	E ; ;$VW:::; ; ; ; ; ; ; ; ; ; ; ; ; ; ;A::::::s   AA!$A!zy_true, y_score, labels, msg)r   g=
ףp=?r+   r&   )r   r   r   )r   r   r   r   )r   r|   r   z9y type must be 'binary' or 'multiclass', got 'continuous'zZNumber of classes in 'y_true' \(4\) not equal to the number of classes in 'y_score' \(3\).)r   r   r   r   z"Parameter 'labels' must be unique.z#Parameter 'labels' must be ordered.)r   r   r+   r&   zSNumber of given labels \(4\) not equal to the number of classes in 'y_score' \(3\).r  z3'y_true' contains labels not in parameter 'labels'.)r|   r   r   )r   r   r   z}`y_true` is binary while y_score is 2d with 3 classes. If `y_true` does not contain all the labels, `labels` must be providedc                     t          j        t          |          5  t          | |d|           d d d            d S # 1 swxY w Y   d S )Nr   r&   r]  )r   r   r   r   )rG   rE   r  rJ  s       rH   test_top_k_accuracy_score_errorrn  m  s    f 
z	-	-	- B BVW&AAAAB B B B B B B B B B B B B B B B B Bs   =AAc                      | g dg dg          }t          j        g dg dg          }t          ||          }|t          j        d          k    sJ d S )Nr   r   )r|   r   r   r  )r0   r   r   r   r1  )r  rG   rE   results       rH   Otest_label_ranking_avg_precision_score_should_allow_csr_matrix_for_y_true_inputrq    sl     ]IIIyyy122Fh344G267CCFV]5))))))))rJ   metricr?  ))FT)r   r+   )r   rY   )zeroonec                 $   t           j                            d          }d|d         }}|                    ||d          }|                    |          } | |||          }t          |t                    rt          j        |          rJ dS |\  }}	}
t          j        |                                          rJ t          j        |	                                          rJ t          j        |
                                          rJ dS )zCheck that the metric works with different types of `pos_label`.

    We can expect `pos_label` to be a bool, an integer, a float, a string.
    No error should be raised for those types.
    *   rT  rL   T)r   replacer  N)	r0   r4   r5   choicerO  
isinstancerP   r   any)rr  r?  rB   r?   rR   rG   y_probarp  metric_1metric_2r   s              rH   #test_ranking_metric_pos_label_typesr~    s
    )


#
#Cwr{yIZZiZ>>Fhhy!!GVFGy999F&%   .8F#######)/&(J8H%%))+++++8H%%))+++++8J''++-------rJ   c                     t           j                            |           }|                    ddd          }|                    d          }t          ||          \  }}}t          j        |d                   sJ dS )zCheck that thresholds do not exceed 1.0 when `y_score` is a probability
    estimate.

    Non-regression test for:
    https://github.com/scikit-learn/scikit-learn/issues/26193
    r   r&   rT  r   N)r0   r4   r5   r   rO  r   isinf)r   rB   rG   rE   rt   r   s         rH   (test_roc_curve_with_probablity_estimatesr    sv     )

 2
3
3C[[AB[''FhhrllG 11Aq*8JqM"""""""rJ   )NF)r   rN  r   )r   r   numpyr0   r   scipyr   sklearnr   sklearn.datasetsr   sklearn.exceptionsr   sklearn.linear_modelr   sklearn.metricsr   r	   r
   r   r   r   r   r   r   r   r   r   r   r   sklearn.metrics._rankingr   r   sklearn.model_selectionr   sklearn.preprocessingr   sklearn.random_projectionr   sklearn.utils._testingr   r   r   r   r   sklearn.utils.extmathr    sklearn.utils.fixesr!   sklearn.utils.validationr"   r#   r$   CURVE_FUNCSrI   rW   rb   rj   r   r   markparametrizer   r   r   r   r   r   r   r   r   r   r   r   r   r  r"  r-  r5  rF  rK  rR  rX  r[  rd  filterwarningsrj  rp  r{  rt  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r[   r  r   r  r
  r  r  r  r  r  r  r  r,  r/  r1  r7  r@  rB  rE  rD  rL  rS  rZ  r^  rh  rj  rl  rn  rq  r~  r  r   rJ   rH   <module>r     su    				                  ; ; ; ; ; ; 5 5 5 5 5 5 3 3 3 3 3 3                                 M L L L L L L L 4 4 4 4 4 4 0 0 0 0 0 0 ; ; ; ; ; ;              * ) ) ) ) ) . . . . . .          #	)# )# )# )#`2 2 2  <  &H H H86 6 6& $//
) 
) 0/
)
" 
" 
") ) )&# # #) ) )) ) )8) ) )6oN oN oNdM M M	) 	) 	). . .&  " 	,,,		+	,,,		&			/			t$	 /X /X /Xd 	,,,		+	&&&	'	'9   : 	,,,		&			t$	yyy!			/	   D      D: : :( 	,hbh|||.D.DiiiP/BH)))**OO	
 BH\\\""	
 1BH)))**OO	
 BH\\\""F	
 BH)))**#J	
 BH\\\""LL	
 BH)))**   	
 ABH)))**OO	
 ABH)))**OO	
 ABH\\\""II	
}CF FN 77P P 87OF FPP ' "%88	
' "%88	
F "B77	
 "c22	
 
4R8=" "F	0 	0G" "F	0& & &* {33# # 43# {33E E 43E0 LMM(>	'JKK(9::' ' ;: LK NM' {33	& 	& 43	& $//    0/ >) ) ), $//` ` 0/`F6 6 6D< < <= = =
= 
= 
== = =*$ $ $2 .	KKK!qc*	NNNQC!-	NNNQC!-	%%%sQC0	KKK			:	NNNMMM999=	NNNMMM999=	%%%}}}iii@	KKK#,	NNNQC!-	NNNQC!-	%%%sQC0	KKK>	NNNNNNLLLA	NNNNNNLLLA	%%%~~~|||D	KKKMMM:	NNNIII}}}=	NNNIII}}}=	%%%yyy---@) 2' '3 2' NNN 
KKK!1!1!1<<<G	KKKiii>	NNN$4$4$4lllEJ	NNNKKKDA	KKKMMM5A	KKKMMM4@	NNNIII}}}eD	NNNIII}}}dC ' ' ' .
Q#saVaV,
Q#saVaV,	$$$sAhA7	$$$sAhA7	$$$qcA3/	$$$iii;	$$$iii=	$$$iii=	 ' ' '   $E$E$EFF2 2 GF2 																 	 	 	 
Q?@	QH?@	KKK!FG	KKK!FG...OO(	

 " " "> > >48S 8S 8Sv  .1 1 14X X X  6# # #N 9:3 3 3 3@ 3)	  "G!RSS  TS C C C m44j11q22  32 21 54  *4 4 4nL L L  	$$$%	%%%&'
))&&&' ( ( (( ( (V .99  :99 9 9 	P 	P 	P) ) )U U U  (  $ $ $= = = u66#V #V 76#VL! ! !* * *- - - 
 
 
4 	q$	q#	q$ 
. 
. 
. 	...	!	!1a(	...	!	!1c*	...	!	!1a(	&&&	'	'A.	&&&	'	'C0	&&&	'	'A.
 
	; 	;
 
	;  	,,,		|||4	,,,		lll3	,,,		lll3	&&&	'	'/C/C/CD	  ,tUm<<. . =< .&+ + +& 	q$	q#	q! 
S 
S 
S 	q	q   $ " OO	 G
	
 LL	 +	
 !  	 !  0
	
 !  	 OO1
	
 LL	 LL 	
 LL	 IIA
	
 F__ooo.W	
KNQ QdB BeQ QdB
 .99* * :9* &	3I9U  CCC . .  .(# # # # #rJ   