
    /iiu                    T   d dl Z d dlZd dlmZ d dlmZmZmZ d dlZ	d dl
Z
d dlmZmZ d dlmZ d dlmZ d dlmZmZ d dlmZ d d	lmZ d d
lmZ d dlmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1 d dl2m3Z3m4Z4m5Z5 d dl6m7Z7 d dl8m9Z9m:Z: d dl;m<Z< d dl=m>Z? d dl=m@Z@mAZA d dlBmCZC d dlDmEZEmFZFmGZGmHZHmIZImJZJ d dlKmLZL d dlMmNZNmOZO d dlPmQZQ ddZRd ZSe
jT        U                    ddd de	jV        g          d             ZWe
jT        U                    dd gdfd dgdfg d dfg          d!             ZXd" ZYd# ZZe
jT        [                    d$          d%             Z\e
jT        [                    d$          d&             Z]e
jT        [                    d$          d'             Z^d( Z_e
jT        U                    d)g d* e	j`        g d+g d,g d-g d.g          fg d/g d0fg          d1             Zae
jT        U                    d)g d2 e	j`        g d.g d.g d3g d3g d4g          fg d5g d6fg          d7             Zbd8 Zcd9 Zdd: Zed; Zfe
jT        U                    d<eN          e
jT        U                    d=eO          d>                         Zgd? Zhe
jT        U                    d@g dA          dB             ZidC ZjdD Zke
jT        U                    dE e	j`        g dF           e	j`        g dF          dGdHf e	j`        g dF           e	j`        g dI          dGdJf e	j`        g dF           e	j`        g dK          dGdLf e	j`        g dI           e	j`        g dF          dGdMfg          dN             Zle
jT        U                    dO e	j`        g dP           e	j`        g dQ          dGdRfg          dS             ZmdT Zne
jT        U                    dUddg          dV             ZodW Zpe
jT        U                    dXdYdZid[dZid\dZd]d^d_d]dZdZd]d^d`d]g          da             Zqe
jT        U                    dbd^d^d]d^fe	jr        dZd]e	jr        fd`dZd]d`fe	jV        e	jV        d]e	jV        fe	jV        e	jV        fg          dc             Zse
jT        U                    dbd^d^d]d^fe	jr        dZd]dZfe	jr        ddd]ddfe	jV        e	jV        d]e	jV        fe	jV        e	jV        fg          de             Ztdf Zudg Zve
jT        U                    dd de	jV        g          e
jT        U                    dhd gd gfg          e
jT        U                    die% ee&dj          e/e0g          dk                                     Zwe
jT        U                    dhd gd gfg          e
jT        U                    die% ee&dj          e/e0g          dl                         Zxdm Zydn Zzdo Z{dp Z|e
jT        U                    dqdrdsg          dt             Z}du Z~e
jT        U                    dvg dw          dx             Zdy Zdz Zd{ Ze
jT        U                    d|g d}fd~dgdfgddg          d             Zd Ze
jT        U                    dg d          d             Zd Zd Zd Zd Zd Zd Zd Zd Zd Ze
jT        [                    d$          d             Zd Zd Zd Zd Zd Zd Zd Ze
jT        U                    dddg          d             Ze
jT        [                    d$          d             Ze
jT        [                    d$          d             Ze
jT        [                    d$          e
jT        U                    dddde	jV        e	jV        fg          d                         Ze
jT        U                    ddg          e
jT        U                    dvg d          e
jT        U                    dd de	jV        g          d                                     Ze
jT        U                    dvg d          d             Ze
jT        U                    dd de	jV        g          d             Zd Zd Ze
jT        U                    dd de	jV        g          d             Ze
jT        U                    ddd de	jV        g          d             Ze
jT        U                    ddd de	jV        g          d             Ze
jT        U                    ddd de	jV        g          d             Zd Zd Zd Zd Ze
jT        U                    d ej        dgd gdgd gg          df ej        d gdgdgdgg          df ej        g dg dg dg          dfg          d             Zd Zd Zd Zd Zd Zd Zd ZdÄ Ze
jT        U                    de	j        e	j        e	j        g          dĄ             Ze
jT        U                    de	j        e	j        e	j        g          dń             Ze
jT        U                    dhg dg dfg ddd gd dgdd ggfg d g dƢg dg dǢgfg          dȄ             ZdɄ Zdʄ Zd˄ Zd̄ Zd̈́ Zd΄ Zdτ Ze
jT        U                    dg dѢg dҢfg dӢg dҢfg dҢg dӢfg          dԄ             Ze
jT        U                    die)e% ee&ddj          e.e/e0e g          e
jT        U                    dg d֢          dׄ                         Ze
jT        U                    d e	j`        d dg           e	j`        dd g          dZf e	j`        d dg           e	j`        d dg          d^f e	j`        d dg           e	j`        d d g          dZf e	j`        d d g           e	j`        d d g          d^fg          dل             Ze
jT        j        e
jT        U                    d e+e%e	jV        ۦ           e+e&de	jV        ܦ           e+e/e	jV        ۦ           e+e0e	jV        ۦ          g          d݄                         Zdބ Zd߄ Zd Zd Zd Zd Ze
jT        U                    dg dg dddfg dg dddfg dg dddfg dg dddfg dg dddfg dddddgddddgddddggddfg dg dgg dg dgddfg dg dg dg dgd dgdfg dg dg dg dgd gdfg	          d             Zd Ze
jT        U                    d eA                      d              Ze
jT        U                    de e*e4e5g          e
jT        U                    dddg          e
jT        U                    dddg          e
jT        U                    d eA                      d                                                 Ze
jT        U                    de e*e4e5g          e
jT        U                    dddg          e
jT        U                    d eA                      d                                     ZdS (      N)partial)chainpermutationsproduct)linalgsparse)hamming)	bernoulli)datasetssvm)config_context)make_multilabel_classification)UndefinedMetricWarning)accuracy_scoreaverage_precision_scorebalanced_accuracy_scorebrier_score_lossclass_likelihood_ratiosclassification_reportcohen_kappa_scoreconfusion_matrixf1_scorefbeta_scorehamming_loss
hinge_lossjaccard_scorelog_lossmake_scorermatthews_corrcoefmultilabel_confusion_matrixprecision_recall_fscore_supportprecision_scorerecall_scorezero_one_loss)_check_targetsd2_brier_scored2_log_loss_score)cross_val_score)LabelBinarizerlabel_binarize)DecisionTreeClassifierdevice)get_namespace)yield_namespace_device_dtype_combinations)MockDataFrame)_array_api_for_testsassert_allcloseassert_almost_equalassert_array_almost_equalassert_array_equalignore_warnings)_nanaverage)CSC_CONTAINERSCSR_CONTAINERS)check_random_stateFc                    | t          j                    } | j        }| j        }|r||dk              ||dk              }}|j        \  }}t          j        |          }t          d          }|                    |           ||         ||         }}t          |dz            }t
          j
                            d          }t
          j        ||                    |d|z            f         }t          j        ddd          }	|	                    |d|         |d|                                       ||d                   }
|r|
ddd	f         }
|	                    ||d                   }||d         }|||
fS )
zMake some classification predictions on a toy dataset using an SVC

    If binary is True restrict to a binary classification problem instead of a
    multiclass classification problem
    N   %   r      linearT)kernelprobabilityrandom_state   )r   	load_irisdatatargetshapenparanger:   shuffleintrandomRandomStatec_randnr   SVCfitpredict_probapredict)datasetbinaryXy	n_samples
n_featuresprnghalfclfy_pred_probay_predy_trues                o/var/www/html/bet.cuttalo.com/ml/venv/lib/python3.11/site-packages/sklearn/metrics/tests/test_classification.pymake_predictionrb   G   sz    $&&AA "Qx1q51GIz
	)A
R
 
 CKKNNNQ41qAy1}D )


"
"C
a9cJ&67778A 't!
D
D
DC771UdU8QuuX..<<QtuuXFFL * $AAAqD)[[455""FtuuXF6<''    c            
         t          j                    } t          | d          \  }}}ddddddd	d
ddddddddddddddddddd}t          ||t	          j        t          | j                            | j        d          }|                                |                                k    sJ |D ]}|dk    r2t          ||         t                    sJ ||         ||         k    sJ :||                                         ||                                         k    sJ ||         D ]*}t          ||         |         ||         |                    +t          |d         d         t                    sJ t          |d         d         t                    sJ t          |d         d         t                    sJ t          |d         d         t                    sJ d S ) NFrT   rU   g7Mo?gUUUUUU?ghQ?   )	precisionrecallf1-scoresupportUUUUUU?gc1Ƹ?g433333?   g)k??$I$I?   gCFQ?gc?gƢ?K   )ri   rg   rh   rj   g?gD~WG?g]3p?)setosa
versicolor	virginica	macro avgaccuracyzweighted avgT)labelstarget_namesoutput_dictru   rq   rg   rt   rj   )r   rD   rb   r   rH   rI   lenrw   keys
isinstancefloatr3   rK   )irisr`   r_   _expected_reportreportkeymetrics           ra   ,test_classification_report_dictionary_outputr   w   sd   D'UCCCFFA
 -)*	
 
 -*+	
 
 -)+	
 
 ++'	
 
 '++(	
 
5   OD #yT.//00&  F ;;==O00222222 W W*fSk511111#;/#"666666#;##%%)=)B)B)D)DDDDD)#. W W#OC$8$@&+fBUVVVVW oh/<eDDDDDok2;?GGGGGoh/	:C@@@@@ok29=sCCCCCCCrc   zero_divisionwarnrC   c                 B   g dg d}}t          j        d          5 }t          j        dd           t          ||| d           | d	k    r5t	          |          d
k    sJ |D ]}d}|t          |j                  v sJ n|rJ d d d            d S # 1 swxY w Y   d S )Nabc)r   r   dTrecordalwaysz.+Use `zero_division`)message)r   rx   r   rC   z7Use `zero_division` parameter to control this behavior.)warningscatch_warningsfilterwarningsr   ry   strr   )r   r`   r_   r   itemmsgs         ra   0test_classification_report_zero_division_warningr      s&   $__oooFF		 	-	-	-  	2IJJJJF-T	
 	
 	
 	
 F""v;;???? 0 0Oc$,///////0                  s   A)BBBzlabels, show_micro_avgTr   rC   r<   c                 t    ddgddg}}t          ||| d          }|rd|v sJ d|vsJ dS d|v sJ d|vsJ dS )a3  Check the behaviour of passing `labels` as a superset or subset of the labels.
    WHen a superset, we expect to show the "accuracy" in the report while it should be
    the micro-averaging if this is a subset.

    Non-regression test for:
    https://github.com/scikit-learn/scikit-learn/issues/27927
    r   rC   T)rv   rx   z	micro avgru   Nr   )rv   show_micro_avgr`   r_   r   s        ra   1test_classification_report_labels_subset_supersetr      s     VaVFF"66&dSSSF )f$$$$''''''V####&((((((rc   c                  F   t          j        g dg dg          } t          j        g dg dg          }t          | |          dk    sJ t          | |           dk    sJ t          ||          dk    sJ t          |t          j        |                    dk    sJ t          | t          j        |                     dk    sJ t          | t          j        | j                            dk    sJ t          |t          j        | j                            dk    sJ d S )Nr   rC   rC   rC   r   rC   r   r   rC         ?rC   r   )rH   arrayr   logical_notzerosrG   y1y2s     ra   .test_multilabel_accuracy_score_subset_accuracyr      s)   	999iii(	)	)B	999iii(	)	)B"b!!S(((("b!!Q&&&&"b!!Q&&&&"bnR0011Q6666"bnR0011Q6666"bhrx0011Q6666"bhrx0011Q666666rc   c            	         t          d          \  } }}t          | |d           \  }}}}t          |ddgd           t          |ddgd           t          |d	d
gd           t          |ddg           i ddifD ]}t	          j                    5  t	          j        d           t          | |fi |}t          |dd           t          | |fi |}	t          |	dd           t          | |fi |}
t          |
d
d           t          t          | |fddi|d|z  |	z  d|z  |	z   z  d           d d d            n# 1 swxY w Y   d S )NTrU   averageg\(\?g333333?r<   g)\(?g(\?皙?gRQ?   r   rU   errorbeta      )rb   r!   r4   r5   r   r   simplefilterr"   r#   r   r3   r   )r`   r_   r~   rZ   rfskwargspsrsfss              ra   %test_precision_recall_f1_score_binaryr      s   't444FFA 1NNNJAq!Qa$q111a$q111a$q111q2r(###
 	8,-  $&& 	 	!'*** ::6::B%b$222ff7777B%b$222&&33F33B%b$222FF===f==R"$r	B7  	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 s   B D>>E	E	z1ignore::sklearn.exceptions.UndefinedMetricWarningc                  f   dt          ddgddg          k    sJ dt          ddgddg          k    sJ dt          ddgddg          k    sJ dt          ddgddgd          k    sJ dt          ddgddg          k    sJ dt          ddgddg          k    sJ dt          ddgddg          k    sJ dt          ddgddgt	          d                    k    sJ t          ddgddgt	          d                    t          j        t          ddgddgd                    k    sJ d S )	N      ?rC   r   r           infg     j@)r"   r#   r   r   r|   pytestapprox rc   ra   +test_precision_recall_f_binary_single_classr     s   
 /1a&1a&111111,1v1v......(Aq6Aq6******+q!fq!f15555555/2r(RH555555,Bx"b222222(B8b"X......+r2hRuU||DDDDDDDBx"be===RHr2hS111B B      rc   c                     g d} g d}t          | t          j        d                    }t          |t          j        d                    }| |f||fg}t          |          D ]\  }\  } }t	          | |g dd           }t          g d|           t	          | |g dd          }t          t          j        g d          |           d	D ]C}|d
k    r|dk    rt          t	          | |g d|          t	          | |d |                     DdD ]}t          j	        t                    5  t	          ||t          j        d          |           d d d            n# 1 swxY w Y   t          j	        t                    5  t	          ||t          j        dd          |           d d d            n# 1 swxY w Y   t          j        g dg dg          } t          j        g dg dg          }t          | |d
ddg          \  }}	}
}t          t          j        ||	|
g          t          j        g d                     d S )N)rC      r   r<   )rC   rC   r   r<   r   classes)r   rC   r<   r   r   rv   r   )r   r   r   r   r   macro)microweightedsamplesr   r   )Nr   r   r      r   r   r   rC   r   r   rC   rC   rC   r   rC   r   rv   )      ?rC   竪?)r*   rH   rI   	enumerater#   r4   meanr3   r   raises
ValueErrorr   r!   )r`   r_   
y_true_bin
y_pred_binrE   iactualr   rZ   r   r   r~   s               ra   $test_precision_recall_f_extra_labelsr   &  s.    \\F\\F	!===J	!===JVz:67D(  FFff___dSSS!";";";VDDD ff___gVVV!"'*C*C*C"D"DfMMM 8 	 	G)##QVVOOOWUUUVVD'JJJ   	 7  ]:&& 	W 	WZ	!gVVVV	W 	W 	W 	W 	W 	W 	W 	W 	W 	W 	W 	W 	W 	W 	W]:&& 	 	JryQ/?/?   	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 Xyyy))),--FXyyy))),--F0	1a&  JAq!Q !Q++RX6G6G6G-H-HIIIIIs$   2&E$$E(	+E(	'F;;F?	F?	c                  \   g d} g d}t          | t          j        d                    }t          |t          j        d                    }| |f||fg}t          |          D ]\  }\  } }t	          t
          | |ddg          }t	          t
          | |d           }t          dd	g |d 
                     t          d |d
                     t          d |d
                     t          d |d
                     dD ]} ||
           ||
          k    sJ d S )N)rC   rC   r<   r   )rC   r   r   r   r   r   rC   r   rv   r   r   r   r   r   UUUUUU?r   r   )r   r   r   )r*   rH   rI   r   r   r#   r4   r3   )	r`   r_   r   r   rE   r   	recall_13
recall_allr   s	            ra   &test_precision_recall_f_ignored_labelsr   T  s{    \\F\\F	!===J	!===JVz:67D( M MFFL&&!QHHH	\66$GGG
!3*ii.E.E.EFFFOYYw-G-G-GHHH3YYz5R5R5RSSSGYYw%?%?%?@@@ 6 	M 	MG9W---G1L1L1LLLLLL	MM Mrc   c            	      .   t          j        g dg dg dg dg dg dg          } t          j        g dg dg dg d	g d
g dg          }d}t          j        t          |          5  t          | |d           ddd           dS # 1 swxY w Y   dS )z:Test multiclass-multiouptut for `average_precision_score`.)r<   r<   rC   rC   r<   r   r   rC   r<   rC   r<   r   rC   ffffff?皙?皙?皙?333333?r   r   r   r   r   r   r   )r   r   r   )r   r   r   z.multiclass-multioutput format is not supportedmatchr<   	pos_labelN)rH   r   r   r   r   r   )r`   y_scoreerr_msgs      ra   -test_average_precision_score_non_binary_classr   k  s   XIIIIIIIIIIII	
	 	F hOOOOOOOOOOOO	
	 	G ?G	z	1	1	1 > >1====> > > > > > > > > > > > > > > > > >s   *B

BBzy_true, y_scorer   r   rC   r<   r   r   r   r   )r   r   r   r   rC   rC   rC   rC   rC   rC   rC   )r   r   r   r   r   333333?r   rm   rm   rC   rC   c                 2    t          | |          dk    sJ dS )a(  
    Duplicate values with precision-recall require a different
    processing than when computing the AUC of a ROC, because the
    precision-recall curve is a decreasing curve
    The following situation corresponds to a perfect
    test statistic, the average_precision_score should be 1.
    rC   Nr   r`   r   s     ra   -test_average_precision_score_duplicate_valuesr     s%    8 #6733q888888rc   )r<   r<   rC   rC   r   )r   r   r   )r   r   r   r   )r   r   r   c                 2    t          | |          dk    sJ d S )Nr   r   r   s     ra   (test_average_precision_score_tied_valuesr     s%    : #6733s::::::rc   c                      d} t          j        t          |           5  t          g dg ddd           d d d            d S # 1 swxY w Y   d S )NzNote that pos_label \(set to 2\) is ignored when average != 'binary' \(got 'macro'\). You may use labels=\[pos_label\] to specify a single positive class.r   r   rC   r<   r<   r<   r   r   r   )r   warnsUserWarningr!   r   s    ra   (test_precision_recall_f_unused_pos_labelr    s    
	  
k	-	-	- 
 
'IIyyyAw	
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
s   AA
Ac                      t          d          \  } }}d } || |            |d | D             d |D                        d S )NTr   c                 n   t          | |          }t          |ddgddgg           |                                \  }}}}||z  ||z  z
  }t          j        ||z   ||z   z  ||z   z  ||z   z            }|dk    rdn||z  }	t          | |          }
t          |
|	d           t          |
dd           d S )	N   r         r   r<   decimal=
ףp=?)r   r5   flattenrH   sqrtr   r4   )r`   r_   cmtpfpfntnnumdentrue_mccmccs              ra   testz*test_confusion_matrix_binary.<locals>.test  s    ff--2Q!R1222BB2gRgrBw27+rBw727CDDq11cCi//!#x;;;;!#tQ777777rc   c                 ,    g | ]}t          |          S r   r   .0rW   s     ra   
<listcomp>z0test_confusion_matrix_binary.<locals>.<listcomp>      	!	!	!Q#a&&	!	!	!rc   c                 ,    g | ]}t          |          S r   r  r  s     ra   r  z0test_confusion_matrix_binary.<locals>.<listcomp>      #;#;#;qCFF#;#;#;rc   rb   r`   r_   r~   r  s       ra   test_confusion_matrix_binaryr$    sr    't444FFA8 8 8 	DD	!	!&	!	!	!#;#;F#;#;#;<<<<<rc   c                      t          d          \  } }}d } || |            |d | D             d |D                        d S )NTr   c                 b    t          | |          }t          |ddgddggddgddggg           d S )Nr
  r	  r   r  r    r5   )r`   r_   r  s      ra   r  z5test_multilabel_confusion_matrix_binary.<locals>.test  sF    (882"a1b' 2b!Wq"g4FGHHHHHrc   c                 ,    g | ]}t          |          S r   r  r  s     ra   r  z;test_multilabel_confusion_matrix_binary.<locals>.<listcomp>  r  rc   c                 ,    g | ]}t          |          S r   r  r  s     ra   r  z;test_multilabel_confusion_matrix_binary.<locals>.<listcomp>  r!  rc   r"  r#  s       ra   'test_multilabel_confusion_matrix_binaryr*    su    't444FFAI I I 	DD	!	!&	!	!	!#;#;F#;#;#;<<<<<rc   c                      t          d          \  } }}dd} || |            |d | D             d |D             d           d S )	NFr   c           	         t          | |          }t          |ddgddggddgddggd	d
gddggg           |rg dng d}t          | ||          }t          |ddgddggd	d
gddggddgddggg           |rg dng d}t          | ||          }t          |ddgddggd	d
gddggddgddggddgddggg           d S )N/   r   r      &   r      r      r   r<      )021r   r<   rC   r   )r3  r4  r5  3)r   r<   rC   r   rp   r   r'  )r`   r_   string_typer  rv   s        ra   r  z9test_multilabel_confusion_matrix_multiclass.<locals>.test  s[   (882q'Ar7#r1gAw%72r(QG9LM	
 	
 	

 %0>YYY(GGG2q'Ar7#r2hB%8B7RG:LM	
 	
 	

 *5F%%%%,,,(GGGa1b'"bAr7#a2q'"a1a&!		
 	
 	
 	
 	
rc   c                 ,    g | ]}t          |          S r   r  r  s     ra   r  z?test_multilabel_confusion_matrix_multiclass.<locals>.<listcomp>  r  rc   c                 ,    g | ]}t          |          S r   r  r  s     ra   r  z?test_multilabel_confusion_matrix_multiclass.<locals>.<listcomp>  r!  rc   T)r8  )Fr"  r#  s       ra   +test_multilabel_confusion_matrix_multiclassr;    s|    'u555FFA
 
 
 
6 	DD	!	!&	!	!	!#;#;F#;#;#;NNNNNNrc   csc_containercsr_containerc                 J   t          j        g dg dg dg          }t          j        g dg dg dg          } ||          } ||          } | |          } | |          }t          j        g d          }dd	gddggdd	gddggd	d
gdd	ggg}	|||g}
|||g}|
D ]'}|D ]"}t          ||          }t          ||	           #(t          ||d          }t          |dd	gddggddgd	dggd	dgd
d	ggg           t          ||d
d	g          }t          |d	d
gdd	ggdd	gddggg           t          ||d
d	gd          }t          |d	d	gddggddgd	d	ggd	dgdd	ggg           t          |||d          }t          |d
d	gd
d
ggddgd	dggd	dgdd	ggg           d S )Nr   r   rC   r   rC   rC   r   r   r   r   )r<   rC   r   rC   r   r<   T
samplewiser   )rv   rB  )sample_weightrB  r   r   )rH   r   r    r5   )r<  r=  r`   r_   
y_true_csr
y_pred_csr
y_true_csc
y_pred_cscrC  real_cmtruespreds
y_true_tmp
y_pred_tmpr  s                  ra   +test_multilabel_confusion_matrix_multilabelrM    s   
 Xyyy)))YYY788FXyyy)))YYY788Fv&&Jv&&Jv&&Jv&&J HYYY''MAA1a&1a&!1QFQF3CDGZ,EZ,E , ,
 	, 	,J,ZDDBr7++++	,
 
%VV	E	E	EBraVaV,1v1v.>!Q!Q@PQRRR 
%VVQF	C	C	CBraVaV,1v1v.>?@@@ 
%VVQFt	T	T	TBraVaV,1v1v.>!Q!Q@PQRRR 
%m
 
 
B raVaV,1v1v.>!Q!Q@PQRRRRRrc   c            	         t          j        g dg dg dg          } t          j        g dg dg dg          }t          j        t          d          5  t          | |d	d
g           d d d            n# 1 swxY w Y   t          j        t          d          5  t          | |g dg dg dg           d d d            n# 1 swxY w Y   d}t          j        t          |          5  t          | |dg           d d d            n# 1 swxY w Y   d}t          j        t          |          5  t          | |dg           d d d            n# 1 swxY w Y   t          j        t          d          5  t          g dg dd           d d d            n# 1 swxY w Y   d}t          j        t          |          5  t          g dg dgg dg dg           d d d            d S # 1 swxY w Y   d S )Nr   r?  r@  r   r   r   inconsistent numbers of samplesr   rC   r<   rC  z)Sample weights must be 1D array or scalarrC   r<   r   )r<   r   r   )r   r   r   z%All labels must be in \[0, n labels\)r   r   r   zSamplewise metricsr   r   TrA  z'multiclass-multioutput is not supported)r<   rC   r   rC   r   r<   )rH   r   r   r   r   r    )r`   r_   r   s      ra   'test_multilabel_confusion_matrix_errorsrS  C  s   Xyyy)))YYY788FXyyy)))YYY788F 
z)J	K	K	K J J#FF1a&IIIIJ J J J J J J J J J J J J J J	z)T	U	U	U 
 
#F999iii*K	
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 7G	z	1	1	1 A A#FFB4@@@@A A A A A A A A A A A A A A A6G	z	1	1	1 @ @#FFA3????@ @ @ @ @ @ @ @ @ @ @ @ @ @ @ 
z)=	>	>	> K K#IIIyyyTJJJJK K K K K K K K K K K K K K K 8G	z	1	1	1 T T#YYY			$:YYY			<RSSST T T T T T T T T T T T T T T T T Tsl   A77A;>A;CC	C	-DDD5EEE;FF"%F"G00G47G4z%normalize, cm_dtype, expected_results))truer   TUU?)predr   rU  )allr   geq?)Nr   r<   c                     g ddz  }t          t          t          g d                     }t          |||           }t	          ||           |j        j        |k    sJ d S )Nr   r   	normalize)listr   r   r   r2   dtypekind)rZ  cm_dtypeexpected_resultsy_testr_   r  s         ra   test_confusion_matrix_normalizera  a  so     YY]F%iii00122F	&&I	>	>	>BB()))8=H$$$$$$rc   c                  H   g d} g d}t          | |d          }|                                t          j        d          k    sJ t	          j                    5  t	          j        dt                     t          | |d          }d d d            n# 1 swxY w Y   |                                t          j        d          k    sJ t	          j                    5  t	          j        dt                     t          || d           d d d            d S # 1 swxY w Y   d S )	N)r   r   r   r   rC   rC   rC   rC   )r   r   r   r   r   r   r   r   rT  rY         @r   rV  r   )r   sumr   r   r   r   r   RuntimeWarning)r`  r_   cm_truecm_preds       ra   ,test_confusion_matrix_normalize_single_classrh  r  s   %%%F%%%Fvv@@@G;;==FM#...... 
	 	"	" E Eg~666"66VDDDE E E E E E E E E E E E E E E ;;==FM#......		 	"	" ; ;g~6666::::; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ;s$   -BBB-DDDc                      g d} g d}t          j        t          d          5  t          ||            ddd           dS # 1 swxY w Y   dS )z8Test `confusion_matrix` warns when only one label found.r   r   r   r   zA single label was found inr   N)r   r  r  r   )r`  r_   s     ra   "test_confusion_matrix_single_labelrk    s    \\F\\F	k)F	G	G	G ) )((() ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) )s   AA	Azparams, warn_msg)rC   rC   rC   r   r   r   r`   r_   z?`positive_likelihood_ratio` is ill-defined and set to `np.nan`.)r   r   r   r   r   r   zdNo samples were predicted for the positive class and `positive_likelihood_ratio` is set to `np.nan`.r   r   r   rC   rC   rC   z?`negative_likelihood_ratio` is ill-defined and set to `np.nan`.z9No samples of the positive class are present in `y_true`.c                     t          j        t          |          5  t          di |  d d d            d S # 1 swxY w Y   d S Nr   r   )r   r  r  r   )paramswarn_msgs     ra   test_likelihood_ratios_warningsrr    s    X 
k	2	2	2 * *))&)))* * * * * * * * * * * * * * * * * *   6::zparams, err_msg)r   rC   r   rC   r   rC   rC   r   r   r<   zeclass_likelihood_ratios only supports binary classification problems, got targets of type: multiclassc                     t          j        t          |          5  t          di |  d d d            d S # 1 swxY w Y   d S ro  )r   r   r   r   )rp  r   s     ra   test_likelihood_ratios_errorsrv    s    $ 
z	1	1	1 * *))&)))* * * * * * * * * * * * * * * * * *rs  c                  F   t          j        dgdz  dgdz  z             } t          j        dgdz  dgdz  z   dgdz  z             }t          | |          \  }}t          |d           t          |d	           t          | |           \  }}t	          |t           j        dz             t          |t          j        d          d
           t          j        dgdz  dgdz  z             }t          | ||          \  }}t          |d           t          |d           d S )NrC   r   r   r
  r<   
   r	  g?g_B{	%?g-q=)rtolr      r   r   rP  gUUUUUU@gqq?)rH   r   r   r2   r5   nanr   )r`   r_   posnegrC  s        ra   test_likelihood_ratiosr~    s*    XqcAgb())FXqcAgb(A37233F&vv66HCC!!!C!!! 'vv66HCsBFQJ'''C!51111
 HcURZ3%!)344M&vv]SSSHCC   C!!!!!rc   raise_warningc                     t          j        ddg          }t          j        ddg          }d}t          j        t          |          5  t          |||            ddd           dS # 1 swxY w Y   dS )zaTest that class_likelihood_ratios raises a `FutureWarning` when `raise_warning`
    param is set.rC   r   zI`raise_warning` was deprecated in version 1.7 and will be removed in 1.9.r   )r  N)rH   r   r   r  FutureWarningr   )r  r`   r_   r   s       ra   0test_likelihood_ratios_raise_warning_deprecationr    s     Xq!fFXq!fF
UC	m3	/	/	/ M MmLLLLM M M M M M M M M M M M M M M M M M   
A**A.1A.c                  r   t          j        g d          } t          j        g d          }t          | |d          \  }}|t          j        d          k    sJ t          j        g d          } t          j        g d          }t          | |d          \  }}|t          j        d          k    sJ dS )zTest that class_likelihood_ratios returns the worst scores `1.0` for both LR+ and
    LR- when `replace_undefined_by=1` is set.r@  r   rC   replace_undefined_byr   r   N)rH   r   r   r   r   )r`   r_   positive_likelihood_ratior~   negative_likelihood_ratios        ra   1test_likelihood_ratios_replace_undefined_by_worstr    s    
 Xiii  FXiii  F#:Q$ $ $ q %c(:(::::: Xiii  FXiii  F#:Q$ $ $ A  %c(:(:::::::rc   r  LR+r   LR-g      )r  r  r   r{  rc  c                     t          j        ddg          }t          j        ddg          }d}t          j        t          |          5  t          |||            ddd           dS # 1 swxY w Y   dS )zTest that class_likelihood_ratios raises a `ValueError` if the input dict for
    `replace_undefined_by` is in the wrong format or contains impossible values.rC   r   zGThe dictionary passed as `replace_undefined_by` needs to be in the formr   r  N)rH   r   r   r   r   r   )r  r`   r_   r   s       ra   6test_likelihood_ratios_wrong_dict_replace_undefined_byr    s     Xq!fFXq!fF
SC	z	-	-	- 
 
F1E	
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
r  zreplace_undefined_by, expectedc                    t          j        g d          }t          j        g d          }t          |||           \  }}t          j        |          rt          j        |          sJ dS |t	          j        |          k    sJ dS )zTest that the `replace_undefined_by` param returns the right value for the
    positive_likelihood_ratio as defined by the user.r@  r   r  NrH   r   r   isnanr   r   )r  expectedr`   r_   r  r~   s         ra   0test_likelihood_ratios_replace_undefined_by_0_fpr  *  s     Xiii  FXiii  F#:-A$ $ $ q 
x Dx12222222(FM(,C,CCCCCCCrc   r   c                    t          j        g d          }t          j        g d          }t          |||           \  }}t          j        |          rt          j        |          sJ dS |t	          j        |          k    sJ dS )zTest that the `replace_undefined_by` param returns the right value for the
    negative_likelihood_ratio as defined by the user.r   r   r  Nr  )r  r  r`   r_   r~   r  s         ra   0test_likelihood_ratios_replace_undefined_by_0_tnr  F  s     Xiii  FXiii  F#:-A$ $ $ A  
x Dx12222222(FM(,C,CCCCCCCrc   c                     t          j        dgdz  dgdz  z             } t          j        dgdz  dgdz  z   dgdz  z   dgdz  z             }t          | |          }t          |dd	
           |t          ||           k    sJ t          j        | dgdz            } t          j        |dgdz            }t          | |ddg          |k    sJ t          t          | |           d           t          j        dgdz  dgdz  z   dgdz  z             } t          j        dgdz  dgdz  z   dgdz  z             }t          t          | |          dd
           t          j        dgdz  dgdz  z   dgdz  z             } t          j        dgdz  dgdz  z   dgdz  z             }t          t          | |          dd
           t          t          | |d          dd
           t          t          | |d          dd
           d S )Nr   (   rC   <   ro   rx  2   gʡE?r   r  r<   r   r   r   .   ,   4          g??g+?r?   weightsg_vO?	quadraticg#?)rH   r   r   r3   append)r   r   kappas      ra   test_cohen_kappar  b  sV    
1#(aS2X%	&	&B	1#(aS2X%b0A38;	<	<Bb"%%Eua0000%b"------ 
2sQw		B	2sQw		BRQF333u<<<<)"b113777 
1#(aS2X%b0	1	1B	1#(aS2X%b0	1	1B)"b1161EEEE 
1#(aS2X%b0	1	1B	1#(aS2X%b0	1	1B)"b1161EEEE)"b(CCCVUVWWWW"b+666     rc   c                     ddg} t          j        dgdz  dgdz  z             }t          j        dgdz            }t          j        t          d          5  t          ||| 	           d
d
d
           d
S # 1 swxY w Y   d
S )zLTest that correct error is raised when users pass labels that are not in y1.rC   r<   r   r   r   rx  z6At least one label in `labels` must be present in `y1`r   r   N)rH   r   r   r   r   r   )rv   r   r   s      ra   (test_cohen_kappa_score_error_wrong_labelr    s    VF	3%!)seai'	(	(B	3%"*		B	R
 
 
 1 1 	"b00001 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1s   A77A;>A;zy_true, y_predr   r   c                    t          j                    5  t          j        d            | |||          }ddd           n# 1 swxY w Y   t          j        |          rt          j        |          sJ dS ||k    sJ dS )zmCheck the behaviour of `zero_division` when setting to 0, 1 or np.nan.
    No warnings should be raised.
    r   r   N)r   r   r   rH   r  )r   r`   r_   r   results        ra   !test_zero_division_nan_no_warningr    s     
	 	"	" E Eg&&&mDDDE E E E E E E E E E E E E E E 
x 'x&&&&&&s   #AA
Ac                     t          j        t                    5   | ||d          }ddd           n# 1 swxY w Y   |dk    sJ dS )ztCheck the behaviour of `zero_division` when setting to "warn".
    A `UndefinedMetricWarning` should be raised.
    r   r  Nr   )r   r  r   )r   r`   r_   r  s       ra   test_zero_division_nan_warningr    s     
,	-	- > >f===> > > > > > > > > > > > > > >S======s   599c                    t           j                            |           }|                    ddd          }|                    ddd          }t	          t          ||          t          j        ||          d         d           d S )Nr   r<   ro   sizer   rC   rx  )rH   rL   rM   randintr3   r   corrcoef)global_random_seedr[   r`   r_   s       ra   -test_matthews_corrcoef_against_numpy_corrcoefr    s    
)

 2
3
3C[[AB[''F[[AB[''F&&))2;vv+F+Ft+Lb    rc   c                   
 t           j                            |           }|                    ddd          }|                    ddd          }|                    d          }t          |||          
t          
          t          
fdt                    D                       }t          
fdt                    D                       }t          j        
fdt                    D                       }|t          j	        ||z            z  }t          |||          }	t          |	|d	           d S )
Nr   r<   ro   r  rP  c           	          g | ]Q}t                    D ]?}t                    D ]-}||f         ||f         z  ||f         ||f         z  z
  .@RS r   range)r  kmlCNs       ra   r  z9test_matthews_corrcoef_against_jurman.<locals>.<listcomp>  s     	
 	
 	
1XX	
 	
 1XX		
 	
  adGa1g!Q$!AqD' 11	
 	
 	
 	
 	
rc   c           	          g | ]Pd d f                                          t          j         fdt                    D                       z  QS )Nc                 T    g | ]$}t                    D ]}|k    ||f         %S r   r  r  r   gr  r  r  s      ra   r  zDtest_matthews_corrcoef_against_jurman.<locals>.<listcomp>.<listcomp>  8    LLL!qLLAQ!VVa1gVVVVrc   rd  rH   r  r  r  r  r  s    @ra   r  z9test_matthews_corrcoef_against_jurman.<locals>.<listcomp>  st     	
 	
 	
  aaadGKKMMfLLLLLLuQxxLLLMMN	
 	
 	
rc   c           	          g | ]Pd d f                                          t          j         fdt                    D                       z  QS )Nc                 T    g | ]$}t                    D ]}|k    ||f         %S r   r  r  s      ra   r  zDtest_matthews_corrcoef_against_jurman.<locals>.<listcomp>.<listcomp>  r  rc   r  r  s    @ra   r  z9test_matthews_corrcoef_against_jurman.<locals>.<listcomp>  st     	
 	
 	
  adGKKMMfLLLLLLuQxxLLLMMN	
 	
 	
rc   rx  )rH   rL   rM   r  randr   ry   rd  r  r  r   r3   )r  r[   r`   r_   rC  cov_ytypcov_ytytcov_ypyp
mcc_jurmanmcc_oursr  r  s             @@ra   %test_matthews_corrcoef_against_jurmanr    s    )

 2
3
3C[[AB[''F[[AB[''FHHRLLM}EEEAAA	
 	
 	
 	
 	
1XX	
 	
 	
 H 	
 	
 	
 	
 	
 1XX	
 	
 	
 H v	
 	
 	
 	
 	
 1XX	
 	
 	
 H BGHx$7888J }MMMH*b11111rc   c           	      R   t           j                            |           }d |                    ddd          D             }t	          t          ||          d           d |D             }t	          t          ||          d           t          |d	d
g          }t          j        |d	d
          }t	          t          ||          d           t	          t          g dg d          d           t	          t          |d	gt          |          z            d           g d}g d}t	          t          ||          d           dgdz  dgdz  z   }t          j
        t                    5  t	          t          |||          d           d d d            d S # 1 swxY w Y   d S )Nc                 "    g | ]}|d k    rdndS )r   r   r   r   r  r   s     ra   r  z*test_matthews_corrcoef.<locals>.<listcomp>  s$    IIIQ!VVccIIIrc   r   r<   ro   r  r   c                 "    g | ]}|d k    rdnd S )r   r   r   r  s     ra   r  z*test_matthews_corrcoef.<locals>.<listcomp>  s$    ;;;qc##s;;;rc   r   r   r   r   rj  r   )rC   r   rC   rC   r   rC   rC   rC   r   rC   rC   rC   rC   rC   rC   rC   r   rC   rC   rC   )rC   rC   rC   r   r   rC   rC   rC   rC   r   rC   rC   rC   r   rC   rC   rC   r   rC   rC   rC   rx  rP  )rH   rL   rM   r  r3   r   r*   wherery   r   r   AssertionError)r  r[   r`   
y_true_invy_true_inv2y_1y_2masks           ra   test_matthews_corrcoefr    s   
)

 2
3
3CIIckk!QRk.H.HIIIF )&&993??? <;F;;;J)&*==rBBB #s<<<K(;S11K)&+>>CCC ),,,EEsKKK )&3%#f++2EFFLLL G
F
FC
F
F
FC)#s33S999 38qcBhD 
~	&	& R R-c3dKKKSQQQR R R R R R R R R R R R R R R R R Rs   .!FF #F c                 B  
 t           j                            |           }t          d          
d}
fd|                    d|d          D             }t          t          ||          d           g d}g d	}t          t          ||          d
           g d}g d}t          t          ||          dt          j        d          z             g d}g d}t          t          ||          d           g d}g d}t          t          ||          d           g d}g d}t          t          ||          d           g d}g d}g d}	t          t          |||	          d           g d}g d}g d}	t          t          |||	          d           d S )Nr   r   c                 4    g | ]}t          |z             S r   )chr)r  r   ord_as     ra   r  z5test_matthews_corrcoef_multiclass.<locals>.<listcomp>  s#    IIIc%!)nnIIIrc   r   ro   r  r   )r   r   rC   rC   r<   r<   )r<   r<   r   r   rC   rC   g      )rC   rC   r   r   r   r   ii  r   )r   r   r   r   	r   rC   r<   r   rC   r<   r   rC   r<   )	rC   rC   rC   r<   r<   r<   r   r   r   )r   r   rC   rC   r<   rt  rC   rC   rC   rC   r   rP  r   r   rC   rC   r   r   )rH   rL   rM   ordr  r3   r   r  )r  r[   	n_classesr`   
y_pred_bad
y_pred_minr_   r  r  rC  r  s             @ra   !test_matthews_corrcoef_multiclassr    s   
)

 2
3
3CHHEIIIIIckk!YRk&H&HIIIF )&&993???  F###J)&*==tDDD  F###J)&*==sRWWEUEU?UVVV YYFYYF)&&993??? YYFYYF)&&993??? &
%
%C
%
%
%C)#s33S999 __F__F#OOM&&FFF   \\F\\F LLM&&FFF    rc   n_pointsd   i'  c                    t           j                            |          d }fd}t          j        ddg|           }t	          t          ||          d           t          j        g d|           }t	          t          ||          d            ||           \  }}t	          t          ||          d           t	          t          ||           |||                     d S )Nc                     t          | |          }|d         }|d         }|d         }t          |           }||z   |z  }||z   |z  }||z  ||z  z
  }	||z  d|z
  z  d|z
  z  }
|	t          j        |
          z  S )NrC   rC   )rC   r   r  rC   )r   ry   rH   r  )r`   r_   conf_matrixtrue_pos	false_pos	false_negr  pos_rateactivitymcc_numeratormcc_denominators              ra   mcc_safez1test_matthews_corrcoef_overflow.<locals>.mcc_safeI  s    &vv66t$%	%	v;;y(H4y(H4 8+h.AA"X-X>!h,Orw7777rc   c                                          |           }|d                     |           dz
  z  z   }|dk    }|dk    }||fS )Nr   r   )random_sample)r  x_truex_predr`   r_   r[   s        ra   	random_ysz2test_matthews_corrcoef_overflow.<locals>.random_ysU  sT    ""8,,#!2!28!<!<s!BCC##v~rc   r   r   )r   r   rc  )rH   rL   rM   repeatr3   r   )r  r  r  r  arrr`   r_   r[   s          @ra   test_matthews_corrcoef_overflowr  D  s     )

 2
3
3C
8 
8 
8     )S#J
)
)C)#s33S999
)OOOX
.
.C)#s33S999Yx((NFF)&&993???)&&9988FF;S;STTTTTrc   c                  N   t          d          \  } }}t          | |d           \  }}}}t          |g dd           t          |g dd           t          |g dd           t          |g d           t	          | |d	d
          }t          |dd           t          | |d
          }t          |dd           t          | |d
          }	t          |	dd           t	          | |d          }t          |dd           t          | |d          }t          |dd           t          | |d          }	t          |	dd           t	          | |d          }t          |dd           t          | |d          }t          |dd           t          | |d          }	t          |	dd           t          j        t                    5  t	          | |d           d d d            n# 1 swxY w Y   t          j        t                    5  t          | |d           d d d            n# 1 swxY w Y   t          j        t                    5  t          | |d           d d d            n# 1 swxY w Y   t          j        t                    5  t          | |dd           d d d            n# 1 swxY w Y   t          | |g dd           \  }}}}t          |g dd           t          |g dd           t          |g dd           t          |g d           d S )NFr   r   )(\?Q?gzG?r<   )HzG?g
ףp=
?rm   )Q?333333?r  )rf   rl   ro   rC   r   r  g(\?r   r   gRQ?r   gGz?r   r   r   r   r6  r   )r  g=
ףp=?r  )r  rm   r   )r  r  r   )rf   ro   rl   )rb   r!   r4   r5   r"   r#   r   r   r   r   r   )
r`   r_   r~   rZ   r   r   r   r   r   r   s
             ra   )test_precision_recall_f1_score_multiclassr  f  s}   'u555FFA 1NNNJAq!Qa!3!3!3Q777a!3!3!3Q777a!3!3!3Q777q,,,''' 
1g	F	F	FBb$***	ffg	6	6	6Bb$***	&&'	2	2	2Bb$***		9	9	9Bb$***	ffg	6	6	6Bb$***	&&'	2	2	2Bb$***		<	<	<Bb$***	ffj	9	9	9Bb$***	&&*	5	5	5Bb$***	z	"	" ; ;	::::; ; ; ; ; ; ; ; ; ; ; ; ; ; ;	z	"	" 8 8VVY77778 8 8 8 8 8 8 8 8 8 8 8 8 8 8	z	"	" 4 433334 4 4 4 4 4 4 4 4 4 4 4 4 4 4	z	"	" A AFFIC@@@@A A A A A A A A A A A A A A A 1yyy$  JAq!Q a!3!3!3Q777a!3!3!3Q777a!3!3!3Q777q,,,'''''sH   G++G/2G/H..H25H2I11I58I5J55J9<J9r   )r   r   r   r   Nc                 $   t          j        g dg          }t          j        g dg          }t          ||g dg |           \  }}}}t          |d           t          |d           t          |d           | t          |g d           d S d S )Nr  r   r   rC   rC   )r   r   rC   r<   )rv   warn_forr   r   r   rC   rC   r   )rH   r   r!   r5   )r   r`   r_   rZ   r   r   r   s          ra   ;test_precision_refcall_f1_score_multilabel_unordered_labelsr    s     X|||n%%FX|||n%%F0|||b'  JAq!Q q!q!q!1lll+++++ rc   c                  T   t          j        g d          } t          j        g d          }t          | |d           \  }}}}t          | |d          \  }}}}|t          j        |          k    sJ |t          j        |          k    sJ |t          j        |          k    sJ t          | |d          \  }}}}t          j        |           }	|t          j        ||	          k    sJ |t          j        ||	          k    sJ |t          j        ||	          k    sJ d S )N)r   rC   r   r   rC   rC   r   rC   r   r   rC   r   rC   r   rC   )rC   rC   r   rC   r   rC   rC   rC   rC   r   rC   r   rC   r   rC   r   r   r   r  )rH   r   r!   r   bincountr   )
r`   r_   r   r   r   r~   rZ   r   r   rj   s
             ra   .test_precision_recall_f1_score_binary_averagedr
    sD   XCCCDDFXCCCDDF 4FFDQQQMBB0QQQJAq!Q0TTTJAq!Qk&!!G
2w///////
2w///////
2w/////////rc   c                     t          j        d          } 	 t          j        g d          }t          j        g d          }t          t	          ||d          dd           t          t          ||d          dd           t          t          ||d          dd           t          j        d	i |  d S # t          j        d	i |  w xY w)
Nraise)rW  )r   rC   r<   r   rC   r<   )r<   r   rC   rC   r<   r   r   r   r   r<   r   )rH   seterrr   r3   r"   r#   r   )old_error_settingsr`   r_   s      ra   test_zero_precision_recallr    s     w///	(,,,--,,,--OFFGLLLcSTUUULIII3PQRRRHVVWEEEsANNN 		''&'''''	''&''''s   BB9 9Cc                  X   t          d          \  } }}t          | |ddg          }t          |ddgddgg           t          | |d	dg          }t          |d
d	gddgg           t          j        |           dz   }t          | |d	|g          }t          |d
dgddgg           d S )NFr   r   rC   r   r.  r   r   r<   r2  rf   )rb   r   r5   rH   max)r`   r_   r~   r  extra_labels        ra   .test_confusion_matrix_multiclass_subset_labelsr    s    'u555FFA 
&&!Q	8	8	8BrRGaV,--- 
&&!Q	8	8	8BrRGb!W-... &..1$K	&&![1A	B	B	BBrRGaV,-----rc   zlabels, err_msgz+'labels' should contain at least one label.r   r   z.At least one label specified must be in y_truez
empty listzunknown labels)idsc                     t          d          \  }}}t          j        t          |          5  t	          |||            d d d            d S # 1 swxY w Y   d S )NFr   r   r   )rb   r   r   r   r   )rv   r   r`   r_   r~   s        ra   test_confusion_matrix_errorr    s     (u555FFA	z	1	1	1 8 877778 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8s   AAAc            	         g d} t          j        t          |                     }t          | |           }|j        t           j        k    sJ t           j        t           j        t           j        fD ]@}t          | | |	                    |d                    }|j        t           j        k    sJ At           j
        t           j        d t          fD ]@}t          | | |	                    |d                    }|j        t           j        k    sJ At          j        t          |           dt           j                  }t          | | |          }|d         dk    sJ |d         d	k    sJ t          j        t          |           d
t           j                  }t          | | |          }|d         d
k    sJ |d         dk    sJ d S )Nr   F)copyrP  l    r\  r   r   r  l    l    )rH   onesry   r   r\  int64bool_int32uint64astypefloat32float64objectfulluint32)rW   weightr  r\  s       ra   test_confusion_matrix_dtyper(    s   		AWSVV__F	!Q		B8rx(BHbi0 $ $a&--E-2R2RSSSx28#####*bj$7 & &a&--E-2R2RSSSx2:%%%%% WSVVZry999F	!Qf	5	5	5Bd8z!!!!d8z!!!! WSVV0AAAF	!Qf	5	5	5Bd8*****d8r>>>>>>rc   r\  )Int64Float64booleanc                    t          j        d          }t          j        g d          }|                    ||           }|                    g dd          }t          ||          }t          ||          }t          ||           dS )zkChecks that confusion_matrix works with pandas nullable dtypes.

    Non-regression test for gh-25635.
    pandas)	rC   r   r   rC   r   rC   rC   r   rC   r  )	r   r   rC   rC   r   rC   rC   rC   rC   r  N)r   importorskiprH   r   Seriesr   r5   )r\  pd	y_ndarrayr`   y_predictedoutputexpected_outputs          ra   %test_confusion_matrix_pandas_nullabler5    s     
	X	&	&B44455IYYyY..F))777w)GGKfk22F&y+>>Ov/////rc   c            	          t          j                    } t          | d          \  }}}d}t          ||t	          j        t          | j                            | j                  }||k    sJ d S )NFre   a|                precision    recall  f1-score   support

      setosa       0.83      0.79      0.81        24
  versicolor       0.33      0.10      0.15        31
   virginica       0.42      0.90      0.57        20

    accuracy                           0.53        75
   macro avg       0.53      0.60      0.51        75
weighted avg       0.51      0.53      0.47        75
rv   rw   r   rD   rb   r   rH   rI   ry   rw   r}   r`   r_   r~   r   r   s         ra   %test_classification_report_multiclassr:    s    D'UCCCFFA
O #yT.//00&	  F _$$$$$$rc   c                  J    g dg d}} d}t          | |          }||k    sJ d S )N)	r   r   r   rC   rC   rC   r<   r<   r<   r  a|                precision    recall  f1-score   support

           0       0.33      0.33      0.33         3
           1       0.33      0.33      0.33         3
           2       0.33      0.33      0.33         3

    accuracy                           0.33         9
   macro avg       0.33      0.33      0.33         9
weighted avg       0.33      0.33      0.33         9
r   )r`   r_   r   r   s       ra   .test_classification_report_multiclass_balancedr<  9  sG    0002M2M2MFF
O #6622F_$$$$$$rc   c                      t          j                    } t          | d          \  }}}d}t          ||          }||k    sJ d S )NFre   a|                precision    recall  f1-score   support

           0       0.83      0.79      0.81        24
           1       0.33      0.10      0.15        31
           2       0.42      0.90      0.57        20

    accuracy                           0.53        75
   macro avg       0.53      0.60      0.51        75
weighted avg       0.51      0.53      0.47        75
)r   rD   rb   r   r9  s         ra   :test_classification_report_multiclass_with_label_detectionr>  K  sX    D'UCCCFFA
O #6622F_$$$$$$rc   c            	          t          j                    } t          | d          \  }}}d}t          ||t	          j        t          | j                            | j        d          }||k    sJ d S )NFre   a|                precision    recall  f1-score   support

      setosa    0.82609   0.79167   0.80851        24
  versicolor    0.33333   0.09677   0.15000        31
   virginica    0.41860   0.90000   0.57143        20

    accuracy                        0.53333        75
   macro avg    0.52601   0.59615   0.50998        75
weighted avg    0.51375   0.53333   0.47310        75
r   )rv   rw   digitsr8  r9  s         ra   1test_classification_report_multiclass_with_digitsrA  _  s    D'UCCCFFA
O #yT.//00&  F _$$$$$$rc   c                     t          d          \  } }}t          j        g d          |          } t          j        g d          |         }d}t          | |          }||k    sJ d}t          | |g d          }||k    sJ d S )NFr   )bluegreenreda|                precision    recall  f1-score   support

        blue       0.83      0.79      0.81        24
       green       0.33      0.10      0.15        31
         red       0.42      0.90      0.57        20

    accuracy                           0.53        75
   macro avg       0.53      0.60      0.51        75
weighted avg       0.51      0.53      0.47        75
a|                precision    recall  f1-score   support

           a       0.83      0.79      0.81        24
           b       0.33      0.10      0.15        31
           c       0.42      0.90      0.57        20

    accuracy                           0.53        75
   macro avg       0.53      0.60      0.51        75
weighted avg       0.51      0.53      0.47        75
r   rw   rb   rH   r   r   )r`   r_   r~   r   r   s        ra   7test_classification_report_multiclass_with_string_labelrH  z  s    'u555FFAX...//7FX...//7F
O #6622F_$$$$
O #66PPPF_$$$$$$rc   c                      t          d          \  } }}t          j        g d          }||          } ||         }d}t          | |          }||k    sJ d S )NFr   )u   blue¢u   green¢u   red¢u                precision    recall  f1-score   support

       blue¢       0.83      0.79      0.81        24
      green¢       0.33      0.10      0.15        31
        red¢       0.42      0.90      0.57        20

    accuracy                           0.53        75
   macro avg       0.53      0.60      0.51        75
weighted avg       0.51      0.53      0.47        75
rG  r`   r_   r~   rv   r   r   s         ra   8test_classification_report_multiclass_with_unicode_labelrK    sp    'u555FFAX:::;;FF^FF^F
O #6622F_$$$$$$rc   c                      t          d          \  } }}t          j        g d          }||          } ||         }d}t          | |          }||k    sJ d S )NFr   )rC  greengreengreengreengreenrE  a                             precision    recall  f1-score   support

                     blue       0.83      0.79      0.81        24
greengreengreengreengreen       0.33      0.10      0.15        31
                      red       0.42      0.90      0.57        20

                 accuracy                           0.53        75
                macro avg       0.53      0.60      0.51        75
             weighted avg       0.51      0.53      0.47        75
rG  rJ  s         ra   <test_classification_report_multiclass_with_long_string_labelrN    sp    'u555FFAX22233FF^FF^F
O #6622F_$$$$$$rc   c                      g d} g d}g d}d}t          j        t          |          5  t          | |ddg|           d d d            d S # 1 swxY w Y   d S )	Nr   r   r<   r   r   r   r<   r<   r   r   zclass 0zclass 1zclass 2z6labels size, 2, does not match size of target_names, 3r   r   r<   r7  )r   r  r  r   )r`   r_   rw   r   s       ra   =test_classification_report_labels_target_names_unequal_lengthrS    s    __F__F444L
BC	k	-	-	- X XffaV,WWWWX X X X X X X X X X X X X X X X X Xs   AAAc                      g d} g d}g d}d}t          j        t          |          5  t          | ||           d d d            d S # 1 swxY w Y   d S )NrP  rQ  rR  zaNumber of classes, 2, does not match size of target_names, 3. Try specifying the labels parameterr   rF  )r   r   r   r   )r`   r_   rw   r   s       ra   @test_classification_report_no_labels_target_names_unequal_lengthrU    s    __F__F444L	. 
 
z	1	1	1 I Iff<HHHHI I I I I I I I I I I I I I I I I Is   A

AAc                      d} d}t          d|| d          \  }}t          d|| d          \  }}d}t          ||          }||k    sJ d S )Nr   r  rC   r   )rY   rX   r  rB   a                precision    recall  f1-score   support

           0       0.50      0.67      0.57        24
           1       0.51      0.74      0.61        27
           2       0.29      0.08      0.12        26
           3       0.52      0.56      0.54        27

   micro avg       0.50      0.51      0.50       104
   macro avg       0.45      0.51      0.46       104
weighted avg       0.45      0.51      0.46       104
 samples avg       0.46      0.42      0.40       104
)r   r   )r  rX   r~   r`   r_   r   r   s          ra   %test_multilabel_classification_reportrW    s    II.	YQ  IAv /	YQ  IAvO #6622F_$$$$$$rc   c                  F   t          j        g dg dg          } t          j        g dg dg          }t          | |          dk    sJ t          | |           dk    sJ t          ||          dk    sJ t          |t          j        |                    dk    sJ t          | t          j        |                     dk    sJ t          | t          j        | j                            dk    sJ t          |t          j        | j                            dk    sJ d S )Nr   r   r   r   r   rC   )rH   r   r$   r   r   rG   r   s     ra   $test_multilabel_zero_one_loss_subsetrY    s)   	999iii(	)	)B	999iii(	)	)BR  C''''R  A%%%%R  A%%%%R^B//00A5555R^B//00A5555RXbh//00A5555RXbh//00A555555rc   c                  h   t          j        g dg dg          } t          j        g dg dg          }t          j        ddg          }t          | |          dk    sJ t          | |           dk    sJ t          ||          dk    sJ t          |d|z
            dk    sJ t          | d| z
            dk    sJ t          | t          j        | j                            dk    sJ t          |t          j        | j                            d	k    sJ t          | ||
          dk    sJ t          | d|z
  |
          dk    sJ t          | t          j        |           |
          dk    sJ t          | d         |d                   t          | d         |d                   k    sJ d S )Nr   r   r   rC   r   UUUUUU?r   r   r   rP  gUUUUUU?gUUUUUU?)rH   r   r   r   rG   
zeros_like
sp_hamming)r   r   ws      ra   test_multilabel_hamming_lossr_    s   	999iii(	)	)B	999iii(	)	)B
!QAB5((((B1$$$$B1$$$$AF##q((((AF##q((((BHRX..//58888BHRX..//36666Ba000H<<<<AF!444	AAAABM"--Q???7JJJJ1r!u%%BqE2a5)A)AAAAAAArc   c                     t          j        g d          } t          j        g d          }d}t          j        t          |          5  t          | |dd           d d d            n# 1 swxY w Y   t          j        g dg dg          } t          j        g d	g d
g          }d}t          j        t          |          5  t          | |dd           d d d            n# 1 swxY w Y   t          j        g d          } t          j        g d          }d}t          j        t          |          5  t          | |d           d d d            n# 1 swxY w Y   d}t          j        t          |          5  t          | |d           d d d            n# 1 swxY w Y   d}t          j        t          |          5  t          | |dd           d d d            d S # 1 swxY w Y   d S )N)r   rC   r   rC   rC   z>pos_label=2 is not a valid label. It should be one of \[0, 1\]r   rU   r<   r   r   r   r   r   r   Target is multilabel-indicator but average='binary'. Please choose another average setting, one of \[None, 'micro', 'macro', 'weighted', 'samples'\].r   )r   rC   rC   r   r<   r  Target is multiclass but average='binary'. Please choose another average setting, one of \[None, 'micro', 'macro', 'weighted'\].r   zJSamplewise metrics are not available outside of multilabel classification.r   zNote that pos_label \(set to 3\) is ignored when average != 'binary' \(got 'micro'\). You may use labels=\[pos_label\] to specify a single positive class.r   r   )rH   r   r   r   r   r   r  r  )r`   r_   r   msg1msg2msg3r   s          ra   test_jaccard_score_validationrg  #  s~   Xooo&&FXooo&&FOG	z	1	1	1 E Effh!DDDDE E E E E E E E E E E E E E E Xyyy))),--FXyyy))),--F	6 	
 
z	.	.	. F Fffh"EEEEF F F F F F F F F F F F F F F Xooo&&FXooo&&F	 	
 
z	.	.	. 8 8ffh77778 8 8 8 8 8 8 8 8 8 8 8 8 8 8WD	z	.	.	. 9 9ffi88889 9 9 9 9 9 9 9 9 9 9 9 9 9 9	  
k	-	-	- D DffgCCCCD D D D D D D D D D D D D D D D D DsZ   
A**A.1A.C&&C*-C*:EE EF  F$'F$G))G-0G-c           	         t          j        g dg dg          }t          j        g dg dg          }t          ||d          dk    sJ t          ||d          dk    sJ t          ||d          dk    sJ t          |t          j        |          d          dk    sJ t          |t          j        |          d          dk    sJ t          |t          j        |j                  d          dk    sJ t          |t          j        |j                  d          dk    sJ t          j        g dg d	g          }t          j        g d
g dg          }t          t          ||d          d           t          t          ||d          d           t          t          ||d          d           t          t          ||dddg          d           t          t          ||dddg          d           t          t          ||d           t          j        g d                     t          j        g dg dg          }t          j        g d
g dg          }t          t          ||d          d           t          t          ||d          d           d}t          j	        t          |          5  t          ||dgd           d d d            n# 1 swxY w Y   d}t          j	        t          |          5  t          ||dgd           d d d            n# 1 swxY w Y   d}t          j        t          |          5  t          t          j        ddgg          t          j        ddgg          d          dk    sJ 	 d d d            n# 1 swxY w Y   d}t          j        t          |          5  t          t          j        ddgddgg          t          j        ddgddgg          d          dk    sJ 	 d d d            n# 1 swxY w Y   t          |           rJ d S )Nr   r   r   r   r   r   rC   r   r   r   r   r   r   r   g?r<   r   r   )r   r   r   r   r   g      ?z	Got 4 > 2r   r   r   z
Got -1 < 0r   zXJaccard is ill-defined and being set to 0.0 in labels with no true or predicted samples.zXJaccard is ill-defined and being set to 0.0 in samples with no true or predicted labels.)rH   r   r   r   r   rG   r3   r5   r   r   r   r  r   r[  )recwarnr   r   r`   r_   re  rf  r   s           ra   test_multilabel_jaccard_scorerj  K  s   	999iii(	)	)B	999iii(	)	)B
 R333t;;;;R333q8888R333q8888R^B//CCCqHHHHR^B//CCCqHHHHRXbh//CCCqHHHHRXbh//CCCqHHHHXyyy))),--FXyyy))),--FffgFFFPPPffgFFFPPPffiHHH(SSSffiAGGG   ffiAGGG   ffd333RX>U>U>U5V5V   Xyyy))),--FXyyy))),--FffgFFFPPPffjIII7SSSD	z	.	.	. C CffaS'BBBBC C C C C C C C C C C C C C CD	z	.	.	. D DffbT7CCCCD D D D D D D D D D D D D D D	- 
 
,C	8	8	8 
 
"(QF8,,bhAx.@.@'RRR    
 
 
 
 
 
 
 
 
 
 
 
 
 
 
	, 
 
,C	8	8	8 
 
1a&1a&)**1a&1a&)**!  
     
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 G}}sJ   K//K36K3L88L<?L< AN00N47N4A	P..P25P2c           	         g d}g d}g d}t                      }|                    |           |                    |          }|                    |          }t          t          ||          }t          t          ||          }ddgddgddgdgdgdgd g}	ddgdd	gd	dgdgdgd	gd g}
d
D ]>}t          |	|
          D ]+\  }}t           |||           |||                     ,?t          j        ddgddgddgg          }t          j        ddgddgddgg          }t                      5  t	          ||d          dk    sJ 	 d d d            n# 1 swxY w Y   t          |           rJ d S )N)antrl  catrm  rl  rm  birdrn  )rm  rl  rm  rm  rl  rn  rn  rm  )rl  rn  rm  rl  rn  rm  r   rC   r<   )r   r   r   Nr   r   r   )r)   rQ   	transformr   r   zipr3   rH   r   r6   r[  )ri  r`   r_   rv   lbr   r   multi_jaccard_scorebin_jaccard_scoremulti_labels_listbin_labels_listr   m_labelb_labels                 ra   test_multiclass_jaccard_scorerx    sA   GGGFGGGF###F			BFF6NNNf%%Jf%%J!-@@z:FF						 1v1v1vsQC!dCO 8   #$5 G G 	 	GW##GGDDD!!''BBB   	 X1v1v1v.//FX1v1v1v.//F			 F FVVZ@@@AEEEEEF F F F F F F F F F F F F F F G}}s   3EE Ec                    t          dgdgd          dk    sJ d}t          j        t          |          5  t          ddgddgd          dk    sJ 	 d d d            n# 1 swxY w Y   t          dgdgdd          d	k    sJ t	          j        g d
          }t	          j        g d          }t          t          ||d          d           t          t          ||dd          d           t          |           rJ d S )NrC   r   rU   r   r   zOJaccard is ill-defined and being set to 0.0 due to no true or predicted samplesr   r  r   )rC   r   rC   rC   r   )rC   r   rC   rC   rC   r   ra  r   )r   r   r  r   rH   r   r3   r[  )ri  r   r`   r_   s       ra   !test_average_binary_jaccard_scorerz    s   !qc8444;;;;	'  
,C	8	8	8 F FaVaVX>>>#EEEEEF F F F F F F F F F F F F F F !qcQAAASHHHHXooo&&FXooo&&FffhGGGQQQffh!DDDg   G}}s   A""A&)A&c                  6   t          j        g dg dg          } t          j        g dg dg          }d}t          j        t          |          5  t          | |dd          }|t          j        d          k    sJ 	 d d d            d S # 1 swxY w Y   d S )	Nr   r   r   r   zJaccard is ill-defined and being set to 0.0 in samples with no true or predicted labels. Use `zero_division` parameter to control this behavior.r   r   r   r   r   r   )rH   r   r   r  r   r   r   )r`   r_   r   scores       ra   (test_jaccard_score_zero_division_warningr    s     Xyyy))),--FXyyy))),--F	C 
 
,C	8	8	8 + +ffivVVVc*******+ + + + + + + + + + + + + + + + + +s   .BBBzzero_division, expected_scorer  )rC   r   c                 R   t          j        g dg dg          }t          j        g dg dg          }t          j                    5  t          j        dt
                     t          ||d|           }d d d            n# 1 swxY w Y   |t          j        |          k    sJ d S )Nr   r|  r   r   r}  )	rH   r   r   r   r   r   r   r   r   )r   expected_scorer`   r_   r~  s        ra   *test_jaccard_score_zero_division_set_valuer    s     Xyyy))),--FXyyy))),--F		 	"	" 
 
g'=>>>FI]
 
 

 
 
 
 
 
 
 
 
 
 
 
 
 
 

 FM.11111111s   .BB	Bc                  Z   t          j        g dg dg dg          } t          j        g dg dg dg          }t          | |d           \  }}}}t          |g dd           t          |g dd           t          |g d	d           t          |g d
d           t	          | |dd           }|}t          |g dd           t          | |d          \  }}}}t          |d           t          |d           t          |d           |J t          t	          | |dd          t          j        |                     t          | |d          \  }}}}t          |d           t          |d           t          |d           |J t          t	          | |dd          d|z  |z  d|z  |z   z             t          | |d          \  }}}}t          |d           t          |d           t          |d           |J t          t	          | |dd          t          j        ||                     t          | |d          \  }}}}t          |d           t          |d           t          |d           |J t          t	          | |dd          d           d S )NrC   r   r   r   r   rC   r   r   r  rC   r   rC   r   r   )r   r   r   r   r<   )r   r   r   r   )r   r   rC   r   )rC   rC   rC   rC   r   r   )r   r  rC   r   r   g      ?r   g?r   r   r   r   r  r   rH   r   r!   r4   r   r3   r   r   r`   r_   rZ   r   r   r   f2rj   s           ra   +test_precision_recall_f1_score_multilabel_1r    s$   
 X|||\\\<<<@AAFX|||\\\<<<@AAF0NNNJAq!Q a!5!5!5q999a!5!5!5q999a!7!7!7;;;aq111	VV!T	:	:	:BGb///1555 1QQQJAq!Q7###3+,,,999FFG<<<bgbkk  
 1QQQJAq!Q333999FFG<<<	!a1q519%   1TTTJAq!Q7###3+,,,999FFJ???

2w'''   1SSSJAq!Q333999FFINNNPSTTTTTrc   c                  \   t          j        g dg dg dg          } t          j        g dg dg dg          }t          | |d           \  }}}}t          |g dd           t          |g d	d           t          |g d
d           t          |g dd           t	          | |dd           }|}t          |g dd           t          | |d          \  }}}}t          |d           t          |d           t          |d           |J t          t	          | |dd          d|z  |z  d|z  |z   z             t          | |d          \  }}}}t          |d           t          |d           t          |d           |J t          t	          | |dd          t          j        |                     t          | |d          \  }}}}t          |d           t          |d           t          |d           |J t          t	          | |dd          t          j        ||                     t          | |d          \  }}}}t          |d           t          |d           t          |d           |J t          t	          | |dd          dd           d S )Nr  r  r  r   r   r   rC   r  r   )r   r   r   r   r<   )r   r   r   r   )r   gQ?r   r   rC   r<   rC   r   r  )r   皙?r   r   r         ?r   r   r   g      ?r[  r   r   rk   r  r   g&S?r  r  s           ra   +test_precision_recall_f1_score_multilabel_2r  3  s/    X|||\\\<<<@AAFX|||\\\<<<@AAF 1NNNJAq!Qa!5!5!5q999a!5!5!5q999a!6!6!6:::aq111	VV!T	:	:	:BGb///15550QQQJAq!Q4   4   0111999FFG<<<	!a1q519%  
 1QQQJAq!Q4   5!!!6"""999FFG<<<bgbkk   1TTTJAq!Q5!!!5!!!=)))999FFJ???

2w'''  
 1SSSJAq!Q 5!!!5!!!=)))999FFI>>>    rc   z%zero_division, zero_division_expected)r   r   r  c           	         t          j        g dg dg dg          }t          j        g dg dg dg          }t          ||d |           \  }}}}t          ||dddgd	           t          |dd
d|gd	           d}t          ||dd|gd	           t          |g dd	           t	          ||d	d |           }	|}
t          |	|dd|gd	           t          ||d|           \  }}}}t          j        |          rdn|}dt          j        |           z   }t          |d	|z   |z             t          |d|z   |z             d}t          ||           |J t          t	          ||d	d|           t          |	d                      t          ||d|           \  }}}}t          |d           t          |d
           t          |d           |J t          t	          ||d	d|           d|z  |z  d|z  |z   z             t          ||d|           \  }}}}t          ||dk    rdnd           t          |d
           d}t          |d|z             |J t          t	          ||d	d|           t          |	|
                     t          ||d          \  }}}}t          |d           t          |d           t          |d           |J d }t          t	          ||d	d|           |d	           d S )!Nr  r  r  rj  r  r}  r   r   r<   r   r   r   rC   r  r   r   r   r  r   r         ?g?r  r   rn   r   r   r   r   g@r   r   rk   gZd;O?)rH   r   r!   r4   r   r  r3   r7   )r   zero_division_expectedr`   r_   rZ   r   r   r   
expected_fr  rj   value_to_sumvalues_to_averageexpected_results                 ra   7test_precision_recall_f1_score_with_an_empty_predictionr  t  s    X|||\\\<<<@AAFX|||\\\<<<@AAF 1M  JAq!Q a"8#sC!H!LLLa#sC1G!H!LLLJa*gq*!EqIIIaq111	VV!T	W	W	WBGb:tQ
"CQGGG0}  JAq!Q !788T11>TL*@!A!AABA,0AABBBC,.2CCDDD J:&&&999'	
 	
 	
 	B%%%	 	 	 1}  JAq!Q 5!!!30111999FG=	
 	
 	
 
!a1q519%	   1
-  JAq!Q $:a$?$?55SIII3M->>???999FJm	
 	
 	
 	B(((	   1SSSJAq!Q 5!!!5!!!5!!!999OFI]	
 	
 	
 		    rc   r   )r   r   r   r   c                 X   t          j        d          }t          j        |          }t          j                    5  t          j        d           t          |||| |          \  }}}}t          ||| ||          }	d d d            n# 1 swxY w Y   |J t          j        |          r!||||	fD ]}
t          j        |
          sJ d S t          |          }t          ||           t          ||           t          ||           t          |	t          |                     d S )Nro   r   r   r   r   r   r  )rH   r   r\  r   r   r   r!   r   r  r|   r3   )r   r   r   r`   r_   rZ   r   r   r   fbetar   s              ra   "test_precision_recall_f1_no_labelsr    s    XgF]6""F		 	"	" 
 
g&&&4'
 
 

1a '
 
 

 
 
 
 
 
 
 
 
 
 
 
 
 
 
" 999 
x !Q& 	$ 	$F8F######-((M=)))=)))=)))u]3344444   AB

BBc                    t          j        d          }t          j        |          }t          }t	          j        t                    5   |||| d          \  }}}}d d d            n# 1 swxY w Y   t          |d           t          |d           t          |d           |J t	          j        t                    5  t          ||| d          }d d d            n# 1 swxY w Y   t          |d           d S )Nr  r   r  r   )	rH   r   r\  r!   r   r  r   r3   r   )	r   r`   r_   funcrZ   r   r   r   r  s	            ra   1test_precision_recall_f1_no_labels_check_warningsr    s   XgF]6""F*D	,	-	- E ET&&'DDD
1aE E E E E E E E E E E E E E E 111999	,	-	- G GFFG#FFFG G G G G G G G G G G G G G G q!!!!!s$   	A**A.1A.C""C&)C&c                    t          j        d          }t          j        |          }t          j                    5  t          j        d           t          ||d d|           \  }}}}t          ||dd |           }d d d            n# 1 swxY w Y   t          j        |           } t          || | | gd           t          || | | gd           t          || | | gd           t          |g dd           t          || | | gd           d S )Nr  r   r   r  r  r<   r|  )
rH   r   r\  r   r   r   r!   r   r#  r4   )r   r`   r_   rZ   r   r   r   r  s           ra   /test_precision_recall_f1_no_labels_average_noner    sx   XgF]6""F 
	 	"	" 
 
g&&&4'
 
 

1a Fd-
 
 

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 J}--Ma-!NPQRRRa-!NPQRRRa-!NPQRRRaA...em]M%RTUVVVVVr  c                  .   t          j        d          } t          j        |           }t          j        t
                    5  t          | |d d          \  }}}}d d d            n# 1 swxY w Y   t          |g dd           t          |g dd           t          |g dd           t          |g dd           t          j        t
                    5  t          | |dd           }d d d            n# 1 swxY w Y   t          |g dd           d S )Nr  rC   r  r|  r<   r  )	rH   r   r\  r   r  r   r!   r4   r   )r`   r_   rZ   r   r   r   r  s          ra   4test_precision_recall_f1_no_labels_average_none_warnr  7  s   XgF]6""F 
,	-	- 
 
4FDq
 
 

1a
 
 
 
 
 
 
 
 
 
 
 
 
 
 

 aA...aA...aA...aA...	,	-	- B BFFDAAAB B B B B B B B B B B B B B B eYYY22222s$   A''A+.A+C77C;>C;c            	      r   t           t          }} dD ]}d}t          j        ||          5   | g dg d|           d d d            n# 1 swxY w Y   d}t          j        ||          5   | g dg d|           d d d            n# 1 swxY w Y   d}t          j        ||          5   | t	          j        d	d
gd	d
gg          t	          j        d	d
gd
d
gg          d           d d d            n# 1 swxY w Y   d}t          j        ||          5   | t	          j        d	d
gd
d
gg          t	          j        d	d
gd	d
gg          d           d d d            n# 1 swxY w Y   d}t          j        ||          5   | t	          j        d	d	gd	d	gg          t	          j        d
d
gd
d
gg          d           d d d            n# 1 swxY w Y   d}t          j        ||          5   | t	          j        d
d
gd
d
gg          t	          j        d	d	gd	d	gg          d           d d d            n# 1 swxY w Y   d}t          j        ||          5   | d	d	gddgd           d d d            n# 1 swxY w Y   d}t          j        ||          5   | ddgd	d	gd           d d d            n# 1 swxY w Y   t          j        d          5 }t          j        d           t          d
d
gd
d
gd           d}t          |
                                j                  |k    sJ d}t          |
                                j                  |k    sJ d}t          |
                                j                  |k    sJ 	 d d d            d S # 1 swxY w Y   d S )NNr   r   zPrecision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.r   r   rC   rC   r<   r   zRecall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.zPrecision is ill-defined and being set to 0.0 in samples with no predicted labels. Use `zero_division` parameter to control this behavior.rC   r   r   zRecall is ill-defined and being set to 0.0 in samples with no true labels. Use `zero_division` parameter to control this behavior.Precision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior.r   zRecall is ill-defined and being set to 0.0 due to no true samples. Use `zero_division` parameter to control this behavior.r   rU   Tr   r   F-score is ill-defined and being set to 0.0 due to no true nor predicted samples. Use `zero_division` parameter to control this behavior.)r!   r   r   r  rH   r   r   r   r   r   popr   )r   r^  r   r   r   s        ra   test_prf_warningsr  S  s   *,BqA. 5 5 	 \!3''' 	5 	5AiiiG4444	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	 \!3''' 	5 	5AiiiG4444	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5
	  
as	#	#	# U U	"(QFQF#
$
$bhAA/?&@&@)TTTTU U U U U U U U U U U U U U U	  
as	#	#	# U U	"(QFQF#
$
$bhAA/?&@&@)TTTTU U U U U U U U U U U U U U U
	  
as	#	#	# S S	"(QFQF#
$
$bhAA/?&@&@'RRRRS S S S S S S S S S S S S S S	  
as	#	#	# S S	"(QFQF#
$
$bhAA/?&@&@'RRRRS S S S S S S S S S S S S S S
	  
as	#	#	# . .	1a&2r(H----. . . . . . . . . . . . . . .	  
as	#	#	# . .	2r(QFH----. . . . . . . . . . . . . . . 
	 	-	-	- 0h''''AAIIII 	
 6::<<'((C//// 	 6::<<'((C//// 	 6::<<'((C/////-0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0s   A

A	A	-BB	B	0?C;;C?C??E))E-0E-?GGG:?II	I	(JJJ*K		KK)B5N,,N03N0c           	      <   t          j                    5  t          j        d           dD ]0}t          g dg d||            t          g dg d||            1t          t	          j        ddgddgg          t	          j        ddgddgg          d|            t          t	          j        ddgddgg          t	          j        ddgddgg          d|            t          t	          j        ddgddgg          t	          j        ddgddgg          d	|            t          t	          j        ddgddgg          t	          j        ddgddgg          d	|            t          ddgd
d
gd|            t          d
d
gddgd|            d d d            n# 1 swxY w Y   t          j        d          5 }t          j        d           t          ddgddgd|            t          |          dk    sJ 	 d d d            d S # 1 swxY w Y   d S )Nr   r  r   r  r}  rC   r   r   r   r   rU   Tr   r   )r   r   r   r!   rH   r   ry   )r   r   r   s      ra   )test_prf_no_warnings_if_zero_division_setr    sM   		 	"	" 2
 2
g&&& 3 	 	G+		999g]    ,		999g]    
 	(Hq!fq!f%&&Hq!fq!f%&&'		
 	
 	
 	
 	(Hq!fq!f%&&Hq!fq!f%&&'		
 	
 	
 	
 	(Hq!fq!f%&&Hq!fq!f%&&'		
 	
 	
 	
 	(Hq!fq!f%&&Hq!fq!f%&&'		
 	
 	
 	
 	(FRHhm	
 	
 	
 	
 	(Hq!fhm	
 	
 	
 	
a2
 2
 2
 2
 2
 2
 2
 2
 2
 2
 2
 2
 2
 2
 2
h 
	 	-	-	-  h''''FQFHM	
 	
 	
 	
 6{{a                                   s%   FF""F&)F&AHHHc           	      0   t          j                    5  t          j        d           t          t	          j        ddgddgg          t	          j        ddgddgg          d|            d d d            n# 1 swxY w Y   t          j        d          5 }t          j        d           t          t	          j        ddgddgg          t	          j        ddgddgg          d|            | d	k    r-t          |                                j                  d
k    sJ nt          |          dk    sJ t          ddgddg           | d	k    r,t          |                                j                  d
k    sJ d d d            d S # 1 swxY w Y   d S )Nr   rC   r   r   r}  Tr   r   r   r  )
r   r   r   r#   rH   r   r   r  r   ry   r   r   s     ra   test_recall_warningsr    sM   		 	"	" 
 
g&&&Hq!fq!f%&&Hq!fq!f%&&'		
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
	 	-	-	- h'''Hq!fq!f%&&Hq!fq!f%&&'		
 	
 	
 	
 F""FJJLL()) ." " " " " " v;;!####aVaV$$$F""FJJLL()) ." " " " "+                 s%   AA88A<?A<C&FFFc           	      0   t          j        d          5 }t          j        d           t          t	          j        ddgddgg          t	          j        ddgddgg          d|            | dk    r-t          |                                j                  d	k    sJ nt          |          dk    sJ t          ddgddg           | dk    r,t          |                                j                  d	k    sJ d d d            n# 1 swxY w Y   t          j                    5  t          j        d
           t          t	          j        ddgddgg          t	          j        ddgddgg          d|            d d d            d S # 1 swxY w Y   d S )NTr   r   rC   r   r   r}  r   r  r   )
r   r   r   r"   rH   r   r   r  r   ry   r  s     ra   test_precision_warningsr  !	  sM   		 	-	-	- h'''Hq!fq!f%&&Hq!fq!f%&&'		
 	
 	
 	
 F""FJJLL()) ." " " " " " v;;!####AA'''F""FJJLL()) ." " " " "+              6 
	 	"	" 
 
g&&&Hq!fq!f%&&Hq!fq!f%&&'		
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
s%   C&DDD&AFFFc           
      0   t          j        d          5 }t          j        d           t          t	          t
          d          fD ]3} |t          j        ddgddgg          t          j        ddgddgg          d| 	           t          |          dk    sJ  |t          j        ddgddgg          t          j        ddgddgg          d| 	           t          |          dk    sJ  |t          j        ddgddgg          t          j        ddgddgg          d| 	           | d
k    r.t          |
                                j                  dk    sJ t          |          dk    sJ 5	 d d d            d S # 1 swxY w Y   d S )NTr   r   r<   r   rC   r   r   r}  r   r  )r   r   r   r   r   r   rH   r   ry   r   r  r   )r   r   r~  s      ra   test_fscore_warningsr  I	  s6   		 	-	-	- "(h'''! < < <= 	( 	(EE1a&1a&)**1a&1a&)**+	    v;;!####E1a&1a&)**1a&1a&)**+	    v;;!####E1a&1a&)**1a&1a&)**+	    &&

,-- 2- - - - - - 6{{a'''''?	("( "( "( "( "( "( "( "( "( "( "( "( "( "( "( "( "( "(s   E'FFFc                     g d} g d}d}t          j        g dg dg dg          }t          j        g dg dg dg          }d}| ||f|||ffD ]q\  }}}t          t          t          t          t          d	
          fD ]A}	t          j        t          |          5   |	||           d d d            n# 1 swxY w Y   Brd S )N)rC   r<   r   r   rC   r<   r   rC   rc  r   r   r   r?  rb  r<   r   r   )
rH   r   r"   r#   r   r   r   r   r   r   )
	y_true_mc	y_pred_mcmsg_mc
y_true_ind
y_pred_indmsg_indr`   r_   r   r   s
             ra   'test_prf_average_binary_data_non_binaryr  p	  sV   II	1 
 999iii;<<J999iii;<<J	<  
Iv&	Z)  ' '
 Ka(((	
 	' 	'F z555 ' 'vv&&&' ' ' ' ' ' ' ' ' ' ' ' ' ' '	'	' 's   B66B:=B:c                     d} d}d}d}d}d}| t          j        g dg dg d	g          f| t          j        d
dgdd
gddgg          f|g df|g df|g df|t          j        dgdgdgg          f|t          j        d
gdgdgg          f|t          j        dgdgdgg          f|t          j        d
dgddgddgg          f|t          j        ddgddgddgg          fg
}i | | f| ||f|||f||| fd || fd ||f|||fd ||fd ||fd | |fd ||fd ||fd ||fd ||fd | |fd ||fd ||fd ||fd | |fd ||fd ||fd i}t          |d          D ]\  \  }}	\  }
}	 |||
f         }n# t          $ r ||
|f         }Y nw xY w|t	          j        t                    5  t          |	|           d d d            n# 1 swxY w Y   ||
k    rZd                    ||
          }t	          j        t          |          5  t          |	|           d d d            n# 1 swxY w Y   |||| fvrXd                    |          }t	          j        t          |          5  t          |	|           d d d            n# 1 swxY w Y   7t          |	|          \  }}}}||k    sJ |	                    d          r|j        dk    sJ |j        dk    sJ nDt          |t          j        |	                     t          |t          j        |                     t	          j        t                    5  t          |	d d         |           d d d            n# 1 swxY w Y   ddg}	d d!g}d"}t	          j        t          |          5  t          |	|           d d d            d S # 1 swxY w Y   d S )#Nmultilabel-indicator
multiclassrU   
continuouszmulticlass-multioutputzcontinuous-multioutputr   r   r   r   rC   )r<   r   rC   )r   r  r   r<   r   r   r  r   r   rc  g?g      @)r  z@Classification metrics can't handle a mix of {0} and {1} targetsr   z{0} is not supported
multilabelcsrr   )rC   r<   )r   r<   r   )r<   )r   r<   zYou appear to be using a legacy multi-label data representation. Sequence of sequences are no longer supported; use a binary array or sparse matrix instead - the MultiLabelBinarizer transformer can convert to this format.)rH   r   r   KeyErrorr   r   r   r%   format
startswithr5   squeeze)INDMCBINCNTMMCMCNEXAMPLESEXPECTEDtype1r   type2r   r  r   merged_typey1outy2outr~   r   s                      ra   test__check_targetsr  	  sl    !C	B
C
C
"C
"C 
bh			999iii899:	bhAAA/001	YYY	iii	ooo	RXsQC!o&&'	bhaS1#''(	bhuse,--.	bhAAA/001	bhc
S#Jc
;<<=H	c
C	R" 
c
C 
S	4	
 
c
D 
b	2 
c
D 
c
D 
c
D 
c
D 
S	4 
c
D 
c
D 
c
D  
c
D!" 
S	4#$ 
c
D%& 
c
D	c
D	S	4	c
D- H2 %,HQ$?$?$? !, !, [eR	.u-HH 	. 	. 	.u-HHH	.z** ' 'r2&&&' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ~~--3VE5-A-A  ]:W=== + +"2r***+ + + + + + + + + + + + + + + b#..4;;EBBGzAAA / /&r2.../ / / / / / / / / / / / / / / ,:"b+A+A(Kq(****%%l33 :|u,,,,|u,,,,,"5"*R..999"5"*R..999z** , ,r#2#w+++, , , , , , , , , , , , , , , )	B
B	3  
z	-	-	-  r2                 sl   .
E99FF/GG	G	H++H/	2H/	.JJ	J	M''M+	.M+	N77N;>N;c                      d} t          j        t          t          j        |                     5  t          t          j        g           t          j        g                      d d d            d S # 1 swxY w Y   d S )NzIFound empty input array (e.g., `y_true` or `y_pred`) while a minimum of 1r   )r   r   r   reescaper%   rH   r   r  s    ra   *test__check_targets_raises_on_empty_inputsr  	  s    
UC	z3	8	8	8 3 3rx||RXb\\2223 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3s   5A22A69A6c                  N    ddg} ddg}t          | |          d         dk    sJ d S )Nr   rC   r   r  )r%   rl  s     ra   Atest__check_targets_multiclass_with_both_y_true_and_y_pred_binaryr  	  s:    VFWF&&))!,<<<<<<rc   zy, target_typerU   r<   r  r   r?  r@  r  c                    |dv rFt          j        t          d          5  t          | |            ddd           dS # 1 swxY w Y   dS t          | |           \  }}}}|dk    sJ |j        dk    sJ |j        dk    sJ dS )z?Check correct behaviour when different target types are sparse.)rU   r  z+Sparse input is only supported when targetsr   Nr  r  )r   r   	TypeErrorr%   r  )rW   target_typey_type
y_true_out
y_pred_outr~   s         ra   !test__check_targets_sparse_inputsr  	  s     ...]J
 
 
 	! 	! 1a   	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! -;1a,@,@)
J///// E)))) E))))))s   >AAc                     t          j        g d          } t          j        g d          }t          | |          dk    sJ t          j        g d          } t          j        g d          }t          | |          dk    sJ d S )N)r   rC   rC   r   )g      !r   r  g333333ӿr   )r   r<   r<   r   )rH   r   r   r`   pred_decisions     ra   test_hinge_loss_binaryr  
  s    Xnnn%%FH33344Mfm,,7777Xlll##FH33344Mfm,,777777rc   c            
         t          j        g dg dg dg dg dg dg          } t          j        g d          }t          j        d| d         d         z
  | d         d         z   d| d         d         z
  | d         d	         z   d| d	         d	         z
  | d	         d
         z   d| d
         d         z
  | d
         d	         z   d| d         d
         z
  | d         d	         z   d| d         d	         z
  | d         d
         z   g          }t          j        |dd |           t          j        |          }t	          ||           |k    sJ d S )N
ףp=
?(\ſ(\gGz)HzGgGz׿Q޿r  333333r  RQؿr  )r  r  r  r  gzGgHzGgHzGѿgQ?)r   rC   r<   rC   r   r<   rC   r   r<   r   r   r   outrH   r   clipr   r   )r  r`   dummy_lossesdummy_hinge_losss       ra   test_hinge_loss_multiclassr   
  s   H((((((((((((((((((	
	 	M X((())F8a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99	
	 	L GL!T|4444w|,,fm,,0@@@@@@@rc   c                      t          j        g d          } t          j        g dg dg dg dg          }d}t          j        t          |          5  t          | |           d d d            d S # 1 swxY w Y   d S )N)r   rC   r<   r<   )gRQ?g rh?g(\gffffffr  r  zDPlease include all labels in y_true or pass labels as third argumentr   )rH   r   r   r   r   r   )r`   r  error_messages      ra   :test_hinge_loss_multiclass_missing_labels_with_labels_noner  ;
  s    Xlll##FH((((((((((((		
 M 	O  
z	7	7	7 * *6=)))* * * * * * * * * * * * * * * * * *   A22A69A6c            
         t          j        g d          } t          j        g d          }d}t          j        t          t          j        |                    5  t          | |           d d d            n# 1 swxY w Y   t          j        ddgddgddgddgddgddgddgg          }g d	}d
}t          j        t          t          j        |                    5  t          | ||           d d d            d S # 1 swxY w Y   d S )N)r<   rC   r   rC   r   rC   rC   )r   rC   r<   rC   r   r<   rC   zThe shape of pred_decision cannot be 1d arraywith a multiclass target. pred_decision shape must be (n_samples, n_classes), that is (7, 3). Got: (7,)r   r  r   rC   r<   r   zThe shape of pred_decision is not consistent with the number of classes. With a multiclass target, pred_decision shape must be (n_samples, n_classes), that is (7, 3). Got: (7, 2))r`   r  rv   )rH   r   r   r   r   r  r  r   )r`   r  r  rv   s       ra   <test_hinge_loss_multiclass_no_consistent_pred_decision_shaper  L
  s    X+++,,FH22233M	  
z=)A)A	B	B	B ? ?&>>>>? ? ? ? ? ? ? ? ? ? ? ? ? ? ? Hq!fq!fq!fq!fq!fq!fqRSfUVVMYYF	  
z=)A)A	B	B	B N N&fMMMMN N N N N N N N N N N N N N N N N Ns$   A::A>A>!DDDc            	         t          j        g dg dg dg dg dg          } t          j        g d          }t          j        g d          }t          j        d| d         d         z
  | d         d         z   d| d         d         z
  | d         d         z   d| d         d         z
  | d         d	         z   d| d	         d         z
  | d	         d         z   d| d
         d         z
  | d
         d	         z   g          }t          j        |dd |           t          j        |          }t	          || |          |k    sJ d S )Nr  皙r  r  r  r  )r   rC   r<   rC   r<   r   rC   r<   r   rC   r   r<   r   r   r  r   r  r  r`   rv   r  r  s        ra   .test_hinge_loss_multiclass_with_missing_labelsr  h
  sz   H(((((((((((((((	
 M Xooo&&FXlll##F8a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99	
 L GL!T|4444w|,,fmF;;;?OOOOOOOrc   c            	         t          j        g dg dg dg dg dg          } t          j        g d          }t          j        g d          }t          j        d| d         d         z
  | d         d         z   d| d         d	         z
  | d         d         z   d| d	         d	         z
  | d	         d         z   d| d
         d         z
  | d
         d	         z   d| d         d	         z
  | d         d         z   g          }t          j        |dd |           t          j        |          }t	          t          || |          |           d S )N)r  r  r  )g333333ÿr  r  )r  r  r  )r  g(\gzGڿ)r   r<   r<   r   r<   r   rC   r   r<   r   r   r  r   )rH   r   r  r   r3   r   r  s        ra   @test_hinge_loss_multiclass_missing_labels_only_two_unq_in_y_truer
  
  s   
 H!!!!!!!!!!!!!!!	
 M Xooo&&FXiii  F8a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99	
 L GL!T|4444w|,,6=888:J    rc   c            
      V   g d} g dg dg dg dg dg dg}t          j        d|d         d         z
  |d         d         z   d|d         d         z
  |d         d         z   d|d         d         z
  |d         d	         z   d|d	         d         z
  |d	         d         z   d|d
         d	         z
  |d
         d         z   d|d         d         z
  |d         d	         z   g          }t          j        |dd |           t          j        |          }t	          | |          |k    sJ d S )N)rC  rD  rE  rD  whiterE  r  r  r  r  rC   r   r<   r   r   r   r  r  )r`   r  r  r  s       ra   +test_hinge_loss_multiclass_invariance_listsr  
  sw    ?>>F$$$$$$$$$$$$$$$$$$M 8a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99	
	 	L GL!T|4444w|,,fm,,0@@@@@@@rc   c            	         g d} t          j        ddgddgddgddgddgd	d
gg          }t          | |          }t          j        t	          j        t          j        |           dk    |d d df                              }t          ||           g d} g dg dg dg}t          | |d          }t          |d           | dz  } |dz  }t          | |d          }t          |d           g d} ddgddgddgg}t          j        t                    5  t          | |           d d d            n# 1 swxY w Y   g d} g dg dg dg}g d}d }t          j        t          t          j        |          !          5  t          | ||"           d d d            n# 1 swxY w Y   g d#} ddgddgddgddgg}t          | |          }t          |d$           ddg} d%d&gddgg}t          j        ddgddgg          }d'}t          j        t          t          j        |          !          5  t          | |           d d d            n# 1 swxY w Y   d%d&gddgddgg}d(}t          j        t          t          j        |          !          5  t          | |           d d d            n# 1 swxY w Y   t          j        t          j        |d d df                              }t          | |ddg"          }t          ||           g d)} g d*g dg d+g}	t          | |	g d,"          }t          |t          j        d                      d S )-Nnor  r  yesr  r  r   r   rm   {Gz?Gz?r   r  gMbP?g+?r  rC   rR  r   r   r   r   r   r   r   r   r   TrY  g躕ʀ?r<   Fg.L`@r   r   r   r   r   )rm   r   r   )r   rm   r   r   r   r   )r   r   r   zPy_true contains values {'b'} not belonging to the passed labels ['a', 'c', 'd'].r   r   hamspamr  r  CT?r   r   zy_true contains only one label (2). Please provide the list of all expected class labels explicitly through the labels argument.zBFound input variables with inconsistent numbers of samples: [3, 2]r   )r   r   r   r   r   r   rQ  )rH   r   r   r   r
   logpmfr2   r   r   r   r  r  log)
r`   r_   loss	loss_truerv   	error_strr   true_log_losscalculated_log_lossy_score2s
             ra   test_log_lossr%  
  s   444FX
sc3Z$SzD$<%QVX F FF##D)"(6*:*:e*CVAAAqD\RRSSSID)$$$ YYFoo@FFFd333DD)$$$ aKF
aKFFFe444DD-((( YYFCj3*sCj1F	z	"	" ! !   ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! __Foo@F__F	"  
z9)=)=	>	>	> 0 0////0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 ,++FCj3*sCj3*=FFF##DD)$$$ VFCj3*%Fhc
S#J/00G	H  
z9)=)=	>	>	> ! !   ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! Cj3*sCj1FTI	z9)=)=	>	>	> ! !   ! ! ! ! ! ! ! ! ! ! ! ! ! ! !
 WRVGAAAqDM22333M"67Aq6BBB'777 YYF///BHFHYYY777DD26#;;,'''''sH   D55D9<D9F!!F%(F%3IIIJ22J69J6c                     t          j        ddg|           }t          j        ddg|           }t          ||          }t          j        |          sJ dS )zCheck the behaviour internal eps that changes depending on the input dtype.

    Non-regression test for:
    https://github.com/scikit-learn/scikit-learn/issues/24315
    r   rC   r  N)rH   r   r   isfinite)r\  r`   r_   r  s       ra   test_log_loss_epsr(  
  s_     Xq!fE***FXq!fE***FFF##D;trc   c                     t          j        g d          }t          j        ddgddgddgddgg|           }t          j        t          d	
          5  t          ||           ddd           dS # 1 swxY w Y   dS )zGCheck that log_loss raises a warning when y_pred values don't sum to 1.r  r   r   r   r   r   r   r  z$The y_prob values do not sum to one.r   N)rH   r   r   r  r  r   )r\  r`   r_   s      ra   'test_log_loss_not_probabilities_warningr*    s     Xlll##FXSzC:SzC:FeTTTF	k)O	P	P	P ! !   ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! !r   r   r   c                 V    t          | |          t          j        d          k    sJ dS )z6Check that log_loss returns 0 for perfect predictions.r   N)r   r   r   rl  s     ra   !test_log_loss_perfect_predictionsr,  "  s/     FF##v}Q'7'7777777rc   c                  n   t          j        g d          } t          j        ddgddgddgddgg          }t          t          fg}	 ddlm}m} |                    ||f           n# t          $ r Y nw xY w|D ];\  }} ||            ||          }}t          ||          }	t          |	d           <d S )	Nr  r   r   r   r   r   )	DataFramer/  r  )
rH   r   r0   r-  r.  r/  r  ImportErrorr   r2   )
y_try_prtypesr.  r/  TrueInputTypePredInputTyper`   r_   r  s
             ra   test_log_loss_pandas_inputr5  0  s   822233D8c3Z#sc3Z#sDEED]+,E,,,,,,,,fi())))   (- ) )$}&t,,mmD.A.A''i((((	) )s   A' '
A43A4c                      t          j        d          } t          j        t          |           5  t          g dg dg dg dgg d           d d d            d S # 1 swxY w Y   d S 	NzLabels passed were ['spam', 'eggs', 'ham']. But this function assumes labels are ordered lexicographically. Pass the ordered labels=['eggs', 'ham', 'spam'] and ensure that the columns of y_prob correspond to this ordering.r   eggsr  r  r   r?  r   )r  r9  r  r   )r  r  r   r  r  r   expected_messages    ra   test_log_loss_warningsr<  B  s    y	=  
k)9	:	:	: 
 
###YY			999-***	
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
    AA!$A!c                     t          j        g d          } t          j        g d          }t          j        | |z
            dz  t	          |           z  }t          t          | |           d           t          t          | |          |           t          t          d| z   |          |           t          t          d| z  dz
  |          |           t          j        d|z
  |f          }t          j        d| z
  | f          }t          t          | |          |           t          t          ||          |           t          t          | |d          |           t          t          | |d	          |           t          t          | |d
          d|z             t          t          dgdg          d           t          t          dgdg          d           t          t          dgdg          d           t          t          dgdgd          d           t          t          dgdgd          d           d S )Nr   rC   rC   r   rC   rC   r   r   rm   r   r   gffffff?r<   r   r   rC   auto)scale_by_halfTFr   r   g|Gz?r   r  foobarr   )rH   r   r   normry   r3   r   column_stack)r`   y_prob
true_scorey_prob_reshapedy_true_reshapeds        ra   test_brier_score_loss_binaryrK  Q  se   X((())FX55566FVf_--2S[[@J(88#>>>(88*EEE(vv>>
KKK(Va@@*MMM oq6z6&:;;Ooq6z6&:;;O(AA:NNN(/JJJWWW v>>>
   t<<<j   u===q:~  
 ("u55v>>>(!se44f===(!se44nEEE(%3%5III6RRR%3%5999    rc   c            	      V   t          t          g dg dg dg dgg d          d           t          t          g dg dg d	g d
g          d           t          t          g dg dg dg dg          d           t          t          g dg dg dg dg          d           d S )Nr8  r  r  )r9  r  r  yamsr   r   rR  r  r  r  gt?r   )r   r   r   )r   r   r   )r   r   r   r   r<   )r3   r   r   rc   ra    test_brier_score_loss_multiclassrN  x  s#   ###\\<<<6222	
 	
 	

 	   II///J	
 	
 		   II///J	
 	
 	
	   II///J	
 	
 	
	    rc   c                     t          j        g d          } t          j        g d          }t          j        t                    5  t          | |dd                     d d d            n# 1 swxY w Y   t          j        t                    5  t          | |dz              d d d            n# 1 swxY w Y   t          j        t                    5  t          | |dz
             d d d            n# 1 swxY w Y   t          j        g d          } t          j        g dg dg dg          }t          j        t                    5  t          | |dd                     d d d            n# 1 swxY w Y   t          j        t                    5  t          | |dz              d d d            n# 1 swxY w Y   t          j        t                    5  t          | |dz
             d d d            n# 1 swxY w Y   t          j        g d	          } t          j        g d
          }t          j        d          }t          j        t          |          5  t          | |           d d d            n# 1 swxY w Y   g d} ddgddgddgg}d}t          j        t          t          j        |                    5  t          | |           d d d            n# 1 swxY w Y   g d} g dg dg dg}g d}d}t          j        t          t          j        |                    5  t          | ||           d d d            n# 1 swxY w Y   dg} ddgg}d}t          j        t          t          j        |                    5  t          | |           d d d            n# 1 swxY w Y   t          t          | |ddg          d           d S )Nr?  r@  rC   r   rR  r  r  r  )r   rC   r<   r   r   r   r   r   zpThe type of the target inferred from y_true is multiclass but should be binary according to the shape of y_prob.r   r   r   zy_true and y_prob contain different number of classes: 3 vs 2. Please provide the true labels explicitly through the labels argument. Classes found in y_true: [0 1 2]r8  r   r?  )r9  r  r  rM  zwThe number of classes in labels is different from that in y_prob. Classes found in labels: ['eggs' 'ham' 'spam' 'yams']r   r9  rm   r   zy_true contains only one label (eggs). Please provide the list of all expected class labels explicitly through the labels argument.r  r  )	rH   r   r   r   r   r   r  r  r3   )r`   rG  r  rv   s       ra   $test_brier_score_loss_invalid_inputsrQ    sR   X((())FX55566F	z	"	" - -,,,- - - - - - - - - - - - - - - 
z	"	" / /#.../ / / / / / / / / / / / / / / 
z	"	" / /#.../ / / / / / / / / / / / / / /
 Xiii  FXIJJF	z	"	" - -,,,- - - - - - - - - - - - - - - 
z	"	" / /#.../ / / / / / / / / / / / / / / 
z	"	" / /#.../ / / / / / / / / / / / / / /
 Xlll##FX***++FI	A M 
z	7	7	7 ) )((() ) ) ) ) ) ) ) ) ) ) ) ) ) ) YYF!fq!fq!f%F	  
z=)A)A	B	B	B ) )((() ) ) ) ) ) ) ) ) ) ) ) ) ) ) %$$FiiIII.F,,,F	/ 
 
z=)A)A	B	B	B 8 877778 8 8 8 8 8 8 8 8 8 8 8 8 8 8 XFCj\F	 
 
z=)A)A	B	B	B ) )((() ) ) ) ) ) ) ) ) ) ) ) ) ) ) (PPPRVWWWWWs   A++A/2A/B//B36B3C33C7:C7
E//E36E3F33F7:F7G77G;>G;I::I>I>K  K$'K$-MMMN++N/2N/c                      t          j        d          } t          j        t          |           5  t          g dg dg dg dgg d           d d d            d S # 1 swxY w Y   d S r7  )r  r  r   r  r  r   r:  s    ra   test_brier_score_loss_warningsrS    s    y	=  
k)9	:	:	: 	
 	
###						
 +**	
 	
 	
 	
	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
r=  c                      d} t          j        t          |           5  t          g dg d           d d d            d S # 1 swxY w Y   d S )Nz%y_pred contains classes not in y_truer   r|  r   )r   r  r  r   r  s    ra   #test_balanced_accuracy_score_unseenrU    s    
1C	k	-	-	- 6 6			9995556 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6s   A  AAzy_true,y_pred)r   r   r   r   )r   r   r   r   )r   r   r   r   c                    t          | |dt          j        |                     }t                      5  t	          | |          }d d d            n# 1 swxY w Y   |t          j        |          k    sJ t	          | |d          }t	          | t          j        | | d                             }|||z
  d|z
  z  k    sJ d S )Nr   r   T)adjustedr   rC   )r#   rH   uniquer6   r   r   r   	full_like)r`   r_   macro_recallbalancedrW  chances         ra   test_balanced_accuracy_scorer]     s     	&0A0A  L 
		 ; ;*66::; ; ; ; ; ; ; ; ; ; ; ; ; ; ; v}\222222&vvEEEH$VR\&&)-L-LMMF6)a&j9999999s   AAAr   ))FTr  )r   r   )zeroonec                 \   t           j                            d          }d|d         }}|                    ||d          }| t          u r|                    |          }n|                                } | |||          }t          j        t          j        |                    rJ dS )	zCheck that the metric works with different types of `pos_label`.

    We can expect `pos_label` to be a bool, an integer, a float, a string.
    No error should be raised for those types.
    *   rx  r   T)r  replacer  r   N)	rH   rL   rM   choicer   uniformr  anyr  )r   r   r[   rX   r   r`   r_   r  s           ra   *test_classification_metric_pos_label_typesrf    s    * )


#
#Cwr{yIZZiZ>>F!!!),,VFFi888Fvbhv&&'''''''rc   zy_true, y_pred, expected_scorec                 Z    t          | |d          t          j        |          k    sJ dS )zCheck the behaviour of `zero_division` for f1-score.

    Non-regression test for:
    https://github.com/scikit-learn/scikit-learn/issues/26965
    r   r  N)r   r   r   )r`   r_   r  s      ra   2test_f1_for_small_binary_inputs_with_zero_divisionrh  6  s4     FF#666&-:W:WWWWWWWrc   scoringr  )r   r   c                     t          j        d          \  }}t          dd                              ||          }t	          |||| dd           dS )	aZ  Check that we validate `np.nan` properly for classification metrics.

    With `n_jobs=2` in cross-validation, the `np.nan` used for the singleton will be
    different in the sub-process and we should not use the `is` operator but
    `math.isnan`.

    Non-regression test for:
    https://github.com/scikit-learn/scikit-learn/issues/27563
    r   )rB   r   )	max_depthrB   r<   r  )ri  n_jobserror_scoreN)r   make_classificationr+   rQ   r(   )ri  rV   rW   
classifiers       ra   9test_classification_metric_division_by_zero_nan_validatonrp  J  s^    ( 'Q777DAq'!!DDDHHANNJJ1gaWUUUUUUrc   c            	         g d} g d}t          j        ddgddgddgddgdd	gd
dgg          }t          j        ddgddgddgddgddgddgg          }t          | |          }t          | |d          }t          | |d          }d||z  z
  }|t	          j        |          k    sJ t          j        g d          }|d d                                         |                                z  |d d df<   |dd                                          |                                z  |d d df<   t          | ||          }t          | ||d          }t          | ||d          }d||z  z
  }|t	          j        |          k    sJ t          j        ddgddgddgddgddgddgg          }t          | |          }d|cxk     rdk     sn J t          ||          }	|	t	          j        |          k    sJ t          j        ddgddgddgddgddgddgg          }t          | |          }|dk     sJ t          ||          }	|	t	          j        |          k    sJ g d} t          j        ddgddgddgddgddgddgg          }t          | |          }|dk    sJ t          ||          }	|	dk    sJ g d} g d}t          j        ddgddgddgddgg          }t          | |          }|dk    sJ t          ||          }	|	dk    sJ g d}t          | ||          }
|
dk    sJ g d} g d}t          j        g d g d g d!g d"g          }t          | |          }d|cxk     rdk     sn J t          | ||          }d|cxk     rdk     sn J t          j        g d#g d$g d"g d%g          }t          | |          }|dk     sJ t          | ||          }|dk     sJ d S )&Nrm  r  r   rm   r   r   r   gffffff?g?r  r  rl  F)r`   r_   rZ  rC   )r<   rC   r   r   r   rC   r   r   )r`   r_   rC  )r`   r_   rC  rZ  r   r   r   r   r  )r   rC   rC   rC   )r  r  r  r  )r<   r<   r<   r<   rP  )highrr  lowneutral)ffffff?r   r   r   )r   r   r   r   r  )r   r   r   r  r  )rH   r   r'   r   r   r   rd  )r`   y_true_stringr_   y_pred_nulld2_scorelog_likelihoodlog_likelihood_nulld2_score_truerC  d2_score_stringd2_score_with_sample_weights              ra   test_d2_log_loss_scorer~  c  s   F;;;MX#J#J#J#J4L4L	
	 	F (#J#J#J#J#J#J	
	 	K !v>>>HVFeLLLN"&PUVVV)<<<Mv}]333333 H///00M%bqb)--//-2C2C2E2EEK1%abb)--//-2C2C2E2EEK1 fM  H #	  N ##	   )<<<Mv}]333333 X#J#J#J#J#J#J	
	 	F !00HC'v>>OfmH555555 X#J#J#J#J4L#J	
	 	F !00Ha<<<<'v>>OfmH555555  FX#J#J#J#J#J#J	
	 	F !00Hq===='v>>Oa \\F///MXd|dD\D$<$NOOF 00Hq===='v>>Oa LLM"3m# # # '!++++ 0//F(((MXOOOOOOOO		
 F !00HC }MMMHCXOOOOOOOO		
 F !00Ha<<<< }MMMHa<<<<<<rc   c                     g d} g d}g d}t          j        g dd          }t          | |||          }t          j        g dd          }t          | |||          }d||z  z
  }t          | |||          }t	          ||           d	S )
zCheck that d2_log_loss_score works when not all labels are present in y_true

    non-regression test for https://github.com/scikit-learn/scikit-learn/issues/30713
    r<   r   r<   r   r   )ru  r   r   r   r   r   rC   )rC  rv   )r   r   r   rC   N)rH   tiler   r'   r2   )	r`   rv   rC  r_   log_loss_obsrw  log_loss_nullexpected_d2_scorerx  s	            ra   %test_d2_log_loss_score_missing_labelsr    s    
 \\FYYF(((MWYYY''FFF-PVWWWL '---00K=  M L=88 mF  H H/00000rc   c                      g d} t          j        g dd          }t          | |g d          }t          | |g d          }t          ||           dS )zGCheck that d2_log_loss_score doesn't depend on the order of the labels.r  r   r  r   r   r6  N)rH   r  r'   r2   )r`   r_   rx  d2_score_others       ra   "test_d2_log_loss_score_label_orderr    sf    \\FWYYY''F 			BBBH&vviiiHHHNHn-----rc   c                     g d} ddgddgddgg}d}t          j        t          |          5  t          | |           d	d	d	           n# 1 swxY w Y   g d} ddgddgddgg}g d}d
}t          j        t          |          5  t          | ||           d	d	d	           n# 1 swxY w Y   g d} g dg dg}d}t          j        t          |          5  t          | |           d	d	d	           n# 1 swxY w Y   dg} ddgg}d}t          j        t
          |          5  t          | |           d	d	d	           n# 1 swxY w Y   g d} ddgddgddgg}d}t          j        t          |          5  t          | |           d	d	d	           n# 1 swxY w Y   g d} dg}ddgddgddgg}d}t          j        t          |          5  t          | ||           d	d	d	           d	S # 1 swxY w Y   d	S )zPTest that d2_log_loss_score raises the appropriate errors on
    invalid inputs.r   r   r   r   r   r   z#contain different number of classesr   Nz(number of classes in labels is differentr   )r   r   r   )r   r   r   rO  rC   zscore is not well-definedr   y_true contains only one labelz.The labels array needs to contain at least two)r   r   r   r'   r  r   )r`   r_   errrv   s       ra   test_d2_log_loss_score_raisesr  !  s    YYFCj3*sCj1F
/C	z	-	-	- * *&&)))* * * * * * * * * * * * * * *
 YYFCj3*sCj1FYYF
4C	z	-	-	- 9 9&&88889 9 9 9 9 9 9 9 9 9 9 9 9 9 9 YYFoo/F
+C	z	-	-	- * *&&)))* * * * * * * * * * * * * * * SFCj\F
%C	,C	8	8	8 * *&&)))* * * * * * * * * * * * * * * YYFCj3*sCj1F
*C	z	-	-	- * *&&)))* * * * * * * * * * * * * * *
 YYFSFCj3*sCj1F
:C	z	-	-	- 9 9&&88889 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9sk   A

AAB$$B(+B(C55C9<C9%EE	E9FFFG00G47G4c                      g d} g d}g d}g d}g d}t          ||          }t          ||          }t          ||          }d||z  z
  }t          j        |          |k    sJ g d}t          ||          }|dk    sJ t          ||d	
          }|dk    sJ g d}t          |||           }|dk    sJ t          ||| d	          }|dk    sJ g d} g d}g d}g dg dg dg dg dg dg dg dg dg dg
}t          |||           }|dk    sJ t          |||           }|dk    sJ g dg dg dg dg dg dg dg dg dg dg
}t          |||           }|dk    sJ t          |||           }|dk    sJ dS )zeTest that d2_brier_score gives expected outcomes in both the binary and
    multiclass settings.
    )r<   r<   r   rC   rC   rC   )r   rC   rC   r   r   rC   )r  r  r  r  r  r  )r   r   r   r   rm   r   )r   r   r   r   r   r   )r`   y_probarC   r   r  )r`   r  r   )r   r   r   r   r   r   )r`   r  rC  )r`   r  rC  r   )
r<   rC   r   rC   rC   r<   rC   r   rC   r   )
r   r   r<   r<   r<   rC   rC   rC   rC   r   )
ddr  ccr  r  bbr  r  r  aa)r   r   r  r   )r   r   r   r   )r   r   r   r   )r   r   r   r   r   r   r   r   r   N)r&   r   r   r   )	rC  r`   rv  r  y_proba_refrx  brier_score_modelbrier_score_refd2_score_expecteds	            ra   test_d2_brier_scorer  R  s   
 '&&MF;;;M -,,G000KVW===H(HHH&fkJJJO-??=""&77777 -,,GVW===Hq====]GuUUUHq====
 -,,Gwm  H q====#	  H q==== 322M+++FPPPM 	G wm  H q====#  H
 q====
 	G wm  H c>>>>#  H
 c>>>>>>rc   c                  :   g d} g d}g dg dg dg dg}t          | ||          }|dk    sJ g d}t          | ||          }|t          j        |          k    sJ g dg dg dg dg}t          | ||          }t          j        |          d	k    sJ d
S )zGTest that d2_brier_score gives expected outcomes when labels are passed)r   r<   r   r<   r   )r   r   r   )r`   r  rv   r   r   r   r   N)r&   r   r   )r`   rv   r  rx  new_d2_scoreneg_d2_scores         ra   test_d2_brier_score_with_labelsr    s    
 \\FYYF	G VWVLLLHq==== YYF!PPPL6=222222 										G "PPPL=&&",,,,,,rc   z!y_true, y_pred, labels, error_msg)rC   r<   rC   r   rP  z7inferred from y_true is multiclass but should be binaryr  r  r  r  zpos_label is not specified)r   rC   r   r   rC   rC   r   z.variables with inconsistent numbers of samples)r   rC   r   rC   )g?r   r   r   z%y_prob contains values greater than 1)gr   r   r   z"y_prob contains values less than 0r   r  r  )r<   r   r   r<   )r   r   r   r   )r   r   r   r   z(Multioutput target data is not supportedr   )r   r   r   z"not belonging to the passed labelsr|  z*labels array needs to contain at least twoc                     t          j        |           } t          j        |          }t          j        t          |          5  t          | ||           ddd           dS # 1 swxY w Y   dS )zMTest that d2_brier_score raises the appropriate errors
    on invalid inputs.r   r   N)rH   asarrayr   r   r   r&   )r`   r_   rv   	error_msgs       ra   test_d2_brier_score_raisesr    s    | ZFZF	z	3	3	3 6 6vvf55556 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6s   A$$A(+A(c                      t          j        dg          } t          j        dg          }d}t          j        t          |          5  t          | |           ddd           dS # 1 swxY w Y   dS )zQTest that d2_brier_score emits a warning when there are less than
    two samplesrC   r   z+not well-defined with less than two samplesr   N)rH   r   r   r  r   r&   )r`   r_   warning_messages      ra   4test_d2_brier_score_warning_on_less_than_two_samplesr    s     Xqc]]FXse__FCO	,O	D	D	D ' 'vv&&&' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' 's   A&&A*-A*zarray_namespace, device, _c                    t          | |          }|                    g d|          }|                    g d|          }|                    g d|          }t          d          5  t          |||          }t	          |          d         t	          |          d         k    sJ t          |          t          |          k    sJ 	 ddd           dS # 1 swxY w Y   dS )	zTest that `confusion_matrix` works for all array types when `labels` are passed
    such that the inner boolean `need_index_conversion` evaluates to `True`.rQ  r,   )r   r   r   Tarray_api_dispatchr   r   N)r1   r  r   r   r.   array_api_device)array_namespacer-   r~   xpr`   r_   rv   r  s           ra   test_confusion_matrix_array_apir  #  sK    
ov	6	6BZZ			&Z11FZZ			&Z11FZZ			&Z11F	4	0	0	0 D D!&&@@@V$$Q'=+@+@+CCCCC''+;F+C+CCCCCCD D D D D D D D D D D D D D D D D Ds   ,A#CC!$C!prob_metric
str_y_trueuse_sample_weightz$array_namespace, device_, dtype_namec                    t          ||          }|rt          j        g d          nd}i }|r9t          j        g d          }	t          j        |	          }
d| j        v rd|d<   n-t          j        g d          }	|                    |	|          }
t          j        g d	|
          }|                    ||          } | |	|fd|i|}t          d          5   | |
|fd|i|}ddd           n# 1 swxY w Y   |t          j        |          k    sJ |r+t          j        g d          }	t          j        |	          }
n-t          j        g d          }	|                    |	|          }
t          j        g dg dg dg dg|
          }|                    ||          } | |	|          }t          d          5   | |
|          }ddd           n# 1 swxY w Y   |t          j        |          k    sJ dS )zTest that :func:`brier_score_loss`, :func:`log_loss`, func:`d2_brier_score`
    and :func:`d2_log_loss_score` work correctly with the array API for binary
    and mutli-class inputs.
    r  Nr  brierr  r   r  r,   )r   r   r   r   r  rC  Tr  )r   r   r   r   r  r  )r   r   r   r   )r   r   r   r   )r   r   r   r   )r1   rH   r   r  __name__r   r   r   )r  r  r  r  device_
dtype_namer  rC  extra_kwargs	y_true_npy_true_xp_or_np	y_prob_np	y_prob_xpmetric_score_npmetric_score_xps                  ra   $test_probabilistic_metrics_array_apir  5  s$    
ow	7	7B.?IBH\\\***TM L 	@H77788	*Y//k*** ).L%H\\\**	**Yw*??---Z@@@I

9W
55I!k9 ,9=I O 
4	0	0	0 
 
%+Y
 
6C
GS
 

 
 
 
 
 
 
 
 
 
 
 
 
 
 

 fmO<<<<<<  @H11122	*Y//H\\\**	**Yw*??            		
   I 

9W
55I!k)Y77O	4	0	0	0 B B%+oyAAB B B B B B B B B B B B B B B fmO<<<<<<<<s$    C99C= C=G&&G*-G*c                    t          ||          }|rt          j        g d          nd}t          j        g dg dg dg dg|          }|                    ||          }t          j        g d	g d
g dg dg|          }	|                    |	|          }
 | ||	|          }t	          d          5   | ||
|          }ddd           n# 1 swxY w Y   |t          j        |          k    sJ dS )zTest that :func:`brier_score_loss`, :func:`log_loss`, func:`d2_brier_score`
    and :func:`d2_log_loss_score` work correctly with the array API for
    multi-label inputs.
    r  Nr  r  r  )rC   rC   r   rC   r  r,   )r   gHzG?gq=
ףp?gQ?)r  gRQ?Q?gq=
ףp?)r  gQ?gQ?g)\(?)gQ?gףp=
?gp=
ף?g(\?rP  Tr  )r1   rH   r   r  r   r   r   )r  r  r  r  r  r  rC  r  	y_true_xpr  r  r  r  s                ra   /test_probabilistic_metrics_multilabel_array_apir  y  s    
ow	7	7B.?IBH\\\***TMLLLLLLLL		
   I 

9W
55I$$$$$$$$$$$$		
   I 

9W
55I!k)YmTTTO	4	0	0	0 Y Y%+i-XXXY Y Y Y Y Y Y Y Y Y Y Y Y Y Y fmO<<<<<<<<s   ;CCC)NF)r  r   	functoolsr   	itertoolsr   r   r   numpyrH   r   scipyr   r   scipy.spatial.distancer	   r]  scipy.statsr
   sklearnr   r   sklearn.baser   sklearn.datasetsr   sklearn.exceptionsr   sklearn.metricsr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   sklearn.metrics._classificationr%   r&   r'   sklearn.model_selectionr(   sklearn.preprocessingr)   r*   sklearn.treer+   sklearn.utils._array_apir-   r  r.   r/   sklearn.utils._mockingr0   sklearn.utils._testingr1   r2   r3   r4   r5   r6   sklearn.utils.extmathr7   sklearn.utils.fixesr8   r9   sklearn.utils.validationr:   rb   r   markparametrizer{  r   r   r   r   r   r   r   r   r   r   r   r   r  r$  r*  r;  rM  rS  ra  rh  rk  rr  rv  r~  r  r  r  r   r  r  r  r  r  r  r  r  r  r  r  r  r  r
  r  r  r  r(  r5  r:  r<  r>  rA  rH  rK  rN  rS  rU  rW  rY  r_  rg  rj  rx  rz  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  
csr_matrixr  r  r  r  r  r  r
  r  r%  r#  r"  float16r(  r*  r,  r5  r<  rK  rN  rQ  rS  rU  r]  rf  rh  thread_unsaferp  r~  r  r  r  r  r  r  r  r  r  r  r   rc   ra   <module>r     s=   				        2 2 2 2 2 2 2 2 2 2                      8 8 8 8 8 8 ! ! ! ! ! ! ! ! ! ! ! ! ! ! ' ' ' ' ' ' ; ; ; ; ; ; 5 5 5 5 5 5                                             .         
 4 3 3 3 3 3 @ @ @ @ @ @ @ @ / / / / / /             1 0 0 0 0 0                . - - - - - > > > > > > > > 7 7 7 7 7 7)( )( )( )(`>D >D >DB 61a*@AA  BA& T{aVUOiii=OP ) ) )(7 7 7  D PQQ  RQ$ PQQ*J *J RQ*JZ PQQM M RQM,> > >6  LLBH#OO#OO#OO#OO	 
	
 .--===	
 (9 9) (9  OOBH#OO#OO#OO#OO#OO 	
 IIOO	
 *; ;+ *;
 
 
 = = =*	= 	= 	= O  O  OF .99.99%S %S :9 :9%SPT T T< +   % % %; ; ;&) ) ) 
 #"(#5#5#566""(#5#5#566  N	
 #"(#5#5#566""(#5#5#566 
B		
 #"(#5#5#566""(#5#5#566  N	
 #"(#5#5#566""(#5#5#566  H	
;$' 'P* *Q' 'P*  #"(???33""(???33 
<		
 * * *" " "4 4-88M M 98M; ; ;0 		S!!E""C  C  
 


 


 


 $S	!	!3'	$	$bf-S	!	!3'	'	'0		 	D D	 	D$ $S	!	!3'	$	$c*	$	$c*	'	'0		 	D D	 	D$  >1 1 1 1a.99)aS1#J<88!$$$	 ' '  98 :9' )aS1#J<88!$$$	    98  $2 $2 $2N R  R  RF5 5 5p c5\22U U 32UB7( 7( 7(t $S$S$STT, , UT,0 0 0"( ( (". . .& 	:;
QAB 
'(   8 8 8  6 "A"A"ABB0 0 CB0"% % %4% % %$% % %(% % %6 %  %  %F% % %,% % %.X X XI I I PQQ% % RQ%<6 6 6B B B(%D %D %DPJ J JZ" " "J  .+ + + 868:LMM	2 	2 NM	2 PQQ?U ?U RQ?UD PQQ= = RQ=@ PQQ+&&2626"23 [ [	  RQ
[| !%%$M$M$MNN1a.99"5 "5 :9 ON &%"5J $M$M$MNN" " ON"& 1a.99 W  W :9 WF3 3 38e0 e0 e0P 1a.99:  :  :9: z 61a*@AA$ $ BA$N 61a*@AA$
 $
 BA$
N 61a*@AA#( #( BA#(L' ' '>_ _ _D3 3 3= = = 		QC!qcA3/	0	0(;		QC!qcA3/	0	0,?		IIIyyy)))<	=	=|L * * * 8 8 8A A A6* * *"N N N8P P P4  @A A A8I( I( I(X 2:rz2:"FGG
 
 HG
 2:rz2:"FGG! ! HG! 	III	aVaVaV,-	YYY			99956 8 8 8) ) )$
 
 
$ $ $N     FIX IX IXX
 
 
&6 6 6 			3334			3334			3334 
: 
: 
: #&&&'  CCC ( (  ($ $	1a&		828QF++S1	1a&		828QF++S1	1a&		828QF++S1	1a&		828QF++S1	 X X X HBF333Karv>>>O26:::L777	 V V  VS S Sl1 1 1:. . ..9 .9 .9b` ` `F- - -@ ' LL   E		
 '&&   (		
 "!!   <		
 LL   3		
 LL!!!0		
 II3Z#sc3Z0,		
 \\<<<(!!!#7#7#786		
 II__ooo?F0		
 II__ooo?C8		
c7: :v6 6w: :v6' ' '  "K"K"M"M D D D $h@QR  t}55,udm<<*,U,U,W,W 9= 9=  =< 65 9=x $h@QR  ,udm<<*,U,U,W,W != !=  =< != != !=rc   