siiZ dZddlZddlZddlZddlZddlZddlmZddlm Z ddl m Z m Z m Z mZmZmZmZmZmZmZmZmZmZddlmZddlZddlZddlm ZddlmZdd l m!Z!d d l"m#Z#m$Z$m%Z%d d l&m'Z'm(Z(d d l)m*Z*m+Z+d dl,m-Z.e rddl/m0Z1ne Z1ej2dZ3e34ddZ5de6de6deeeej7ej7fee8e8ffddffdZ9defdZ:defdZ;deddfdZe5j=deej7ej7ffdZ?e5j=deej7ej7ffdZ@e5j=deej7ej7ffdZAe5j=dee1ej7ffdZBe5j=d eCdee!jDej7ej7e!jDej7ej7e!jDej7ej7f fd!ZE dWd"d#d$d%e6de6d&e6d'eFd(eFd)e6deeej7eej7eej7ffd*ZGee!jDejHejIejHejIfZJe Gd+d,ZKGd-d.eZLGd/d0ZMd1ejHejIdeejHejHejHffd2ZN dXd4e!jDd5ejHejId6ejHejId7eOdejHejPf d8ZQd9ee!jDejHejIejHejIfd:ejHejPdeKfd;ZRdejHejId?ejHejTdee!jDejHejIejHejIejHejIff d@ZUdAe dBee$dCeCddfdDZVe5j=de6de6dEeOdFeFdeee!jDej7ff dGZWdHe6dIe6deeCfdJZXdKdLd"d#ejTdMdNde6de6dOe6dPeFdEeOdQeOdReFd)e6dSej jYdCeCdee*ej7ffdTZZGdUdVe#Z[dS)YzUtilities for data generation.N)ThreadPoolExecutor) dataclass) TYPE_CHECKINGAnyCallableDict GeneratorList NamedTupleOptionalSequenceSetTupleTypeUnion)request)typing)r )sparse)DataIterDMatrixQuantileDMatrix)is_pd_cat_dtypepandas_pyarrow_mapper) ArrayLike XGBRanker)train) DataFramejoblibz ./cachedir)verbose n_samples n_featuresreturnc#0Ktjd}tjd}|dd||z||}tjtjtj tj tj tj tj tjtjtjtjtjtjtjtjtjtjtjtjtjg}|D]H}tj||}||fV||fVI|D]H}tj||}||}||} || fVI|dd||z ||}tjt@fD]} tj|| }||fVtjt@fD]H} tj|| }||}||} || fVId S) z*Enumerate all supported dtypes from numpy.pandasrlowhighsizedtypeg?r+N)!pytest importorskipnprandom RandomStaterandintreshapeint32int64byteshortintcint_longlonguint32uint64ubyteushortuintcuint ulonglongfloat16float32float64halfsingledoublearraytolistrbinomialbool_bool) r!r"pdrngorigdtypesr-Xdf_origdfdtype1dtype2s Z/var/www/html/bet.cuttalo.com/ml/venv/lib/python3.11/site-packages/xgboost/testing/data.py np_dtypesrZ/sL  X & &B )   % %C ;;13Y-C; D D L L:  D                    )F,(( HT ' ' 'Ag kkmmQXXZZ''''' HT ' ' ',,t$$ \\!__rk <<3Y%;< < < D D:  D8T" HT ( ( (Ag 8T" HT ( ( (,,t$$ \\!__rk c #\Ktjd}|||||||| g}tj }| dd|dgdd|dgdtj }tj d|jfD]/}|D]*}| dd|dgdd|dgd|}||fV+0tj }||g}| d d |d gd d |d gdtj }tj d|jfD]s}|D]n}| d d |d gd d |d gd|}||fV|d }|d }t#||jsJt#||jsJ||fVot|d }|jD]*}||jt.||<+tj d|jfD]<}| dd|dgdd|dgd|}||fV=d|jfD]u}dd|dgdd|dgd} | | | tjn|}| | |}||fVvdS)z/Enumerate all supported pandas extension types.r%r.rf0f1r,N?g@g@r`categoryTF)r0r1 UInt8Dtype UInt16Dtype UInt32Dtype UInt64Dtype Int8Dtype Int16Dtype Int32Dtype Int64Dtyper2nanrrFNA Float32Dtype Float64Dtype isinstanceSeriesastypecolumnscatrename_categoriesintCategoricalDtyperN BooleanDtype) rPrSNullrRr-rVser_origsercdatas rY pd_dtypesr~js  X & &B         F%'FD <<1dAq!T1o66bj   Dru%  E1dAq!T1o>>eB(NNNN   6Doo!2!2 3F <<S$$S#tS,ABB"*   Dru%    ES$,S#tS4IJJRWB(NNNDzHT(Cc29-- - - -h 22 2 2 2C-      ;;z " "D \55q'+//44Qru% \\q$?1aq/ : :%%''  Bhru UD$/tT47PQQ||DDLbooFWFW|XX \\$boo&7&7\ 8 8Bh r[c #Ktjd}tjd}t}d|jdfD]}|D]}|ds|dr-||s|dkr|n t j}|dd|d gd d |dgd t j }|dd|d gd d |dgd | }||fV|jd fD]}|dd |dgd d|dgd | }|dd |dgd d|dgd | |  }||fVdS)z*Pandas DataFrame with pyarrow backed type.r%pyarrowNrrErOr.rr]r^r_r,FT) r0r1rrm startswithisnar2rlrrFrx ArrowDtyperN)rPparSryr- orig_nullrRrVs rYpd_arrow_dtypesrs  X & &B  Y ' 'B#F.rua   E ** e.>.>v.F.F $&GGDMMKdaiiRVI<<1i+Aq)Q3GHHj D 1dAq!T1o>>eB(NNNN    ||%t,UD$4M N N//##  \\%t,UD$4M N N-- ++  Bh  r[rQc|ddd}|d}tj|d<t jt d5t||dddn #1swxYwYt jt d5t||ddddS#1swxYwYdS) zValidate there's no inf in X. r/r^)rzInput data contains `inf`matchN) r3r6r2infr0raises ValueErrorrr)rQrTys rY check_infrsG  ##Aq))A  AfAdG z)D E E E1 z)D E E E1 s$+BB B .C  CCcdtjdtjd}dt t dt t dt t dtjffd }d t d t dtjffd }||d d gddgddg|ddgddgddg|dd|dd|dd|dd |d!d"|d#d$|d%d&d' }||j d(g }|d( }||fS))zSynthesize a dataset similar to the sklearn California housing dataset. The real one can be obtained via: .. code-block:: import sklearn.datasets X, y = sklearn.datasets.fetch_california_housing(return_X_y=True) iPir%meanssigmasweightsr#c t|dz|d|d}|jdz |d|d}tj||gdS)Nr)r+locscaler.axis)normalrvshaper2 concatenate)rrrl0l1r!rQs rY mixture_2compz-get_california_housing..mixture_2compsZZi'!*,--E!HF1I  ZZi"(1+5E!HFSTIZ V V~r2hQ////r[meanstdc6||fS)Nrrr+)r)rrr!rQs rYnormz$get_california_housing..norm szzd#YLzAAAr[g5ŀ]g~(Fv^gr-|E?g3mE^1?gDi-T?gÅv-W?gXcB@g& @@g6 ?g](?g8W nx?gd?g|["@g2{e?)rrgVb<@g>+)@gZK@g@g)P=?g˧ ^T?g/E@g@gI@gtbO$@gg9h@gk}v?) LongitudeLatitudeMedIncHouseAgeAveRooms AveBedrms PopulationAveOccup MedHouseValr) r2r3 default_rngr0r1r floatndarrayrrs differenceto_numpy)rPrrrVrTrr!rQs @@rYget_california_housingrs I )   % %C  X & &B0E{0$(K0:>u+0 0000000B5BuBBBBBBBB &}-#%78Z( & k*#%78Z( d 28JKKK"4:LMMM"39JKKK#4:MNNN$$6d?d@d$dAd|dB<|dCdDd d"dEd|dF<|d@dGdGdHdI|dJ<dKtdLtdtdj ffdM }|dNdOd|dP<|dQdRd|dS<|dTdUd|dV<|dWdXd|dY<|dZd[d|d\<|d]d^d|d_<|d`dad|db<|dcddd|de<|dfdgd|dh<|didjd|dk<t|j}|||}t jfl}|jD]e}t#||jjr3|||jjtjz }U|||jz }f|dm|z z}|dn|z z }||fS)oamGet a synthetic version of the amse housing dataset. The real one can be obtained via: .. code-block:: from sklearn import datasets datasets.fetch_openml(data_id=42165, as_frame=True, return_X_y=True) Number of samples: 1460 Number of features: 20 Number of categorical features: 10 Number of numerical features: 10 rNr%r&i name_probadensityr#c t d|z z}tjd|z dko|dk}|rd|z }||tj<t |}t |}|dxxdtj|z z cc< | |} | td|}|S) Nr.rbư>r)r+pc,t|tSN)rpstr)xs rYz5get_ames_housing..synth_cat..vsAs!3!3r[r,) rvr2absrllistkeysvaluessumchoicerqrwfilter) rrn_nullshas_nanrrrrseriesr!rPrQs rY synth_catz#get_ames_housing..synth_catdsi1w;/00&w''$.>7Q;  *W}H!)Jrv JOO%%&& ""$$ % % "rvayy  JJt)qJ 1 1 %%33T::   r[gqu]?gqh.?gsmB.synth_nums JJ3c J : :i1w;/00 6#- 4 ' 'GaKKzz)'5zIIH&AhKyy"*y---r[gmtF @gOfKhttps://s3-us-west-2.amazonaws.com/xgboost-examples/MQ2008.zipz MQ2008.zip)urlfilenamer)pathNzMQ2008/Fold1/train.txtzMQ2008/Fold1/test.txtzMQ2008/Fold1/vali.txtTF)query_id zero_based) r0r1osr%joinexistsr urlretrievezipfileZipFile extractallload_svmlight_files)r rsrcrfx_trainy_train qid_trainx_testy_testqid_testx_validy_valid qid_valids rY get_mq2008r;s}"#566H JC W\\% . .F 7>>& ! !6f5555  % %! %    !!!!!!!!!!!!!!! $$ GLL 8 9 9 GLL 7 8 8 GLL 7 8 8  %      sB$$B(+B(Fr&) vary_sizern_samples_per_batch n_batchesuse_cupyr<rcg}g}g}|r1ddl} | jtj|} ntj|} t |D]} |r|| dzzn|} | | |} | | }| dd| }|| |||||||fS)zMake batches of dense data.rN r.r() cupyr3r4r2r?rrandnuniformappend)r=r"r>r?r<rrTrwrBrQrr!_X_y_ws rY make_batchesrJ:s A A A2 k%%bi &=&=>>i##L11 9  4=V'!b&00CV YYy* - - YYy ! ! [[QQY[ 7 7      a7Nr[ceZdZUdZejed<eje j ed<eje j ed<eje j ed<eje j ed<eje j ed<dS) ClickFoldzCA structure containing information about generated user-click data.rTrqidscoreclickposN) __name__ __module__ __qualname____doc__r csr_matrix__annotations__nptNDArrayr2r7rFr8r[rYrLrL[sMM  {28 RX  ;rz """" ;rx    RX r[rLc>eZdZUdZeed<eed<eed<defdZdS) RelDataCVzPSimple data struct for holding a train-test split of a learning to rank dataset.rtestmax_relr#c|jdkS)z6Whether the label consists of binary relevance degree.r.)r]selfs rY is_binaryzRelDataCV.is_binaryns|q  r[N) rQrRrSrTRelDatarVrvrOrarYr[rYr[r[gsRZZ NNN MMM LLL!4!!!!!!r[r[ceZdZdZdeddfdZdejej dejej dejej fdZ dS) PBMaSimulate click data with position bias model. There are other models available in `ULTRA `_ like the cascading model. References ---------- Unbiased LambdaMART: An Unbiased Pairwise Learning-to-Rank Algorithm etar#Nctjgd|_tjgd}tj|||_dS)N)rg{Gz?Q?gp= ף?rb) g(\?gQ?gQ?g(\?rg皙?g)\(?rg{Gz?gQ?)r2rK click_probpower exam_prob)r`rerks rY__init__z PBM.__init__}sM(#?#?#?@@H H H H  )S11r[labelspositionctj|d}tj|j}d||dk<d||t |jk<|j|}tj|j}|j|jksJtj|d}d|||jjk<|j|}tj d}||jdtj }tj|jtj }d||||zk<|S) zSample clicks for one query based on input relevance degree and position. Parameters ---------- labels : relevance_degree T)copyrrr&)r+r-r,r.) r2rKrrlenrir+rkr3rrFr7) r`rmrnrirkranksrQprobclickss rYsample_clicks_for_queryzPBM.sample_clicks_for_querys&t,,,Xfl++ vz13vT_---._V, HV\** } ++++---.0et~**+N5) i##D))zzv|AbjzAA(*RX(N(N(N01ti*,,- r[) rQrRrSrTrrlrWrXr2r7r8rurYr[rYrdrdss2E2d2222!k"(+!7:{287L! RX !!!!!!r[rdrc tj|}|j}tjdtjtj|dd|ddddzf}tjtj||f}||}tj|tj|jg}|||fS)zzRun length encoding using numpy, modified from: https://gist.github.com/nvictus/66627b580c13068589957d6ab0919e66 rr.NrT) equal_nan) r2asarrayr+r_ flatnonzeroisclosediffrErK)rrstartslengthsrindptrs rYrlencoders 1 A A U1bnbj122#2#$&O&O&O%OPPSTTT UFgbeFAI&''G vYF Yvrx11 2 2F 7F ""r[rrTrrM sample_ratectjd}t|jd|z}tjd|jdtj}|||d|}||}||}||} tj| } || }|| }| | } tdd} | ||| | |} | S) zWe use XGBoost to generate the initial score instead of SVMRank for simplicity. Sample rate is set to 0.1 by default so that we can test with small datasets. r&rr,Nz rank:ndcghist) objective tree_method)rM) r2r3rrvraranger?rargsortrfitpredict) rTrrMrrQr!indexX_trainr3r4 sorted_idxltrscoress rYinit_rank_scorers )   % %CAGAJ,--I1agaj BBBEKK *9* EhGhGE II&&Jj!Gj!G*%I kv > > >CGGGW)G,,,[[^^F Mr[fold scores_foldc|\}}}|jtjksJtj|}tj|jftj}tj|jftj}td}|D]u} | |k} | | j d} || } tj | ddd} | || <|| } | | | }||| <v|j d|j dksJ|j |j f|j d|j dksJ|j |j ft||||||S)zSimulate clicks for one fold.r,rb)rerNr) r-r2r7uniqueemptyr+r8rdr6rrrurL)rrX_foldy_foldqid_foldqidsrnrtpbmqqid_mask query_scoresquery_positionrelevance_degrees query_clickss rYsimulate_one_foldrsz $FFH >RX % % % % 9X  Dxbh777H Xv{nBH 5 5 5F #,,,C ( (=##HN1$566"8, L11$$B$7+"8,223DnUU 'x <?hnQ/ / / /&,1O / / / <?fl1o - - - fl/K - - - VVX{FH M MMr[cv_datac v tt|j|j\}}}t jdgd|Dzt jtdksJtj |}t j |}t j |}t|||fdtdj D}ggggggf\ tj dz D]}t||||||f||} | j| j| j| j| j| jʈfdtj dz D} tdD](}| |||ksJ)t dkr;t/ dddddd} d} n0 fd tt D\} } | | fS) z6Simulate click data using position biased model (PBM).rc(g|]}|jdS)rr).0vs rY z#simulate_clicks..s333AQWQZ333r[r]cBg|]}|dz |S)r.rY)rrr scores_fulls rYrz#simulate_clicks..s/ T T Tk&Q-&)34 T T Tr[r.c g|] }| SrYrY)rrs_lsts rYrz#simulate_clicks..s???1eAh???r[rNc 3K|]<}t||||||V=dSr)rL)rrX_lstc_lstp_lstq_lstry_lsts rY z"simulate_clicks..$sa   eAha%(E!HeAha Q Q      r[)rziprr\r2rKcumsumrqrvstackrrrr+rrErTrrMrNrOrPallrL)rrTrrMX_fully_fullqid_fullrrrscores_check_1rr\rrrrrrrrs @@@@@@@@rYsimulate_clicksrsS 5566IAq#Xqc333333 4 4F Yv  F v;;%     ]1  F ^A  F~c""H"&&(;;K T T T T TeAv{>S>S T T TF/12r2r2/E,E5%u 6;? # # !A$!c!f!5vayAA TV TV TX TZ    TZ    TX????fkAo(>(>???N 1XX66q!VAY.33555555 5zzQ%(E!HeAha%(ERSHUU         3u::&&   t $;r[rtrPc tj|}||}||}||}||}t|\}}}td|jD]^}||dz } ||} | | ks J| | ftj|| | jdks J| | f|| | } | dksJ| | | jdz ks?J| | j|tj|| | ftj| }|| | ||| | <|| | ||| | <|| | ||| | <|| | ||| | <`||||f} | S)z,Sort data based on query index and position.r.r)r2rrrr+rminmax) rTrrMrtrPrr_rbegend query_posr}s rYsort_ltr_samplesr+sCJ * A J F j/C j/CC==LFAq 1fk " "00QUmQiSyyy3*yyyySW&&+q0003*000CL }}!###Y]]__###}}).1"4444 MMOO N Ic#c'l # # 7 444 Z ** s3wZ +#c'  S/*5s3ws3wZ +#c' 3s7|J/CG fa D Kr[DTypeDMatrixTdevicechtj}||dddtjdd}t|dr|jdddf}n |dddf}|}|||| }tj td 5td |d |dddn #1swxYwYt|ds,|| dd}||ksJ|jjjdusJ|jjjdusJ||j || dd}||jksJ|}||| } ||d|j| } | | ksJ|tj}||| } | | ksJ|dddd}tj td 5||ddddS#1swxYwYdSdS)zRun tests for base margin.rrbdr/2rilocN) base_marginz.*base_margin.*rr)rrFTr.r)r2r3rrrrrFr6hasattrrr0rrtrain_fnget_base_marginrTflags c_contiguous f_contiguousset_infoset_base_marginr+rG) rrrrQrTrrXygotbm_colbm_rowbm_f64s rYrun_base_margin_infor\s )   ! !C cjjCcj**11"*==EEb!LLMMAq& F111a4L aaadGK !QK 0 0 0B z); < < <@@6::B???@@@@@@@@@@@@@@@ 1f  ,eB&&((00Q7788{"'')))))}"/58888}"/47777 ...eB&&((00B7788{}$))+++++  ;'''##%% ;..q+2BCCDDD##%%& %%'''''"((44  ;'''##%%& %%'''''ii1a++ ]:-? @ @ @ , ,   { + + + , , , , , , , , , , , , , , , , , ,7,,s$ C**C.1C.L%%L),L)ras_densecttjdsWtjd}t jdz |d}|dd}||fSt tjdtd tj ffd }g}t 5} tD]+} | | || , d d d n #1swxYwYg} g} |D]C} | \}}| || |Dt!| ksJt j| d }tj| }||jd|jdfj}tj|d}|jdksJ|jdksJ|jdksJ|rQ|}|jdksJ|jdksJtj||dk<||fS||fS)zMake sparse matrix. Parameters ---------- as_dense: Return the matrix as np.ndarray with missing values filled by NaN rr&rbcsr)mrrrformatrt_idr#ctjd|z} z}| dz kr ||zz }n|}tj |d z |}tj df}t |jdD]b}|j|dz|j|z }|dkr<||dd|f | dfzdzz }c||fS)Nr&r.rb)rrrrrrh) r2r3rrtocscrrrrtoarray) rrQ thread_sizen_features_tlocrTrrr+r"r! n_threadsrs rY random_cscz*make_sparse_regression..random_cscsi##D4K00 I- 9q= (4++==OO)O M(N    %'' Hi^ $ $qwqz"" J JA8AE?QXa[0DqyyQqqq!tW__&&YN)C)CCcII!t r[) max_workersNrrr.r)rr2r3r4rrrmultiprocessing cpu_countrv csc_matrixrrrEsubmitresultrqhstackrxr6rrrrrl)r!r"rrrQrTrrfuturesexecutorr X_results y_resultsr1rarrrs``` @rYmake_sparse_regressionrs 29m , , i##D)) M(N     JJ3c J : :!t O-//<>Y & & & &#]9UCCCC 9A 171:qwqz*++-A qqA 9Q<9 $ $ $ $ 9Q<: % % % % 71: " " " "kkmmy|y((((y|z))))C1H Av 6Ms ? ? ? ? ? Jtvdgtw > > > ? ? ? ? ? ? ? ? ? ? ? ? ? ? ?  %%''&/&&((-1V=46$'?'')))     1 ts.A55A9<A9cd|_dS)Nr)rr_s rYresetzIteratorForTest.resetzs r[cDt|jdtjrtj|jd}nt j|jd}t j|jd}|jrt j|jd}nd}|||fS)zReturn concatenated arrays.rrrrN) rprTrrUrr2rrrF)r`rTrrFs rY as_arrayszIteratorForTest.as_arrays}s dfQi!2 3 3 / dfU333AAtvA...A N46 * * * 6 tvA...AAA!Qwr[)r#N)rQrRrSrTr r rrOrvrlrr&r(rrr2rrrUrr* __classcell__)rs@rYrrQsMM.2       H   }   'sm        *xD"  uRZ!223Y@SS T        r[r)F)r)\rTr$rr(rr,concurrent.futuresr dataclassesrrrrrrr r r r r rrrrurllibrnumpyr2r0rW numpy.randomRNGscipyrcorerrrr}rrsklearnrrtrainingrrr%r DataFrameTr1rMemorymemoryrvrrrZr~rrrrrrrrrrUr;rOrJrXr7rbrLr[rdrrrFrrrr8rrrr DTypeLikerrrYr[rYr:s $$  111111!!!!!! ))))))555555555599999999********((((((.......J  X & & |Q / /88 #8uU2:rz12E$*4EEFdRS8888v>9>>>>B77777t 3 4    4bj"*&< =4444n"E"*bj01""""8E"*bj018888  E"*bj01     s% BJ 67ssssl5 5  JJ JJ JJ 5555x      4 T"*-tBJ/? ?@< !3;rx#8#+bh:OO P   ! ! ! ! ! ! ! !44444444n # BH% #% S[#+0U*V # # # #& !! ! {28! RX ! !  [ !!!!HN !3;rx#8#+bh:OO PNRZ(NNNNNF(Y(5HY