sii0dZddlmZmZddlmZmZmZmZm Z m Z m Z m Z m Z ddlZddlZddlmZddlmZddlmZdd lmZmZmZdd lmZd d lmZd ej dej!fdZ"eddZ#e#ddddddZ$de e ej!de ej!fdZ%deej&deej&e'e(gdfddfdZ)GddeZ*dej&defd Z+d!ee'e ej!fd"e e,d#ee'efd$e ed%ee'efdef d&Z-deej&d'e e e'd"e e,d(e(d)ee'efd*e(d+e(de ee effd,Z. d2d.ed!ed/e ed0e(dej!f d1Z/dS)3z*Utilities for processing spark partitions.) defaultdict namedtuple) AnyCallableDictIteratorListOptionalSequenceTupleUnionN) csr_matrix) ArrayLikeconcat)DataIterDMatrixQuantileDMatrix)XGBModel) get_loggerseriesreturncZ|d}tj|}|S)zStack a series of arrays.F)copy)to_numpynpstack)rarrays X/var/www/html/bet.cuttalo.com/ml/venv/lib/python3.11/site-packages/xgboost/spark/data.py stack_seriesr"s' OOO ' 'E HUOOE LAlias)datalabelweightmarginvalidqidvaluesr&r' baseMarginvalidationIndicatorr*seqc(|rt|SdS)z&Concatenate the data if it's not None.Nr)r.s r!concat_or_noner0s c{{ 4r#iteratorappendcpdtjdtddffd }d}|D]}|tj|jv}|durtj|jvsJ|rD|j|tjddf}|j|tjddf}n|d}}||d| ||ddS)znExtract partitions from pyspark iterator. `append` is a user defined function for accepting new partition.partis_validrNc|tj||tj||tj||tj||tj|dS)N)aliasr%r&r'r(r*)r4r5r2s r! make_blobz#cache_partitions..make_blob+swtUZ***tU[(+++tU\8,,,tU\8,,,tUY)))))r#TF)pd DataFrameboolr7r)columnsloc)r1r2r8has_validationr4trainr)s ` r!cache_partitionsr@%s * ********&*N##  !"[DL8N T ! !;$,....  &Hd5;//23EHT%+.12EE5E %   IeT " " "##r#ceZdZdZdeeefdeede ddffd Z dee e j dee j fdZd edefd Zd d ZxZS) PartIterz7Iterator for creating Quantile DMatrix from partitions.r% device_idkwargsrNc d|_||_||_||_t ddS)NrT) release_data)_iter _device_id_data_kwargssuper__init__)selfr%rCrD __class__s r!rLzPartIter.__init__HsB #   d+++++r#c|sdS|jLddl}ddl}|jj|j|||jS||jSNr)rHcudfcupycudaruntime setDevicer:rG)rMr%rQcps r!_fetchzPartIter._fetchRsn 4 ? & KKK     GO % %do 6 6 6>>$tz"233 3DJr# input_datac |jt|jtjkrdS|d||jtj||jtjd||jtjd||jtj d||jtj dd|j |xjdz c_dS)NFr%r&r' base_marginr*rT) rGlenrIr7r%rWgetr&r'r(r*rJ)rMrXs r!nextz PartIter.nextas :TZ 344 4 45  TZ 344++djnnU[$??@@;;tz~~elDAABB DJNN5<$F$FGG DJNN59d;;<<   l     a tr#cd|_dSrP)rG)rMs r!resetzPartIter.resetos  r#)rN)__name__ __module__ __qualname____doc__rstrr r intrrLr r9r:rWrr;r_ra __classcell__)rNs@r!rBrBEsAA,dO,08 ,IL, ,,,,,,  8HR\$:;  @V      x D    r#rBr4cgdgg}}}d}t|j|j|j|jD]\}}}}|dkrt |} |} |} n1t |} tj| tj } |} |dkr| }|| ksJ| | | |dt | z| | tj |} tj |} tj |}t|| | ft ||fS)Nr)dtype)shape)zipfeatureVectorTypefeatureVectorSizefeatureVectorIndicesfeatureVectorValuesrgr]rarangeint32r2r concatenater)r4csr_indices_listcsr_indptr_listcsr_values_list n_featuresvec_type vec_size_ vec_indices vec_valuesvec_size csr_indices csr_valuescsr_indptr_arrcsr_indices_arrcsr_values_arrs r!)_read_csr_matrix_from_unwrapped_spark_vecrssg9;aS"oJ8;   !  99++4)[* q==9~~H%K#JJ :H)HBH===K#J ??!JX%%%% ,,,r2S5E5EEFFFz****Xo..Nn%566O^O44N  .9#d))ZAX   r#r% dev_ordinalmetarefparamsc|s#ttjd|St||fi|}t|fi|d|i}|S)z+Handle empty partition for QuantileDMatrix.rr)rr)rremptyrB)r%rrrritms r!make_qdmrse :rx//S9999 $ , ,t , ,B..f..#...A Hr# feature_colsuse_qdmrDenable_sparse_data_optimhas_validation_colc bttttddtjdtdt ddffd }dtjdtdt ddffd }d t tttj fd t ttfdtfd } |r|} d vr d d ksJn|} dtt ttft ttttt fffffd } | \} } &|r$t!|| t#|| d| }ne|st!|| | }nD&|r$t!|| t#|| d| }nt!|| | }|r'|rt#|| || }n|r | nd}nd}|,||ksJ||fS)a~Create DMatrix from spark data partitions. Parameters ---------- iterator : Pyspark partition iterator. feature_cols: A sequence of feature names, used only when rapids plugin is enabled. dev_ordinal: Device ordinal, used when GPU is enabled. use_qdm : Whether QuantileDMatrix should be used instead of DMatrix. kwargs : Metainfo for DMatrix. enable_sparse_data_optim : Whether sparse data should be unwrapped has_validation: Whether there's validation data. Returns ------- Training DMatrix and an optional validation DMatrix. rr4namer5rNc|tjks ||jvr|tjkr" |jddkr |}nA||jddkr(||}|tjkrt |}nd}|tjkr(|&dkr |jd|jdksJ|dS|r||dS||dSdSNrr)r7r%r<rlr"r2)r4rr5r rrx train_data valid_datas r!append_mz0create_dmatrix_from_partitions..append_ms( 5:  !5!5 "" ,&,Q/!33.2<.@d!!$q((T 5:%%(//Euz!!e&7??!&QJ!U[^3333} /4 ''.....4 ''.....5"6!5r#cN|tjks ||jvr|tjkr6t|}dkr |jd|jdksJn||}|r||dS||dSdSr)r7r%r<rrlr2)r4rr5r rxrrs r!append_m_sparsez7create_dmatrix_from_partitions..append_m_sparses 5:  !5!5uz!!A$GG??!&QJ!U[^33333T  /4 ''.....4 ''....."6!5r#r+rDc vt|dkrBtddtddt jdi|St |tj}t | tj d}t | tj d}t | tj d}t | tj d}td|||||d|S)NrXGBoostPySparkz_Detected an empty partition in the training data. Consider to enable repartition_random_shuffler%rrZr\)r]rwarningrrrr0r7r%r^r&r'r(r*)r+rDr%r&r'r(r*s r!makez,create_dmatrix_from_partitions..makes v;;!   ' ( ( 0 0.    ;; 0 0;F;; ;fUZ011vzz%+t<<== 5< > >?? 5< > >??VZZ 48899 U6v3  RX   r#missinggcjd}i}i}D]\}}||vr|||<|||<||fS)N)max_binrsilentnthreadenable_categorical)items) non_data_keysnon_data_paramsrkvrDs r! split_paramsz4create_dmatrix_from_partitions..split_paramss_  LLNN  DAqM!!%&""Q_$$r#)rlistr9r:rfr;rr rndarrayrrr r rgfloatr@rnum_col)r1rrrrDrrrrr append_fnrrrdtraindvalidrxrrs ` ` @@@r!create_dmatrix_from_partitionsrsD/:$.?.?J.9$.?.?JJ/r|/3/$/4/////////</bl/#//$////////" T#tBJ//0 $sCx. W    $ # F""vi'8C'?'?'?'?'? %%S#XS%UD@P:Q5Q0R RS%%%%%%* <>>LD&G9---":{D$OO  !' !9---j&))  ' 9---*k4vFF9---j&))  N(0Kvv))FF2DMTT*f---FF ~~6>>#3#33333 6>r#Fmodelr[ strict_shapec |d}t|||j|j|j|j|j}||dd||S)z4Predict contributions with data with the full model.N)r[rr feature_typesfeature_weightsrTF) pred_contribsvalidate_featuresiteration_ranger) _get_iteration_rangerrn_jobsrrr get_boosterpredict)rr%r[rr data_dmatrixs r!rrUs0066O   )- 3L      & &'! '  r#)NF)0re collectionsrrtypingrrrrr r r r r numpyrpandasr9 scipy.sparser_typingrcompatrcorerrrsklearnrutilsrSeriesrr"r$r7r0r:rfr;r@rBrrgrrrr\r#r!rs00////////XXXXXXXXXXXXXXXXXXXXXX######5555555555rz  7QRR h<9NPUVV"*!568BJ;O#r|$#.6 c47PRV7V.W# ####@+++++x+++\(BL(Z((((V  sD$$ %  #   sCx.   '   cN       er|$e8C=)e# e  e cN e#ee 7HW% %&eeeeV(,   )$  Z r#