
    sii              	          d Z ddlZddlZddlZddlZddlZddlmZ ddlm	Z	m
Z
mZmZmZmZmZ ddlZddlmZmZmZmZmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddl m!Z! ddl"m#Z# ddl$m%Z% dede&fdZ'de
dee&         dee&e	f         fdZ( G d de          Zd3de&de)de)defdZ*dede)defdZ+dede&fdZ,defdZ-d4d e&d!eee&e)f                  dej.        fd"Z/d e&dee)         fd#Z0d$ede)fd%Z1d$ede2fd&Z3dede2fd'Z4d(ede)fd)Z5de&fd*Z6d+e&d,e
g e#f         de#fd-Z7d.e!de&fd/Z8d+e&de!fd0Z9d1ee&         de2fd2Z:dS )5z;Xgboost pyspark integration submodule for helper functions.    N)Thread)AnyCallableDictOptionalSetTypeUnion)BarrierTaskContext	SparkConfSparkContext
SparkFilesTaskContext)SparkSession   )CommunicatorContext)Config)_Args)_ArgVals)Booster)XGBModel)RabitTrackerclsreturnc                 $    | j          d| j         S )zReturn the class name..)
__module____name__)r   s    Y/var/www/html/bet.cuttalo.com/ml/venv/lib/python3.11/site-packages/xgboost/spark/utils.pyget_class_namer       s    n--s|---    funcunsupported_setc                     t          j        |           }i }|j                                        D ](}|j        |j        ur|j        |vr|j        ||j        <   )|S )zReturns a dictionary of parameters and their default value of function fn.  Only
    the parameters with a default value will be included.

    )inspect	signature
parametersvaluesdefaultemptyname)r"   r#   sigfiltered_params_dict	parameters        r   _get_default_params_from_funcr/      so     
D
!
!C^**,, E E	 Y_44o553<3D 0r!   c                   0     e Zd ZdZdededdf fdZ xZS )r   z&Context with PySpark specific task ID.contextargsr   Nc                     t          |                                          |d<    t                      j        di | d S )Ndmlc_task_id )strpartitionIdsuper__init__)selfr1   r2   	__class__s      r   r9   zCommunicatorContext.__init__5   sD    "7#6#6#8#899^  4     r!   )r   r   __qualname____doc__r   CollArgsValsr9   __classcell__)r;   s   @r   r   r   2   sY        00! 2 !L !T ! ! ! ! ! ! ! ! ! !r!   r   host	n_workersportc                 
   d|i}t          || d|          }|                                 t          |j                  }d|_        |                                 |                    |                                           |S )z"Start Rabit tracker with n_workersrA   task)rA   host_ipsortbyrB   )targetT)r   startr   wait_fordaemonupdateworker_args)r@   rA   rB   r2   trackerthreads         r   _start_trackerrO   :   sz    !9-DYVRVWWWGMMOOO7+,,,FFM
LLNNNKK##%%&&&Kr!   confc                 d    | j         J | j        dn| j        }t          | j         ||          }|S )z3Get rabit context arguments to send to each worker.Nr   )tracker_host_iptracker_portrO   )rP   rA   rB   envs       r   _get_rabit_argsrU   F   s>    +++!)11t/@D
-y$
?
?CJr!   r1   c                 N    d |                                  D             }|d         S )zLGets the hostIP for Spark. This essentially gets the IP of the first worker.c                 N    g | ]"}|j                             d           d         #S ):r   )addresssplit).0infos     r   
<listcomp>z _get_host_ip.<locals>.<listcomp>P   s-    RRR4DL&&s++A.RRRr!   r   )getTaskInfos)r1   task_ip_lists     r   _get_host_ipr`   N   s+    RR7;O;O;Q;QRRRL?r!   c                      t           j                                        t          d          t          j                                        S )z`Get or create spark session. Note: This function can only be invoked from driver
    side.

    Nz<_get_spark_session should not be invoked from executor side.)pysparkr   getRuntimeErrorr   buildergetOrCreater5   r!   r   _get_spark_sessionrg   T   sE    
   ,J
 
 	
 ++---r!   r+   levelc                    t          j        |           }||                    |           n4|j        t           j        k    r|                    t           j                   |j        stt          j                    j        s\t          j        t          j	                  }t          j
        d          }|                    |           |                    |           |S )zGGets a logger by name, or creates and configures it for the first time.Nz<%(asctime)s %(levelname)s %(name)s: %(funcName)s %(message)s)logging	getLoggersetLevelrh   NOTSETINFOhandlersStreamHandlersysstderr	FormattersetFormatter
addHandler)r+   rh   loggerhandler	formatters        r   
get_loggerry   a   s    t$$F <7>))OOGL)))? #7#4#6#6#? #'
33%J
 
	 	Y''''"""Mr!   c                 f    t          j        |           }|j        t           j        k    rdn|j        S )z+Get the logger level for the given log nameN)rj   rk   rh   rm   )r+   rv   s     r   get_logger_levelr{   u   s,    t$$F<7>1144v|Cr!   spark_contextc                    | j                                                                         dk    rh| j                                                             | j                                                                                             d                    S | j                                                                         S )z0Gets the current max number of concurrent tasks.z3.1r   )_jscscversionmaxNumConcurrentTasksresourceProfileManagerresourceProfileFromIdr|   s    r   _get_max_num_concurrent_tasksr   {   s     &&((E11!$$&&<<!!##::<<RRSTUU
 
 	
   ""88:::r!   c                 X    | j                                                                         S )zWhether it is Spark local mode)r~   r   isLocalr   s    r   	_is_localr      s%       ""**,,,r!   c                     |                      d          }|d uo)|                    d          p|                    d          S )Nzspark.masterzspark://zlocal-cluster)rc   
startswith)rP   masters     r   _is_standalone_or_localclusterr      sI    XXn%%F *%%K):):?)K)Kr!   task_contextc                     | t          d          |                                 }d|vrt          d          t          |d         j        d                                                   S )z&Get the gpu id from the task resourcesNz3_get_gpu_id should not be invoked from driver side.gpuzDCouldn't get the gpu id, Please check the GPU resource configurationr   )rd   	resourcesint	addressesstrip)r   r   s     r   _get_gpu_idr      sq    PQQQ&&((IIR
 
 	
 y)!,2244555r!   c                      t          j                    } t          j                            | d          }t          j                            |          st          j        |           |S )Nzxgboost-tmp)r   getRootDirectoryospathjoinexistsmakedirs)root_dirxgb_tmp_dirs     r   _get_or_create_tmp_dirr      sQ    *,,H',,x77K7>>+&& !
K   r!   modelxgb_model_creatorc                      |            }|                     t          |                     d                               |S )zH
    Deserialize an xgboost.XGBModel instance from the input model.
    utf-8)
load_model	bytearrayencode)r   r   	xgb_models      r   deserialize_xgb_modelr      s@     "!##I5<<#8#899:::r!   boosterc                 *   t           j                            t                      t	          j                     d          }|                     |           t          |d          5 }|                                }ddd           n# 1 swxY w Y   |S )z
    Serialize the input booster to a string.

    Parameters
    ----------
    booster:
        an xgboost.core.Booster instance
    .jsonr   encodingN)	r   r   r   r   uuiduuid4
save_modelopenread)r   tmp_file_namefser_model_strings       r   serialize_boosterr      s     GLL!7!9!9djll;Q;Q;QRRM}%%%	mg	.	.	. $!6688$ $ $ $ $ $ $ $ $ $ $ $ $ $ $s   'BBBc                 J   t                      }t          j                            t	                      t          j                     d          }t          |dd          5 }|                    |            ddd           n# 1 swxY w Y   |	                    |           |S )zN
    Deserialize an xgboost.core.Booster from the input ser_model_string.
    r   wr   r   N)
r   r   r   r   r   r   r   r   writer   )r   r   r   r   s       r   deserialize_boosterr      s     iiGGLL!7!9!9djll;Q;Q;QRRM	mS7	3	3	3 q	              }%%%Ns   !BB
Bdevicec                 
    | dv S )z&Whether xgboost is using CUDA workers.)cudar   r5   )r   s    r   use_cudar      s    _$$r!   )r   )N);r=   r%   rj   r   rq   r   	threadingr   typingr   r   r   r   r   r	   r
   rb   r   r   r   r   r   pyspark.sql.sessionr   
collectiver   CCtxr   r   CollArgsr   r>   corer   sklearnr   rM   r   r6   r    r/   r   rO   rU   r`   rg   Loggerry   r{   r   boolr   r   r   r   r   r   r   r   r5   r!   r   <module>r      s   A A   				 



        B B B B B B B B B B B B B B B B B B  X X X X X X X X X X X X X X , , , , , , 4 4 4 4 4 4       * * * * * * 1 1 1 1 1 1             " " " " " ". . . . . .
 
 %(X 	#s(^       &! ! ! ! !$ ! ! !	 	 	 	C 	 	 	 	 	& S X    ,     
.L 
. 
. 
. 
. S %S/!: gn    (D3 D8C= D D D D; ;# ; ; ; ;-\ -d - - - - t    6k 6c 6 6 6 6    #+BL#9   w 3    "
s 
w 
 
 
 
%Xc] %t % % % % % %r!   