
    Dii                        U d Z ddlmZ ddlmZ ddlmZ ddlmZm	Z	 ddl
Z
ddlmZmZmZmZmZ ddlmZ i Zeed	f         ed
<   g dZ G d d	e          ZdededdfdZdedededede
j        dedefdZdedededede
j        dedefdZ ede            ede           dedede	ee
j        f         dedef
dZdefdZ ededed         fd             Z!dS )!z
This is an experimental new API for PyTorch Distributed. This is actively in development and subject to change or deletion entirely.

This is intended as a proving ground for more flexible and object oriented distributed APIs.
    )	Generator)contextmanager)	timedelta)ProtocolUnionN)_current_process_group_set_process_groupProcessGroupReduceOpStore)
rendezvousProcessGroupFactory	_BACKENDS)r
   r   r   register_backend	new_groupcurrent_process_groupprocess_groupc                   @    e Zd ZdZdededededej        de	de
fd	Zd
S )r   z%Protocol for process group factories.storerank
world_sizetimeoutdevicekwargsreturnc                     d S )N )selfr   r   r   r   r   r   s          ^/var/www/html/bet.cuttalo.com/ml/venv/lib/python3.11/site-packages/torch/distributed/_dist2.py__call__zProcessGroupFactory.__call__'   s	     s    N)__name__
__module____qualname____doc__r   intr   torchr   objectr
   r    r   r!   r   r   r   $   st        //  	
    
     r!   namefuncr   c                 R    | t           v rt          d|  d          |t           | <   dS )z
    Register a new process group backend.

    Args:
        name: The name of the backend.
        func: The function to create the process group.
    Backend z already registeredN)r   
ValueError)r)   r*   s     r   r   r   2   s6     y=D===>>>IdOOOr!   r   r   r   r   r   r   c                 b   ddl m} t          |          dk    s
J d             || |||          }|                                 t	          | ||          }|                    t          j        j                   |                    |t          j        j        |           |                    t          j
        d          t          j        j        |           t          j                                        r8|                    t          j
        d          t          j        j        |           |S )Nr   )ProcessGroupGlooz'Gloo backend received unexpected kwargscpucuda)torch.distributedr/   len_set_sequence_number_for_groupr
   _set_default_backendBackendTypeGLOO_register_backendr'   r   r1   is_available)	r   r   r   r   r   r   r/   backend_classpgs	            r   _gloo_factoryr<   @   s&    322222v;;!F$$UD*gFFM00222	eT:	.	.BL49::: !9!>NNNU\5:M   z   

L  ,":"?	
 	
 	
 Ir!   c                    ddl m} |                                }||_        |                                D ]8\  }}	t          ||          st          d|           t          |||	           9 || |||          }
|
                                 |
	                    |           t          | ||          }|                    t          j        j                   |                    |t          j        j        |
           |S )Nr   )ProcessGroupNCCLzUnknown option )r2   r>   Options_timeoutitemshasattrKeyErrorsetattrr4   eager_connect_single_devicer
   r5   r6   NCCLr8   )r   r   r   r   r   r   r>   optskvr:   r;   s               r   _nccl_factoryrJ   ^   s    322222##%%DDM  1tQ 	20Q00111a$$UD*dCCM00222--f555	eT:	.	.BL49:::!9!>NNNIr!   glooncclbackendc                    | t           vrt          d|  d          t          j        |          }t	          t          t          d                              \  }}}|                    |           t          |          |||||fi |S )aF  
    Create a new process group with the given backend and options. This group is
    independent and will not be globally registered and thus not usable via the
    standard torch.distributed.* APIs.

    Args:
        backend: The backend to use for the process group.
        timeout: The timeout for collective operations.
        device: The device to use for the process group.
        **kwargs: All remaining arguments are passed to the backend constructor.
                  See the backend specific documentation for details.

    Returns:
        A new process group.
    r,   z not registeredzenv://)r   r-   r'   r   nextiterr   set_timeout)rM   r   r   r   r   r   r   s          r   r   r   ~   s    * i<G<<<===\&!!F"4
8(<(<#=#=>>E4	gWeT:wQQ&QQQr!   c                      t                      S )zn
    Get the current process group. Thread local method.

    Returns:
        The current process group.
    )r   r   r!   r   r   r      s     "###r!   r;   )NNNc              #      K   t                      }t          |            	 dV  t          |           dS # t          |           w xY w)zs
    Context manager for process groups. Thread local method.

    Args:
        pg: The process group to use.
    N)r   r	   )r;   prev_pgs     r   r   r      sX       $%%Gr$7#####7####s	   6 A)"r%   collections.abcr   
contextlibr   datetimer   typingr   r   r'   torch._C._distributed_c10dr   r	   r
   r   r   torch.distributed.rendezvousr   r   dictstr__annotations____all__r   r   r&   r   r(   r<   rJ   r   r   r   r   r!   r   <module>r_      s     & % % % % % % % % % % %       " " " " " " " "               4 3 3 3 3 3 /1	4**+ 0 0 0      (   3 &9 d    
  	
 L     <
  	
 L     8   ' ' '   ' ' 'RRR #u|#$R 	R
 R R R R@$| $ $ $ $ $l $y1A'B $ $ $ $ $ $r!   