
    tniuO                     
   d Z ddlZddlZddlZddlmZ ddlmZmZm	Z	m
Z
 ddlmZ ddlZddlZddlmZ ddlmZ ddlmZ  ej                      G d	 d
e          Z G d de          Zd Zd Zd Z G d de          ZdS )z' Tests for runpod | serverless| worker     N)mock)patch	mock_openMock	MagicMock)IsolatedAsyncioTestCase)RunPodLogger)_handle_uncaught_exception)_signal_handlerc                   z    e Zd ZdZd Zd Zd Zd Zd Z e	d           e	d          d	                         Z
d
S )
TestWorkerz#Tests for Runpod serverless worker.c                 `   K   t          j        d          | _        | j        dd id| _        d S )Ntest)return_value
test_input)handlerrp_args)r   r   mock_handlermock_configselfs    ^/var/www/html/gpu-tools/venv/lib/python3.11/site-packages/tests/test_serverless/test_worker.py
asyncSetUpzTestWorker.asyncSetUp   s>       I6:::($d+
 
    c                    t          d          5 }d|j        j        _        t          j        j                            di i          du sJ t          j        j                            dddii          du sJ d|j        j        _        t          j        j                            | j                  du sJ 	 ddd           dS # 1 swxY w Y   dS )z 
        Test _is_local
        runpod.serverless.worker.osNr   Tr   	somethingF	r   environgetr   runpod
serverlessworker	_is_localr   r   mock_oss     r   test_is_localzTestWorker.test_is_local    s6    011 	QW/3GO,$+55y"oFF$NNNN!(22{ ;<      0;GO,$+55d6FGG5PPPPP	Q 	Q 	Q 	Q 	Q 	Q 	Q 	Q 	Q 	Q 	Q 	Q 	Q 	Q 	Q 	Q 	Q 	Qs   B&CCCc                 8   t          dt          d                    5 }|                     t                    5  t          j                            d| j        i           |j        sJ 	 ddd           n# 1 swxY w Y   ddd           dS # 1 swxY w Y   dS )z(
        Test basic start call.
        zbuiltins.openz{"input":{"number":1}})	read_datar   N)	r   r   assertRaises
SystemExitr!   r"   startr   called)r   	mock_files     r   
test_startzTestWorker.test_start1   s"    Y1IJJJ
 
 	$))*55	$ 	$ ##Y0A$BCCC#####	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$s4   B0A7+B7A;	;B>A;	?BBBc                 P   t          d          5 }d|j        j        _        t          j        j                            | j                  du sJ d|j        j        _        t          j        j                            | j                  du sJ 	 ddd           dS # 1 swxY w Y   dS )z(
        Test _is_local_testing
        r   NTr   Fr   r%   s     r   test_is_local_testingz TestWorker.test_is_local_testing=   s     011 	QW/3GO,$+55d6FGG4OOOO/:GO,$+55d6FGG5PPPPP		Q 	Q 	Q 	Q 	Q 	Q 	Q 	Q 	Q 	Q 	Q 	Q 	Q 	Q 	Q 	Q 	Q 	Qs   A=BB"Bc                    t          j                    }d|_        d|_        d|_        d|_        d|_        d|_        d|_        t          d          5 }t          d          5 }|g f|_
        t          j                            d	| j        i           |j        sJ 	 ddd           n# 1 swxY w Y   ddd           dS # 1 swxY w Y   dS )
z+
        Test local FastAPI setup.
        NT@     	localhost{"test": "test"}(argparse.ArgumentParser.parse_known_argsz.runpod.serverless.modules.rp_fastapi.WorkerAPIr   )argparse	Namespacerp_log_levelrp_debuggerrp_serve_apirp_api_portrp_api_concurrencyrp_api_hostr   r   r   r!   r"   r,   r   r-   )r   
known_argsmock_parse_known_argsmock_worker_apis       r   test_local_apizTestWorker.test_local_apiG   sd    '))
"&
!%
"&
!%
()
%!,
 2
6
 
 		*"E<%
 %
		* 1;R!.##Y0A$BCCC")))))		* 		* 		* 		* 		* 		* 		* 		* 		* 		* 		* 		* 		* 		* 		* 		* 		* 		* 		* 		* 		* 		* 		* 		* 		* 		* 		* 		* 		* 		* 		* 		* 		*s6   C$9B*C*B.	.C1B.	2CC	Czrunpod.serverless.logzrunpod.serverless.sys.exitc                 T    t          dd           |j        sJ |j        j        sJ dS )z&
        Test signal handler.
        N)r   r-   info)r   	mock_exitmock_loggers      r   test_signal_handlerzTestWorker.test_signal_handler_   s;     	d###&&&&&&r   N)__name__
__module____qualname____doc__r   r'   r/   r1   rC   r   rH    r   r   r   r      s        --
 
 
Q Q Q"
$ 
$ 
$Q Q Q* * *0 U"##
U'((' ' )( $#' ' 'r   r   c                       e Zd ZdZd Zd ZdS )TestWorkerTestInput%Tests for runpod | serverless| workerc                 `   K   t                      | _        i | j        _        d| j        _        d S )Nr   )r   r   r   r   s    r   r   zTestWorkerTestInput.asyncSetUpo   s.       FF)+&)/&&&r   c                    t          j                    }d|_        d|_        d|_        d|_        d|_        d|_        d|_        d|_	        t          d          5 }|                     t                    5  |g f|_        t          j                            d	| j        i           t%                      }|j        dk    sJ 	 ddd           n# 1 swxY w Y   ddd           dS # 1 swxY w Y   dS )
z 
        Test sys args.
        WARNTNr3   r4   r5   r6   r7   r   )r8   r9   r:   r;   r<   r=   r>   r?   r   test_outputr   r*   r+   r   r!   r"   r,   r   r	   level)r   r@   rA   logs       r   test_worker_bad_localz)TestWorkerTestInput.test_worker_bad_localu   s~    '))
"(
!%
"&
!%
()
%!,
 2
!3
6
 
 		'"D$5$5j$A$A		' 		' 2<R!.##Y0A$BCCC ..C9&&&&&		' 		' 		' 		' 		' 		' 		' 		' 		' 		' 		' 		' 		' 		' 		' 		' 		' 		' 		' 		' 		' 		' 		' 		' 		' 		' 		' 		' 		' 		' 		' 		' 		's7   C&6ACC&C	C&C	C&&C*-C*N)rI   rJ   rK   rL   r   rW   rM   r   r   rO   rO   l   s8        //0 0 0' ' ' ' 'r   rO   c              #   8   K   t          |            dV  dV  dS ) 
    Test generator_handler
    test1test2N)printjobs    r   generator_handlerr_      s)       
#JJJ
MMM
MMMMMr   c              #   f   K   t          |            dV  t          d           t                      )rY   rZ   zRaise exception)r\   	Exceptionr]   s    r   generator_handler_exceptionrb      s5       
#JJJ
MMM	

++r   c                      ddi} t          |           }t          |          }|dk    s
J d            	 t          |           J d            # t          $ r Y d	S w xY w)
z Test generator_handler_exceptionidtest_jobrZ   zFirst output should be 'test1'Fz"Expected an exception to be raisedTz Exception was caught as expectedN)rb   nextra   )r^   genoutputs      r    test_generator_handler_exceptionri      s    
C
%c
*
*C #YYFW>8S			::::: 8 8 87778s   A 
AAc                      e Zd ZdZd Zd Z ed           ed           ed           ed           ed          d	                                                             Z ed           ed           ed           ed          d
                                                 Z ed           ed           ed           ed          d                                                 Z	 ed           ed           ed           ed          d                                                 Z
 ed           ed           ed           ed           ed          d                                                             Z ed           ed           ed           ed           ed          d                                                             Z ed           ed          d                         Z ed           ed          d                         Z ed           ed           ed          d                                     ZdS )TestRunWorkerrP   c                 \   K   dt           j        d<   t                      ddddd| _        d S )Nzhttps://test.comRUNPOD_WEBHOOK_GET_JOBTDEBUGr;   r:   r   refresh_workerr   )osr   r   configr   s    r   r   zTestRunWorker.asyncSetUp   s<      /A
+, !{{"'+WEE
 
r   c                 6   K   t           j        t           _        d S N)sys__excepthook__
excepthookr   s    r   asyncTearDownzTestRunWorker.asyncTearDown   s      +r   z5runpod.serverless.modules.rp_scale.AsyncClientSessionz*runpod.serverless.modules.rp_scale.get_jobz(runpod.serverless.modules.rp_job.run_jobz.runpod.serverless.modules.rp_job.stream_resultz,runpod.serverless.modules.rp_job.send_resultc                 0  K   dddidgg g|_         dddii|_        t          j                            | j                   |                     |j        d           |                                 |                                 |j	        rJ |j	        sJ dS )	z)Test run_worker with synchronous handler.123numberr4   rd   inputrh   resultoddN)
side_effectr   r!   r"   r,   rs   assertEqual
call_countassert_called_oncer-   )r   mock_send_resultmock_stream_resultmock_run_jobmock_get_jobmock_sessions         r   test_run_workerzTestRunWorker.test_run_worker   s        XqM223$
  &.%/@$A! 	,,, 	0!444'')))++---%,,,,""""""r   c                    K   dddidg|_         t          dd}t          j                            |           |j        sJ |j        rJ |j        d         \  }}}|d         g ddk    sJ d	S )
a  
        Test run_worker with generator handler.

        Args:
            mock_stream_result (_type_): _description_
            mock_run_job_generator (_type_): _description_
            mock_run_job (_type_): _description_
            mock_get_job (_type_): _description_
        zgenerator-123r|   r4   r}   Tr   rq   r   rh   stopPodNr   r_   r!   r"   r,   r-   
mock_callsr   r   r   r   r   generator_config_argss           r   !test_run_worker_generator_handlerz/TestRunWorker.test_run_worker_generator_handler   s      " -<xQRm%T%T$U! (9DQQ 0111!((((&&&& &03
4AwRD99999999r   c                   K   t                                          d           dddidg|_        t          j                            t          dd           |j        dk    sJ |j        rJ |j        dk    sJ |j	        D ]M}|\  }}|r2t          |          dk    rd|d         v sd	|d         v s
J d
            ;d|v sd	|v s
J d            NdS )a)  
        Test run_worker with generator handler.

        This test verifies that:
        - `stream_result` is called before an exception occurs.
        - `run_job` is never called since `handler` is a generator function.
        - An error is correctly reported back via `send_result`.
        rn   zgenerator-123-exceptionr|   r4   r}   Tr   errorr   z!Expected error or result in args.z#Expected error or result in kwargs.N)r	   	set_levelr   r!   r"   r,   rb   r   r-   call_args_listlen)r   r   r   r   r   callr   kwargss           r   +test_run_worker_generator_handler_exceptionz9TestRunWorker.test_run_worker_generator_handler_exception  s6      	  ))) -!}EE%
! 	3tLL	
 	
 	

 ",1111  &&&&  *a//// %3 	f 	fDLD& fD		A$q'))Xa-@-@-@Be-@-@-@ &((H,>,>,>@e,>,>,>	f 	fr   c                    K   dddidg|_         t          ddd}t          j                            |           |j        sJ |j        sJ |j        rJ |j        d         \  }}}|d         dd	gdd
k    sJ dS )aG  
        Test run_worker with generator handler.

        Args:
            mock_send_result (_type_): _description_
            mock_stream_result (_type_): _description_
            mock_run_job (_type_): _description_
            mock_get_job (_type_): _description_
            mock_session (_type_): _description_
        zgenerator-123-aggregater|   r4   r}   T)r   return_aggregate_streamrq   r   rZ   r[   r   Nr   r   s           r   +test_run_worker_generator_aggregate_handlerz9TestRunWorker.test_run_worker_generator_aggregate_handler.  s      & -!}EE%
! )'+"
 
 	 0111&&&&!((((&&&& &03
4Awgw%7DIIIIIIIIr   c                   K   dddidg|_         dddii|_         d }| j                                        }||d	<   t          j                            |           |                     |j        dd
|j                    |                     |j        dd|j                    |                     |j        dd|j                    |                     |j	        d           | 
                    |j	        d           |j        D ]J}|\  }	}
d|	d         v r$|                     d|	d         d         d           5|                     d           KdS )a  
        Test run_worker with synchronous handler, ensuring that concurrency behavior
        is respected and that the calls to `get_job`, `run_job`, and `send_result`
        follow expected patterns.

        Args:
            mock_send_result: Mock for send_result function
            mock_stream_result: Mock for stream_result function
            mock_run_job: Mock for run_job function
            mock_get_job: Mock for get_job function
            mock_session: Mock for AsyncClientSession
        r{   r|   r4   r}   rh   r   r   c                     | S ru   rM   )current_concurrencys    r   concurrency_modifierzGTestRunWorker.test_run_worker_concurrency.<locals>.concurrency_modifiers  s    &&r   r   z/Expected at least one call to get_job, but got z/Expected at least one call to run_job, but got z3Expected at least one call to send_result, but got z5stream_result should not be called in this test case.z3Expected the mock_session to be used at least once.z8Expected 'result' to be part of the 'output' dictionary.z@The 'output' key was not found in the arguments for send_result.N)r   rs   copyr!   r"   r,   assertGreaterEqualr   assertFalser-   
assertTruer   assertInfail)r   r   r   r   r   r   r   config_with_concurrencyr   r   r   s              r   test_run_worker_concurrencyz)TestRunWorker.test_run_worker_concurrencyU  s     4 -2XqM%J%J$K!%-%/@$A!	' 	' 	' #'+"2"2"4"4:N 67 	 7888 	#QWl>UWW	
 	
 	

 	#QWl>UWW	
 	
 	

 	'_BRB]__	
 	
 	

 	%C	
 	
 	

 	A	
 	
 	
 %3 		^ 		^DLD&47""d1gh/N   
 		\]]]]		^ 		^r   c           	      H  K   dddidg|_         dddii|_         t          j                            | j                   |                     |j        dd           |                     |j        dd	           |                     |j        dd
           |                     |j        d           | 	                    |j        d           t          dd}t          j                            |           | 	                    |j        d           t          d          5 }t                      dddddddd}||_         t          j                            |           | 	                    |j        d           t          |j                   ddd           dS # 1 swxY w Y   dS )zf
        Test run_worker with multi-processing enabled for both async and generator handlers.
        r{   r|   r4   r}   rh   r   r   z&Expected at least one call to get_job.z&Expected at least one call to run_job.z*Expected at least one call to send_result.z(Expected stream_result to not be called.z#Expected mock session to be called.Tr   z>Expected stream_result to be called for the generator handler."runpod.serverless._set_config_argsNr3   r5   )r;   r<   r=   r>   r?   rp   z'Expected _set_config_args to be called.)r   r!   r"   r,   rs   r   r   r   r-   r   r_   r   r   r\   r   )	r   r   r   r   r   r   r   mock_set_config_argslimited_configs	            r    test_run_worker_multi_processingz.TestRunWorker.test_run_worker_multi_processing  s     $ -2XqM%J%J$K!%-%/@$A! 	,,, 	 7<deee 7<deee 0 ;Q@lmmm 	+24^___ 	+-RSSS (9DQQ 0111 	*13sttt 788 	7<P66"&#'$(#'*+#. 
 
N 1? -##N333 OO079bccc&5666%	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7s   A.FFFc                   K   dddidg|_         dddii|_         g dd	d
t                      ddddd}fd}t          d|          5  t          j                            |           ddd           dS # 1 swxY w Y   dS )a  
        Test run_worker with multi processing enabled, the scale-up and scale-down
        behavior with concurrency_controller.

        Args:
            mock_send_result (_type_): _description_
            mock_stream_result (_type_): _description_
            mock_run_job (_type_): _description_
            mock_get_job (_type_): _description_
            mock_session (_type_): _description_
        r{   r|   r4   r}   rh   r   r   )FFFFFFTTTTTr   )behaviorcounterFTrn   ro   rp   c                  @    d         dk     } dxx         dz  cc<   | S )Nr   
   r4   rM   )resscale_behaviors    r   mock_is_alivezPTestRunWorker.test_run_worker_multi_processing_scaling_up.<locals>.mock_is_alive  s3     +b0C9%%%*%%%Jr   z5runpod.serverless.modules.rp_scale.JobScaler.is_alive)wrapsN)r   r   r   r!   r"   r,   )r   r   r   rs   r   r   s        @r   +test_run_worker_multi_processing_scaling_upz9TestRunWorker.test_run_worker_multi_processing_scaling_up  s'     " -2XqM%J%J$K!%-%/@$A!   
 
& !{{#'+WEE
 
	 	 	 	 	
 C=
 
 
 	, 	, ##F+++	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	,s    A;;A?A?zrunpod.serverless.signal.signalz/runpod.serverless.worker.rp_scale.JobScaler.runc                 r    t           j                            i            t          j        t
          k    sJ d S ru   )r!   r"   r,   rv   rx   r
   )r   r   __s      r   test_start_sets_excepthookz(TestRunWorker.test_start_sets_excepthook  s5     	###~!;;;;;;;r   z<runpod.serverless.modules.rp_fastapi.WorkerAPI.start_uvicornr   c                     | j         |_        | j                             ddddddi           t          j                            | j                    t          j        t          k    sJ d S )Nr   Tr5   r3   r4   )r<   r?   r=   r>   )	rs   r   updater!   r"   r,   rv   rx   r
   )r   r   r   r   s       r   "test_start_does_not_set_excepthookz0TestRunWorker.test_start_does_not_set_excepthook"  s{     -1K)I &"#	(
 (
  	 	 	 	,,,~!;;;;;;;r   N)rI   rJ   rK   rL   r   ry   r   r   r   r   r   r   r   r   r   r   rM   r   r   rk   rk      s       //
 
 
, , , UBCC
U788
U566
U;<<
U9::# # ;: =< 76 98 DC
#6 U788
U566
U;<<
U9::: : ;: =< 76 98:4 U788
U566
U;<<
U9::'f 'f ;: =< 76 98'fR U788
U566
U;<<
U9::!J !J ;: =< 76 98!JF UBCC
U788
U566
U;<<
U9::F^ F^ ;: =< 76 98 DC
F^P UBCC
U788
U566
U;<<
U9::87 87 ;: =< 76 98 DC
87t U788
U5667, 7, 76 987,r U,--
U<==< < >= .-< U,--
UIJJ
U/00
< 
< 10 KJ .-
< 
< 
<r   rk   )rL   r8   rr   rv   unittestr   unittest.mockr   r   r   r   r   nest_asyncior!   #runpod.serverless.modules.rp_loggerr	   "runpod.serverless.modules.rp_scaler
   runpod.serverlessr   applyr   rO   r_   rb   ri   rk   rM   r   r   <module>r      s   - -  				 



       ; ; ; ; ; ; ; ; ; ; ; ; , , , , , ,      < < < < < < I I I I I I - - - - - -     S' S' S' S' S'( S' S' S'l '  '  '  '  '1  '  '  'F    8 8 8"|< |< |< |< |<+ |< |< |< |< |<r   