
    tni+                         d Z ddlZddlZddlZddlZddlmZ ddlmZm	Z	m
Z
 ddlmZ ddlZddlmZ ddlmZ dd	iZd
edefdZdede
eef         fdZdede	eee         f         dee         fdZdedefdZdS )z
PodWorker | modules | download.py

Called when inputs are images or zip files.
Downloads them into a temporary directory called "input_objects".
This directory is cleaned up after the job is complete.
    N)ThreadPoolExecutor)ListUnionDict)urlparse)RequestException)SyncClientSessionz
User-Agentz:runpod-python/0.0.0 (https://runpod.io; support@runpod.io)	file_sizereturnc                 &    | dk    rdS | dk    rdS dS )z;
    Calculates the chunk size based on the file size.
    i   i   i   @i    )r
   s    `/var/www/html/gpu-tools/venv/lib/python3.11/site-packages/runpod/serverless/utils/rp_download.pycalculate_chunk_sizer      s+     Kt&&&{    content_dispositionc                 \    d |                      d          D             }d |D             }|S )Nc              3   >   K   | ]}|                                 V  d S )N)strip).0ps     r   	<genexpr>z-extract_disposition_params.<locals>.<genexpr>&   s*      ??1QWWYY??????r   ;c                     i | ]k}d |v |                     d d          \  }}|                                                                |                                                    d          lS )=   ")splitr   lower)r   partkeyvalues       r   
<dictcomp>z.extract_disposition_params.<locals>.<dictcomp>(   sl       $;;::c1--JC 			U[[]]0055;;r   )r   )r   partsparamss      r   extract_disposition_paramsr%   %   sI    ?? 3 9 9# > >???E   F Mr   job_idurlsc                 (   t           j                            t           j                            d| d                    t          j        d           t          j        t
          j        t          d          dt          dt          d	t          fd
            dt          d	t          ffd}t          |t                    r|g}t                      5 }t          |                    ||                    }ddd           n# 1 swxY w Y   |S )z
    Accepts a single URL or a list of URLs and downloads the files.
    Returns the list of downloaded file absolute paths.
    Saves the files in a directory called "downloaded_files" in the job directory.
    jobsdownloaded_filesTexist_ok   )	max_triesurlpath_to_saver   c                    t                                          | t          dd          5 }|                                 |j                            d          }d}|rHt          |          }t          j                            |                    dd                    d         }|s7t          j                            t          |           j                  d         }t          |j                            dd	                    }t          |          }t          ||z   d
          5 }|                    |          D ]}	|	r|                    |	           	 d d d            n# 1 swxY w Y   |cd d d            S # 1 swxY w Y   d S )NT   )headersstreamtimeoutContent-Disposition filenamer   zContent-Lengthr   wb)
chunk_size)r	   getHEADERSraise_for_statusr3   r%   ospathsplitextr   intr   openiter_contentwrite)
r/   r0   responser   file_extensionr$   r
   r:   	file_pathchunks
             r   download_filez/download_files_from_urls.<locals>.download_file;   s      $$S'$PQ$RR 	"V^%%'''"*"2"6"67L"M"MN" Q34GHH!#!1!1&**Z2L2L!M!Ma!P " I!#!1!1(3--2D!E!Ea!HH,001A1EEFFI-i88J l^3T:: /i%22j2II / /E /!...// / / / / / / / / / / / / / /
 "+	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	"s6   C>F )1E'F 'E+	+F .E+	/F  FFc                 0   | d S t          j                     }t          j                            |          }	  | |          }n-# t
          $ r }t          d|  d|            Y d }~d S d }~ww xY wt          j                            | |           S )NzFailed to download z: )uuiduuid4r>   r?   joinr   printabspath)r/   	file_nameoutput_file_pathrF   errdownload_directoryrI   s        r   download_file_to_pathz7download_files_from_urls.<locals>.download_file_to_pathT   s    ;4z||%	7<<(:IFF	*]30@AANN 	 	 	444s4455544444	 w"2DNDDEEEs   A 
A2A--A2N)r>   r?   rO   rM   makedirsbackoffon_exceptionexpor   str
isinstancer   listmap)r&   r'   rT   executorr*   rS   rI   s        @@r   download_files_from_urlsr^   2   su    ffFX)Y)YZZK"T2222',(8AFFF"3 "c "c " " " GF"0F3 F3 F F F F F F F $ v			 K-BD I IJJK K K K K K K K K K K K K K K s   $DDDfile_urlc                 `   t          j        dd           t                                          | t          d          }|j                            d          }d}|r%t          |          }|                    dd          }|s3t          |           j        }t           j        	                    |          }t           j        
                    |          d	                             d
d          }t          j                     }t           j                            d| d
|           }t          |d          5 }	|	                    |j                   ddd           n# 1 swxY w Y   |dk    rt           j                            d|          }
t          j        |
d           t%          j        |d          5 }|                    |
           ddd           n# 1 swxY w Y   t           j                            |
          }
nd}
t           j                            |          |||
dS )a  
    Downloads a single file from a given URL, file is given a random name.
    First checks if the content-disposition header is set, if so, uses the file name from there.
    If the file is a zip file, it is extracted into a directory with the same name.

    Returns an object that contains:
    - The absolute path to the downloaded file
    - File type
    - Original file name
    	job_filesTr+      )r3   r5   r6   r7   r8   r   .r9   Nzipr)rG   typeoriginal_nameextracted_path)r>   rU   r	   r;   r<   r3   r%   r   r?   basenamer@   replacerK   rL   rM   rB   rD   contentzipfileZipFile
extractallrO   )r_   download_responser   original_file_namer$   download_path	file_typerP   rQ   output_fileunzipped_directoryzip_refs               r   filerv   l   s    Kd++++)++//'SU/VV+3778MNN 8+,?@@#ZZ
B77 = **/W--m<<  !344Q7??RHHI:<<!Iw||KI1K1K	1K1KLL		%	% 5+34445 5 5 5 5 5 5 5 5 5 5 5 5 5 5 EW\\+yAA
&6666_-s33 	3w1222	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3W__-?@@! W__%566+,	  s$   :E!!E%(E%=GG#&G#)__doc__r>   rerK   rl   concurrent.futuresr   typingr   r   r   urllib.parser   rV   requestsr   runpod.http_clientr	   r<   rA   r   rY   r%   r^   dictrv   r   r   r   <module>r      sb    
			 				   1 1 1 1 1 1 $ $ $ $ $ $ $ $ $ $ ! ! ! ! ! !  % % % % % % 0 0 0 0 0 0U
V	C 	C 	 	 	 	
C 
DcN 
 
 
 
7S 7c49n0E 7$s) 7 7 7 7t13 14 1 1 1 1 1 1r   