IriddlZddlZddlZddlZddlZddlmZddlmZddlm Z ddl m Z ddl m Z ddlmZejejeZejedgZdad Zejd ZGd d eZd ZddddddZddddddZdZdZ dZ!dZ"GddeZ#Gdde Z$dS)N)Path)knobs) GPUTarget) GPUDriver) _allocation)compile_module_from_srcincludec: ddl}|dkrdSddl ddlm}m}m}m m m}G fdd j } |||||||} dj }n#t$rYdSwxYw| g|_||_d dz} fd } ||| | r't#j | SdS) NrLinux)c_charc_intc_size_tc_void_pc_char_pPOINTERc eZdZdfdfgZdS)8_find_already_mmapped_dylib_on_linux..DlPhdrInfo dlpi_addr dlpi_nameN)__name__ __module__ __qualname___fields_)rrsr/var/www/html/bestrading.cuttalo.com/models/btc_v9/venv/lib/python3.11/site-packages/triton/backends/amd/driver.py DlPhdrInfors& ( # ( # rz libc.so.6ic |jj}ttj|}|jvr4||tt|dSdS)Nrr) contentsrrosfsdecodenamememmoveminlen)infosizedatarpctypeslib_namemax_path_lengths rcallbackz6_find_already_mmapped_dylib_on_linux..callback4saM+ Y'' ( ( qv   NN4CY,P,P Q Q Q1qr)platformsystemr*r r rrrr Structure CFUNCTYPECDLLdl_iterate_phdr Exceptionargtypesrestypecreate_string_bufferr r! string_at)r+r.r r rrr callback_tr3pathr-rrr*r,s` @@@@r$_find_already_mmapped_dylib_on_linuxr;sOOOG##t MMMKKKKKKKKKKKKKKKK        V%   !!%)<)O>OQXQXY_Q`Q`aaJ ++k22B tt!+H5O#OO  & &': ; ;Dzz(++T223{6++D11222 4sB B+*B+c~ dtjjx}rK|r!tj|r|Std|dt}|r6tj|r|Std|dg}tj tj td}tj|r|S| |ddl }|}|}|jr|g|z}|D]\}tj |dd}tj|r|cS| |]t jd }|rp|d D]Z} tj | } tj| r| cS| | [t jd } | rWtj | d} tj| r| S| |  t'jd d g} | rWtj | d} tj| r| S| | n#t&jt0f$rYnwxYwt jd}|rWtj |d}tj|r|S| |t'jddgd}fd|D}|D]:}tj|r|cS| |;tj d}tj|r|S| |tdd|)Nzlibamdhip64.sozTRITON_LIBHIP_PATH 'z' does not point to a valid zmemory mapped 'z'' in process does not point to a valid librtorchLD_LIBRARY_PATH:HIP_PATH hipconfigz--path ROCM_PATHz/sbin/ldconfigz-pignore)errorscg|]C}|)|dDS))stripendswithsplit).0liner+s r z2_get_path_to_hip_runtime_dylib..sB ^ ^ ^djjll>S>ST\>]>] ^DJJLL  ^ ^ ^rz/opt/rocm/lib/zcannot locate z after attempted paths )ramd libhip_pathrIr r:exists RuntimeErrorr;joindirname__file__appendsitegetsitepackagesgetusersitepackagesENABLE_USER_SITEgetenvrJ subprocess check_outputdecoderHCalledProcessErrorFileNotFoundError splitlines)env_libhip_path mmapped_pathpaths local_librV site_packages user_siter:env_ld_library_pathdf env_hip_path hip_lib_pathhip_root env_rocm_path rocm_lib_pathlibslocsloccommon_install_pathr+s @r_get_path_to_hip_runtime_dylibrsBsH )//k  # #H - - #"'..2Q2Q #" "i/ii_giijjj8AALn 7>>, ' '  l\llbjllmmm E RW__X66xHHI w~~i   LLKKK((**M((**I 4" m3 w||D'5(;; 7>>$   KKK T)$566$**3//  A Q))Aw~~a    LLOOOO9Z((L#w||L%BB 7>>, ' '   \"""  *K+BCCJJLLRRTT  '7<<%BBLw~~l++ $## LL & & &  )+< =     Ik**M$ ]E8DD 7>>- ( ( !  ]###  "$4d#; < < C C8 C T TD _ ^ ^ ^):): ^ ^ ^D 7>>#   JJJ S',,'7BB w~~)**#"" LL$%%% PPPPP Q QQs0A=M.MMMc$eZdZfdZdZxZS)HIPUtilsct|ds-tt|||_|jS)Ninstance)hasattrsuperru__new__rw)cls __class__s rrzzHIPUtils.__new__s<sJ'' = 3//77<"%(%>"-r)rrrrzr __classcell__r|s@rrurusG . . . . . . .rrucp|drdS|dkrdSdddddd d d d d d d d d d d|S)N*hipDeviceptr_t tensordesc TDMDescriptorint8_tint16_tint32_tint64_tuint8_tuint16_tuint32_tuint64_tdouble)i1i8i16i32i64u1u8u16u32u64fp16bf16fp32f32fp64 startswith)tys r ty_to_cpprsr }}S  \    !  rrrr)rrrrr pack_fp16 pack_bf16 pack_fp32 pack_fp64 piiiKKOOOOOcfd}fdfdfddt||D}dfd|D}t|z}dt |}t t t|d}d t|D}t|d kr4d d d | Dznd}g}| D]b\} } | d kr | tvr&| t| d| ;| t| d| cd |} g} | D]\} } | dr| d| d4| dr| d| b| tvr| d| d| d kr| d| d} d| D}d| D}d| D}t}t t!t|}d| D}| d| dd|dt| d krd | znddd |d |d!|d"|d#d$fd%| Dd&|d'|d(| |d| |d| |d)t| d krd d | zndd*}|S)+Ncg}d}|D]k}t|tr=|dr' r |nd}|dz }tjd|}|d}|d}|ddz}|X|d|ztd|zD]} |d|d n|dt|D]} |d t|D]} |dV||m|S) Nrrrztensordesc<([^[>]*)\[([^]]*)\],rrrr) isinstancestrrrematchgroupcountrUrange) signatureoutputtensordesc_idxsigmetardtypeshapendim_tensordesc_metas r_expand_signaturez(make_launcher.._expand_signatures # #C#s## #|(D(D #:IS~66t!#!CSII A A{{3''!+<MM#+..."1t8__-- e,,,,MM$''''MM,///t))AMM%((((t))AMM%(((() c"""" rcxt|tr#dt|S|S)Nr)rtuplerRmap)r_serialize_signatures rrz+make_launcher.._serialize_signatures7 c5 ! ! <88C 4c::;; ; rct|tr)dt|}d|dS|ds|drdS|dkrdSt |S)Nr[]rrz PyObject* constexprrrrRrrr)rval_extracted_types rrz&make_launcher.._extracted_type s b%  ((33344Cs:::  ==   |!.format_ofs b%  ''#i,,--Cs:::  ==   |!z!make_launcher..(sWWWFCaWWWrrc&g|] }|Srr)rKrrs rrMz!make_launcher..*s!FFFR99R==FFFrrci|]\}}|| Srr)rKrrs rrz!make_launcher...s777$!QA777rr, c3&K|] \}}d|V dS)z&_argNrrKrrs r z make_launcher../s, L LB L L L L L Lrrz argrptr_infoz.dev_ptrrz*desc_arg_storagez c \g|])\}}|dd|d|d|d|d *S)rzDevicePtrInfo ptr_infoz = getPointer(_argrz); if (!ptr_infoz.valid) return NULL;rrs rrMz!make_launcher..Gsa Ar ==  eeeaee1eeaeeerc Vg|]&\}}|dd|d|d|d'S)rzTDMDescriptor* descz = getTDMDescriptor(_argr);rrs rrMz!make_launcher..LsYHM2 == & &CaCCCCaCCCrc ng|]2\}}|tvt|d|dt|d|d3S) _argz _storage = z(_argr)FLOAT_STORAGE_TYPEFLOAT_PACK_FUNCTIONrs rrMz!make_launcher..Ps_ Ar # # # b !YYYY6I"6MYYTUYYY # # #rc*g|]\}}|dk d|S)rz&argrrs rrMz!make_launcher..Zs, M M MUQ2;L;LjQjj;L;L;Lrz&global_scratchz&profile_scratcha #define __HIP_PLATFORM_AMD__ #include #include #include #include #include #include typedef struct { uint32_t group0_0; uint32_t group0_1; uint32_t group0_2; uint32_t group0_3; uint32_t group1_0; uint32_t group1_1; uint32_t group1_2; uint32_t group1_3; uint32_t group1_4; uint32_t group1_5; uint32_t group1_6; uint32_t group1_7; } TDMDescriptor; typedef struct { PyObject_HEAD; TDMDescriptor desc; } PyTDMDescriptorObject; // The list of paths to search for the HIP runtime library. The caller Python // code should substitute the search path placeholder. static const char *hipLibSearchPaths[] = {"a2"}; // The list of HIP dynamic library symbols and their signature we are interested // in this file. #define HIP_SYMBOL_LIST(FOR_EACH_ERR_FN, FOR_EACH_STR_FN) \ FOR_EACH_STR_FN(hipGetLastError, true) \ FOR_EACH_STR_FN(hipGetErrorString, true, hipError_t hipError) \ FOR_EACH_ERR_FN(hipDrvLaunchKernelEx, false, \ const HIP_LAUNCH_CONFIG *config, \ hipFunction_t f, \ void **kernelParams, \ void **extra) \ FOR_EACH_ERR_FN(hipModuleLaunchKernel, true, hipFunction_t f, \ unsigned int gridDimX, unsigned int gridDimY, \ unsigned int gridDimZ, unsigned int blockDimX, \ unsigned int blockDimY, unsigned int blockDimZ, \ unsigned int sharedMemBytes, hipStream_t stream, \ void **kernelParams, void **extra) \ FOR_EACH_ERR_FN(hipModuleLaunchCooperativeKernel, true, hipFunction_t f, \ unsigned int gridDimX, unsigned int gridDimY, \ unsigned int gridDimZ, unsigned int blockDimX, \ unsigned int blockDimY, unsigned int blockDimZ, \ unsigned int sharedMemBytes, hipStream_t stream, \ void **kernelParams, void **extra) \ FOR_EACH_ERR_FN(hipPointerGetAttribute, true, void *data, \ hipPointer_attribute attribute, hipDeviceptr_t ptr) // The HIP symbol table for holding resolved dynamic library symbols. struct HIPSymbolTable { #define DEFINE_EACH_ERR_FIELD(hipSymbolName, required, ...) \ hipError_t (*hipSymbolName)(__VA_ARGS__); #define DEFINE_EACH_STR_FIELD(hipSymbolName, required, ...) \ const char *(*hipSymbolName)(__VA_ARGS__); HIP_SYMBOL_LIST(DEFINE_EACH_ERR_FIELD, DEFINE_EACH_STR_FIELD) }; static struct HIPSymbolTable hipSymbolTable; bool initSymbolTable() { // Use the HIP runtime library loaded into the existing process if it exits. void *lib = dlopen("libamdhip64.so", RTLD_NOLOAD); // Otherwise, go through the list of search paths to dlopen the first HIP // driver library. if (!lib) { int n = sizeof(hipLibSearchPaths) / sizeof(hipLibSearchPaths[0]); for (int i = 0; i < n; ++i) { void *handle = dlopen(hipLibSearchPaths[i], RTLD_LAZY | RTLD_LOCAL); if (handle) { lib = handle; } } } if (!lib) { PyErr_SetString(PyExc_RuntimeError, "cannot open libamdhip64.so"); return false; } typedef hipError_t (*hipGetProcAddress_fn)( const char *symbol, void **pfn, int hipVersion, uint64_t hipFlags, hipDriverProcAddressQueryResult *symbolStatus); hipGetProcAddress_fn hipGetProcAddress; dlerror(); // Clear existing errors const char *error = NULL; *(void **)&hipGetProcAddress = dlsym(lib, "hipGetProcAddress"); error = dlerror(); if (error) { PyErr_SetString(PyExc_RuntimeError, "cannot query 'hipGetProcAddress' from libamdhip64.so"); dlclose(lib); return false; } // Resolve all symbols we are interested in. int hipVersion = HIP_VERSION; uint64_t hipFlags = 0; hipDriverProcAddressQueryResult symbolStatus; hipError_t status = hipSuccess; #define QUERY_EACH_FN(hipSymbolName, required, ...) status = hipGetProcAddress(#hipSymbolName, (void **)&hipSymbolTable.hipSymbolName, hipVersion, hipFlags, &symbolStatus); if (required && status != hipSuccess) { PyErr_SetString(PyExc_RuntimeError, "cannot get address for '" #hipSymbolName "' from libamdhip64.so"); dlclose(lib); return false; } HIP_SYMBOL_LIST(QUERY_EACH_FN, QUERY_EACH_FN) return true; } static inline void gpuAssert(hipError_t code, const char *file, int line) { if (code != HIP_SUCCESS) { const char* prefix = "Triton Error [HIP]: "; const char* str = hipSymbolTable.hipGetErrorString(code); char err[1024] = {0}; snprintf(err, 1024, "%s Code: %d, Messsage: %s", prefix, code, str ); PyErr_SetString(PyExc_RuntimeError, err); } } #define HIP_CHECK(ans) { gpuAssert((ans), __FILE__, __LINE__); } static void _launch(int gridX, int gridY, int gridZ, int num_warps, int num_ctas, int launch_cooperative_grid, int shared_memory, hipStream_t stream, hipFunction_t function, hipDeviceptr_t profile_scratchzl) { if (gridX * gridY * gridZ == 0) return; hipDeviceptr_t global_scratch = 0; void *params[] = { a }; if(num_ctas > 1) { if (!hipSymbolTable.hipDrvLaunchKernelEx) { PyErr_SetString(PyExc_RuntimeError, "missing hipDrvLaunchKernelEx symbol; please update HIP runtime"); return; } hipLaunchAttribute attributes[2]; // Attribute0: Cluster dimensions attributes[0].id = 4; int *cluster_dims = (int*)attributes[0].val.pad; cluster_dims[0] = num_ctas; cluster_dims[1] = 1; cluster_dims[2] = 1; // Attribute1: Cooperative launch attributes[1].id = hipLaunchAttributeCooperative; attributes[1].val.cooperative = launch_cooperative_grid; HIP_LAUNCH_CONFIG config = { gridX * num_ctas, gridY, gridZ, // Grid size a[ * num_warps, 1, 1, // Block size shared_memory, stream, attributes, 2 // Number of attributes }; HIP_CHECK(hipSymbolTable.hipDrvLaunchKernelEx(&config, function, params, 0)); return; } else if (launch_cooperative_grid) { HIP_CHECK(hipSymbolTable.hipModuleLaunchCooperativeKernel(function, gridX, gridY, gridZ, z*num_warps, 1, 1, shared_memory, stream, params, 0)); return; } else { HIP_CHECK(hipSymbolTable.hipModuleLaunchKernel(function, gridX, gridY, gridZ, a *num_warps, 1, 1, shared_memory, stream, params, 0)); } } typedef struct _DevicePtrInfo { hipDeviceptr_t dev_ptr; bool valid; } DevicePtrInfo; static PyObject* data_ptr_str = NULL; static PyObject* py_tdm_descriptor_type = NULL; static inline DevicePtrInfo getPointer(PyObject *obj, int idx) { DevicePtrInfo ptr_info; hipError_t status = hipSuccess; ptr_info.dev_ptr = 0; ptr_info.valid = true; if (PyLong_Check(obj)) { ptr_info.dev_ptr = (hipDeviceptr_t)PyLong_AsUnsignedLongLong(obj); return ptr_info; } if (obj == Py_None) { // valid nullptr return ptr_info; } PyObject *ret = PyObject_CallMethodNoArgs(obj, data_ptr_str); if (!ret) { PyErr_SetString(PyExc_TypeError, "Pointer argument must be either uint64 or have data_ptr method"); ptr_info.valid = false; goto cleanup; } if (!PyLong_Check(ret)) { PyErr_SetString(PyExc_TypeError, "data_ptr method of Pointer object must return 64-bit int"); ptr_info.valid = false; goto cleanup; } ptr_info.dev_ptr = (hipDeviceptr_t)PyLong_AsUnsignedLongLong(ret); if (!ptr_info.dev_ptr) goto cleanup; uint64_t dev_ptr; status = hipSymbolTable.hipPointerGetAttribute(&dev_ptr, HIP_POINTER_ATTRIBUTE_DEVICE_POINTER, ptr_info.dev_ptr); if (status == hipErrorInvalidValue) { PyErr_Format(PyExc_ValueError, "Pointer argument (at %d) cannot be accessed from Triton (cpu tensor?)", idx); ptr_info.valid = false; // Clear and ignore HIP error (void)hipSymbolTable.hipGetLastError(); } ptr_info.dev_ptr = (hipDeviceptr_t)dev_ptr; cleanup: Py_DECREF(ret); return ptr_info; } static inline TDMDescriptor* getTDMDescriptor(PyObject* obj, int idx) { if (Py_TYPE(obj) != (PyTypeObject*)py_tdm_descriptor_type) { PyErr_Format(PyExc_TypeError, "object must be of type PyTDMDescriptor, got %s", Py_TYPE(obj)->tp_name); return NULL; } TDMDescriptor* desc = &((PyTDMDescriptorObject*)obj)->desc; return desc; } static uint16_t pack_fp16(double f) { uint16_t result; // from https://github.com/python/pythoncapi-compat/blob/5e317108f872c904eb726cb8d560dcadbdf88a72/pythoncapi_compat.h#L482-L492 #if 0x030600B1 <= PY_VERSION_HEX && PY_VERSION_HEX <= 0x030B00A1 && !defined(PYPY_VERSION) _PyFloat_Pack2(f, (unsigned char*)&result, 1); #else PyFloat_Pack2(f, (char*)&result, 1); #endif return result; } static uint16_t pack_bf16(double f) { float f32 = (float)f; uint32_t u32 = *(uint32_t*)&f32; return (uint16_t)(u32 >> 16); } static uint32_t pack_fp32(double f) { float f32 = (float)f; return *(uint32_t*)&f32; } static uint64_t pack_fp64(double f) { return *(uint64_t*)&f; } static PyObject* launch(PyObject* self, PyObject* args) { int gridX, gridY, gridZ; uint64_t _stream; uint64_t _function; int launch_cooperative_grid; PyObject *profile_scratch_obj = NULL; PyObject *launch_enter_hook = NULL; PyObject *launch_exit_hook = NULL; PyObject *kernel_metadata = NULL; PyObject *launch_metadata = NULL;  c8g|]\}}|d|dS)rz; r)rKrrrs rrMz!make_launcher..rs8 O O OEAr## / /! / / / O O Orz if(!PyArg_ParseTuple(args, "a,", &launch_cooperative_grid, &gridX, &gridY, &gridZ, &_stream, &_function, &profile_scratch_obj, &kernel_metadata, &launch_metadata, &launch_enter_hook, &launch_exit_hook a)) { return NULL; } // extract kernel metadata int num_warps, num_ctas, shared_memory; if (!PyArg_ParseTuple(kernel_metadata, "iii", &num_warps, &num_ctas, &shared_memory)) { return NULL; } // extract launch metadata if (launch_enter_hook != Py_None){ PyObject* ret = PyObject_CallOneArg(launch_enter_hook, launch_metadata); if (!ret) return NULL; Py_DECREF(ret); } hipDeviceptr_t profile_scratch = 0; if (profile_scratch_obj != Py_None) { DevicePtrInfo profile_scratch_info = getPointer(profile_scratch_obj, -1); if (!profile_scratch_info.valid) { return NULL; } profile_scratch = profile_scratch_info.dev_ptr; } // raise exception asap z _launch(gridX, gridY, gridZ, num_warps, num_ctas, launch_cooperative_grid, shared_memory, (hipStream_t)_stream, (hipFunction_t)_function, (hipDeviceptr_t)profile_scratcha); if(launch_exit_hook != Py_None){ PyObject* ret = PyObject_CallOneArg(launch_exit_hook, launch_metadata); if (!ret) return NULL; Py_DECREF(ret); } if(PyErr_Occurred()) { return NULL; } Py_RETURN_NONE; } static PyMethodDef ModuleMethods[] = { {"launch", launch, METH_VARARGS, "Entry point for all kernels with this signature"}, {NULL, NULL, 0, NULL} // sentinel }; static struct PyModuleDef ModuleDef = { PyModuleDef_HEAD_INIT, "__triton_launcher", NULL, //documentation -1, //size ModuleMethods }; PyMODINIT_FUNC PyInit___triton_launcher(void) { if (!initSymbolTable()) { return NULL; } PyObject *m = PyModule_Create(&ModuleDef); if(m == NULL) { return NULL; } data_ptr_str = PyUnicode_InternFromString("data_ptr"); if(data_ptr_str == NULL) { return NULL; } PyObject* driver_mod = PyImport_ImportModule("triton.backends.amd.driver"); if (driver_mod == NULL) { return NULL; } py_tdm_descriptor_type = PyObject_GetAttrString(driver_mod, "PyTDMDescriptor"); if (py_tdm_descriptor_type == NULL) { return NULL; } PyModule_AddFunctions(m, ModuleMethods); return m; } ) enumeratevaluesrR_BASE_ARGS_FORMATrlistfilterboolrJr%itemsrrUrrrsr) constantsr warp_sizerr args_formatformat args_list arg_decl_listrr arg_declsinternal_args_listnewline ptr_declstensor_desc_declsfloat_storage_declsrOparamsrrrrs ` @@@r make_launcherrs-> *XWi0A0A)BRBRBTBT0U0U&V&VWWWI''FFFF93C3C3E3EFFFGGK  ,F193C3C3E3EFFGGIVD)//#"6"67788I77)I"6"6777IPST]P^P^abPbPbtyy L L)//:K:K L L LLLLLhjIM""<<2     # # #  $6r$:!C!C!C!C D D D D  IbMM!:!:q!:!: ; ; ; ; -((I""222 ==   2  % %&<&<&<&< = = = = ]]< ( ( 2  % %kakk 2 2 2 2 % % %  % %&8Q&8&8&8 9 9 9 9 ;    % %jQjj 1 1 1G__&&I QZQ`Q`QbQb__&& 122K%I'' ( (F M Mioo&7&7 M M MF MM#$$$ MM$%%%k >.9?k k ZbefobpbpstbtbtNRU^N^N^z|[k k byy((ck k J Kk k Z_h[k k bT]ck k j88 O O O OY__=N=N O O OPPkk k l!'mk k rS\sk k h  <<!""i k k j  << k k k l  <<#$$m k k n UXYkUlUlopUpUpmqtxt}t}~PtQtQmQmQvxo k k k CX Jrc L|/|jg|j|j|jdk|j|jS|j}|j}|j}d|vrd|vsJ|d}|d}d\}} |dg} | r9t | dkrt | dd ksJ| d\}} |d} tjj j } t| tsJ| j ||| || |||} | g||S) a Translate a tensor descriptor argument into the appropriate list of kernel arguments. If `tensordesc_metadata` is provided, we will create a TDMDescriptor object. Otherwise, we decompose the tensor descriptor into base pointer, shape, strides, and padding flag. In both cases, we append the shape and strides at the end to match the expected kernel signature. Nnan elem_bits block_size)rrinterval_padding_pairsrrr)baserstridespaddingdata_ptrgetr%tritonruntimedriveractiver HIPDriverutilsr)argkernel_metadatatensordesc_metadatarrrrr pad_interval pad_amountr num_warpsrdescs rmake_tensordesc_argr(sk"c39cs{cCK54Hc39cWZWbcc IEkG 8    D - - -,BU2U2U2U2U#K0I$\2J#L*0445MrRR=)**a//C8Nq8Q4R4RVW4W4W4W4W#9!#< j"I ^ " )F fi ( (((( < - -iYP\^hjo.5t = =D  #5 #7 ##rcZtd|D}|sStdt|Dr"t t ksJsdgt zfd}|S)a Wrap a kernel launcher function to handle tensor descriptor arguments. Use the provided `tensordesc_metadata` to determine whether to create TDMDescriptor objects or decompose the tensor descriptors. Args: launcher (callable): The original kernel launcher function. signature (Dict[int, str]): The kernel signature mapping argument indices to types. tensordesc_metadata (List[Dict] or None): The list of tensor descriptor metadata, following the order of tensor descriptor arguments. If None, decompose tensor descriptors. Returns: launcher (callable): The wrapped kernel launcher function. c3jK|].}t|to|dV/dS)rNrrr)rKrs rrz)wrap_handle_tensordesc..s>rrX[jc22Ss~~l7S7Srrrrrrrclg|]1\}}t|t|d/|2S)rr+)rKrrs rrMz*wrap_handle_tensordesc.. s@pppvq#*S#:N:NpSVSaSabnSoSopppprNc\|dtt}|ttd}g}d}t|D]V\}}| vr8t||d |}|||dz }A||Wg||RS)Nrr)r%rrr(extendrU) args meta_argsraw_kernel_args final_argsrrr!tensordesc_argslaunchertensordesc_indicesr#s rinnerz%wrap_handle_tensordesc..inners0#/0001 s#455667 00 ' 'FAs&&&"5c9Q<6I.6Y#[#[!!/222!#!!#&&&&x00Z0000r)anyrsetrr%)r5rr#has_tensor_desc_argr7r6s` ` @rwrap_handle_tensordescr;srr_h_o_o_q_qrrrrr pp9#3#3#5#566ppprr"Yc*=&>&>#FXBYBY&Y&Y&Y&Y ?#fs+='>'>> 1 1 1 1 1 1 1 LrceZdZdZdZdS) HIPLauncherctdrjn t}fdfd|D}djD}t |dd}t |||j|tdt}t|j |||_ |j |_ |j |_ |j|_dS)Nrcrt|tr jj|fn|SN)rrfn arg_namesindex)xrs rz&HIPLauncher.__init__..$s2Z3=O=OVSV-33A6699UVrc.i|]\}}||Srr)rKrvaluearg_idxs rrz(HIPLauncher.__init__..%s'MMMZS%WWS\\5MMMrci|]\}}|| Srr)rKrrGs rrz(HIPLauncher.__init__..&sHHHJCS%HHHrr__triton_launcherr)rxrdictrrgetattrrrrrr;launchlaunch_cooperative_gridprofile_scratch_sizeprofile_scratch_align)rrmetadatarrrrrHs ` @rrzHIPLauncher.__init__"s%,S+%>%>JCMMDFF VVVVMMMM9??;L;LMMM HH#-2E2E2G2GHHH !(,=tDDIy(2DoVV%#4GVbccc,SZOTT '/'G$$,$A!%-%C"""rc fd}||j|jtj}|j|j||g|RdS)Ncp|dkr.zz}||z}|}||| SdSNr)r) r'align allocator grid_size alloc_sizealloc_fngridXgridYgridZstreams rallocate_scratchz.HIPLauncher.__call__..allocate_scratch1sJaxx!EME1 &- $==??x E6:::4r)rOrPr_profile_allocatorrMrN) rrZr[r\r]functionr0r^profile_scratchs ```` r__call__zHIPLauncher.__call__/s        +*4+DdF`+6+IKK  D0%vxYhpkopppppprN)rrrrrbrrrr=r= s: D D D q q q q qrr=cheZdZfdZdZedZdedefdZdZ dZ d Z d Z d Z xZS) rctt|_t|_dSr@)ryrrur r= launcher_cls)rr|s rrzHIPDriver.__init__As2 ZZ 'rcddl}|jSrT)r>cudarr>s rget_device_interfacezHIPDriver.get_device_interfaceFs zrc| ddl}|jo |jjduS#t $rYdSwxYw)NrF)r>rg is_availableversionhip ImportError)r>s r is_activezHIPDriver.is_activeJsU  LLL:**,,P%-2C42O P   55 s *- ;;rreturnc t|Sr@)r)rrs rmap_python_to_cpp_typez HIPDriver.map_python_to_cpp_typeRs}}rc|}|j|}tjjp|d}|d}t d|dd|S)NarchwarpSizermr@r)get_current_devicer rrr override_archrrJ)rdevicedevice_propertiesrtrs rget_current_targetzHIPDriver.get_current_targetUsi((** J<>>rcZddl}|d|S)Nrrg)r>rxrvrhs rget_active_torch_devicez!HIPDriver.get_active_torch_device\s+ ||FD$;$;$=$=>>>rcddlm}|S)Nr)do_bench)triton.testingr~)rr~s rget_benchmarkerzHIPDriver.get_benchmarkeras++++++rchddl}d}|t|dz|jdS)Nrirg)rrx)r>emptyint)rr> cache_sizes rget_empty_cache_for_benchmarkz'HIPDriver.get_empty_cache_for_benchmarkes8 ' {{3zQ//uy{PPPrc.|dSr@)zero_)rcaches r clear_cachezHIPDriver.clear_cachels r)rrrrri staticmethodrorrrrzr|rrrrrs@rrr?s((((( \?????? QQQrr)% functoolsr r[rrpathlibrrtriton.backends.compilerrtriton.backends.driverrtriton.runtimertriton.runtime.buildrr:rSrealpathrTrRrrr; lru_cachersobjectrurrrrrr(r;r=rrrrrs8  ......,,,,,,&&&&&&888888 '//"'**844 5 5 Wi001 ---`\R\R\R~.....v....   2            "fffR&$&$&$R(((Vqqqqq&qqq>..... .....r