Gii} UddlZddlmZmZmZddlmZddlmZm Z m Z m Z m Z ddl mZddlmZddlmZdd lmZdd lmZdd lmZddlZd d gZe dZedZejjZdZiZ e!eefe"d<dZ#d;de e eefge eefffdZ$e$ej%ddde&fdZ'e$ej(dd%Z?dd&dee@e@e&d'fe@e&d'fe@e&d'fe e@e&d'fffd(ZAdd&dee@e@e&d'fe@e&d'fe@e&d'fe e@e&d'fffd)ZBe$ejCd*+ddde&fd,ZDe$ejEd*+de&fd-ZFd.ZGe$ejHejIejJgddde&fd/ZKe$ejLd*+de&fd0ZMe$ejNd*+de&fd1ZOiej%e'ej(e)ej*e+ej,e-ej.e/ej3e7ej4e7ej5e7ej6e7ej8e9ej;e>ej<e>ej=e>ejHeKejIeKejJeKejCeDejEeFejLeMejNeOiZ d2ZPgd3ZQd4ZRd5ZSd6ZTd7ZUGd8d ZVGd9d:eZWdS)>N)tree_map tree_flattentree_unflatten) ModuleTracker)AnyOptionalUnionTypeVarCallable)Iterator) ParamSpec) defaultdict)TorchDispatchModeprodwrapsFlopCounterModeregister_flop_formula_T_PcHt|tjr|jS|SN) isinstancetorchTensorshape)is ^/var/www/html/bet.cuttalo.com/ml/venv/lib/python3.11/site-packages/torch/utils/flop_counter.py get_shaper!s"!U\""w H flop_registrycBtddfd }|S)N)out_valcPtt|||f\}}}|d|i|S)N out_shape)rr!)r%argskwargsr'fs r nfzshape_wrapper..nfs:"*9tVW6M"N"Nfiq$6)6v666r"rr*r+s` r shape_wrapperr-s@ 1XX777777X7 Ir"Freturnc|dtttfdtttfffd }|S)N flop_formular.cstfd}tjj|S)Nct|tjjs"t d|dt ||t vrtd|t |<dS)Nzlregister_flop_formula(targets): expected each target to be OpOverloadPacket (i.e. torch.ops.mylib.foo), got z which is of type zduplicate registrations for )rr_opsOpOverloadPacket ValueErrortyper# RuntimeError)targetr0s r registerz=register_flop_formula..register_fun..register(sfej&ABB A @@@15f@@AAA&&"#J&#J#JKKK$0M& ! ! !r")r-rutils_pytree tree_map_)r0r9get_rawtargetss` r register_funz+register_flop_formula..register_fun$sU 7(66L 1 1 1 1 1  %%h888r")r rr)r>r=r?s`` r rr#sO8BF#3R8H& r")r'c<|\}}|\}}||ksJ||zdz|zS)zCount flops for matmul.) a_shapeb_shaper'r(r)mkk2ns r mm_floprI9s5 DAq EB 7777 q519q=r"c "t||S)zCount flops for addmm.rI self_shaperCrDr'r)s r addmm_floprNDs 7G $ $$r"c Z|\}}}|\}}} ||ksJ||ksJ||z| zdz|z} | S)z"Count flops for the bmm operation.rArB) rCrDr'r)brErFb2rGrHflops r bmm_floprSIsO GAq!IBA 7777 7777 q519q=1 D Kr"c "t||S)z&Count flops for the baddbmm operation.rSrLs r baddbmm_floprVVs GW % %%r"c  "t||S)zCount flops for _scaled_mm.rK) rCrD scale_a_shape scale_b_shape bias_shapescale_result_shape out_dtypeuse_fast_accumr'r)s r _scaled_mm_flopr^]s 7G $ $$r"x_shapew_shaper' transposedc|d}|r|n|dd}|^}}} t|t|z|z|z|zdz} | S)aCount flops for convolution. Note only multiplication is counted. Computation for bias are ignored. Flops for a transposed convolution are calculated as flops = (x_shape[2:] * prod(w_shape) * batch_size). Args: x_shape (list(int)): The input shape before convolution. w_shape (list(int)): The filter shape. out_shape (list(int)): The output shape after convolution. transposed (bool): is the convolution transposed Returns: int: the number of flops rrANr) r_r`r'ra batch_size conv_shapec_outc_in filter_sizerRs r conv_flop_countrhnsj(J'6''Y;J 'E4+   d;// /* .ts(a%(#d59oo55r"rrFrj)r!rh)grad_out_shaper_r`rkrlrmrnra_output_padding_groups output_maskr'rs flop_countgrad_input_shapegrad_weight_shapes r conv_backward_flopr{s666JDL1~a$Yq\22ong?OU_Q_``` 1~q%il33  q /!!N*;*;QQwZZK\I]I]joppp pJJ /!!G**aa6G6GK\I]I]joppp pJ r"c|\}}}}|\}}} } |\} } } }||cxkr| kr%nn"||cxkr| krnn|| kr | | kr|| ksJd}|t||z||f||z|| fz }|t||z|| f||z| |fz }|S)z^ Count flops for self-attention. NB: We can assume that value_shape == key_shape rrU) query_shape key_shape value_shaperPhs_qd_q_b2_h2s_k_d2_b3_h3_s3d_v total_flopss r sdpa_flop_countrs !NAq#s"Cc3$Cc3 ????s?????qC33#::#**QTX[Q[Q[Q[Q[K8QUC-AsC/@AAAK8QUC-AsC/@AAAK r"c$t|||S)Count flops for self-attention.r)r}r~rr'r(r)s r sdpa_floprs ; ; ? ??r"cddlm}ddlm}t |||fs6|jjdkr&|S|g| ddz zS)z If the offsets tensor is fake, then we don't know the actual lengths. In that case, we can just assume the worst case; each batch has max length. r) FakeTensor)FunctionalTensormetar) torch._subclasses.fake_tensorr#torch._subclasses.functional_tensorrrdevicer6difftolistsize)offsetsmax_lenrrs r _offsets_to_lengthsrs 988888DDDDDD g ,<= > >'7>CVZ`C`C`||~~$$&&& 9 Q!+ ,,r")grad_out.c#K|t|jdksJt|jdksJ||j|jksJ|j\}} } |j\}} } |j\}} }|J|J|j|jksJt||}t||}t||D]%\}}d| || f}d| || f}d| ||f}||nd}||||fV&dS|j|j|j||jndfVdS)a; Given inputs to a flash_attention_(forward|backward) kernel, this will handle behavior for NestedTensor inputs by effectively unbinding the NestedTensor and yielding the shapes for each batch element. In the case that this isn't a NestedTensor kernel, then it just yields the original shapes. Nrlenrrzip)querykeyvaluer cum_seq_q cum_seq_kmax_qmax_k_h_qrh_kd_kh_vr seq_q_lengths seq_k_lengths seq_q_len seq_k_lennew_query_shape new_key_shapenew_value_shapenew_grad_out_shapes r %_unpack_flash_attention_nested_shapesr)st$ 39~~""""5;1$$$$8>U[#@#@#@#@k 3i 3k 3$$$$$$)/1111+Iu== +Iu== &)-&G&G V V "Y  #y#6OY4M #y#6O4<4Hd !=/CUU U U U U +sy%+AUx~~[_ ______r"c# K|t|jdksJt|jdksJ||j|jksJ|j\}}} } |j\}}} } |j\}}} }|J|J|j|jksJt||}t||}t||D]%\}}d| || f}d| || f}d| ||f}||nd}||||fV&dS|j|j|j||jndfVdS)a? Given inputs to a efficient_attention_(forward|backward) kernel, this will handle behavior for NestedTensor inputs by effectively unbinding the NestedTensor and yielding the shapes for each batch element. In the case that this isn't a NestedTensor kernel, then it just yields the original shapes. Nrr)rrrr cu_seqlens_q cu_seqlens_k max_seqlen_q max_seqlen_krrrrrrr seqlens_q seqlens_klen_qlen_krrrrs r )_unpack_efficient_attention_nested_shapesrWs{$39~~""""5;1$$$$8>U[#@#@#@#@1c31c31c3''''''!\%77777' lCC ' lCC  955 V VLE5 #uc2OUC0M #uc2O4<4Hd !=/CUU U U U U +sy%+AUx~~[_ ______r"T)r=c `t|||||||} td| DS)r)rrrrrrrc3BK|]\}}}}t|||VdSrr.0r}r~rrs r z0_flash_attention_forward_flop..J 2KK  Y <<r"rsum) rrrrrrrr'r(r)sizess r _flash_attention_forward_floprs]" 2    E 6;  r"c `t|||||||} td| DS)r)rrrrrrrc3BK|]\}}}}t|||VdSrrrs r rz4_efficient_attention_forward_flop..rr"rr) rrrbiasrrrrr(r)rs r !_efficient_attention_forward_floprs]" 6 !!!!   E 6;  r"cd}|\}}}}|\} } } } |\} }}}|\}}}}|| cxkr | cxkr|kr nn|| cxkr |cxkr|kr nn|| ksJ||kr | |kr||ksJd}|t||z||f||z|| fz }|t||z||f||z|| fz }|t||z| |f||z||fz }|t||z|| f||z| |fz }|t||z||f||z|| fz }|SNrrU)rtr}r~rrrPrrrrrrrrrrr_b4_h4_s4_d4s r sdpa_backward_flop_countrsK NAq#s"Cc3$Cc3'Cc3  ! ! ! !s ! ! ! !c ! ! ! ! !a3&<&<&<&<#&<&<&<&<&<&<&<&<&< #::#**K8QUC-AsC/@AAAK8QUC-AsC/@AAAK8QUC-AsC/@AAAK8QUC-AsC/@AAAK8QUC-AsC/@AAAK r"c&t||||S)z(Count flops for self-attention backward.r)rtr}r~rr'r(r)s r sdpa_backward_floprs $NKK X XXr"c bt|||||||| } td| DS)N)rrrrrrrrc3DK|]\}}}}t||||VdSrrrr}r~rrts r rz1_flash_attention_backward_flop..L ?KK !iUUr"r) rrrrout logsumexprrrrr(r)shapess r _flash_attention_backward_floprs`"3    F CI  r"c bt|||||||| } td| DS)N)rrrrrrrrc3DK|]\}}}}t||||VdSrrrs r rz5_efficient_attention_backward_flop..%rr"r) rrrrrrrrrrr(r)rs r "_efficient_attention_backward_flopr s`"7 !!!!   F CI  r"c6t|ts|fS|Sr)rtuple)xs r normalize_tuplerBs a  t Hr")KMBTc tdtttdz tt |dz dz}t|S)NrrrAr)maxminrsuffixesstr)numberindexs r get_suffix_strrKsJ 3s8}}q(3s6{{+;+;a+?A*EFF G GE E?r"cjt|}|d|zz d}|t|zS)Niz.3f)rr)rsuffixrrs r convert_num_with_suffixrRs6 NN6 " "E % + +E 8E? ""r"c |dkrdS||z dS)Nr0%z.2%rB)numdenoms r convert_to_percent_strrYs  zztEk  r"c<tfd}|S)NcRt|\}}|}t||Sr)rr)r( flat_argsspecrr*s r r+z)_pytreeify_preserve_structure..nf_s/&t,, 4amc4(((r"rr,s` r _pytreeify_preserve_structurer^s3 1XX))))X) Ir"c eZdZdZ ddeeejje ejjfde de dee e e fffd Zd e fd Zd e ee e e fffd Zdd ZdZdZdZxZS)ra ``FlopCounterMode`` is a context manager that counts the number of flops within its context. It does this using a ``TorchDispatchMode``. It also supports hierarchical output by passing a module (or list of modules) to FlopCounterMode on construction. If you do not need hierarchical output, you do not need to use it with a module. Example usage .. code-block:: python mod = ... with FlopCounterMode(mod) as flop_counter: mod.sum().backward() NrATmodsdepthdisplaycustom_mappingcRttd|_||_||_d|_|i}|tjdditd| D|_ t|_ dS)Nc*ttSr)rintrBr"r z*FlopCounterMode.__init__..s+VYJZJZr"zz,FlopCounterMode.__init__..s<nnntqRSqwq*e44J!!-:J:Jnnnr") super__init__r flop_countsrr modewarningswarnr#itemsr mod_tracker)selfrrr r  __class__s r rzFlopCounterMode.__init__|s 6ABZBZ6[6[  04  !N   MXef g g g g  nnWeWkWkWmWmnnn )??r"r.cZt|jdS)NGlobal)rrvaluesrs r get_total_flopszFlopCounterMode.get_total_flopss$4#H-4466777r"cHd|jDS)aReturn the flop counts as a dictionary of dictionaries. The outer dictionary is keyed by module name, and the inner dictionary is keyed by operation name. Returns: Dict[str, Dict[Any, int]]: The flop counts as a dictionary. c4i|]\}}|t|SrB)dictrs r rz3FlopCounterMode.get_flop_counts..s$@@@tq!477@@@r")rrr#s r get_flop_countszFlopCounterMode.get_flop_countss(A@t'7'='='?'?@@@@r"c@ |j}|d}ddl}d|_gd}g} t  d fd}t jD]L}|dkr |dd z}||kr(|||d z }| |Mdjvr$ s"|D]} d | dz| d<|dd|z}t|dkrgd g}|||d S)Ni?BrT)ModuleFLOPz% TotalFc t j|} |kz d|z}g}|||zt |t |g j|D]L\}}||dzt|zt |t |gM|S)N z - )rrr"appendrrrr) mod_namerrpaddingr"rFr global_flops global_suffixis_global_subsumedrs r process_modz.FlopCounterMode.get_table..process_modsd.x8??AABBK +"= = EkGF MM("' ]CC&{LAA    (288::  1 eOc!ff,+A}==*1l;; Mr"r!.rr-)r!0r)leftrightr8)headerscolalign) rtabulatePRESERVE_WHITESPACEr$rsortedrkeyscountextendr) rrr;headerr"r4mod mod_depth cur_valuesrr1r2r3s ` @@@r get_tablezFlopCounterMode.get_tables =JE =E'+$...++-- &|44 "        ,$*//1122 & &Ch #*I5  $S)a-88J MM* % % % % t' ' '0B ' * *q>a [1--6F v;;!  +++,F  B\ ]]]r"c|j|jt ||_|j|Sr)rclearr __enter___FlopCounterModerr#s r rHzFlopCounterMode.__enter__sT     ""$$$$T**   r"c|jJ|jj|}d|_|j|jr't ||j|Sr)r__exit__rr printrEr)rr(rPs r rKzFlopCounterMode.__exit__shy$$$ DI  %  !!### < . $..,, - - -r"c||jvrP|j|}||i|d|i}t|jjD]}|j||xx|z cc<|S)Nr%)r#setrparentsr)r func_packetrr(r)flop_count_funcrxpars r _count_flopszFlopCounterMode._count_flopss $, , ,"0=O($F&FF#FFFJ4+344 A A %k222j@2222 r")NrATNr)__name__ __module__ __qualname____doc__r r rnnr*rrr boolr'rrr$rr(rErHrKrS __classcell__)rs@r rrhs>*MQ 7; ++5$ux2G!GHI++ + %T#s(^4 ++++++*88888 Ac4S>&9!: A A A A:^:^:^:^zr"c2eZdZdZdefdZdZdZd dZdS) rITcounterc||_dSr)r\)rr\s r rz_FlopCounterMode.__init__s  r"cddl}||jj}|5||}dddn #1swxYwY||jj}||j_||fS)aExecute a branch function and capture its FLOP counts without affecting self.counter.flop_counts Args: branch_fn: The branch function to execute operands: Arguments to pass to the branch function Returns: Tuple of (result, flop_counts) where result is the branch output and flop_counts is a copy of the FLOP counts after execution rN)copyr\r)r branch_fnoperandsr_checkpointed_flop_countsresultrs r $_execute_with_isolated_flop_countingz5_FlopCounterMode._execute_with_isolated_flop_countings  #'99T\-E#F#F  * *Y)F * * * * * * * * * * * * * * *ii 899 #;  {""s 8<<c||tjjjhvrtS|tjjjur|\}}}}|||\} } | turtS|||\} } | turtSt | t | z} i}| D]}| |}| |}i}t |t |z}|D]A}||d}||d}t||||<B|||<| D]*\}}|j j | |+| SdSr)rops higher_ordercondNotImplementedrdrNr>getrrr\rupdate)rfunctypesr(r)pred true_branch false_branchratrue_outtrue_flop_counts false_outfalse_flop_counts all_mod_keysmerged_flop_counts outer_keytrue_func_countsfalse_func_countsmerged_func_counts all_func_keysfunc_keytrue_val false_val inner_dicts r _handle_higher_order_opsz)_FlopCounterMode._handle_higher_order_opss  .36 6 6! ! 59). . .8< 5D+|X)-)R)RX** &H&>))%%+/+T+Th,, (I(N**%%/446677#>O>T>T>V>V:W:WWL!# ) C C #3I#> $5i$@!%'" #$4$9$9$;$; < IN9AIN?GIN'/IN+3IN)1IN-5IN19IN5=IN(0IN,4IN&.IN)13 3 3 " ! dEJ: ; ; L00udFKK K t|1 1 1d%).BWB_6_6_  "DND3F33N**        *                dD#F##|(()=sD&QQQs<H..H25H2)rBN) rTrUrVsupports_higher_order_operatorsrrrdrrrBr"r rIrIsn&*####(///b"R"R"R"R"R"Rr"rI)Fr)NNNFN)Xrtorch.utils._pytreerrrmodule_trackerrtypingrr r r r collections.abcr typing_extensionsr collectionsrtorch.utils._python_dispatchrmathr functoolsrr__all__rrrfrr!r#r'__annotations__r-rmmr rIaddmmrNbmmrSbaddbmmrV _scaled_mmr^rrrYrh convolution _convolutioncudnn_convolution_slow_conv2d_forwardroconvolution_backwardr{r'_scaled_dot_product_efficient_attention#_scaled_dot_product_flash_attention#_scaled_dot_product_cudnn_attentionrrrrr_flash_attention_forwardr_efficient_attention_forwardrr0_scaled_dot_product_efficient_attention_backward,_scaled_dot_product_flash_attention_backward,_scaled_dot_product_cudnn_attention_backwardr_flash_attention_backwardr_efficient_attention_backwardrrrrrrrrrIrBr"r rs FFFFFFFFFF))))))::::::::::::::$$$$$$''''''######:::::: 5 6 WT]]Yt__ y~   !# tCH~"""XxB?O>PRZ[]_a[aRb>b5c,tw/3# tz""%%#%%%#"%tx    C   !  t|$$&&C&&&%$& t''  % %  % % %(' %( $$ #Y$ #Y$Cy$ $  $$$$L($*;T=SUYUnoppbfOOOuxOOOqpO t011eeee21eN$D@@BCCEI@@@WZ@@@CC@ - - -" +`+`+`eE#s(OU38_eCHoxPUVY[^V^P_G``ab+`+`+`+`f -`-`-`eE#s(OU38_eCHoxPUVY[^V^P_G``ab-`-`-`-``t4dCCC DC>t8$GGG HG>6MIIKLL^bYYYpsYYYLLYt5tDDD ED@t94HHH IH@GWJ  Hh L,   O_   i  y I y 1 0) ,i ,i 9;M 57I  57I!" !#@#$ %'H"$B&(J) .    $ # ####   LLLLLLLL\mRmRmRmRmR(mRmRmRmRmRr"