o 3/i< @sddlZddlmZddlmZddlZddlmmZ ddl Z ddl m Z ddl m Z mZmZddlmZddlmZddlmZddlmZmZddlmZddlmZdd lmZGd d d eZ ed d Z!ej"j#ddhddej$dej$ddfddZ%e%j&dej$dej$ddfddZ'dej$dej$fddZ(GdddeZ)eddZ*ej"j#ddhddej$d ej$dej$ddfd!d"Z+e+j&dej$d ej$dej$ddfd#d$Z,dej$d ej$dej$fd%d&Z-Gd'd(d(ej.j/Z0dej$dej$fd)d*Z1dS)+N)Type)partial)Int64Float32 const_expr)make_fake_tensor) row_reduceonline_softmax_reduce) ReductionBase) jit_cache)torch2cute_dtype_mapc seZdZddeejdedeffdd ZddZ d d Z e j d e j d e j d ejfddZe jd e j d e j de jde jdejef ddZZS)SoftmaxTdtypeNonline_softmaxcs.tj|||s dnd|stntd||_dS)Nstagereduction_dtype)super__init__rrr)selfrrr __class__Z/lsinfo/ai/hellotax_ai/llm_service/venv_vllm/lib/python3.10/site-packages/quack/softmax.pyrs   zSoftmax.__init__cC(|j}dD] \}}||kr|SqdS)N)@i ir)@r"rrrlimitthreadsrrr_threads_per_row#  zSoftmax._threads_per_rowcCsT|j}t|jjdkrgd}ngd}|D]\}}||kr$||_dSqd|_dS)Nr#)r'r)r))r ))r0r)r1r)r3r2)ir rrrwidth cluster_nrr thresholdsr+clusterrrr_set_cluster_n*   zSoftmax._set_cluster_nmXmOstreamc Cs|j|jksJ|ttdd||fD}|jd|d\}}}|j}||||||jt |j d|d|j dg|ddgt|j dkrOd|j dgnd|ddS)Ncs|]}|jjVqdSN element_typer5.0trrr ?z#Softmax.__call__..r"vecsizerrgridblockr9r> rBrr:rmax_get_tiled_copysizekernellaunchcuteceil_divshaper6) rr<r=r>largest_dtype_width tiled_copytiler_mnthreads_per_row num_threadsrrr__call__6s  zSoftmax.__call__rXrWrYc# s|j}tj\}}}tj\}}t|jdkrtdntjd|j} t| } fdd||| fD\} } } t j }|j |j tjdddd}|||\}}||}|| }||}|| }|| d }d d||fD\}}t| dd|jk}|rdn tj|| | dd }ttj|d }t|tjj}|||||dd| dkr|||d dtjtjdt| rt |||j j t||| tj!}t|j" rKt#|tj$j%||dt|jdkr|dndt!j t|jdkrtjj&ndd}t'(t'j)}tj'j*||||d d} t#| tj$j+||dt|jdkrE|dnddd}!nt,|||d|t|jdkr^tjj&ndd d\}}!} | tj-|!}"|.|" |j |dd| dkr|||dSdS)Nrrcg|] }t|fqSrrS local_tilerDmTbidx cluster_yrXrr ]sz"Softmax.kernel..rrorderr#byte_alignment)rNNNcSg|]}t|qSrrSmake_rmem_tensor_likerDthrrrrrdkr+predTis_asyncNNrinit_valhook_fn)fastmath)NNr)rx)ry return_exp_x)/layout_tv_tiledrSarch thread_idx block_idxrr6rUmake_identity_tensorcutlassutils SmemAllocatorallocate_tensorrBmake_ordered_layout#_allocate_reduction_buffer_and_mbar get_slice partition_S partition_D copy_utils predicate_krcopyrP WARP_SIZE_initialize_clustercp_async_commit_groupcp_async_wait_groupfill_oobinf autovec_copyloadtorrr ReductionOpMAX cluster_waitmathlog2eexp2ADDr rcp_approxstore)#rr<r=rXrWrY tv_layouttidx_rUidXgXgOcXsmemsXreduction_buffermbar_ptr thr_copy_XtXgXtXsXtXgOtXcXtXrXtXrO is_even_NtXpXr num_warpsxmax_xlog2_eexp_xdenomyrrarrQKs $ "             zSoftmax.kernel)T)__name__ __module__ __qualname__rrNumericintboolrr-r:rSjitTensorcudaCUstreamr[rQShape TiledCopy Constexpr __classcell__rrrrr s0"   r cs`ttd|jfdd||fD\}}t|}tj|||tjjddddS)Nr"cg|] }t|fqSr fake_tensorrDdtr batch_symdivrrrdsz(_compile_softmax_fwd..Tuse_tvm_ffi_env_stream--enable-tvm-ffioptions) rSsym_intrgcdr5r compileruntimemake_fake_stream)r out_dtyperx_cuteout_cute softmax_oprrr_compile_softmax_fwds  rzquack::_softmax_fwdout) mutates_argsrreturncCsv|dks Jd|jsJd|jtjtjtjfvs Jd|d}dd||fD\}}t|||||dS) zSoftmax forward pass. Args: x: Input tensor of shape (M, N) Returns: Softmax output tensor of same shape as x rzInput must be 2DzTensor must be on CUDA deviceUnsupported dtypercSg|]}t|jqSrr rrCrrrrdrpz _softmax_fwd..N) dimis_cudartorchfloat16bfloat16float32rPr)rrrrrrrr _softmax_fwds  rcCshddlm}|r0t|dtjs2|d}dd||fD\}}t|||t||||dSdSdS)Nr COMPILE_ONLYrcSrrrrCrrrrdrpz%_softmax_fwd_fake..)quack.cache_utilsr isinstancerPrSymIntr_compile_softmax_backward)rrrrrrrrr_softmax_fwd_fakes   rcCst|}t|||Sr@)r empty_liker)rrrrr softmax_fwds  rcseZdZdeejdeffdd ZddZddZ d d Z e j d e j d e j d e j dejfddZe jd e j d e j d e j de jde jdejef ddZZS)SoftmaxBackwardrrcstj||dtddS)Nrr)rrr)rrrrrrrszSoftmaxBackward.__init__cCr)N)rr!r$r&) r"r(r)r*rrrr-r.z SoftmaxBackward._threads_per_rowcCsT|j}t|jjdkrgd}ngd}|D]\}}||kr$||_dSqd|_dS)Nr#r/r4r7rrrr:r;zSoftmaxBackward._set_cluster_ncCs|jdkrdSdS)Nrr"r(r))rrrr _num_threadsszSoftmaxBackward._num_threadsmdYmYmdXr>c Cs|j|jksJ|ttdd|||fD}|jd|d\}}}|j} |||||||jt |j d|d|j dg| ddgt|j dkrQd|j dgnd|ddS)Ncsr?r@rArCrrrrFrGz+SoftmaxBackward.__call__..r"rHrrrJrM) rrrrr>rVrWrXrYrZrrrr[s  zSoftmaxBackward.__call__rXrWrYc' stj\}}}tj\}}t|jdkrtdntjd|j} |j} t| } fdd|||| fD\} } }}t j }|j |j tjdddd}|j |j tjdddd}||| \}}||}|| }||}|| }||}||}||d }d d|||fD\}}}t| dd|jk}|rdn tj||| dd } ttj| d }!t|tjj}"||||"|dd| dkr|!||d d|!||d dtjtjdt||t|||tj}#|tj}$t |#|$tj!j"||dt|jdkr#|nddt|jdkr1tjj#ndd}%|$|#|%}&|$|&|j |dd| dkrV|!||dSdS)Nrrcr\rr]r_rarrrd sz*SoftmaxBackward.kernel..rerfr#rhrjcSrkrrlrnrrrrd5rprqrrTrtrvr{rw)%rSr~rrrr6r}rUrrrrrrBrrrrrrrrrrPrrrrrrrrrrrrr)'rrrrrXrWrYrrrrUrgdYgYgdXrrsdYsYrrthr_copytdYgdYtdYsdYtYgYtYsYtdXgdXrtdYrdYtYrYtdXrdXrrrrdyrdotdxrrarrQsl $              zSoftmaxBackward.kernel)rrrrrrrrr-r:rrSrrrrr[rQrrrrrrrrrs:  rcsfttd|jfdd|||fD\}}}t|}tj||||tjjddddS)Nr"crrrrrrrrdcsz-_compile_softmax_backward..Trrr) rSrrrr5rrrr)ry_dtypedx_dtyperdy_cutey_cutedx_cutesoftmax_backward_oprrrr_s  rzquack::_softmax_backwardr r rcCs|dks Jd|dksJd|j|jksJd|jr$|js(Jd|jtjtjtjfvs7Jd|j|jksAJd|d}d d |||fD\}}}t |||||||d S) zSoftmax backward pass. Args: dy: Upstream gradients tensor of shape (M, N) y: Softmax output tensor of shape (M, N) Returns: Input gradients tensor of same shape as dy and y rz dy must be 2Dz y must be 2Dzdy and y must have same shapezTensors must be on CUDA devicerzdy and y must have same dtypercSrrrrCrrrrdrpz%_softmax_backward..N) rrUrrrrrrrPr)r rr rrrrrrr_softmax_backwardqs  rcCs`ddlm}|r,t|dtjs.|d}dd|||fD\}}}t||||dSdSdS)NrrrcSrrrrCrrrrdrpz*_softmax_backward_fake..)rrrrPrrr)r rr rrrrrrrr_softmax_backward_fakes  rcCst|}t||||Sr@)rrr)r rr rrr softmax_bwds  rc@s$eZdZeddZeddZdS)SoftmaxFunctioncCst|}|||Sr@)rsave_for_backward)ctxrrrrrforward zSoftmaxFunction.forwardcCs|j\}t||}|Sr@) saved_tensorsr)rr rr rrrbackwardrzSoftmaxFunction.backwardN)rrr staticmethodrrrrrrrs  rcCs t|S)zSoftmax forward pass with automatic differentiation support. Args: x: Input tensor of shape (M, N) Returns: Softmax output tensor of same shape as x )rapply)rrrrsoftmaxs r )2rtypingr functoolsrrcuda.bindings.driverbindingsdriverrr cutlass.cuterSrrr quack.utilsrquack.copy_utilsrquack.compile_utilsrr quack.reducerr quack.reduction_baser rr quack.cute_dsl_utilsr r rlibrary custom_oprr register_fakerrrrrrrautogradFunctionrr rrrrsF           ""