o :/iz @sUddlZddlZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z m Z m Z ddlmZddlmZddlmZddlmZddlZddlmZddlmZdd lmZddlmZdd lm Z m!Z!m"Z"dd l#m$Z$dd l%m&Z&m'Z'dd l(m)Z)ddl*m+Z+ddl,m-Z-ddl.m/Z/m0Z0ddl1m2Z2ddl3m4Z4m5Z5m6Z6m7Z7m8Z8ddl9m:Z:ddl;mm?Z?m@Z@ddlAmBZBe)eCZDdeEeFdeEejGdBde defde deffddZHde de4fd d!ZIGd"d#d#ZJGd$d%d%eKZLejMGd&d'd'ZNd(ejOdePfd)d*ZQd+eRejOeFfd,eEeFddfd-d.ZSd/ejTd0eEeUdeVejTeEeNffd1d2ZWd3aXd4ed5e"de d6ePd7ePdef d8d9ZYGd:d;d;ejjZZ[da^ePe]d?<edGd@eUdAePde dBfdCdDZ_GdEdFdFZ`dS)HN) defaultdict)Callable GeneratorSequence)contextmanager)deepcopy)partialAny) dynamo_timed)trace_structured)CompilationConfig CUDAGraphMode VllmConfig)DynamicShapesType)Range hash_factors) init_logger)lazy)current_platform) instrumentinstrument_manual)resolve_obj_by_qualname)CompilerInterface EagerAdaptorInductorAdaptorInductorStandaloneAdaptoris_compile_cache_enabled)compilation_counter)inductor_partition_rule_context should_split) InductorPass pass_context)PostGradPassManagersym_tensor_indices input_buffers callable_fn.returncsdtdtffdd }|S)aCreate a wrapper that copies inputs to static buffers before calling. This is used for cudagraph input copying where we need to copy dynamic tensors to static buffers before invoking the compiled graph. Args: sym_tensor_indices: Indices of tensors with symbolic shapes input_buffers: List of static buffers (can contain None for lazy init) callable_fn: The compiled function to call Returns: A wrapper function that copies inputs and calls the compiled function argsr(csnt|}tD]*\}}||}|jd}|dur!||<|d|}|||||<q|SNr)list enumerateshapeclonecopy_)r) list_argsiindexruntime_tensor runtime_shape static_tensorr'r&r%f/lsinfo/ai/hellotax_ai/llm_service/venv_vllm/lib/python3.10/site-packages/vllm/compilation/backends.py copy_and_callGs     z)make_copy_and_call..copy_and_callr )r%r&r'r9r7r6r8make_copy_and_call4sr:compilation_configcCstjr tjs Jd|jdkr*tjr"ttjdr"tdt |j Stdt S|jdkr7tdt Std|jt t}t|tsLJ|S) NzCVLLM_USE_MEGA_AOT_ARTIFACT=1 requires VLLM_USE_STANDALONE_COMPILE=1inductorstandalone_compilezUsing InductorStandaloneAdaptorzUsing InductorAdaptoreagerzUsing EagerAdaptorzUsing custom backend: %s)envsVLLM_USE_MEGA_AOT_ARTIFACTVLLM_USE_STANDALONE_COMPILEbackendhasattrtorch _inductorloggerdebugrcompile_cache_save_formatrrrrget_compile_backend isinstancer)r;compilerr7r7r8 make_compilerYs(     rLc@seZdZdZdeddfddZdedefdd Ze d e de d fd d Z d%dede deddfddZd&ddZdejdeeded e dedefdBf ddZedd  d'dejdeed!eeefded e ded"edefd#d$ZdS)(CompilerManagera A manager to manage the compilation process, including caching the compiled graph, loading the compiled graph, and compiling the graph. The cache is a dict mapping `(runtime_shape, graph_index, backend_name)` to `any_data` returned from the compiler. When serializing the cache, we save it to a Python file for readability. We don't use json here because json doesn't support int as key. r;r(NcCs(t|_d|_||_t||_i|_dSNF)dictcacheis_cache_updatedr;rLrKloaded_artifacts)selfr;r7r7r8__init__s   zCompilerManager.__init__ vllm_configcCs |j|SN)rK compute_hash)rSrUr7r7r8rWs zCompilerManager.compute_hash compile_rangeNNNc cst|2|jjr$t|jj dVWdn1swYn dVWddSWddS1s:wYdS)zProvide compilation context for the duration of compilation to set any torch global properties we want to scope to a single Inductor compilation (e.g. partition rules, pass context).N)r#r;use_inductor_graph_partitionr splitting_ops)rSrXr7r7r8compile_contexts "zCompilerManager.compile_contextF cache_dir disable_cacheprefixcs||_||_tj|d|_|sZtj|jrZt|j}t | }Wdn1s.wYdt dt ddfdddt dt tttfffd d fd d |D|_|jj|||d dS)a Initialize the cache directory for the compiler. The organization of the cache directory is as follows: cache_dir=/path/to/hash_str/rank_i_j/prefix/ inside cache_dir, there will be: - vllm_compile_cache.py - computation_graph.py - transformed_code.py for multiple prefixes, they can share the same base cache dir of /path/to/hash_str/rank_i_j/ , to store some common compilation artifacts. zvllm_compile_cache.pyNvaluetyr(cSs,t||std|dt|d|dS)Nz Expected z but got z for )rJ TypeErrortype)rarbr7r7r8 check_types z4CompilerManager.initialize_cache..check_typekeycsd|\}}}|t|tt|tr(|\}}|t|tt||d}|t|||fS)N)startend)intstrrJtupler)rf range_tuple graph_index compiler_namergrh)rer7r8 parse_keys         z3CompilerManager.initialize_cache..parse_keycsi|] \}}||qSr7r7).0rfra)ror7r8 sz4CompilerManager.initialize_cache..)r^r_r`)r_r^ospathjoincache_file_pathexistsopenast literal_evalreadr rdrkrrirjitemsrPrKinitialize_cache)rSr^r_r`frPr7)reror8r|s    z CompilerManager.initialize_cachecCsd|js|jsdStjdd}||j}t|jd}||WddS1s+wYdS)N)indentw) r_rQpprint PrettyPrinterpformatrPrwruwrite)rSprinterdatar}r7r7r8 save_to_files    "zCompilerManager.save_to_filegraphexample_inputsrm.c Cs|||jjf|jvr dSdtdttttftffdd}z||j|||jjf\}}Wn ty7YdSw|j|||||}||j|<t d|t||jj||S)Nrar(cSsJt|tsJ|d}t|dtsJt|dtsJ|d}||fS)N graph_handlerr cache_key)rJrOrj)rahandlerr7r7r8 parse_values z)CompilerManager.load..parse_valuezGDirectly load the %s-th graph for compile range %sfrom %s via handle %s) rKnamerPr rkrj ExceptionloadrRrFrG) rSrrrmrXrrrcompiled_graphr7r7r8rs,"   zCompilerManager.loadz Compile graph span_namerradditional_inductor_config num_graphsc s|dkrtatjd7_d}||||}|dur=||dkr;tt} |j| 7_tjdt || dd|St j t rFd} nd} | |j d|j7} | d|7} |dtjjjjfd d } dd lm} tjjjd d U| d| @zj ||||| \}} Wn)tydusJjYWdWdWdSwWdn1swYWdn1swYdur|dur|j<Wdn1swY|dusJdt|r:| dur:| dj||j jf<tjd7_d _|dkr-tdt |t d|t |j j| ||dkrYtt} |j| 7_tjdt || dd|S)NrrzTDirectly load the compiled graph(s) for compile range %s from the cache, took %.3f slocalscopeartifact_compile_range__ _subgraph_cs6|i|}|dur dS|djvrt|Sr*)rR StopCompiling)r)kwargsresultrorigrSr7r8autograd_cache_keyDs z3CompilerManager.compile..autograd_cache_key)patchT)autograd_cache_normalize_inputsz@torch._functorch._aot_autograd.autograd_cache.autograd_cache_keyzFailed to compile the graph)rrz1Cache the graph of compile range %s for later usez?Store the %s-th graph for compile range%s from %s via handle %sz3Compiling a graph for compile range %s takes %.2f s)!time perf_countercompilation_start_timernum_backend_compilationsrcompilation_timerF info_oncerjrJrKrrgrhr\rD _functorch _aot_autogradautograd_cacher unittest.mockrconfigcompilerrRrrPrnum_cache_entries_updatedrQrG)rSrrrr;rXrmrrelapsed maybe_keyrrrr7rr8rs           3 9  zCompilerManager.compile)Fr]r(N)rr)__name__ __module__ __qualname____doc__r rTrrjrWrrrr\boolr|rfx GraphModuler+r rirrrrOrr7r7r7r8rMus` 4 (  rMc@s eZdZdS)rN)rrrr7r7r7r8rsrc@s0eZdZUeed<eed<eed<ejed<dS) SplitItem submod_namegraph_idis_splitting_graphrN) rrrrj__annotations__rirrrr7r7r7r8rs rnodecCs|jdkr |jdkS|jdkrdS|j}|tjtjtjfvr dSt|tjjr+|j }nt|tjj r7| }ndS| dpB| dS)N call_method new_empty call_functionFTz aten::emptyzaten::new_empty) optargetrDempty empty_like empty_stridedrJ_opsOpOverloadPacket_qualified_op_name OpOverloadr startswith)rr packet_namer7r7r8_is_empty_allocation_nodes    rnode_to_subgraph_idsplit_op_graphsc stt}D] \}}|||qt|}dtdd}t|dD]@}||g}|s3q(||v}t |dkoBt |d} d} | rbdurb|d} t fdd| j Drb| <d } | sh|rh|q(dS) z Merge a partition that only contains an empty allocation op into the previous partition. This avoids generating standalone empty submodules, which can lead to empty cudagraph captures. N)defaultrrFc3s&|]}|jdkp|kVqdS) placeholderNr)rp input_noderprev_non_splitting_subgraph_idr7r8 s   z._merge_empty_only_subgraphs..T) rr+r{appendsetmaxvaluesrangegetlenrallall_input_nodes) rrnodes_by_subgraph_idr subgraph_idsplitting_subgraphsmax_subgraph_idnodesis_non_splitting_subgraphis_empty_only_subgraphmerged empty_noder7rr8_merge_empty_only_subgraphss0   rrr[c s\d}ig}|jjD]P}|jdvrq |jdkr4|jtjkr4|jd}|jdkr4|vs-J||<q t||rV|d7}||<||t|j |rQ|d8}q |d7}q ||<q t |t j j jj|dfdddd }g}d d |D}|D]$} d | vs| d krq}t|| } t| dd } |t| | | |v| q}|jddd||fS)Nr)outputrrrrcs|SrVr7rrr7r8zsplit_graph..T)keep_original_ordercSsg|]\}}|qSr7r7)rprmoduler7r7r8 szsplit_graph...r]submod_cSs|jSrV)r)xr7r7r8rs)rf)rrrroperatorgetitemr)r!rnextrrDrpasses split_module named_modulesgetattrrireplacersort) rr[rrrrsplit_gmoutputsnamesrrrr7rr8 split_graphsD               rgpiecewise_backendrUis_first_graph is_last_graphc CsJ|jr|jr |Sddlm}tt}|||tj ||| |ddS)a? Wrap a piecewise backend with CUDA graph wrapper if needed. This function is shared between VllmBackend and construct_serializable_fn_from_inductor_cache. Args: piecewise_backend: The backend to wrap vllm_config: The vLLM configuration compilation_config: The compilation configuration is_first_graph: Whether this is the first graph in the sequence is_last_graph: Whether this is the last graph in the sequence Returns: The wrapped backend if CUDA graphs are enabled, otherwise the original backend r)CUDAGraphOptions)debug_log_enable gc_disableweak_ref_output)runnablerU runtime_modecudagraph_options) cudagraph_modehas_piecewise_cudagraphsrZ cuda_graphrrrget_static_graph_wrapper_clsr PIECEWISE)rrUr;rrrstatic_graph_wrapper_classr7r7r8wrap_with_cudagraph_if_needed!s& rc seZdZdZdejjdeede ddddf fd d Z e d d d e de ffdd Z dejjjd eejjjdfdeee fde fddZZS)PiecewiseCompileInterpreteraCode adapted from `torch.fx.passes.shape_prop.ShapeProp`. It runs the given split graph interpreter, and for each submodule in `compile_submod_names`, creates a PiecewiseBackend and compiles all ranges up front. NOTE: the order in `compile_submod_names` matters, because it will be used to determine the order of the compiled piecewise graphs. The first graph will handle logging, and the last graph has some special cudagraph output handling. Note: This class shares similar logic with reconstruct_serializable_fn_from_mega_artifact in caching.py. Both create PiecewiseBackend instances and wrap them with cudagraph. The key difference is: - reconstruct_serializable_fn_from_mega_artifact: PiecewiseBackend receives pre-compiled runnables (compiled_runnables is set, graph is None) - this class: PiecewiseBackend receives the FX graph to compile (graph is set, compiled_runnables is None) If modifying the backend creation/wrapping logic, consider updating both. rcompile_submod_namesrU vllm_backend VllmBackendr(Ncs0t|||_|j|_||_||_d|_dSrN)superrTrr;rUrextra_traceback)rSrrrUr __class__r7r8rTos  z$PiecewiseCompileInterpreter.__init__zInductor compilationrr)cs tj|SrV)rrun)rSr)rr7r8r~s zPiecewiseCompileInterpreter.runr.rc Cst|tsJt|j|}|jjd}t|dd}||j vrl|j |}| |}ddt |D} ddl m} ddlm} | ||j|t|j | |j| ||d } t| |j|j| j| j|jj|<tjd7_|S) NrcSs |jdS)N example_value)metarr7r7r8r z9PiecewiseCompileInterpreter.call_module..cSs g|] \}}t|tjr|qSr7)rJrDSymIntrpr1rr7r7r8rs z;PiecewiseCompileInterpreter.call_module..)graph_returns_tuplerPiecewiseBackend)r)rJrjrrr output_noder)rmap_argrr2 fetch_attrr,torch._inductor.compile_fxr$rr&rUrrrr;rr__dict__r$num_piecewise_capturable_graphs_seen) rSrr)rgmrrr2submodsym_shape_indicesr$r&rr7r7r8 call_modules>        z'PiecewiseCompileInterpreter.call_module)rrrrrDrrr+rjrrTrr rrTargetrkArgumentrOr0 __classcell__r7r7rr8rWs0 rbackbone model_tagFmodel_is_encodertag is_encoderrYccsP|tksJd|dtdt}t}|a|az dVW|a|adS|a|aw)z%Context manager to set the model tag.z Model tag z is the same as the current tag rN)r5r6)r7r8old_tagold_is_encoderr7r7r8 set_model_tags r;c @seZdZUdZeed<eed<dZeed<e j ed<e j ed<e e ed<e d efed <ee d efed <eed <eeefed <  ddedededdfddZdeeeee efdBeeefdBffddZd ddZddZedde j deedefddZdS)!raThe compilation backend for `torch.compile` with vLLM. It is used for compilation mode of `CompilationMode.VLLM_COMPILE`, where we customize the compilation. The major work of this backend is to split the graph into piecewise graphs, and pass them to the piecewise backend. This backend also adds the PostGradPassManager to Inductor config, which handles the post-grad passes. rUr;F_calledrrpiecewise_graphs.returned_callablepost_grad_passescompiler_managerinductor_configr]r`r8r(NcCsX|pt|_|pt|_tt|_tj|_||_ |j |_ t |j |_ t |j j|_dSrV)r5r`r6r8rrget_pass_manager_cls pass_managerpass_keyrUr;rMr@rinductor_compile_configrA)rSrUr`r8r7r7r8rTs  zVllmBackend.__init__c CstjsdSddlm}ddlm}|}i}i}|jD]B\}}t|j|}t |dr/|j n|} t | |s7q|} | j || <| j || <| D]\} } || | | td| | t| qIqtd|||tdt|j|||fS) aCollect inductor cache artifacts from all piecewise backends. Returns: tuple: (standalone_compile_artifacts, sym_shape_indices_map, returns_tuple_map) - standalone_compile_artifacts: StandaloneCompiledArtifacts with compiled artifacts - sym_shape_indices_map: dict mapping submod_name to sym_shape_indices - returns_tuple_map: dict mapping submod_name to returns_tuple rYr)StandaloneCompiledArtifactsr%r z-collected artifact for %s shape %s (%d bytes)z=collected artifacts: %d entries, %d artifacts, %d bytes totalz$standalone compile artifact keys: %s)r?r@cachingrFrr&rnamed_childrenrrCr rJr/ returns_tupleto_bytesr{insertrFrGrinfo num_entries num_artifacts size_bytesr+submodule_byteskeys) rSrFr&standalone_compile_artifactssym_shape_indices_mapreturns_tuple_maprrchildrr shape_str bytes_datar7r7r8$collect_standalone_compile_artifactssF         z0VllmBackend.collect_standalone_compile_artifactscCst|j|j|j|jvr1t|j|jtrtdt|jj |jt s&J|j |jj |j|j|j|j<dS)Nz9PostGradPassManager can not be kept in CompilationConfig.) rC configurerUrDrArJr$ ValueErrorr;rEr"addrSr7r7r8configure_post_passNs   zVllmBackend.configure_post_passcs^jjdtdBdtfddfddtDtdd d fd d d dS) z4Log vLLM compilation config for TORCH_TRACE/tlparse.lstNr(cSs |durdSddd|DS)Nr]z, css|]}t|VqdSrV)rjrprr7r7r8rlzKVllmBackend._log_compilation_config..list_to_str..rt)r^r7r7r8 list_to_strisz8VllmBackend._log_compilation_config..list_to_strcs0g|]}tt|jtrt|jr|jqSr7)rJrrr)rpr})pass_cfgr7r8ros z7VllmBackend._log_compilation_config..artifactcSs dddS)Nvllm_compilation_configjson)rencodingr7r7r7r7r8rwsz5VllmBackend._log_compilation_config..csvtjjjjtjjj j tj j j jtjtjjjjdS)N)modelr`moderB custom_opsr[r compile_sizescompile_ranges_endpointsrZinductor_passesenabled_passesdynamic_shapes_typedynamic_shapes_evaluate_guards)rfdumpsrU model_configrhr`rjrirBrjr[rrkrlrZr+rmrQdynamic_shapes_configrdevaluate_guardsr7)ccrnrbrSr7r8r{s&  metadata_fn payload_fn)r; pass_configr+rj dataclassesfieldsr r\r7)rurnrbrcrSr8_log_compilation_configcs  z#VllmBackend._log_compilation_configrrc, s*ddlm}j}t}t|}|}j|}t t j j t dtfddg} D]:} | | | dkrCq7zt| } | | Wdn1sZwYWq7ttfyqt d| Yq7wtd| } j j j js||| |g} tt| dd }tjtj d |}|j _j j}tj!|d d |j _|j"j#}|j"j$}tj|d |d|j%}tj!|d d |j _&t'j( }|j)duo|j)*}|p|}|rt j+dddnt j+d|ddj,||j%t d|||| |zCt dtt-t.j/|dd|tj|d}tj0|sVt|d} t1j2||| |d| dd dWdn 1sQwYWnt3yit jd|d dYnwt4j5d7_5ddl6m7}t89|}t j+d|ddj j:|7_:t;|d }d!|i}t?j j@rg}nj jApg}tB||\_C_Dd}tjErtFjC}d$d%lGmH}|d&j>|d'jCtId(d)dfd*dd+t4jJtKjD7_Jd,d-jDD}g|j>jLd.d/D] }|jMd0q fd1d-tND}tOjC|jjP|t89} jQt89| }!|!dkrIt j+d2|!ddd$d3lRmS}"|"}#j jTjUrj jTjVtWjXkrd$d4lYmZ}$|#j[j\]D]\}%}&|&j^dkr|$d$|&j_|#j[j\|%<qmtj|d5}'tj0|'sd6jCj`d7d8}(|(ad9d:}(t|'d } | b|(Wdn 1swYt jcd;|'ddd _=tjEr|nj>})j jdtejfks܈j jgs||)j%jCjhd<Sd$d=limjfd>d-tN|D}*tk|*fd?d-|*DjC}+||)j%|+jh|*d@S)ANr)VllmSerializableFunctionz9Traced files (to be considered for compilation cache): %scs dS)N rar7)forward_code_filesr7r8rr!z&VllmBackend.__call__..zzFailed to read file %sr~ torch_compile_cacheT)exist_okrank_rz'vLLM's torch.compile cache is disabled.rrz2Using cache directory: %s for vLLM's torch.compilezAtorch.compile cache factors: env=%s cfg=%s comp=%s code=%s dir=%sz2Compile env factors (raw): %s Vllm config hash: %sx)widthzcache_key_factors.jsonr)env config_hash code_hash compiler_hash)r sort_keyszCould not write compile cache metadata at %s; continuing without metadata. Compiled cache remains valid; diagnostics may be limited.)exc_info)torch_compile_start_timez&Dynamo bytecode transform time: %.2f sgeAzdynamo.time_secondszDynamo bytecode transformz#VllmBackend can only be called oncer)lazy_format_graph_codez before splitz after split graph_dumpcSsddiS)Nrvllm_piecewise_split_graphr7r7r7r7r8rArcsjjddS)NF print_output)rprint_readabler7r\r7r8rBsrvcSsg|]}|js|jqSr7)rr)rpitemr7r7r8rFs z(VllmBackend.__call__..rrrcs(g|]\}}t|tjr|n|qSr7)rJrDTensor)rpr1t)all_fake_valuesr7r8rQsz-Saved compiler manager cache in %.2f seconds.)detect_fake_mode) ValueRangeszcomputation_graph.pyz0from __future__ import annotations import torch FrzrzComputation graph saved to %s)r8r is_symboliccs>g|]\}}t|tjjjrtfdd|Dr|qS)c3s|]}|VqdSrVr7)rpdrr7r8rr`z2VllmBackend.__call__...)rJrD _subclasses fake_tensor FakeTensoranysizer#rr7r8rscsg|]}|qSr7)r.r_)rr7r8rs)r8rr%)lrGr}rUr|r?compile_factorsrrWr@r+sortedr; traced_filesrFrGrrrwrzOSErrorUnicodeDecodeErrorwarninghashlibsha256rtencode hexdigestclearr^rjrrrsVLLM_CACHE_ROOTmakedirsparallel_configrankdata_parallel_indexr`local_cache_dirrrAspeculative_config use_ngram_gpurr|rrrrvrfdumprrnum_graphs_seenmonitorrrrrrirr<rr]rZr[rrr=r@rtorch._dynamo.utilsrr num_piecewise_graphs_seenr find_nodesr r,rrr torch._guardsrrsrtrdrBACKEDtorch.utils._sympy.value_rangesr shape_env var_to_ranger{lowerupperrrr debug_oncerrNONEcudagraph_copy_inputsr8%torch.fx.experimental.symbolic_shapesrr:),rSrrr}rU env_factorsenv_hashrr hash_contentfilepathr}rfactorshash_keyr^rdp_rankrr_is_ngram_gpu_enabled meta_pathr dynamo_time start_time attributes fx_split_opsoriginal_split_gmrsubmod_names_to_compiler1 fake_argstime_before_savingrr fake_modersr graph_pathsrcgraph_to_serializer%r9r7)rrrrrSr8__call__s                                    zVllmBackend.__call__)r]Fr)rrrrrrr r<rrrr+rrr rrMrOrjrTrkrirXr]r|r rr7r7r7r8rs<       '( >."r)F)arxrzrrfrrrrr collectionsrcollections.abcrrr contextlibrcopyr functoolsrtypingr rDtorch.fxrrr torch._logging._internalr vllm.envsr? vllm.configr rrvllm.config.compilationrvllm.config.utilsrr vllm.loggerrvllm.logging_utilsrvllm.platformsr vllm.tracingrrvllm.utils.import_utilsrcompiler_interfacerrrrrcounterrpartition_rulesr r!passes.inductor_passr"r#passes.pass_managerr$rrFr+rirr:rLrM BaseExceptionr dataclassrNoderrrOrrrjrkrrr Interpreterrr5rr6r;rr7r7r7r8s                     %  * D 6 ^