o :/iF@sddlZddlZddZddZeddlZddlmZddlm Z ee Z dej d <dej d <d ej j_d d ZdeefddZd dedefddZd!ddZddZee drddlmZddlmZddlmZmZeegdej jjd<ee_ee_ dSdS)"NcCszTtjd}|s WdS|jrtj|j}n |jr |jd}nWdStj|d}tj |s3WdStj d|}|r?|j sBWdStj |}|j |t|ddWSty^YdSw)zPeripheral function to _maybe_set_cuda_compatibility_path(). PyTorch version must not be determined by importing directly because it will trigger the CUDA initialization, losing the chance to set the LD_LIBRARY_PATH beforehand. torchNrz version.pyz torch.versioncuda) importlibutil find_specoriginospathdirnamesubmodule_search_locationsjoinexistsspec_from_file_locationloadermodule_from_spec exec_modulegetattr Exception)spec torch_root version_pathver_specmoduler^/lsinfo/ai/hellotax_ai/llm_service/venv_vllm/lib/python3.10/site-packages/vllm/env_override.py_get_torch_cuda_versions*       rc s:tjdddv}|sdStjdd}|r tj|s8tjdd}tj|d}|r8tj|r8|}|r@tj|sSt}|rSd |d }tj|rS|}|r[tj|s]dStj |tjd d}|rr| tj ng}|r|d rtj |d krdSgfd d|D}tj |tjd <dS)aSet LD_LIBRARY_PATH for CUDA forward compatibility if enabled. Must run before 'import torch' since torch loads CUDA shared libraries at import time and the dynamic linker only consults LD_LIBRARY_PATH when a library is first loaded. CUDA forward compatibility is only supported on select professional and datacenter NVIDIA GPUs. Consumer GPUs (GeForce, RTX) do not support it and will get Error 803 if compat libs are loaded. VLLM_ENABLE_CUDA_COMPATIBILITY0)1trueNVLLM_CUDA_COMPATIBILITY_PATH CONDA_PREFIXz cuda-compatz/usr/local/cuda-z/compatLD_LIBRARY_PATHrcs$g|]}|rtj|kr|qSr)rr normpath).0p norm_pathrr Osz6_maybe_set_cuda_compatibility_path..) renvirongetstriplowerr isdirr rr$splitpathsep) enablecuda_compat_path conda_prefix conda_compattorch_cuda_version default_pathexistingld_paths new_pathsrr'r"_maybe_set_cuda_compatibility_path's4     r:) init_loggeris_torch_equalrPYTORCH_NVML_BASED_CUDA_CHECKTORCHINDUCTOR_COMPILE_THREADSc sddlmmddlm}m}m}m}m}ddl m dt t ffdd }t jj|r@jjjdur@|jjjj}nj}|jrrt |jd|rr|jdjj|vrr|j|jrrt |jd|rr|jdjj|vsY|g}g} tt|jD]/} |j| } t | |r| |d|j| <qt | |r||qt | |r| |q| |t|dksJdS)Nr)EnterSubgraphLineExitSubgraphLineMemoryPlanningLineMemoryPlanningStateSubgraphPythonWrapperCodegenVreturncsddl}g}|d}|d}|D]3}t|jr)|jjdt|qt|jr>|jjdt|q|| q|S)Nr_none_shape) itertoolscount isinstanceNoneAsConstantBufferappendgraphnamenextShapeAsConstantBufferget_name) graph_outputsrKnames shape_counter none_counternoderGirrrget_output_namess    z3memory_plan_reuse_patched..get_output_names)torch._inductor.ir _inductorr[torch._inductor.codegen.wrapperrArBrCrDrEtorch._inductor.virtualizedrGliststrrMrP wrapper_codepartition_signatures output_nodesr\linesrYrQpoprangelenplanrO) selfrArBrCrDrEr\ out_namesplanning_statespast_planning_statesilinerrZrmemory_plan_reuse_patchedtsF         rrskip_cudagraphscsddlm}ddlm}mmddlmddlm }g}|j } dt dtffdd tt|t|D]\}} |} |D] } | | jqI| |} |jd d |D} |fd d | j| jBD| }|fd d|D}||D]} | jqfdd | D}||fdd|D}fdd|D}fdd |D}| ||fdd| D} fdd | D}fdd |D}||}|||||| |}||||| }q@|dddS)z Gets signature for each graph partition, including input nodes, output nodes, and whether deallocating an input within graph partition. r) dependencies)GraphPartitionSignatureMutationOutput NoneLayoutrF) OrderedSetbuf_namerHcsVj|d}|dur dSt|jjr)t|jr'j|d}r'|SdSdS)z Checks if buf_name is NoneLayout. Buffers with NoneLayout is not allocated so graph partition should not take it as inputs or outputs. NFT) name_to_bufr+rMrYlayoutmutation_real_name)rybuf real_name)rvrwis_none_layoutrlrrrs z=get_graph_partition_signature_patched..is_none_layoutcSsg|]}|jqSr) read_writes)r%rYrrrr)sz9get_graph_partition_signature_patched..csg|] }|js|jqSr)rQ)r%x)rrrr)s c3|] }j||VqdSNr|r+r%rQrlrr  z8get_graph_partition_signature_patched..csg|]}|vr|qSrrr name_to_noderrr)s csi|] }|vr||qSrrrrrr  z9get_graph_partition_signature_patched..csi|] }|vr||vqSrrrbuffer_names_to_freerrrrrcs g|] }|vr|vr|qSrrrrrrr)s c3rrrrrrrr#rcsg|] }|s|qSrrr)rrrrr)'s csg|] }|jjvr|qSr)rP constantsrrFrrr)-sNr])torch._inductorrtr^rurvrwrarGtorch.utils._ordered_setrxrPr\get_name_to_nodesrcboolzipreversedupdateoutputs_by_namekeys intersection ReadWrites merge_listreadswrites last_usage!get_graph_partition_symbol_inputsrOunion)rl partitionsrsrtrurx signaturesunmet_output_names partitionskip_cudagraph output_namesrYreturned_output_namesrpartition_input_namesextra_input_names input_nodesinput_deallocationextra_output_namesrfconstant_names symbol_inputspartition_signaturer)rvrwrGrrrrlr%get_graph_partition_signature_patcheds                     rF should_logrHcsddlmm}ddlm}m}ddlm}m}m }|j } t | t jjj rY| j} rY| } t | t jjr>| d| jn| } | t jjjvsN| t jjjvrYt | t jjsWJdSt jjjjsf|jdurfdSdtd|dBd dfd d } |rw|n| }t ||rtfd d |jDS|j dusJ|s|d|ddSt |j |jr|d|ddSt |j |jr|d|ddSt|j ddr|d|ddS||j r|d|ddSdS)zBReturn True if we should partition the inductor graph on this noderN)BaseSchedulerNodeFusedSchedulerNode)&_unstable_customized_partition_wrapperis_cudagraph_unsafe_opmaybe_log_cudagraph_partition.TmsgrYrHcSsdSrr)rrYrrrnoop_logsz*should_partition_patched..noop_logc3s|]}|VqdSr)should_partition)r%snoderrrrsz+should_partition_patched..z non gpu ops)rYzDeviceCopy opszConditional opsunbacked_bindingszunbacked binding opszCUDAGraph-unsafe custom opsF)r^r_r[torch._inductor.schedulerrrtorch._inductor.utilsrrrrYrMrFallbackKernel op_overloadrQ_ops OpOverload _overloadnameconfigcustom_should_partition_opstriton cudagraphswrapperrcanysnodesis_gpu DeviceCopy Conditionalr)rlrYrr[rrrrrir_nodeopop_overload_packet_nameop_overload_namerlog_partition_reasonrrrshould_partition_patchedZsV           rcCshddlmm}ddlm}t|_t|_| dd||j |_ WddS1s-wYdS)z (Re)initializes the scheduler member. When initializing the scheduler, no CUBIN files should be generated (to avoid biasing any benchmarks and pessimizing fusion decisions). rN) Schedulerztriton.store_cubinF) torch._inductor.configr_rrrrrrget_graph_partition_signaturepatch operations scheduler)rlrrrrr_update_scheduler_patcheds "rcCsNddlm}|ds|dr%ddl}ttjdr#ddlm}||_dSdSdS)z;Workaround for TorchInductor autotune get_raw_stream() bug.rr<2.9.0z2.9.1N_cuda_getCurrentRawStream)r) vllm.utils.torch_utilsr=builtinshasattrr_Ctorch._Crget_raw_stream)r=r_get_raw_streamrrr_patch_get_raw_stream_if_neededs    rr)PythonWrapperCodegen) GraphLowering)_Config _ConfigEntry)defaultr)F)rHN)!importlib.utilrrrr:r vllm.loggerr;rr=__name__loggerr*r_rcompile_threadsrrrbrrrrrr`rtorch._inductor.graphrtorch.utils._config_modulerr_configmemory_plan_reuse_update_schedulerrrrrs>.     D # S