o :/i=@sddlZddlZddlZddlZddlmZddlmZddlmZddlm Z ddl m Z ddl m Z dd lmZmZdd lmZdd lmZzddlZWn ey[e d ZYnwejd ddeedeededededef ddZdeeefdededefddZdedBdeedededede defdd Z d!d"d#e d$ed%ed&e!ed'ffd(d)Z"d*e dedBfd+d,Z#d$ed-ed#e fd.d/Z$d ddeedeeded$ed0ed#e dedefd1d2Z%d$ed%ed3ee!eefd4efd5d6Z&dedBdeed$ed%ed3ee!eefd*e d7edefd8d9Z'deedeedeededed:ed0ed3ee!eefd#e d7edefd;d<Z(eGd=d>d>Z)d?e)fd@dAZ*d?ej+fdBdCZ,e-dDkroej.e)j/dEZ0e)1e0e,e02dSdS)FN)contextmanager) dataclass)datetime)Path)ClassVar)PlaceholderModule)ParameterSweepParameterSweepItem) ServerProcess)sanitize_filenamepandas,)server_ready_timeout serve_cmdafter_bench_cmd show_stdoutserve_overridesdry_runrccs||}tdtd|td||r#dVtddSt|||d}|j|d|VWdn1s>wYtddS)Nz[BEGIN SERVER]zServer overrides: zServer command: z [END SERVER])r)timeout) apply_to_cmdprintr wait_until_ready)rrrrrr server_cmdserverrh/lsinfo/ai/hellotax_ai/llm_service/venv_vllm/lib/python3.10/site-packages/vllm/benchmarks/sweep/serve.py run_servers   rrun_databench_overrides run_numbercCs ||d<|||||S)Nr )update)rrrr rrr_update_run_data5s  r"r bench_cmd output_pathc Csg||ddddt|jd|j}tdtd|td|td |td ||rftd td |d }t|} t | |||WdS1sawY|dury|sst d|tddS|jj ddd| || |d  }t|} Wdn1swYt | |||} |d}tj| |ddWdn1swYtd| S)Nz--percentile-metricszttft,tpot,itl,e2elz --save-resultz --result-dirz--result-filenamez[BEGIN BENCHMARK]zBenchmark overrides: z Run Number: zBenchmark command: z Output file: zFound existing results.z[SKIPPED BENCHMARK]rbzCannot find results at z[END BENCHMARK]T)parentsexist_okwindent)rstrparentnamerexistsopenjsonloadr" ValueErrormkdirrun_subcommand after_benchdump) rr#rrr r$r benchmark_cmdfrrrr run_benchmarkBsp          r:r) extra_partsexperiment_dir serve_comb bench_combr;.cCsRtt}|r|d|jf|r|d|jf|r |||td|S)NzSERVE-zBENCH--)listr,extendr.r join)r<r=r>r;partsrrr_get_comb_base_paths  rD base_pathcCs |dur|dS|d|dS)Nz summary.jsonzrun=z.jsonr)rEr rrr_get_comb_run_pathsrF bench_combscCs0|D]}t|||}t|ddsdSqdS)Nr TF)rDrFr/)r=rGr<r>rErrr_comb_needs_servers  rI bench_paramscCs(t|||s tSt||||||dS)N)rrrr)rI contextlib nullcontextr)rrrr=rJr<rrrrr server_ctxs rM link_varsreturncstfdd|DS)Nc3s4|]\}}|vo|vo||kVqdSNr).0 serve_key bench_keyr>r=rr s z!_comb_is_valid..)all)r=r>rNrrTr_comb_is_validsrWnum_runsc Cst|||sdSttttf}t|D]} t||||| t|| |d} | dur.|| q|r3dSt|dd d} t j || ddWd|S1sPwY|S)N)rrr r$rrHr(r)r*) rWr@dictr,objectranger:rFappendr0r1r7) rr#r=r>rNrErXr comb_datar rr9rrrrun_combs0    r^ serve_paramsc Csttttf} |D]>} t|||| ||| |d(} |D]}t|| |}t| || |||| | d}|dur9| |qWdn1sDwYq | rNdStj | }| |d|S)N)rr=rJr<rr)r=r>rNrErXrz summary.csv) r@rYr,rZrMrDr^rApd DataFrame from_recordsto_csv)rr#rrrr_rJrNr<rXrall_datar=rr>rEr] combined_dfrrr run_combssH    rfc@seZdZUeeed<eeed<eeed<eed<eed<eed<eed<ee eefed<e ed <eed <eed <eed <eed <dZ e eed<dZ e eed<edejfddZedejdejfddZededee eeffddZde fddZede fdd Zd!S)"SweepServeArgsrr#rrrr_rJrN output_direxperiment_namerXrresumeserve parser_namez2Run vLLM server benchmark under multiple settings. parser_helpargsc Cst|j}t|j}|jdurgnt|j}|jr#t|j}ntig}|j r3t|j }ntig}| |j }|j rF|j }nt d}|j} | dkrXtd|||||j|||t|j|| |j|j|jd S)Nz %Y%m%d_%H%M%Srz `num_runs` should be at least 1.) rr#rrr_rJrNrhrirXrrjr)shlexsplitrr#rr_r read_jsonrbrJparse_link_varsrNrirnowstrftimerXr3rrrhrrjr) clsrnrr#rr_rJrNrirXrrr from_cli_argsCs@     zSweepServeArgs.from_cli_argsparserrOcCs|jdtddd|jdtddd|jdtddd |jd d d d |jdtddd |jdtddd |jdtddd |jdtddd |jddtddd |jddtddd |jdtd d!d |jd"d d#d |jd$d d%d |S)&Nz --serve-cmdTz4The command used to run the server: `vllm serve ...`)typerequiredhelpz --bench-cmdz=The command used to run the benchmark: `vllm bench serve ...`z--after-bench-cmdzlAfter a benchmark run is complete, invoke this command instead of the default `ServerWrapper.clear_cache()`.)rxdefaultrzz --show-stdout store_truez^If set, logs the standard output of subcommands. Useful for debugging but can be quite spammy.)actionrzz--server-ready-timeoutrz:Timeout in seconds to wait for the server to become ready.z--serve-paramsaPath to JSON file containing parameter combinations for the `vllm serve` command. Can be either a list of dicts or a dict where keys are benchmark names. If both `serve_params` and `bench_params` are given, this script will iterate over their Cartesian product.z --link-varszComma-separated list of linked variables between serve and bench, e.g. max_num_seqs=max_concurrency,max_model_len=random_input_lenz--bench-paramsa Path to JSON file containing parameter combinations for the `vllm bench serve` command. Can be either a list of dicts or a dict where keys are benchmark names. If both `serve_params` and `bench_params` are given, this script will iterate over their Cartesian product.z-oz --output-dirresultsz0The main directory to which results are written.z-ez--experiment-namezwThe name of this experiment (defaults to current timestamp). Results will be stored under `output_dir/experiment_name`.z --num-runsz)Number of runs per parameter combination.z --dry-runzFIf set, prints the commands to run, then exits without executing them.z--resumezResume a previous execution of this script, i.e., only run parameter combinations for which there are still no output files under `output_dir/experiment_name`.) add_argumentr,int)rurwrrr add_cli_argsrs   zSweepServeArgs.add_cli_argsscCsD|sgSg}|dD]}|d\}}|||fq |S)N,=)rpr\strip)rpairsitemabrrrrrszSweepServeArgs.parse_link_varscCsF|j|j}|jr|std||S|r!td||S)Nz/Cannot resume from non-existent experiment_dir=z)Cannot overwrite existing experiment_dir=)rhrirjr/r3)selfr<rrrresolve_experiment_dirs z%SweepServeArgs.resolve_experiment_dirr<c cs^|jrdVtd|dSz dVtd|WdSty.}ztd|d}~ww)NzExperiment will be saved at: zExperiment has been saved at: z`The script was terminated early. Use `--resume` to continue the script from its last checkpoint.)rr BaseException RuntimeError)rr<excrrrrun_ctxs zSweepServeArgs.run_ctxN)__name__ __module__ __qualname__r@r,__annotations__boolrr tuplerrlrrm classmethodargparse NamespacervArgumentParserr staticmethodrrrrrrrrrrg0s2   .d   rgrncCsh|}||!t|j|j|j|j|j|j|j |j ||j |j d WdS1s-wYdS)N) rr#rNrrrr_rJr<rXr) rrrfrr#rNrrrr_rJrXr)rnr<rrrrun_mains  $rcCstt|dSrP)rrgrv)rnrrrmainsr__main__) description)3rrKr1ror dataclassesrrpathlibrtypingrvllm.utils.import_utilsr param_sweepr r rr utilsr r r` ImportErrorr@r,rrrrYrZr"r:rrDrFrIrMrWr^rfrgrrrrrrmrwr parse_argsrrrrs0                M       '    4N