)j dddlZddlZddlZddlZddlZddlZddlmZddlmZddl m Z m Z m Z m Z mZmZmZddlmZddlmZddlmZddlmZddlmZdd lmZmZmZm Z m!Z!m"Z"m#Z#m$Z$dd l%m&Z&dd l'm(Z(dd l)m*Z*m+Z+d Z,dZ-dZ.dZ/dZ0dZ1dZ2dZ3dZ4dZ5dZ6dZ7dZ8dZ9ej:ej;Zdee ej?fdZ@eGddZAdZBdddddddddddd dejCdej>d eDd!ee ejCgejCfd"ee e ejCejCgejCfd#eeDd$ee d%eDd&eeDd'eDd(eDd)ee eDeDgdfd*eejCd+e eejCejCfddffd,ZEd-ddddd.dddd/ dejCdej>d0ej>d1eDd eDd!ee ejCgejCfd"ee e ejCejCgejCfd$ee d%eDd&eeDd'eDd(eDd+e eejCejCeFfddffd2ZG dNdej>d3eee(fdeeHejCe eDfd eDd0eej>d+e eAddff d4ZI dOdej>d3eee(fdeeHe eDfd6eFd+eHf d7ZJdMd8ZKdMd9ZLeGd:d;ZMeGd<d=ZNeGd>d?ZOd@ZPdAZQdBZRGdCdDZS dPdFe e eDdGee e e d eeDe eDfd6eFdHeFd"ee e ejCejCgejCfd+eNfdIZTdJZUeVdKkreWdLeUdSdS)QN) dataclass)partial)AnyCallable GeneratorListOptionalTupleUnion) tree_reduce)PreTrainedTokenizercache) ArraysCache BatchKVCacheBatchRotatingKVCache CacheListKVCacheQuantizedKVCacheRotatingKVCacheload_prompt_cache) make_sampler)TokenizerWrapper)#does_model_support_input_embeddingsloadhellodg?z(mlx-community/Llama-3.2-3B-Instruct-4bitic.|dvS)N)falsef)lower)strings Y/lsinfo/ai/hellotax_ai/base_platform/venv/lib/python3.11/site-packages/mlx_lm/generate.pystr2boolr&7s <<>> //ctjd}|dtdtdd|dd d |d td |dtddd|ddd|ddt d|ddd|ddt td|d ttd!|d"ttd#|d$ttd%|d&t td'|d(ttd)|d*td+d,|d-t td.|d/t td0|d1d d2 |d3d d4 |d5d6d7|d8t d9d:|d;t d|d?d@d dA |dBt dCd|dDt dEdF|dGdHt t"I|dJtdKd|dLt dMdN|S)Oz&Set up and return the argument parser.zLLM inference script) descriptionz--modelz[The path to the local model directory or Hugging Face repo. If no model is specified, then z is used.N)typehelpdefaultz--trust-remote-code store_truez)Enable trusting remote code for tokenizer)actionr+z--adapter-pathz9Optional path for the trained adapter weights and config.)r*r+z--extra-eos-token+z:Add tokens in the list of eos tokens that stop generation.)r*r,nargsr+z--system-promptz.System prompt to be used for the chat template)r,r+z--promptz-pz;Message to be processed by the model ('-' reads from stdin)z--prefill-responsez1Prefill response to be used for the chat templatez --max-tokensz-mz$Maximum number of tokens to generate)r*r,r+z--tempzSampling temperaturez--top-pzSampling top-pz--min-pzSampling min-pz--top-kzSampling top-kz--xtc-probabilityz5Probability of XTC sampling to happen each next tokenz--xtc-thresholdrzDThresold the probs of each next token candidate to be sampled by XTCz--min-tokens-to-keepz*Minimum tokens to keep for min-p sampling.z--seedz PRNG seedz--ignore-chat-templatez9Use the raw prompt without the tokenizer's chat template.z--use-default-chat-templatezUse the default chat templatez--chat-template-configzAdditional config for `apply_chat_template`. Should be a dictionary of string keys to values represented as a JSON decodable string.)r+r,z --verboseTzTLog verbose output when 'True' or 'T' or only print the response when 'False' or 'F'z --max-kv-sizez$Set the maximum key-value cache sizez--prompt-cache-filez;A file containing saved KV caches to avoid recomputing themz--quantize-activationsz-qazSQuantize activations using the same quantization config as the corresponding layer.z --kv-bitszFNumber of bits for KV cache quantization. Defaults to no quantization.z--kv-group-sizez%Group size for KV cache quantization.@z--quantized-kv-startzLWhen --kv-bits is set, start quantizing the KV cache from this step onwards.)r+r*r,z --draft-modelz,A model to be used for speculative decoding.z--num-draft-tokensz:Number of tokens to draft when using speculative decoding.)argparseArgumentParser add_argumentstr DEFAULT_MODELDEFAULT_PROMPTintDEFAULT_MAX_TOKENSfloat DEFAULT_TEMP DEFAULT_TOP_P DEFAULT_MIN_P DEFAULT_TOP_KDEFAULT_XTC_PROBABILITYDEFAULT_MIN_TOKENS_TO_KEEP DEFAULT_SEEDr&DEFAULT_QUANTIZED_KV_START)parsers r%setup_arg_parserrF;s  $1G H H HF   G.; G G G  8    H    I   =    J    @   " 3  ul9O };K };K ]9I  ' D     S    * 9          H  % ,   I     c     3     J      b     U     4   " *    ;     I  Mr'modelstreamsc#NKtjs dVdS#wxYwtd|d}tjd}|d|zkr |dz}|dz}t d|d|d tj|} dV||D]}tj|ntjtj|dS#||D]}tj|ntjtj|wxYw) a A context manager to temporarily change the wired limit. Note, the wired limit should not be changed during an async eval. If an async eval could be running pass in the streams to synchronize with prior to exiting the context manager. NcNt|tjr ||jzn|SN) isinstancemxarraynbytes)accxs r%zwired_limit..s!Z28-D-DM3>>#r'r max_recommended_working_set_sizeg?iz0[WARNING] Generating with a model that requires z6 MB which is close to the maximum recommended size of z MB. This can be slow. See the documentation for possible work-arounds: https://github.com/ml-explore/mlx-lm/tree/main#large-models)rMmetal is_availabler device_infoprintset_wired_limit synchronize)rGrH model_bytes max_rec_sizemodel_mb max_rec_mb old_limitss r% wired_limitr`s 8 " "*  EEE DDDDD! M MuVW  ~''(JK |+ + +"e+H%.J N8NNEONNN    &|44  * EEE" &&AN1%%%%&    y ) ) ) ) ) " &&AN1%%%%&    y ) ) ) )s(*CAD$ceZdZUdZeed<eed<ejed<e ed<eed<e ed<eed<e ed <e ed <d Z e eed <d S) GenerationResponsea The output of :func:`stream_generate`. Args: text (str): The next segment of decoded text. This can be an empty string. token (int): The next token. from_draft (bool): Whether the token was generated by the draft model. logprobs (mx.array): A vector of log probabilities. prompt_tokens (int): The number of tokens in the prompt. prompt_tps (float): The prompt processing tokens-per-second. generation_tokens (int): The number of generated tokens. generation_tps (float): The tokens-per-second for generation. peak_memory (float): The peak memory used so far in GB. finish_reason (str): The reason the response is being sent: "length", "stop" or `None` texttokenlogprobs from_draft prompt_tokens prompt_tpsgeneration_tokensgeneration_tps peak_memoryN finish_reason) __name__ __module__ __qualname____doc__r7__annotations__r:rMrNboolr<rlr r/r'r%rbrb s III JJJh#'M8C='''''r'rbc|dSt|D]:\}}t|dr%|j|kr|||||<;dS)N to_quantized) group_sizebits) enumeratehasattroffsetrt) prompt_cachequantized_kv_start kv_group_sizekv_bitsecs r%maybe_quantize_kv_cacher'so,''UU1 1n % % U!(6H*H*Hnn GnTTLOUUr'r2) max_tokenssamplerlogits_processors max_kv_sizerzprefill_step_sizer}r|r{prompt_progress_callbackinput_embeddingspromptrrrrrzrr}r|r{rrreturnc #DK| tstdt|dkrPt|t| kr0tdt| dt|dn"t|dkrtddtj|| pd } t jt| | | pd d tj d ttj ffd dd tj d ttj ffd } tj t5| t| nt|}d}| ||||z dkr||z dz }t||}|d|d| | d|dndtjdD||z }| ||||d}| | |dn| } tj||z dk| || \}}dddn #1swxYwYtj||d} ||kr#| |\}}tj|||dkr tj|| ||||krdS||fV|dzdkrtj||}}|dz })a A generator producing token ids based on the given prompt from the model. Args: prompt (mx.array): The input prompt. model (nn.Module): The model to use for generation. max_tokens (int): The maximum number of tokens. Use``-1`` for an infinite generator. Default: ``256``. sampler (Callable[mx.array, mx.array], optional): A sampler for sampling a token from a vector of log probabilities. Default: ``None``. logits_processors (List[Callable[[mx.array, mx.array], mx.array]], optional): A list of functions that take tokens and logits and return the processed logits. Default: ``None``. max_kv_size (int, optional): Maximum size of the key-value cache. Old entries (except the first 4 tokens) will be overwritten. prompt_cache (List[Any], optional): A pre-computed prompt cache. Note, if provided, the cache will be updated in place. prefill_step_size (int): Step size for processing the prompt. kv_bits (int, optional): Number of bits to use for KV cache quantization. None implies no cache quantization. Default: ``None``. kv_group_size (int): Group size for KV cache quantization. Default: ``64``. quantized_kv_start (int): Step to begin using a quantized KV cache. when ``kv_bits`` is non-None. Default: ``0``. prompt_progress_callback (Callable[[int, int], None]): A call-back which takes the prompt tokens processed so far and the total number of prompt tokens. input_embeddings (mx.array, optional): Input embeddings to use instead of or in conjunction with prompt tokens. Default: ``None``. Yields: Tuple[mx.array, mx.array]: One token and a vector of log probabilities. Nz(Model does not support input embeddings.rz8When providing input_embeddings, their sequence length (z0) must match the sequence length of the prompt (z), or the prompt must be empty.z=Either input_embeddings or prompt (or both) must be provided.)rcdSrKr/_s r%rRzgenerate_step..ustr'r{r|r}c.tj|dSNaxisrMargmaxrQs r%rRzgenerate_step..~BIab$9$9$9r' input_tokensrc>|||S|S)N)rrrr/)rrrGrzs r% _model_callz"generate_step.._model_callsA  '5LCS 5\::: :r'ctjt5|d||dnd}|dddddf}r>t|dkr+ tj |gn| D]}| |} |tj|dz } |}||dfcdddS#1swxYwYdS)NrrrrT)keepdims)rMstreamgeneration_streamlenconcat logsumexpsqueeze) rrlogits processorresampledrrrzquantize_cache_fnrtokenss r%_stepzgenerate_step.._stepszY( ) ) 0 0 [)$/.>.J$T**PTFAAAr111H%F  7S%6%6%:%:)Iv|4555% "377I&Yvv66FF  l + + + Vd C C CCHgh''GH,,Q//// 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0sB2CC!Crrcg|] }|j Sr/state.0rs r% z!generate_step..333QW333r'TrrK)r ValueErrorrrmake_prompt_cache functoolsrrrMrNr rrmineval clear_cache async_evalitem)rrGrrrrrzrr}r|r{rrrtotal_prompt_tokensprompt_processed_tokens remaining n_to_processyrennext_y next_logprobsrrrs ` `` ` @@@r% generate_stepr/s^#2599 GHH H [[1__V4D0E0E!E!E(3O_K`K`((ADV(((  V   K   F. #   8LOO!)-# :99G;"(;hrx>P;;;;;;;00BH08J000000000008 $ % %TT%5%AC ! ! !s6{{ #$  !8:MNNN!$;;a??,/FF!KI0)<Q R R RLMM*F$/!//%  N   +"$;;a??.eBRSSS 8;TTTTTTTTTTTTTTT>M!X A  ??$)E!HH !FM M&- 0 0 0 66 GAJJJ $ $%8:M N N N ?? Effhh    s7a<< N   m8 Q s#C;I**I.1I.) num_draft_tokensrrrrzrr}r|r{ draft_modelrc #K|tj} d|)tj|tjn8|dt |j|t |jdpdtjt| | | fddfd fd} fd}fd }tj t5| | }| || } dddn #1swxYwYd }d }d } t||z |}|||}dj | j z |z dztj| |g} || |dz\}}tj|||}|}d }||kr=||||||}}}||krn|dz }|dz }||d fV||krn||k=||kr|dz }||||d fV||krntj||gtj} | }||kr;tjtj|d dtj|g}dt%||z d ||| |||dS#|||wxYw)au A generator producing token ids based on the given prompt from the model. Args: prompt (mx.array): The input prompt. model (nn.Module): The model to use for generation. draft_model (nn.Module): The draft model for speculative decoding. num_draft_tokens (int, optional): The number of draft tokens for speculative decoding. Default: ``2``. max_tokens (int): The maximum number of tokens. Use``-1`` for an infinite generator. Default: ``256``. sampler (Callable[[mx.array], mx.array], optional): A sampler for sampling a token from a vector of log probabilities. Default: ``None``. logits_processors (List[Callable[[mx.array, mx.array], mx.array]], optional): A list of functions that take tokens and logits and return the processed logits. Default: ``None``. prompt_cache (List[Any], optional): A pre-computed prompt cache. Note, if provided, the cache will be updated in place. The cache must be trimmable. prefill_step_size (int): Step size for processing the prompt. kv_bits (int, optional): Number of bits to use for KV cache quantization. None implies no cache quantization. Default: ``None``. kv_group_size (int): Group size for KV cache quantization. Default: ``64``. quantized_kv_start (int): Step to begin using a quantized KV cache. when ``kv_bits`` is non-None. Default: ``0``. Yields: Tuple[mx.array, mx.array, bool]: One token, a vector of log probabilities, and a bool indicating if the token was generated by the draft model Nc.tj|dSrrrs r%rRz+speculative_generate_step.. rr'rc|rD]}|||}|tj|ddz }|}||fS)NrTrr)rMr)rrrrerrrs r%_process_and_samplez6speculative_generate_step.._process_and_samplesa  3. 3 3 "6622BLb4HHHH GH  ({r'rc tjt5||d|}|dd| dddf} | rgg}}|dkr|d|dz }t|D]b} tj |gn|  |dd|ddf\}}||||ctj|dtj|dfcdddS d|dcdddS#1swxYwYdS)Nrrrr)rMrrrange concatenateappendr) rGrr n_predictrout_y out_logprobsirerr prev_tokensrs r%rz(speculative_generate_step.._steps Y( ) ) D DU1T7%000FAAA z{{AAA-.F  e $ $ $  D&("|q==,i!m,,-Ay))22A'2 Q'7888 #6"5k6!!!QPQPQPQ'?"S"SKAxLLOOO ''1111~e!444bn q777' D D D D D D D D.+*41B1BCC/ D D D D D D D D D D D D D D D D D DsC(D;D;;D?D?c|jkrl||dd||tjd|D|d}tj|jkl|S)Nrcg|] }|j Sr/rrs r%rz?speculative_generate_step.._prefill..=s,,,QW,,,r')sizerMrr)rGrrrrs r%_prefillz+speculative_generate_step.._prefill9sf((( E!&&&'-U ; ; ; ;  e $ $ $ G,,e,,, - - -#$$%A N    f((( r'ctj||z tjt||z dz ddS)Nrr)rtrim_prompt_cachemax) num_draft num_accept draft_cache model_caches r% _rewind_cachez0speculative_generate_step.._rewind_cacheBsH  Y-CDDD  SZ1G!1KQ-O-OPPPPPr'c|dkrtjgtjSg}t|D];}|\}}tj|||._draft_generateFs >>8B ** * y!!  A5k155DAq M!    IIaLLLL~b!!!r'rTFr)r)astyperMrrrrlayersrrrrrrrrrtolistrNr) rrGrrrrrrzrr}r|r{rrrrdraft_yntoksrr draft_tokensrretndtnlpnrrrrrrs ` `` ` @@@@@@r%speculative_generate_steprsPZ  bi  AK-e44 -k:: "#6S%6%6#67 "3u|#4#4#6#67 :99G!)-# DDDDDDDDD4QQQQQQ""""""" $ % %,,(; Q77 HUK + +,,,,,,,,,,,,,,, EI A+$( (J.0@AAI*?7I>>L&)*UK,z"stream_generate..s>  )8UHe $      r'rrrreA) rcrdrerfrgrhrirjrkrlstoplength)rLrrMrNr7 bos_token startswithencode detokenizerpoprrr`rtime perf_counterrwr eos_token_ids add_tokenrb last_segmentget_peak_memoryfinalize)rGrrrrkwargsrrtoken_generatorticrrdrerf prompt_timerhs r%stream_generater s~8 i!1 2 20$Y// fbh ' '" fc " " U!*!4!<"FDUDU#EEA %%fAS%TTF&!!'K%F< %t,,,'@@@@  > 5 6 6 6 F  8x- 8 8" 7 8 8 8     <85 < <& ; < < <    ;h2;;;;<<< Kr'cztd|Dtjfd|DS)Nc34K|]}t|VdSrKrrps r%rz$_left_pad_prompts..(11AQ111111r'cBg|]}dgt|z z|zSrrrr max_lengths r%rz%_left_pad_prompts..s0FFFaSJQ/014FFFr'rrMrNpromptsrs `r%_left_pad_promptsr"J1111111 8FFFFgFFF G GGr'cztd|Dtjfd|DS)Nc34K|]}t|VdSrKrrs r%rz%_right_pad_prompts..rr'cBg|]}|dgt|z zzSrrrs r%rz&_right_pad_prompts.. s0FFFQ! SVV 344FFFr'rr s `r%_right_pad_promptsr'r#r'cveZdZUdZdZeed<dZeed<dZ eed<dZ eed<dZ eed<dZ eed<dZ eed <d S) BatchStatsa% An data object to hold generation stats. Args: prompt_tokens (int): The number of prompt tokens processed. prompt_tps (float): The prompt processing tokens-per-second. prompt_time (float): The time in seconds spent in prompt processing. generation_tokens (int): The number of generated tokens. generation_tps (float): The tokens-per-second for generation. generation_time (float): The time in seconds spent in generation . peak_memory (float): The peak memory used so far in GB. rrgrhrrirjgeneration_timerkN)rmrnrorprgr:rqrhr<rrirjr*rkr/r'r%r)r)#s  M3JKsNEOUKr'r)cbeZdZUdZeeed<eed<eeee ed<dS) BatchResponsez An data object to hold a batch generation response. Args: texts: (List[str]): The generated text for each prompt. stats (BatchStats): Statistics about the generation. textsstatscachesN) rmrnrorprr7rqr)r rr/r'r%r,r,;sS 9  T$s)_ %%%%%%r'r,ceZdZUeeed<ejed<ejed<eeed<eeed<eeed<eeed<eeed<eejed <d Z d eefd Z d Z dZ dS)Batchuidsrrer num_tokensrsamplersrrc*t|jSrK)rr2selfs r%__len__z Batch.__len__Vs49~~r'keep_idxcfd|D_fd|D_fd|D_fd|D_fd|D_fd|D_fd|D_tj|tj }j |_ j D]}| |dS)Nc*g|]}j|Sr/)r2rkr7s r%rz Batch.filter..Zs444aTYq\444r'c*g|]}j|Sr/)rer<s r%rz Batch.filter..[ <<.\ @@@!4?1-@@@r'c*g|]}j|Sr/)r3r<s r%rz Batch.filter..]rAr'c*g|]}j|Sr/)r4r<s r%rz Batch.filter..^r?r'c*g|]}j|Sr/)rr<s r%rz Batch.filter.._s!!N!N!N$"8";!N!N!Nr'c*g|]}j|Sr/)rr<s r%rz Batch.filter..`s888!t{1~888r') r2rerr3r4rrrMrNint32rrfilter)r7r9rs` r%rGz Batch.filterYs44448444 <<<<8<<< @@@@x@@@@@@@x@@@<<<<8<<< !N!N!N!NX!N!N!N8888x888 8Hbh//!  A HHX      r'cl|j|jtj|j|jg|_|j|j|j|j|j|j|j|j|j |j |j |j t|j |j D]\}}||dSrK) r2extendrMrrrer3rr4rrzipr)r7otherros r%rIz Batch.extendfs $$$ 122 U^,,, u/000 u/000 U^,,, %%e&=>>> 5<((( EK00  DAq HHQKKKK  r'c*fd|jDS)Nc:g|]}|Sr/extract)rridxs r%rz'Batch.extract_cache..ss#3331 #333r'r)r7rQs `r% extract_cachezBatch.extract_cachers3333 3333r'N) rmrnrorr:rqrMrNrr8rGrIrRr/r'r%r1r1Js s)OOO xKKKhS S  93iCy   N tCy       44444r'r1cfdt|dr"|}fd|DSfd|jDSfd|jDS)z[ Convert a list of regular caches into their corresponding batch-aware caches. ct|turtSt|trt j|_|St|tr/|j dkrtdt|j St|trtfd|jDStt|d)Nrz2RotatingKVCache with keep tokens is not supported.c3.K|]}|VdSrKr/)rsub_cto_batch_caches r%rz6_make_cache..to_batch_cache..s-KK~~e44KKKKKKr'z does not yet support batching)r*rrrLrrMrN left_paddingrkeeprrmax_sizerr/)rrXrWs r%rWz#_make_cache..to_batch_cache|s 77g   -- - ; ' ' IXl33ANH ? + + Ivzz !UVVV' LAA A 9 % % IKKKK!(KKKL LQGGGHH Hr' make_cachec&g|] }|Sr/r/)rrrWs r%rz_make_cache..s#111aq!!111r'Nc0g|]}tSr/)r)rrrXrs r%rz_make_cache..s1DE$[,??r'c.g|]}tSr/)r)rrrXs r%rz_make_cache..s!AAAq \**AAAr')rxr[r)rGrXrrrWs `` @r% _make_cacher_vs I I I I I Iul##B  ""111151111  "IN BAAAELAAAAr'cdg}tt|dD]t|ddrA||dfd|D_t t |dd|S)Nrmergec g|] }| Sr/r/rrrs r%rz!_merge_caches..s2H2H2HA1Q42H2H2Hr'z+ does not yet support batching with history)rrrxrrarr*)r/ batch_cachers @r% _merge_cachesresK 3vay>> " " 6!9Q< ) )    vay|112H2H2H2H2H2H2HII J J J Jq ! %%RRR  r'c fd|DS)Nc3BK|]}|VdSrKrOrcs r%rz&_lazy_extract_cache..s- ( (QAIIaLL ( ( ( ( ( (r'r/)rrs `r%_lazy_extract_cacherhs ( ( ( (% ( ( ((r'ceZdZeGddZ d%ded eed eee j ge j fd ee ee j e j ge j fd ed ededeee e eee e fgdfdeee e eeefgdfdeefdZdZdZ d&dee eedfdedzd edzdeezdzfdZd'de edefdZedZdZde j de e dedzd edzd e e j f d!Zd"Zd#Zd$ZdS)(BatchGeneratorcteZdZUeed<eed<ejed<eeed<e ge e fed<dS)BatchGenerator.ResponseuidrdrerlrzN) rmrnror:rqrMrNr r7rrrr/r'r%Responserls[  (}$$$r49}------r'rnN rr stop_tokensrrcompletion_batch_sizeprefill_batch_sizerprompt_checkpoint_callbackrrc ||_g|_||_|p t|_|pd|_|pg|_d|_||_||_ t|||_ | |_ | pd|_ t|_d|_| |_d|_t&jr2t'jt'jd|_dSd|_dS)Nc.tj|dSrrrs r%rRz)BatchGenerator.__init__..sRYqr-B-B-Br'rcdSrKr/rs r%rRz)BatchGenerator.__init__..sPTr'rS)rGunprocessed_promptsrsetrrrr uid_countrrtrrsrurr)_stats _next_countr active_batchrMrTrUrXrV_old_wired_limit) r7rGrrrrrrsrtrrurrs r%__init__zBatchGenerator.__init__s( #% $&/#%%C#B#B !2!8b!2"4%()>@R%S%S"*D'(@(U__% ll &  8 " " )$&$6  !CD%%D ! ! !%)D ! ! !r'c|j;tjttj|jd|_dSdSrK)rrMrYrrXr6s r%closezBatchGenerator.closesF  , N, - - -  t4 5 5 5$(D ! ! ! - ,r'c.|dSrK)rr6s r%__del__zBatchGenerator.__del__s r'r4prompt_checkpointsc g}|t|tr|p|jgt|z}|t|tr|pdgt|z}|dgt|z}t t|D]&}||t j|j||<'|pdgt|z}|p|jgt|z}t||||||D]Y\} } } } } }|j |j | | | | | |f| |j |xj dz c_ Zt|j d|_ |S)Nrrclt|dtd|dDzS)Nrc3>K|]}|VdSrK)rrs r%rz:BatchGenerator.insert.... s*)A)Aq!&&(()A)A)A)A)A)Ar'r3)rrrs r%rRz'BatchGenerator.insert.. s0#ad))c)A)AAaD)A)A)A&A&AAr')key)rLr:rrrrrrGrrJryrr{sorted)r7r!rr/r4rrr2rrmrr_lppcs r%insertzBatchGenerator.inserts  J!Vc'll*Fs7||$$ @ @Aay !3DJ??q 4vG 4-X$2H1ICPWLL1X"% Z3DFX# #   Aq!QB  $ + +T^Q1aR,P Q Q Q KK ' ' ' NNa NNN#)  $AA$ $ $   r'Fr2return_prompt_cachesc i}t|j|j}|r7t|jD]"\}}|vr ||||<#fdt|jD}t |dkr||nd|_ttt |j D]1}|j |dvr|j |2|r|SdS)Nc"g|] \}}|v | Sr/r/)rr~rmr2s r%rz)BatchGenerator.remove..s"QQQfaDr'r) rzr~rwr2rRrrGreversedrryr) r7r2rr/batchr~rmr9rs ` r%removezBatchGenerator.remove s04yy   (%E# 9' 3399FAs$ "'"5"5a"8"8F3KKQQQQ %*(=(=QQQH8}}q   X&&&&$(!%D$< = =>>?? 0 0A'*1-55(,,Q///  M  r'ctd|jD}|j&|td|jjDz }|S)Nc3:K|]}|dD] }|jV dS)r3NrO)rrrs r%rz5BatchGenerator.prompt_cache_nbytes..&s5MM!MM1AHMMMMMMMr'c3$K|] }|jV dSrKrrs r%rz5BatchGenerator.prompt_cache_nbytes..(s$CCaCCCCCCr')sumryr~r)r7totals r%prompt_cache_nbytesz"BatchGenerator.prompt_cache_nbytes$sVMMd&>MMMMM   ( SCC4+<+BCCCCC CE r'c t|\}}}}}}}d|D} t| fd| D} dt| |D}tdt||jxjt | z c_d|D} dt d|Dr t |}t|j| |j |j dkrt|j |j dz } ||ddd| f tjd D|dd| df}| z |fd t|| Dtj|j dkʐnctjfd |D} t%|}t'|D]%}|fd | D| &|j dkrt|j |j dz } ||ddd| f tjdD|dd| df}| z |fdt|| Dtj|j dktjdD| }D]}||j/|fdt/|DdkrD||ddddz f tjdDtj||||| \}}tj||t5t7|||t7|dgt9|zt7|t7|| S)Nc,g|]}t|Sr/rrs r%rz3BatchGenerator._process_prompts..6s***a3q66***r'cg|]}|z Sr/r/)rlrs r%rz3BatchGenerator._process_prompts..8s333a:>333r'c0g|]\}}|dkr||z n| Srr/)rrrs r%rz3BatchGenerator._process_prompts..=s;   ,1ArrAvvQVVB3   r'rc6g|]}tj|Sr/)rMrN)rinps r%rz3BatchGenerator._process_prompts..Ds 222C"(3--222r'rc3JK|]}|dVdS)rN)emptyrs r%rz2BatchGenerator._process_prompts..Js.,,qtzz||,,,,,,r')rrcg|] }|j Sr/rrs r%rz3BatchGenerator._process_prompts..S777Q777r'c g|] \}}||f Sr/r/rrmrprocessed_tokenss r%rz3BatchGenerator._process_prompts..W5'C.7r'c&g|] }| dSrKr/)rrprompt_checkpoints r%rz3BatchGenerator._process_prompts..ds(#K#K#KqA'8&8&9&9$:#K#K#Kr'cg|]}|z Sr/r/)rrrs r%rz3BatchGenerator._process_prompts..lsDDDqQ!22DDDr')lengths right_paddingcg|] }|j Sr/rrs r%rz3BatchGenerator._process_prompts..urr'c g|] \}}||f Sr/r/rs r%rz3BatchGenerator._process_prompts..yrr'cg|] }|j Sr/rrs r%rz3BatchGenerator._process_prompts..rr'c<g|]\}}|t|fSr/)rh)rrrmrzrs r%rz3BatchGenerator._process_prompts..s@3+-@q-Q-QRr'cg|] }|j Sr/rrs r%rz3BatchGenerator._process_prompts..rr')rJrr|rgrallr"r_rGrshaperrrMrrrrNr'repreparerrurwrrr1listr)r7r!r2inputsrr/r4rrrpaddingrr last_inputsrrrerrrzrs @@@@r%_process_promptszBatchGenerator._process_prompts+s_ M       +*6***\\ 33337333   58BT5U5U    3'9#:#:;; !!S\\1!!226222 ,,V,,, , ,7 !&v*EEEF&tz7DO,O    6!!!]l]"23< HHH77,777888<== 01 L0 --+.tW+=+=    ,q/$555,(#K#K#K#KF#K#K#KLLK':FFFF(00L!   DDDDGDDD") ,q/$555"*FLO>O,O    6!!!]l]"23< HHH77,777888<== 01 L0 --+.tW+=+=    ,q/$555 G33l333 4 4 4 F  A JJLLLL  * 6  + +"+D//    q JJvaaa!8#4q#8!889J N N N G33l333 4 4 4 jj L(,=v  8 a""" JJ     C#d))O  NN " # #    r'rrzrc|jd}|||}|dddddf}t|rig}t|D]A} || | dz} || D]} | || | } || Bt j|d}|t j|ddz } t|reg} t|D]<} || p|j}|| | | dz}| |=t j| d}n|| }|t| fS)NrrrrrTr) rrGanyrrrMrrrr)r7rrzr4rr batch_sizerprocessed_logitsr~ sample_logitsrre all_samplessample_samplerrs r%rzBatchGenerator._steps"'* L ==2qqq!  ! ! >! :&& 7 7 &q1q5y 1 !21!5HHI$-IfQi$G$GMM '' 6666^$41===FBLb4HHHH x== -K:&& , ,!)!!< (.!a!e))<==""7++++n[q999GGll8,,GX&&r'c|jj|jjz |j_|jj|jjz |j_tjdz |j_ |jS)Nr) r|rgrrhrir*rjrMrrkr6s r%r.zBatchGenerator.statssY!%!:T[=T!T  K )DK,G G "#%"4"6"6"< {r'c 2tj}d}|j}|rt|nd}|j|z }||jkr5|jd|j}t|dkr|dkrnt|dkr d|_gS|]|s[tj|j |j |j xj tj|z z c_ tj}| |}|j|jd|_d}|j||_n|j|t|j}|t|z}||jk5|j}|j |j }}t|jD].\} } tj| || | dzf|j| </||dddf|j|j|j|j\|_ |_ tj|j |j |j|}tj} |r|j xj| |z z c_n|j xj | |z z c_ g} g} g}tt3||j|j|jD]\}\}}}}d}|dz }||j|<||jvrd}| |n5||krd}| |nd}| ||||}|| ||||||t| r0t| dkr|!| nd|_|xj"dz c_"|j"dzdkrtj#|j xj$t|z c_$|S)NFrTrrrr)%rrr~rrsrtryrMrrrer|r*rrIrwrrrrr4rrrrrJr2r3rrrrrRrnrGr}rri)r7rprompt_processingr num_active num_to_addr!rrertokstocr9end_idx responsesr~trmnum_tokmax_tokrrls r%_nextzBatchGenerator._nextsF!!!!#(/SZZZa /*< D333./H1H/HIG7||q  Z!^^W""$(!  ): 000 ++t/@/B/BS/HH++'))))'22E'+'?'))(D $!%  ($)!!!((///T.//J #e** $J9D333<!gu~8 .. C CGAt ndAa!a%iL-ABBELOO"&** aaagJ K N  # L # #  egu~u|<<< HHJJ!!  5 K # #sSy 0 # # # K ' '39 4 ' ' -6 5:u/1A B B. .  W W )A)3E qLG")E Q D$$$ & q!!!!G## ( q!!!! $ """(++A..   T]]38A; uUU V V V V w<< )8}}q   X&&&&$(! A  c !Q & & N    %%Y7%%r'ctjt5|cdddS#1swxYwYdSrK)rMrrrr6s r%nextzBatchGenerator.next/s Y( ) )  ::<<                  s ;??) roNNNrprqrNNN)NNNNNF)rmrnrorrnr:r rzrrMrNrr rrrrr rrrrrpropertyrrrr.rrr/r'r%rjrjs.......Y.%)<@ %'"#!%  %)%+)+)+)c] +) (BH:rx#789 +) $ 28RX.89 : +) #+) +)+)%- d5c49!4567= >% +)#+ d5c3/0147 8# +)$c]%+)+)+)+)Z)))  37 $)-04%%$s)S$./% + %  $; %!3J-%%%%N49D.X z z z x!'h!'3i!'+ !'  $; !' RX !'!'!'!'F[[[z     r'rjror! prompt_cachesrc t|fdji|} t|} d} |rtd| dd| ||} d| Di| x} r| D]i}|j0|r|j|j<|r| d z } td | d | dd|jd kr%|j |j j| x} | |rtd | d | fd| D}| }|rfd| Dnd }|r[td|j d|jddtd|jd|jddtd|jddt%|||S)a? Generate responses for the given batch of prompts. Args: model (nn.Module): The language model. tokenizer (PreTrainedTokenizer): The tokenizer. prompts (List[List[int]]): The input prompts. prompt_caches (List[List[Any]], optional): Pre-computed prompt-caches for each input prompt. Note, unlike ``generate_step``, the caches won't be updated in-place. verbose (bool): If ``True``, print tokens and timing information. Default: ``False``. max_tokens (Union[int, List[int]): Maximum number of output tokens. This can be per prompt if a list is provided. return_prompt_caches (bool): Return the prompt caches in the batch responses. Default: ``False``. logits_processors (List[Callable[[mx.array, mx.array], mx.array]], optional): A list of functions that take tokens and logits and return the processed logits. Default: ``None``. kwargs: The remaining options get passed to :obj:`BatchGenerator`. See :obj:`BatchGenerator` for more details. rrrz'[batch_generate] Finished processing 0/z ... )r )r/ci|]}|gSr/r/)rrms r% z"batch_generate..`s'''3sB'''r'Nrz%[batch_generate] Finished processing /rcFg|]}|Sr/)decode)rrmresultsrs r%rz"batch_generate..ts+ < < <Y  gcl + + < < .vs 1 1 1SmC 1 1 1r'z[batch_generate] Prompt: rrrz[batch_generate] Generation: z[batch_generate] Peak memory: r)rjrrrWrrrlrzrmrrdrr.rgrhrirjrkr,)rGrr!rrr rrrgen num_samplesfinr2rrr-r.r/rs ` ` @r%batch_generater4sB    +    C g,,K CU I IIItTTTT ::gz-: @ @D''$'''GMxxzz !) / / /A*':+,>M!%(1HCWWWkWWW &((%%ag...xxzz !) /IIKKKK IcIIKIIJJJ = < < < rrrz5Draft model tokenizer does not match model tokenizer.)top_kxtc_probability xtc_thresholdxtc_special_tokens) rr rrrzr}r|r{rr)9rF parse_argsseedrMrandomprompt_cache_filerrLrr}rvrr|rujsonloadsrrGr8rrrextra_eos_token add_eos_tokenchat_template_configrreplacesysstdinreadignore_chat_templatehas_chat_template system_promptrprefill_responseapply_chat_templateindexrr vocab_sizertemptop_pmin_pmin_tokens_to_keeprrrrrrrr rr{rrW)rEargs using_cacherzmetadatar model_pathrGr eos_tokentemplate_kwargsrmessages has_prefill test_promptrdraft_tokenizerrrs r%mainrs   F     D y ty!!!(4K !2  " " " "  h l1o'7 8 8 |'DLLO?? [  - - 5 5eT B BF!'3SY^^   FF  $*)D*   )!)d6HIIJHHHF;;<<<+47  U OO[T=RSS T T T..  #.&1/        <&/HRL ##77'2*5o 8K K--i88::;F!!&U!CC!!&)) #'+D,<'='=$ _  %)= = =TUU U >     j,($++D11D9P4Q4QQ   G ? $%0:\\d (2.H < hr'__main__zCalling `python -m mlx_lm.generate...` directly is deprecated. Use `mlx_lm.generate...` or `python -m mlx_lm generate ...` instead.rK)rNr)NroFFN)Xr4 contextlibrrrr dataclassesrrtypingrrrrr r r mlx.corecorerMmlx.nnnn mlx.utilsr transformersr modelsr models.cacherrrrrrrr sample_utilsrtokenizer_utilsrutilsrrr9r;r=r>r?r@rADEFAULT_XTC_THRESHOLDrBrCr8rDr&rF new_streamdefault_devicercontextmanagerModuleStreamr`rbrrNr:rrrrr7r rr"r'r)r,r1r_rerhrjrrrmrWr/r'r%r2s !!!!!!!!!!!!,,,,,,                    '&&&&&------<<<<<<<<     : !000___F"BM"3""3"5"566 $*$*ry$*8DO+D$*$*$*$*N ((((((( (:UUU8