o di@stddlZddlZddlZddlmZddlmZddlmZm Z m Z m Z m Z m Z ddlZddlmZddlmZddlmZddlmZdd lmZdd lmZd d lmZmZmZd d lmZd dlm Z!ddl"m#Z#ddl$m%Z%m&Z&ddlm'Z'm(Z(m)Z)m*Z*m+Z+m,Z,erddl-m.Z.m/Z/ddl0m1Z1erddl2m Z3ddl2m4Z4ddl5m6Z7ddl8m9Z9ddl:m;Z<erddl=m4Z4ddl>m9Z9ddl?med?e fd@dAZdS)G GPTQQuantizerz, A simple API for GPTQ Quantization N皙?FTrgptqbitsdataset group_size damp_percentdesc_actsymtrue_sequential use_cuda_fp16 model_seqlenblock_name_to_quantize!module_name_preceding_first_block batch_size pad_token_iddisable_exllamaexllama_configmax_input_lengthcache_block_outputsmodules_in_block_to_quantizecheckpoint_formatmetabackendcOs`||_||_||_||_||_||_||_||_||_ |dur%|nd|_ ||_ | |_ | |_ | |_| |_| |_||_||_||_tj|_||_||_gd|_|jdvr[td|jdkri|jdkritdd|jkrwdks|td td |jdurd tji|_n d |jvrtd |jd tjtjfvr|jd }td ||jd |_dS) a Args: bits (`int`): The number of bits to quantize to, supported numbers are (2, 3, 4, 8). dataset (`Union[List[str], str, Any]`, defaults to `None`): The dataset used for quantization. You can provide your own dataset in a list of string or in a list of tokenized data (e.g. [{ "input_ids": [ 1, 100, 15, ... ],"attention_mask": [ 1, 1, 1, ... ]},...]) or just use the original datasets used in GPTQ paper ['wikitext2','c4','c4-new']. group_size (int, defaults to 128): The group size to use for quantization. Recommended value is 128 and -1 uses per-column quantization. damp_percent (`float`, defaults to `0.1`): The percent of the average Hessian diagonal to use for dampening, recommended value is 0.1. desc_act (`bool`, defaults to `False`): Whether to quantize columns in order of decreasing activation size. Setting it to False can significantly speed up inference but the perplexity may become slightly worse. Also known as act-order. sym (`bool`, defaults to `True`): Whether to use symetric quantization. true_sequential (`bool`, defaults to `True`): Whether to perform sequential quantization even within a single Transformer block. Instead of quantizing the entire block at once, we perform layer-wise quantization. As a result, each layer undergoes quantization using inputs that have passed through the previously quantized layers. use_cuda_fp16 (`bool`, defaults to `False`): Whether or not to use optimized cuda kernel for fp16 model. Need to have model in fp16. model_seqlen (`Optional[int]`, defaults to `None`): The maximum sequence length that the model can take. block_name_to_quantize (`Optional[str]`, defaults to `None`): The transformers block name to quantize. If None, we will infer the block name using common patterns (e.g. model.layers) module_name_preceding_first_block (`Optional[List[str]]`, defaults to `None`): The layers that are preceding the first Transformer block. batch_size (`int`, defaults to `1`): The batch size of the dataset pad_token_id (`Optional[int]`, defaults to `None`): The pad token id. Needed to prepare the dataset when `batch_size` > 1. disable_exllama (`bool`, defaults to `False`): Whether to use exllama backend. Only works with `bits` = 4. exllama_config (`Dict[str, Any]`, *optional*): The exllama config. You can specify the version of the exllama kernel through the `version` key. Defaults to `{"version": 2}` if unset. max_input_length (`Optional[int]`, defaults to `None`): The maximum input length. This is needed to initialize a buffer that depends on the maximum expected input length. It is specific to the exllama backend with act-order. cache_block_outputs (`bool`, defaults to `True`): Whether to cache block outputs to reuse as inputs for the succeeding block. It allows optimization of non-standard models (e.g. ChatGLM) but can require more time. modules_in_block_to_quantize (`Optional[List[List[str]]]`, defaults to `None`): List list of module names to quantize in the block specified. This argument is useful to exclude certain linear modules from being quantized. The block to quantize can be specified by setting `block_name_to_quantize`. We will quantize each list sequentially. If not set, we will quantize all linear layers. Example: `inside_layer_modules=[["self_attention.query_key_value"], ["mlp.dense_h_to_4h"]]` checkpoint_format (`str`, *optional*, defaults to `gptq`): GPTQ weight format. `gptq`(v1) is supported by both gptqmodel and auto-gptq. `gptq_v2` is gptqmodel only. meta (`Dict[str, any]`, *optional*): Properties, such as tooling:version, that do not directly contributes to quantization or quant inference are stored in meta. i.e. `meta.quantizer`: ["optimum:_version_", "gptqmodel:_version_"] backend (`str`, *optional*): Controls which gptq kernel to be used. Valid values for gptqmodel are `auto`, `auto_trainable` and more. For auto-gptq, only valid value is None and `auto_trainable`. Ref gptqmodel backends: https://github.com/ModelCloud/GPTQModel/blob/main/gptqmodel/utils/backend.py N) r=r>r?r@rArBrC quant_methodrNrOrP)rz(only support quantize to [2,3,4,8] bits.rz0group_size must be greater than 0 or equal to -1rz"damp_percent must between 0 and 1.r .`exllama_config` needs to have a `version` keyaOnly supported versions are in [ExllamaVersion.ONE, ExllamaVersion.TWO] - not recognized version )r=r>r?r@rArBrClowerrOrPrQrDrErFrGrHrIrJrKrLrr%rRrMrNserialization_keys ValueErrorr3r8r7exllama_version)selfr=r>r?r@rArBrCrDrErFrGrHrIrJrKrLrMrNrOrPrQargskwargsr r0r0r1__init__TsRS      zGPTQQuantizer.__init__ device_mappackc Csptrt|j|j|j|j|j|j||j|d |_ dStd|j|j|j|j p*|j t j k|j p2|j t jkd|_ dS)N) r=r?rArBrOrPrarQrbF) use_tritonrAr?r=rJdisable_exllamav2)rr'r=r?rArBrOrPrQ quant_linearrJr\r3r7r8)r]rarbr0r0r1select_quant_linears(   z!GPTQQuantizer.select_quant_linearcCsi}|jD] }t||||<q|dduri|d<|d}|ddurIdtg|d<tr<|ddt|StrI|ddt|S)z2 Returns the args in dict format. rPN quantizerzoptimum:z gptqmodel:z auto_gptq:) rZgetattrgetoptimum_versionrappendgptqmodel_versionrautogptq_version)r] gptq_dictkeyrPr0r0r1to_dicts zGPTQQuantizer.to_dict config_dictcCs|di|S)a Instantiates a `GPTQQuantizer` using config_dict as kwargs Args: config_dict (`Dict[str,Any]`): quantization config Returns: `GPTQQuantizer`: The quantizer object instantiated from those parameters. Nr0r0)clsrqr0r0r1 from_dict s zGPTQQuantizer.from_dictmodelc s|jdur t||_|j}t||d}|jdurBt|jg}t|D]tfdd|DsAt dd|jd|=q$|j | ddd d | |||S) z Convert the model to a GPTQ model by getting and replacing the layers. Args: model (`nn.Module`): Model to be converted N)prefixc3s|]}|VqdSN)endswith).0layernamer0r1 +z.GPTQQuantizer.convert_model..zQuantization disabled for z$ (only modules_in_block_to_quantize=z are quantized)raFrarb) rFrrrNsumlistkeysanyloggerinforfri_replace_by_quant_layers)r]rtr_ block_namelayers_to_be_replacedlayers_to_keepr0rzr1 convert_models     zGPTQQuantizer.convert_modelcCst||jdjj}|g}|S)z Get the modules that should not be split across multiple devices. Args: model (`nn.Module`): The input model r)rrF __class__r4)r]rtblock_class_nameno_split_module_classesr0r0r1get_no_split_module_classes7sz)GPTQQuantizer.get_no_split_module_classesmodulenamesr{c Cst||jrdSt|D]}t||}|dkr|d|n|}||vrt|}t||t|tjr9|j}|j } nt|tj rF|j }|j } nt|t rW|jjd}|jjd} |jdu} trs|j|j|j|j|j|| | |jjd} n)|jr{|jdkr|j|j|j|| | |j|jjd} n|j|j|j|| | |jjd} || _t||| |q |D]\}} || ||dkr|d|n|qdS) a^ Replaces linear layers in `module` by `QuantLinear` Args: module (`nn.Module`): Module to quantize names (`List[str]`): List of names of the module to quantize name (`str`, defaults to `""`): To keep track of the name of the current module Nr.rr) weight_dtyperV)rDr) isinstanceredirrhrdelattrr Linear in_features out_featuresConv2d in_channels out_channelsrweightshapebiasrr=r?rArBdtyperDdevicesetattrtonamed_childrenr) r]rrr{attrryname1rrrr new_layerchildr0r0r1rCsj            $z&GPTQQuantizer._replace_by_quant_layers tokenizerc% sJ ts ts tdtrtrtdto%ttjdtdkp(t}|s4t j s4td|j s>ts>t d|jdkrJtsJt d|trY|jdkrYd|_d }d }t|d rld }|jj}d |j_t|d rt|jd }d vrt ddvst dvrt|jdkrtdd}|jD]\}} | dkrt||} t| d dt| |d\} }qnd }t|dr|jt jk|_|j durt!dt"||_ t#|} t$|j%trt$|j%dt&s|j%} tdnWt$t&rzt'(Wnt)yt ddw|j%durt dt$|j%t&r-t*|j%|j dd} nt$|j%tr?fdd|j%D} n t d t+|j%d!t,| |j-|j.d"} gg} g|j/durdt0||_/|j1durqt2||j/|_1t||j/} t#| dtsj+dkrd|s}|j1D]}t||} | durt d#|d$| 3|} q| d3|| d<fd%d&}|j4r| dj5|d d'}| D](}|D] \}}t6|||<qz |dEi|Wqt yYqw|7|s| d3| |j1D]}t||} | durt d#|d$qt j 8tt d(r2t j9 r2t j98i}t:t;| d)|j/d*d+D]\}}td,|j/d-|dd.t| |j4s|j5|d d'}| D](}|D] \}}t6|||<qlz |dEi|Wqft yYqfw|7|rt#|t dkrt<r|3d}t=|t#|}ts|j+dkrd}t$|j>trt|j>dkr|j?r|j>}nt@|j>gg}n|j?rd/dAD}ntAg}td0|t;|d d1d2D]}fd3d4|D}ig}|D]*}tB|||<|jCjD|jE|j d d5fd6d7}|F||G||qtHt| D],}t6|||<|D]\}}t6||||<qS||i|qB|D]}|7qq|D]K}td)|d8|dd.t| d9|jI|jJ|jK|jLd:} | d| d| d;}!}"}#|jC|!|"|#f||j/d!|d!|<|Mq{~q|j4rtHt| D]}||i|}$| F|$q|s|3| | |<| g} ngt j 8tt d(rt j9 rt j98qA|jEdd?d@DrB|jOsAtsAtdAd |_On+|jLrZ|jOsZ|jPtQjRkrZtdBd |_On|jOsm|jPtQjSkrmtdCd |_O|jT||dDd |_UtVjB|_W|r||j_|X|j_Y|Z|}t j 8tt d(rt j9 rt j98|S)Fa Quantizes the model using the dataset Args: model (`nn.Module`): The model to quantize tokenizer (Optional[`Any`], defaults to `None`): The tokenizer to use in order to prepare the dataset. You can pass either: - A custom tokenizer object. - A string, the *model id* of a predefined tokenizer hosted inside a model repo on huggingface.co. Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a user or organization name, like `dbmdz/bert-base-german-cased`. - A path to a *directory* containing vocabulary files required by the tokenizer, for instance saved using the [`~PreTrainedTokenizer.save_pretrained`] method, e.g., `./my_model_directory/`. Returns: `nn.Module`: The quantized model zgptqmodel or auto-gptq is required in order to perform gptq quantzation: `pip install gptqmodel` or `pip install auto-gptq`. Please notice that auto-gptq will be deprecated in the future.zeDetected gptqmodel and auto-gptq, will use gptqmodel. The auto_gptq will be deprecated in the future.z auto-gptqz0.4.2zmNo cuda gpu or cpu support using Intel/IPEX found. A gpu or cpu with Intel/IPEX is required for quantization.zpAsymmetric sym=False quantization is not supported with auto-gptq. Please use gptqmodel: `pip install gptqmodel`gptq_v2z_gptq_v2 format only supported with gptqmodel. Please install gptqmodel: `pip install gptqmodel`FconfigT hf_device_mapdiskz4disk offload is not supported with GPTQ quantizationcpurzJCpu offload is not recommended. There might be some issues with the memoryN)recurse)prev_module_hookrirzMGPTQQuantizer dataset appears to be already tokenized. Skipping tokenization.zWe were not able to get the tokenizer using `AutoTokenizer.from_pretrained` with the string that you have passed z. If you have a custom tokenizer, you can pass it as input. For now, we only support quantization for text model. Support for vision, speech and multimodel will come later.z:You need to pass `dataset` in order to quantize your modeltrain)seqlensplitcsg|]}|ddqS)pt)return_tensorsr0)rxdata)rr0r1 sz0GPTQQuantizer.quantize_model..z^You need to pass a list of string, a list of tokenized data or a string for `dataset`. Found: r)rIrHzModule z was not found in modelcsv|d}|durd|vrt|df}ntd|i}|D]\}}|dvr3t|||<q$|t)Nr hidden_statesz'No input value found in the foward pass)r)rr[rkitems)_inputr^r_ other_kwargskv)cur_layer_devicelayer_input_kwargs layer_inputsr0r1store_input_hooks  z6GPTQQuantizer.quantize_model..store_input_hook) with_kwargsr+z Quantizing z blocks )desczStart quantizing block  /cSsg|]}|gqSr0r0)rxror0r0r1r]szModule to quantize z"Quantizing layers inside the block)leaverci|]}||qSr0r0)rxr{layersr0r1 bz0GPTQQuantizer.quantize_model..)r=rB perchannelcsfdd}|S)Ncs|dj|jdS)Nr) add_batchr)rroutput)r<r{r0r1tmpksz.add_batch..tmpr0)r{r)r<rzr1rjsz/GPTQQuantizer.quantize_model..add_batchz in block z...)percdampr?actorderrrTr-c3s|]}|vVqdSrvr0rxd)devicesr0r1r|sz/GPTQQuantizer.quantize_model..rrhpuFound modules on cpu/disk. Using Exllama/Exllamav2 backend requires all the modules to be on GPU. Setting `disable_exllama=True`zUsing Exllama backend with act_order will reorder the weights offline, thus you will not be able to save the model with the right weights.Setting `disable_exllama=True`. You should only use Exllama backend with act_order for inference. zUsing Exllamav2 backend will reorder the weights offline, thus you will not be able to save the model with the right weights.Setting `disable_exllama=True`. You should only use Exllamav2 backend for inference. )rt quantizersr0)[rr RuntimeErrorrwarningr parse importlibmetadatar,r-r.rBr[rOevalr/r use_cacherrvaluesrlenrrrr"r rfloat16rDrEminrrrr>strr from_pretrained ExceptionrtyperrIrHrFrrGrrrMregister_forward_pre_hookrremove empty_cacher+ enumerater r2rrNrCrrr%rg configurer=rkregister_forward_hookrange fasterquantr@r?rAfreerrJr\r3r7r8 pack_model is_quantizedrquantization_methodrpquantization_configpost_init_model)%r]rtrgptq_supports_cpu has_confighas_device_maprhookr{rrr> layer_outputsblocks to_device module_namerhandlerrrriblock block_devicelayers_name_listsubset_name_list subset_layershandlesrjh quant_outputsscalezerog_idx layer_outputr0)rrr<rrrrr1quantize_models                          $&"   &     *    zGPTQQuantizer.quantize_modelcs|jdkr*|js*tjdkstdr*tfdddDr*|js*tdd|_Gd d d t}t rCt |j|j |j |j \}|_|jj_t|jd |jrj|jsj|jtjkrj|jd urjt|jS) z Post-initialization that require device information, for example buffers initialization on device. Args: model (`nn.Module`): The input model rTr-rc3s|]}|jvVqdSrv)rrrtr0r1r|r}z0GPTQQuantizer.post_init_model..rrTc@s eZdZdS)z0GPTQQuantizer.post_init_model..StoreAttrN)r4r5r6r0r0r0r1 StoreAttrsr) use_act_orderN)r=rJrrr/rrrobjectrr(rerOrPquantize_configrAgptq_post_initr\r3r7rLr#)r]rtrrr0rr1rs6    zGPTQQuantizer.post_init_modelrc stdt|fdd|D|j|jdd|||t||jg}|D]J}t|||\||<}}}||j}||d|d|d|d|df\|<}}}|| |||||||q)tddS) a Pack the model by replacing the layers by quantized layers Args: model (`nn.Module`): The model to pack quantizers (`Dict[str,Tuple]`): A mapping of the layer name and the data needed to pack the layer zPacking model...crr0r0)rxnrr0r1rrz,GPTQQuantizer.pack_model..Tr~rz Model packed.N) rrrrfrrrerrrb) r]rtrqlayersr{r r r  layer_devicer0rr1rs    4zGPTQQuantizer.pack_model10GBsave_dirmax_shard_sizesafe_serializationcCstrt||j|j|j|j|j\}}|rd|_tj|dd|j |||dt tj |t ddd}tj||dd Wd d S1sIwYd S) a Save model state dict and configs Args: model (`nn.Module`): Model to be saved. The model can be wrapped or unwraped. save_dir (`str`): Directory to which to save. Will be created if it doesn't exist. max_shard_size (`str`, defaults to `"10GB"`): The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size lower than this size. If expressed as a string, needs to be digits followed by a unit (like `"5MB"`). If a single weight of the model is bigger than `max_shard_size`, it will be in its own checkpoint shard which will be bigger than `max_shard_size`. safe_serialization (`bool`, defaults to `True`): Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). r<T)exist_ok)rrwutf-8encodingr)indentN)rr)rBr=rerOrPosmakedirssave_pretrainedopenpathjoinrjsondumprp)r]rtrrr convertedfr0r0r1save s"zGPTQQuantizer.save)Nr:r;FTTFNNNrNFNNTNr<NN)F)rrv)rT) r4r5r6__doc__intrr rrfloatboolrrrr`dictrfrp classmethodrsr Modulerrrr,no_gradr rrrr+r0r0r0r1r9Os         BB%  ""r9Frt save_folderquant_config_namestate_dict_namera max_memoryroffload_folderoffload_buffersoffload_state_dictrJrKrLc  Cstjs ts tdtststdtstd|dur-dtji}t d| dur7dt j i} nd| vr?t d| dt j t j fvrT| d} t d | z5t|d rgt|jd rg|jj}n!ttj||d d d }t|}Wdn1swYWnty}z t d|d|d|d}~wwt|}| |_| |_|jd|_| |_|j||d}|dur| |}t!||durtj||n||||||| d}|"|}d|_#t$j%|_&|'|S)a Load quantized weights from the save_folder into the converted model and dispatch the weights according to the device_map. Args: model (`nn.Module`): The model can be enpty or not. save_folder (`str`): Directory to which to load the weights. quant_config_name (`str`, defaults to `GPTQ_CONFIG`): Name of the quantization config file state_dict_name (`Optional[str]`, defaults to `None`): Name of the state dict file device_map (`Optional[str]`, defaults to `None`): A map that specifies where each submodule should go. It doesn't need to be refined to each parameter/buffer name, once a given module name is inside, every submodule of it will be sent to the same device. To have Accelerate compute the most optimized `device_map` automatically, set `device_map="auto"`. max_memory (`Optional[Dict]`, defaults to `None`): A dictionary device identifier to maximum memory. Will default to the maximum memory available for each GPU and the available CPU RAM if unset. no_split_module_classes (`Optional[Dict]`, defaults to `None`): A list of layer class names that should never be split across device (for instance any layer that has a residual connection). offload_folder (`Optional[str]`, defaults to `None`): If the `device_map` contains any value `"disk"`, the folder where we will offload weights. offload_buffers (`Optional[str]`, defaults to `None`): In the layers that are offloaded on the CPU or the hard drive, whether or not to offload the buffers as well as the parameters. offload_state_dict (`bool`, defaults to `False`): If `True`, will temporarily offload the CPU state dict on the hard drive to avoid getting out of CPU RAM if the weight of the CPU state dict + the biggest shard does not fit. Will default to `True` if the device map picked contains `"disk"` values. disable_exllama (`Optional[bool]`, defaults to `None`): Whether to use exllama backend. Only works with `bits` = 4. exllama_config (`Optional[Dict[str, Any]]`, defaults to `None`): The exllama config. You can specify the version of the exllama kernel through the `version` key. Defaults to `{"version": 2}` if unset. max_input_length (`Optional[int]`, defaults to `None`): The maximum input length. This is needed to initialize a buffer that depends on the maximum expected input length. It is specific to the exllama backend with act-order. Returns: `nn.Module`: The quantized model zBNo GPU found. A GPU is needed to run quantized model by auto_gptq.zgptqmodel (`pip install gptqmodel`) or auto-gptq (`pip install auto-gptq`) is required in order to load quantized weights. Please notice that auto-gptq will be deprecated in the future.zYou need to install accelerate in order to load and dispatch weights toa quantized model. You can do it with `pip install accelerate`Nrz\The device_map was not initialized.Setting device_map to `{'':torch.cuda.current_device()}`.r rWrXrrrrrz(Failed to load quantization config from z (lookup for traceback): z Tip: If the save directory is saved from a transformers.PreTrainedModel, make sure that `config.json` contains a 'quantization_config' key.)ra) checkpointrar7rr8r9r:T)(r,r-r.rrrrcurrent_devicerrr3r8r[r7r/rrrpr$r!r%r&r'loadrr9rsrJrKr\rLrrr!rrrr%rr)rtr4r5r6rar7rr8r9r:rJrKrLr quantize_config_dictr*errrgr0r0r1load_quantized_model2sx9        rA)Qrr'r!enumrloggingrtypingrrrrrr r, packagingr r tqdm.autor transformersr transformers.pytorch_utilsr&transformers.utils.quantization_configrutilsrrrutils.modeling_utilsrrrj constantsrrrrrrrrrr accelerater r!accelerate.hooksr" auto_gptqrmr#auto_gptq.modeling._utilsr$rauto_gptq.quantizationr%auto_gptq.utils.import_utilsr&r' gptqmodelgptqmodel.quantizationgptqmodel.utils.importergptqmodel.utils.modelr(r)r*gptqmodel.versionrlr4rr2r-r3rr9r2rr/rAr0r0r0r1s                         k