o ~rin@slddlZddlZddlZddlZddlZddlZddlZddlmZddl m Z m Z ddl m Z mZddlZddlZddlZddlZddlmZmZddlmZddlmZzddlZWn eyeYnwedkrpddlZerdd lmZdd l m!Z!dd l"m#Z#dd l$m%Z%dd l&m'Z'm(Z(m)Z)m*Z*m+Z+ddl,m-Z-dddZ.e/ej0ej1Z2e/ej0ej3ej1dZ4ddiddiddiddiddiddiddidZ5dZ6dd7e58dZ9Gd d!d!Z:Gd"d#d#Z;d$ee<dBd%e?fd)d*Z@d8d+ee?fd,d-ZAd.ee?d0e?d%e?@[]^_`{|}~textz5There is a Llama in my lawn, how can I get rid of it?zyWrite a Python function that integrates any Python function f(x) numerically over an arbitrary interval [x_start, x_end].z4How many helicopters can a human eat in one sitting?z4Count to 10 but skip every number ending with an 'e'zWhy aren't birds real?z2Why is it important to eat socks after meditating?z$Which number is larger, 9.9 or 9.11?)llamacode helicopternumbersbirdssocksnumbers2a **TRANSFORMERS CHAT INTERFACE** Chat interface to try out a model. Besides chatting with the model, here are some basic commands: - **!help**: shows all available commands (set generation settings, save chat, etc.) - **!status**: shows the current status of the model and generation settings - **!clear**: clears the current conversation and starts a new one - **!exit**: closes the interface am **TRANSFORMERS CHAT INTERFACE HELP** Full command list: - **!help**: shows this help message - **!clear**: clears the current conversation and starts a new one - **!status**: shows the current status of the model and generation settings - **!example {NAME}**: loads example named `{NAME}` from the config and uses it as the user input. Available example names: `z`, `a#` - **!set {ARG_1}={VALUE_1} {ARG_2}={VALUE_2}** ...: changes the system prompt or generation settings (multiple settings are separated by a space). Accepts the same flags and format as the `generate_flags` CLI argument. If you're a new user, check this basic flag guide: https://huggingface.co/docs/transformers/llm_tutorial#common-options - **!save {SAVE_NAME} (optional)**: saves the current chat and settings to file by default to `./chat_history/{MODEL_ID}/chat_{DATETIME}.yaml` or `{SAVE_NAME}` if provided - **!exit**: closes the interface c@seZdZdededefddZdeedeeeeBdBffd d Z defd d Z d dZ defddZ dedefddZ d#dededefddZd#defddZdefddZd efd!d"ZdS)$ RichInterfacemodel_iduser_idbase_urlcCst|_||_||_||_dSN)r _consoler"r#r$)selfr"r#r$r(c/lsinfo/ai/hellotax_ai/llm_service/venv_embed/lib/python3.10/site-packages/transformers/cli/chat.py__init__ks zRichInterface.__init__streamreturnNc s|jd|jdt|jddg}d}d}|IdH2zT3dHW}|jdjj}t|jdd|}|s6qt dd |}||7}g}| D]}| || d rY| d qG| d qGt d|d d} |j| ddq6Wdn1s}wY|j||fS)Nz [bold blue]:)consolerefresh_per_secondr finish_reasonz <(/*)(\w*)>z\<\1\2\>z``` z z github-dark) code_themeT)refresh)r&printr"rchoicesdeltacontentgetattrresub splitlinesappend startswithrjoinstripupdate) r'r+liverr1tokenoutputslineslinemarkdownr(r(r) stream_outputqs0      *zRichInterface.stream_outputcCs$|jd|jd}|j|S)z!Gets user input from the console. [bold red]: )r&inputr#r5)r'rJr(r(r)rJs zRichInterface.inputcCs|jdS)zClears the console.N)r&clear)r'r(r(r)rKszRichInterface.clearrcCs(|jd|jd||jdS)z%Prints a user message to the console.rIz>:[/ bold red] N)r&r5r#)r'rr(r(r)print_user_messagesz RichInterface.print_user_messagecolorcCs&|jd|d||jdS)z,Prints text in a given color to the console.z[bold ]N)r&r5)r'rrMr(r(r) print_colorzRichInterface.print_colorFmessagedefaultcCsL|rdnd}|jd|d|d}|j|}|s"|S|dvS)zFDisplays a yes/no prompt to the user, returning True for confirmation.zY/nzy/Nz [bold yellow]z (z): >yyes)r&rJr5r@lower)r'rQrR default_hintresponser(r(r)confirms   zRichInterface.confirmminimalcCs&|jt|rtnt|jdS)z'Prints the help message to the console.N)r&r5rHELP_STRING_MINIMAL HELP_STRING)r'rYr(r(r) print_helprPzRichInterface.print_helpmodelc stj|jdddidd}|Gdddt}dd d d d |jjtd dkfdd}t t dt dd|t |jd}|j |ddd}d}t||jddr|D]e}|rj|dskqat|dd} | d} | dkr| dd}nB| dkrt| dd | d!kr| d"} | d#} || } | r| d$krd%nd&}|j|| | d'| d(|d)qa|j|| d*dd+qaWdn1swY|r|jtd,d-n |jtd,d.|jdS)/N/z /load_modelr]T)jsonr+c@seZdZddZdS)z3RichInterface.print_model_load..StatsColumncSs|jstdS|jddkrZtt|j}tt|j}|jr.dtt|jdnd}|j durIdt|j ddt|j dd}nd}t|d |||d d Stt|jd t|jS) Nr0unitbytesz z/s<:02dr^zprogress.download)style) totalrfieldsgetr decimalint completedspeedtime_remaining)r'taskdonetotrletar(r(r)renders" (z:RichInterface.print_model_load..StatsColumn.renderN)__name__ __module__ __qualname__rrr(r(r(r) StatsColumns rvzLoading processorzLoading configzDownloading fileszLoading into memory) processorconfigdownloadweightsFcs"||}rd|S|S)Nu → )rh) stage_key stage_textr]show_model_prefix stage_labelsr(r)_labels z.RichInterface.print_model_load.._labelz[bold]{task.description}() bar_width)r.rw)rfF)r. transientsdata: statusreadycachederrorrQz Unknown errorloadingstageprogressryraitemscurrentrf) descriptionrkrfr`r)rrkrfz_*z was already loaded.*_z is warm.*_)requestspostr$rstripraise_for_statusrr&widthlenrrrradd_taskr iter_linesr>r_loadsrh RuntimeErrorrAr5r)r'r]rWrvrrtask_idrrFeventrrproglabelr`r(rr)print_model_loads\"      zRichInterface.print_model_loadrxcCs6|jd|jd|jd||jdS)zFPrints the status of the model and generation settings to the console.z[bold blue]Model: r2z [bold blue]N)r&r5r")r'rxr(r(r) print_statusszRichInterface.print_status)F)rsrtrustrr*rrtuplerrHrJrKrLrOboolrXr\rr rr(r(r(r)r!js&0 Rr!c@sTeZdZdZdeddedddddddfdeeejd d fd eedBejd d fd ee edBejdd fdeedBej dd fdeedBej dd fdeeej dd fdeedBej dd fdeedBej dd fddfddZ e ddZ deded eeeeeffd!ed"e edee eeff d#d$Zd%d&ZdS)'Chat(Chat with a model from the command line.zhttp://rrcrNz./chat_history/r"z9ID of the model to use (e.g. 'HuggingFaceTB/SmolLM3-3B').)helpr$z7Base url to connect to (e.g. http://localhost:8000/v1).generate_flagsaFlags to pass to `generate`, using a space as a separator between flags. Accepts booleans, numbers, and lists of integers, more advanced parameterization should be set through --generation-config. Example: `transformers chat max_new_tokens=100 do_sample=False eos_token_id=[1,2]`. If you're a new user, check this basic flag guide: https://huggingface.co/docs/transformers/llm_tutorial#common-optionsuserzKUsername to display in chat interface. Defaults to the current user's name. system_promptzSystem prompt. save_folderzFolder to save chat history. examples_pathz"Path to a yaml file with examples.generation_configzPath to a local generation config file or to a HuggingFace repo containing a `generation_config.json` file. Other generation settings passed as CLI arguments will be applied on top of this generation config.r,c Cs||_t|j} | jtdkr| jtdkr||j||_||_||_t |} | j ddd| j d it || |_ |||j d|_|durM|nt|_|rnt|} t| |_Wdn1shwYnt|_tsxtdt|dS) rrrT) do_samplemax_new_tokens)r$r"rxNzHYou need to install rich to use the chat interface. (`pip install rich`)r()r$rrDEFAULT_HTTP_ENDPOINTr check_healthr"rrload_generation_configrAparse_generate_flagsrxto_dictsettings get_usernameropenyaml safe_loadexamplesDEFAULT_EXAMPLESr ImportErrorasynciorun _inner_run) r'r"r$rrrrrrparsedrxfr(r(r)r*$s,#   z Chat.__init__cCsbt|dd}zt|}|jdkrtd|d|jdWd Stjy0td|dw) Nr^healthzThe server running on z returned status code z on health check (/health).zNo server currently running on z. To run a local server, please run `transformers serve` in aseparate shell. Find more information here: https://huggingface.co/docs/transformers/servingT)rhttpxrh status_code ValueError ConnectError)url health_urloutputr(r(r)rjs    zChat.check_health user_input interfacerrxchatc Csd}|dkrt|j}|n|dkr|n|drYt|dkrY|}t|dkr4|dntj |j |j dt dd }t|||jd |jd |d d dn|dr|dd} | } | D]} d| vr|jd| dddn ql|jd"it| nd|drt|dkr|d} | |vr|g}||| d|d|| ddn1d| dt|d} |j| ddn|dkr|j|dnd}|jd |d!dd||||fS)#z Handles all user commands except for `!exit`. May update the chat history (e.g. reset it) or the generation config (e.g. set a new flag). T!clear!help!savechat_%Y-%m-%d_%H-%M-%S.jsonfilenamerrChat saved to !greenrrM!setr-N=(Invalid flag format, missing `=` after `;`. Please use the format `arg_1=value_1 arg_2=value_2 ...`.red!examplerrroler8Example * not found in list of available examples: .!statusrxF'/' is not a valid command. Showing help message.r()new_chat_historyrrKr\r>rsplitospathr?rr"timestrftime save_chatrrOr@rArrLr=listkeysr) r'rrrrxr valid_command split_inputrnew_generate_flagsflag example_name example_errorr(r(r)handle_non_exit_user_commands{sV           z"Chat.handle_non_exit_user_commandsc sdt|j|j|jd}|t|j}|jdd||j|j }t |jd4IdHv}d} zZ|durB|}d}| |n| }|dkrMWnL|dkr\t|j}|Wq1|dkrf|Wq1| drt|d kr|}t|d kr|d ntj|j|jd td d }t|||jd|jd|dddWq1| dr|dd} | } | D]} d| vr|jd| dddn q|jd,it| Wq1| dr)t|d kr)|d } | |jvr|g}| |j| d|d|j| ddnBd| dt|jd} |j| ddn,|d kr6|j |d!Wq1| drM|jd"|d#dd|Wq1|d|d|j!|d|j|"|jd$d%} |#| IdH\}}|d&|d|d'kr|d(d)|$d*rd+}Wq1Wn t%yYnwq2WdIdHdS1IdHswYdS)-N)r"r#r$T)rY)r$z!exitrrrrrrrrrrrrrrr-rrrrrrrrrrrrrrr)rr])r+r] extra_body assistantlengthz2Generation stopped after reaching the token limit.yellowzContinue generating?u'Please continue. Do not repeat text.”r()&r!r"rr$rKrrr\rrxrrLrJr>rrrrr?rrrrrrOr@rArrr=rrrchat_completionto_json_stringrHrXKeyboardInterrupt)r'rrrxclientpending_user_inputrrrrrrrr+ model_outputr1r(r(r)rs                   0zChat._inner_run)rsrtru__doc__rrrtyperArgumentrOptionr* staticmethodrr!dictr rrrr(r(r(r)rs|    ! F  Jrrr,cCsD|durtSd|vrtj|}tj|}t||St|S)Nr)r rrdirnamebasenamefrom_pretrained)rr rr(r(r)r:s    rrcs|dus t|dkr iSdd|D}dd|D}dd|D}dtdtfd d fd d|D}d d d|D}d|d}|dd}|dd}|dd}|dd}|dd}|dd}zt|}W|Stjyt dw)zUParses the generate flags from the user input into a dictionary of `generate` kwargs.NrcSs.i|]}d|ddd|ddqS)"rrr)r).0rr(r(r) Ns.z(parse_generate_flags..cSs*i|]\}}||dvr|n|qS))truefalse)rUrkvr(r(r)rRscSs"i|] \}}||dkr dn|qS)Nonenullr(rr(r(r)rUs"sr,cSs|d}|dddS)N-rr0r) removeprefixreplaceisdigit)rr(r(r) is_numberXs z'parse_generate_flags..is_numbercs*i|]\}}||sd|dn|qS)rr(rrr(r)r]s*z, cSsg|] \}}|d|qS)z: r(rr(r(r) bsz(parse_generate_flags..{}z"null"rz"true"rz"false"rz"[[z]"rNrrczFailed to convert `generate_flags` into a valid JSON object. `generate_flags` = {generate_flags} Converted JSON string = {generate_flags_string}) rrrrr?rr_rJSONDecodeErrorr)rgenerate_flags_as_dictgenerate_flags_stringprocessed_generate_flagsr(rr)rFs2        rrcCs|rd|dgSgS)z Returns a new chat conversation.systemrr()rr(r(r)r|srrrrcCsbtjtj|ddt|d}tj||d|ddWdn1s&wYtj|S)z!Saves the chat history to a file.T)exist_okw)r chat_historyr-)indentN)rmakedirsrr rr_dumpabspath)rrrrr(r(r)rs   rcCs$tdkr tSttjS)z)Returns the username of the current user.r )platformr'rgetloginpwdgetpwuidgetuidpw_namer(r(r(r)rs r__main__z meta-llama/Llama-3.2-3b-Instruct)r"r%)Err_rr/r:stringrcollections.abcrtypingrr urllib.parserrrrrrhuggingface_hubrr transformersr transformers.utilsr readlinerr'r1richr rich.consoler rich.liver rich.markdownr rich.progressrrrrr rich.textrrset ascii_letters whitespaceALLOWED_KEY_CHARSdigitsALLOWED_VALUE_CHARSrrZr?rr[r!rrrrr rrrrrsr(r(r(r)s~              6 6