o Q5iX@sZddlZddlZddlZddlZddlZddlZddlmZddlm Z m Z m Z m Z ddl Z ddlmZddlmZmZddlmZddlmZmZddlmZdd lmZmZmZmZmZm Z dd l!m"Z"m#Z#m$Z$e rrdd l%m&Z&d e'ed e'e(efddZ)ej*iej+ej,ej-ej.ej/dd e'ede0de1e0e fde0de2de2de0de0fddZ3dS)N)asynccontextmanager)AnyOptionalUnion TYPE_CHECKING) EngineArgs)AsyncEmbeddingEngineAsyncEngineArray)MANAGER)docserrors)logger)AudioCorruptionImageCorruptionModalityModelCapabilitesMatryoshkaDimErrorModelNotDeployedError)PostHogStartupTelemetrytelemetry_log_info) DataURIorURLengine_args_listcapabilities_listc CsFtdtj}t||D]\}}tt|t |||dqdS)N<) engine_args num_engines capabilities session_id) timesleepuuiduuid4hexziprcapturerlen)rrrargrr(j/lsinfo/ai/hellotax_ai/llm_service/venv_embed/lib/python3.10/site-packages/infinity_emb/infinity_server.pysend_telemetry_start!s  r*) url_prefix doc_extraredirect_slash preload_onlypermissive_corsapi_keyproxy_root_pathr+r,r-r.r/r0r1c! sddlm}m} mm} mddlm} ddlm } m } ddl m }ddl m}m}mm}m}mm}m}mtd| f fdd }| tjtjtjtjtd d d d ddd||d g}|rvj| dgddgdgdr| dd}||fdt | ffdd }|!|||" #t$j%t$j&j'dd| j(ddtt)t*ffdd} rdd l+m, -d!sJd"'d! fd#d$}j'd%|| j(|d&d'fd(d)}d*t)dd+ffd,d- d.t.d/t/d/fdt/t.t)t0ffd0d1j1d2| j(|d3d'd4|ffd5d6 }j1d7| j(|d8d'd4|ffd9d: }j1d;| j(|dd? }j1d@| j(|dAddBdCd4|ffdDdE }j1dF| j(|dGddHdCd4|ffdIdJ } S)Kz> creates the FastAPI server for a set of EngineArgs. r)DependsFastAPI HTTPException responsesstatus)CORSMiddleware)HTTPAuthorizationCredentials HTTPBearer)Instrumentator) AudioEmbeddingInput ClassifyInputClassifyResultImageEmbeddingInputMultiModalOpenAIEmbeddingOpenAIEmbeddingResultOpenAIModelInfo RerankInput ReRankResultappcs|tdtdddDtt|_tj t dd|jDfd}d|_ | |j IdHttjddd dd rkd tfd d }tddt|ddV|jIdHdS)Nz Creating z engines: cSg|]}|jqSr()served_model_name.0er(r(r) Uz3create_server..lifespan..cSrEr()rrGr(r(r)rJ[rK)targetargsThostport)rNrOprefixsecondscs(t|IdHtttjdS)N)asyncior oskillgetpidsignalSIGINT)rQr(r(r) kill_laterlsz3create_server..lifespan..kill_laterz&Preloaded configuration successfully. z -> exit .)exposer infor&rr from_args engine_array threadingThreadr*daemonstartastartr startup_messagepopintrR create_taskastop)rDthrX)r,rinstrumentatorr.r+r(r)lifespanQs6    zcreate_server..lifespanzMichael Feil, Raphael Wirth)namez/docsz /openapi.jsonz MIT LicenseMIT)rk identifier) titlesummary descriptionversioncontactdocs_url openapi_url license_inforj root_path*T) allow_originsallow_credentials allow_methods allow_headersF) auto_error credentialcs,|dus |jkrjdddiddS)N UnauthorizedzWWW-AuthenticateBearer) status_codedetailheaders) credentialsHTTP_401_UNAUTHORIZED)r})r4r0r6r(r)validate_tokensz%create_server..validate_tokenz/healthhealth) operation_idresponse_classreturncsdtiS)zq health check endpoint Returns: dict(unix=float): dict with unix time stamp unix)rr(r(r(r)_healths zcreate_server.._health)RedirectResponse/z redirect_slash must start with /csd}|S)N)urlr()response)rr-r(r)redirects zcreate_server..redirectz/modelsmodels)response_modelr dependenciesrcs~j}g}|D]1}|j}|t|jt|j|j|j|j d|j |j j |j j |jj |j|j|jj d qt|dS)zget models endpoint)queue_fractionqueue_absoluteresults_pending batch_size) idstatsrbackendembedding_dtypedtyperevisionlengths_via_tokenizedevice)data)r]rappenddictrFoverload_statusrrresults_absoluterrenginerkrrrrr)r]rrr)rDr(r)_modelss0  zcreate_server.._modelsmodelrc sbzj|}Wnty}z tjd|jdd}~ww|r/tjd|djd|S)NzInvalid model: codezmodel z is currently overloaded)r] IndexErrorr OpenAIExceptionHTTP_400_BAD_REQUEST is_overloadedHTTP_429_TOO_MANY_REQUESTS)rrex)rDr6r(r)_resolve_engines z&create_server.._resolve_engineinputsrcSs@t|dr t|g}|St|dr|jg}|Sdd|D}|S)NrNmimetypecSs$g|]}t|dr t|n|jqS)rNhasattrstrr)rHdr(r(r)rJsz?create_server.._resolve_mixed_input..r)r urls_or_bytesr(r(r)_resolve_mixed_inputs   z+create_server.._resolve_mixed_inputz /embeddings embeddingsrc s|jj}|j}|j}zt}|tjkr;t|jt r"|jg}n|j}t dt ||j ||jdIdH\}}n?|tjkr[|j}t dt ||j||jdIdH\}}n|tjkrz|j}t dt ||j||jdIdH\}}t|d} t d | j||j|j|d WSty} ztjd |jd |jd | jdd} ~ wtttfy} ztj| jd| jdd} ~ wty} z tjd| j dd} ~ ww)a^ Encode Embeddings. Supports with multimodal inputs. Aligned with OpenAI Embeddings API. ## Running Text Embeddings ```python import requests, base64 requests.post("http://..:7997/embeddings", json={"model":"openai/clip-vit-base-patch32","input":["Two cute cats."]}) ``` ## Running Image Embeddings ```python requests.post("http://..:7997/embeddings", json={ "model": "openai/clip-vit-base-patch32", "encoding_format": "base64", "input": [ "http://images.cocodataset.org/val2017/000000039769.jpg", # can also be base64 encoded ], # set extra modality to image to process as image "modality": "image" ) ``` ## Running Audio Embeddings ```python import requests, base64 url = "https://github.com/michaelfeil/infinity/raw/3b72eb7c14bae06e68ddd07c1f23fe0bf403f220/libs/infinity_emb/tests/data/audio/beep.wav" def url_to_base64(url, modality = "image"): '''small helper to convert url to base64 without server requiring access to the url''' response = requests.get(url) response.raise_for_status() base64_encoded = base64.b64encode(response.content).decode('utf-8') mimetype = f"{modality}/{url.split('.')[-1]}" return f"data:{mimetype};base64,{base64_encoded}" requests.post("http://localhost:7997/embeddings", json={ "model": "laion/larger_clap_general", "encoding_format": "float", "input": [ url, url_to_base64(url, "audio") ], # set extra modality to audio to process as audio "modality": "audio" } ) ``` ## Running via OpenAI Client ```python from openai import OpenAI # pip install openai==1.51.0 client = OpenAI(base_url="http://localhost:7997/") client.embeddings.create( model="laion/larger_clap_general", input=[url_to_base64(url, "audio")], encoding_format="float", extra_body={ "modality": "audio" } ) client.embeddings.create( model="laion/larger_clap_general", input=["the sound of a beep", "the sound of a cat"], encoding_format="base64", # base64: optional high performance setting extra_body={ "modality": "text" } ) ``` ### Hint: Run all the above models on one server: ```bash infinity_emb v2 --model-id BAAI/bge-small-en-v1.5 --model-id openai/clip-vit-base-patch32 --model-id laion/larger_clap_general ``` u,[📝] Received request with %s input texts ) sentencesmatryoshka_dimNu-[📝] Received request with %s input audios )audiosru-[📝] Received request with %s input images )imagesr[✅] Done in %s msrrencoding_formatusageModelNotDeployedError: model=`z)` does not support `embed` for modality `z `. Reason: r -> InternalServerError: )!rootmodalityrr perf_counterrtext isinstanceinputrr debugr&embed dimensionsaudio audio_embedimage image_embedto_embeddings_responserrrr rvaluerrrr __class__ ExceptionHTTP_500_INTERNAL_SERVER_ERROR) rr data_rootrrainput_ embeddingrrdurationrr@rrr6r(r) _embeddingss|W         z"create_server.._embeddingsz/rerankrerankc s|j}z:tdt|jt}|j|j|j|j |j dIdH\}}t|d}td|j ||j j ||jdWSty[}ztjd|jd|jd d}~wtyq}z tjd |jd d}~ww) aRerank documents. Aligned with Cohere API (https://docs.cohere.com/reference/rerank) ```python import requests requests.post("http://..:7997/rerank", json={ "model":"mixedbread-ai/mxbai-rerank-xsmall-v1", "query":"Where is Munich?", "documents":["Munich is in Germany.", "The sky is blue."] }) ``` %[📝] Received request with %s docs )queryr raw_scorestop_nNrr)scoresrrreturn_documentsrz%` does not support `rerank`. Reason: rr)rr rr& documentsrrrrrrto_rerank_responserrFrrr rrrr)rrrarrrr)rCrr6r(r)_reranks@  zcreate_server.._rerankz /classifyclassify)rrrrc s|j}z4tdt|jt}|j|j|jdIdH\}}t|d}td|j ||j j |dWSt yU}zt jd|jd|jd d}~wtyk}z t jd |jd d}~ww) zScore or Classify Sentiments ```python import requests requests.post("http://..:7997/classify", json={"model":"SamLowe/roberta-base-go_emotions","input":["I am not having a great day."]}) ``` r)rrNrr) scores_labelsrrrz'` does not support `classify`. Reason: rr)rr rr&rrrrrto_classify_responserrFrr rrrr)rrrarrrr)r=rr6r(r) _classifys8  z create_server.._classifyz/embeddings_imageembeddings_imagez;Deprecated: Use `embeddings` with `modality` set to `image`)rrrr deprecatedroc |j}|j}z1tdt|t}|j|dIdH\}}t|d}td|j||j |j |dWSt t fyX}zt j|jd|jdd}~wtyr}zt jd |jd |jdd}~wty}z t jd |jdd}~ww) aEncode Embeddings from Image files Supports URLs of Images and Base64-encoded Images ```python import requests requests.post("http://..:7997/embeddings_image", json={ "model":"openai/clip-vit-base-patch32", "input": [ "http://images.cocodataset.org/val2017/000000039769.jpg", "data:image/png;base64,iVBORw0KGgoDEMOoSAMPLEoENCODEDIMAGE" ] }) ``` %[📝] Received request with %s Urls )rNrrrrrrz*` does not support `image_embed`. Reason: r)rrr rr&rrrrrrrrr rrrrrrrrrrarrrrrr(r)_embeddings_imageD   z(create_server.._embeddings_imagez/embeddings_audioembeddings_audioz;Deprecated: Use `embeddings` with `modality` set to `audio`c r) aBEncode Embeddings from Audio files Supports URLs of Audios and Base64-encoded Audios ```python import requests requests.post("http://..:7997/embeddings_audio", json={ "model":"laion/larger_clap_general", "input": [ "https://github.com/michaelfeil/infinity/raw/3b72eb7c14bae06e68ddd07c1f23fe0bf403f220/libs/infinity_emb/tests/data/audio/beep.wav", "data:audio/wav;base64,iVBORw0KGgoDEMOoSAMPLEoENCODEDAUDIO" ] }) ``` r)rNrrrrrrz*` does not support `audio_embed`. Reason: r)rrr rr&rrrrrrrrr rrrrrrrrr(r)_embeddings_audio'rz(create_server.._embeddings_audio)2fastapir2r3r4r5r6fastapi.middleware.corsr7fastapi.securityr8r9!prometheus_fastapi_instrumentatorr:%infinity_emb.fastapi_schemas.pymodelsr;r<r=r>r?r@rArBrCrr FASTAPI_TITLEFASTAPI_SUMMARYFASTAPI_DESCRIPTION infinity_emb __version__radd_middlewarerr instrumentadd_exception_handlerr ropenai_exception_handlergetORJSONResponserfloatfastapi.responsesr startswithrlistbytespost)!rr+r,r-r.r/r0r1r2r3r5r7r8r9r:r;r<r>r?rArBrjroute_dependencies oauth2_schemerrrrrrrrrr()r=r4r@rCrrrr0rDr,rrir.r-r6r+r) create_server2s  , %    -% 2 2r )4rRrSrVrr^r! contextlibrtypingrrrrrinfinity_emb.argsrinfinity_emb.enginerr infinity_emb.envr infinity_emb.fastapi_schemasr r infinity_emb.log_handlerr infinity_emb.primitivesrrrrrrinfinity_emb.telemetryrrrrrrsetr*r+r-r.r/r0r1rrboolr r(r(r(r)s^