h. RdZddlZddlZddlZddlZddlmZmZmZm Z m Z m Z m Z m Z mZmZddlmZddlZddlmZmZddlmZddlmZddlmZdd lmZdd lmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%er dd l&m'Z(dd l)m*Z*neZ(eZ*Gd deZ+GddZ, dde de eedfde e-effdZ.y)a This contains LLMCachingHandler This exposes two methods: - async_get_cache - async_set_cache This file is a wrapper around caching.py This class is used to handle caching logic specific for LLM API requests (completion / embedding / text_completion / transcription etc) It utilizes the (RedisCache, s3Cache, RedisSemanticCache, QdrantSemanticCache, InMemoryCache, DiskCache) based on what the user has setup In each method it will call the appropriate method from caching.py N) TYPE_CHECKINGAnyAsyncGeneratorCallableDict GeneratorListOptionalTupleUnion) BaseModel) print_verboseverbose_logger)S3Cache)CachedEmbedding)1_assemble_complete_response_from_streaming_chunks)RerankResponse) CallTypes EmbeddingEmbeddingResponse ModelResponseTextCompletionResponseTranscriptionResponseUsage)LoggingCustomStreamWrappercHeZdZUdZdZeeed<dZee ed<dZ e ed<y)CachingHandlerResponsea This is the response object for the caching handler. We need to separate embedding cached responses and (completion / text_completion / transcription) cached responses For embeddings there can be a cache hit for some of the inputs in the list and a cache miss for others N cached_resultfinal_embedding_cached_responseF embedding_all_elements_cache_hit) __name__ __module__ __qualname____doc__r r r__annotations__r!rr"bool[/var/www/Befach/backend/env/lib/python3.12/site-packages/litellm/caching/caching_handler.pyrr>s3 $(M8C='CG#X.?%@G-2$d2r*rceZdZdedeeefdejfdZ d-dedede dejded eeefd e e ed fd e fd Z d-dedede dejded eeefd e e ed fd e fdZd eeefd eefdZdee eefd e efdZde edee ed eeefde dejded e e eeffdZdeded efdZde dedejdejd ef dZde dedejdejdef dZded eeefd e ed fd e efdZ d-deded eeefde ded e ed fde ed e eeeeee e!ffd Z"dedede ded e!f d!Z# d-d"eded eeefd e e ed ffd#Z$ d-d"ed eeefd e e ed ffd$Z%ded eeefd efd%Z&ded efd&Z'd'efd(Z(d'efd)Z) d.de ded eeefded*ed+ede efd,Z*y)/LLMCachingHandleroriginal_functionrequest_kwargs start_timecJg|_g|_||_||_||_yN)async_streaming_chunkssync_streaming_chunksr/r.r0)selfr.r/r0s r+__init__zLLMCachingHandler.__init__Ks. <>#:<",!2$ r*Nmodel logging_obj call_typekwargsargs.returnc Kddlm}|xsd}d} d} d} |jddtj|jdddurv|jdijd ddurQtj@|j | r-t jd |j||| d{} | kt| tsZt jd d} tjj} tj||jdd|jdd|jdd\}}}}|j|||| d||j}|j!| |||||jdd|} |jdddur|j#|| || | tjj$di|}t| t&s t| |rt)| dr|| j*d<t-| S|t.j0j2k(rq| ot| tr_tjOttjjt4s'|j7| | ||||\} } t-| | St jd| t-| | S7&w)aC Internal method to get from the cache. Handles different call types (embeddings, chat/completions, text_completion, transcription) and accordingly returns the cached response Args: model: str: original_function: Callable: logging_obj: LiteLLMLoggingObj: start_time: datetime.datetime: call_type: str: kwargs: Dict[str, Any]: args: Optional[Tuple[Any, ...]] = None: Returns: CachingHandlerResponse: Raises: None rrr)NFcachingTcachezno-cacher.zChecking Async Cache)r9r:r;z Cache Hit!custom_llm_providerapi_baseapi_keyr7rArBrC)r8r7r:r is_asyncrAr r9r:r8r7rAr;streamr8r r0end_time cache_hit_hidden_params cache_keyr )r!r r:r8r0r7)r!r"zCACHE RESULT: )r r!) litellm.utilsrgetlitellmr? _is_call_type_supported_by_cacherdebug_retrieve_from_cache isinstancelistdatetimenowget_llm_provider'_update_litellm_logging_obj_environmentr#(_convert_cached_result_to_model_response!_async_log_cache_hit_on_callbacks!_get_preset_cache_key_from_kwargsr hasattrrKrr aembeddingvaluer(_process_async_embedding_cached_response)r5r7r.r8r0r9r:r;rr!r"r rJrIrA_rLs r+_async_get_cachez"LLMCachingHandler._async_get_cacheXs< 6zrGK'16('+ ZZ 4 ( 0W]]5Nzz)U+t3 JJw # ' ' E :$ F}}(T-R-R"3.S.$$%;<&*&?&?'!'@'! !,Z t5T"((6 $I'00446H7>7O7O#,2JJ7Ld,S!'J!= & 9d ; 84E.1 @@$/#%&3!%,? A!2 : :I$($Q$Q&3"+%$/#,2JJ7Ld,S!%R%Mzz(E2e;>>(3*7'1%-&/ ?!( O O! !I#=)<%m5HI!-1ABDM 44[A1 NN!5!5!;!;;%1"=$7 1& ++WEE8W&3%$/#-# F7828W9Y ~m_=>%',K  _!sB?K*K'H&K*c ddlm}|xsd}|j} | jt |j |d} t j|j|rtdt jjdi| } | Wd| vrnQ|j}|j| |||||jdd|} d } tjj} t j |xsd |jdd|jd d|jd d \}} }}|j#|||| dt%j&|j(| || | fj+t jj,di|}t/| t0s t/| |rt3| dr|| j4d<t7| St7| S)Nrrr)r@zChecking Sync CachedetailrArFTrBrCrDF)r8r7r:r rEtargetr;rKrLrM)rNrcopyupdateconvert_args_to_kwargsr.rPr?rQr get_cacher#rZrOrVrWrXrY threadingThreadsuccess_handlerstartr\rTr r]rKr)r5r7r.r8r0r9r:r;r new_kwargsr rJrIrAdynamic_api_keyrBrLs r+_sync_get_cachez!LLMCachingHandler._sync_get_caches 6zr[[]  "&&  (, == $)N)N/*O*  / 0#MM33AjAM(}, 1 : :I$($Q$Q&3"+%$/#,2JJ7Ld,S!%R%M!%I'00446H  00#kr,2JJ7Ld,S!'J!= & 9d ;  +' @@$/#%&3!& A$$*::+Z9Meg ' O O! !I#=)<%m5HI!-1ABDM 44[A1 NN%MBBr*czt|dtr|dgSt|dtr|dStd)zO Handles the input of kwargs['input'] being a list or a string input input must be a string or a list)rTstrrU ValueError)r5r:s r+handle_kwargs_input_list_or_strz1LLMCachingHandler.handle_kwargs_input_list_or_str*sC fWos +7O$ $ w .'? "?@ @r* non_null_listcj|D].\}}t|ts|jds)|dcSy)a Helper method to extract the model name from cached results. Args: non_null_list: List of (idx, cr) tuples where cr is the cached result dict Returns: Optional[str]: The model name if found, None otherwise r7N)rTdictrO)r5ryracrs r+"_extract_model_from_cached_resultsz4LLMCachingHandler._extract_model_from_cached_results5s5#EAr"d#w'{"#r*r!r c d}g}g} |j|} t|D]/\} } | |j| | | j| | f1||d<t| dkDr|j | } | s|j d} t | dgt| z}d|jd<d}| D]b}|\} } |  | j d }|t|| d |j| <t| | tsMdd l m }||| | d z }dt|d| }||_t|dk(rd}d}t j j#}t%j&||j dd|j dd|j dd\}}}}|j)||||dd|j+|||||||fS||fS)a Returns the final embedding cached response and a boolean indicating if all elements in the list have a cache hit For embedding responses, there can be a cache hit for some of the inputs in the list and a cache miss for others This function processes the cached embedding responses and returns the final embedding cached response and a boolean indicating if all elements in the list have a cache hit Args: final_embedding_cached_response: Optional[EmbeddingResponse]: cached_result: List[Optional[Dict[str, Any]]]: kwargs: Dict[str, Any]: logging_obj: LiteLLMLoggingObj: start_time: datetime.datetime: model: str: Returns: Tuple[Optional[EmbeddingResponse], bool]: Returns the final embedding cached response and a boolean indicating if all elements in the list have a cache hit FNrtrr7)r7dataTrJ embedding)rindexobject) token_counter)textcount_response_tokens prompt_tokenscompletion_tokens total_tokensrArBrCrD)r8r7r:r rE is_embeddingrH)rx enumerateappendlenr}rOrrKrrrTrvrNrrusagerVrWrPrXrYr[)r5r!r r:r8r0r7r"remaining_listrykwargs_input_as_listidxr| model_namervalembedding_datarrrJrIrArqrBs r+r`z:LLMCachingHandler._process_async_embedding_cached_responseDsQ:27( #CCFK /GCz%%&:3&?@$$c2Y/ 0 )w }  !@@OJ#ZZ0 .? Vc"677/ +KO + : :; GM$R>%'VVK%8N%1DM&4"%#.E7<'>s'CDq&&t, SJY!AAF  "AAPP   z ! -/DR!!AAN & E E K K W"((4NRN`N`0PPVV)//OaO % E E K )HHHr*rJctj|j||||tj|j ||||fj y)a Helper function to log the success of a cached result on callbacks Args: logging_obj (LiteLLMLoggingObj): The logging object. cached_result: The cached result. start_time (datetime): The start time of the operation. end_time (datetime): The end time of the operation. cache_hit (bool): Whether it was a cache hit. rfN)asyncio create_taskasync_success_handlerrlrmrnro)r5r8r r0rIrJs r+r[z3LLMCachingHandler._async_log_cache_hit_on_callbackssW$   - -z8Y  ..XyA  %'r*cKtjy|j}|jt |j |d}|t jjk(rt|dtr |dg|d<nt|dts tdg}t|dD]Y\}}tjjdii|d|i} |jtjj!| [t#j$|d{}|$t|trt'd|Drd}|Stjj)dur)tjj di|d{}|Stjj*di|}|S77)w)a Internal method to - get cache key - check what type of cache is used - Redis, RedisSemantic, Qdrant, S3 - async get cache value - return the cached value Args: call_type: str: kwargs: Dict[str, Any]: args: Optional[Tuple[Any, ...]] = None: Returns: Optional[Any]: Raises: None Nrtru)rLc3$K|]}|du ywr2r)).0results r+ z9LLMCachingHandler._retrieve_from_cache..6sB&v~BsTr))rPr?rhrirjr.rr^r_rTrvrUrwr get_cache_keyrasync_get_cachergatherall_supports_asyncrk) r5r9r:r;rpr tasksripreset_cache_keys r+rSz&LLMCachingHandler._retrieve_from_caches( == [[]  "&&  (,  ,,22 2*W-s3'1'':&; 7# 7 3T: !CDDE#Jw$78Q#*==#>#>$00Wa0$  W]]::EU:VW 9 #*..%"88M(Z t-LBMBB$(M  }},,.$6&-mm&C&C&Qj&Q Q !( 7 7 E* E 9!Rs%D"F?$F;%A.F?F=(F?=F?rAcddlm}|tjjk(s|tj jk(rLt |tr<|jdddur|j||||}n||t}|tjjk(s|tjjk(rHt |tr8|jdddur|j||||}n%tdi|}n|tjjk(s|tjjk(r$t |tr||t!d }n|tj"jk(s|tj$jk(rt |tr ||d d }nd|tj&jk(s|tj(jk(r*t |trd |dd } ||t+d| }t-|dr5|j.)t |j.trd|j.d<|S)aH Internal method to process the cached result Checks the call type and converts the cached result to the appropriate model response object example if call type is text_completion -> returns TextCompletionResponse object Args: cached_result: Any: call_type: str: kwargs: Dict[str, Any]: logging_obj: LiteLLMLoggingObj: model: str: custom_llm_provider: Optional[str] = None: args: Optional[Tuple[Any, ...]] = None: Returns: Optional[Any]: r) convert_to_model_response_objectrGFT)r r9r8r7)response_objectmodel_response_objectr)rr response_typeNrerankz whisper-1)r7rArJaudio_transcription)rrr hidden_paramsrKrJr))rNrr acompletionr_ completionrTr{rO_convert_cached_stream_responseratext_completiontext_completionrr^rrarerankratranscription transcriptionrr]rK) r5r r9r:r8r7r;rArrs r+rZz:LLMCachingHandler._convert_cached_result_to_model_response?s+J C ..44 4I00666-zz(E*d2 $ D D"/' + !E! !A$1*7/! 3399 9I55;;;-zz(E*d2 $ D D"/' + !E! !7 G G --33 3I//555-< -&7&9)M **00 0IAQAQAWAW4W-< -&*&M 1177 7I33999-$':!M = -&;&=3+ M M#3 4,,8=77>8>vTT  3 3W=$$&}}66$Y)eg'' 55"2248B ##GMM$A$A&$WJ$WXA sHHc |j}|jt|j|tj y|j |j|r t j j|fi|y)zE Sync internal method to add the result to the cache Nr)rhrirjr.rPr?rr)r5rr:r;rps r+sync_set_cachez LLMCachingHandler.sync_set_cachesz[[]  "&&  ==   - -"44Z .  MM # #F 9j 9r*ctjduxrstjjduxrUt|jtjjvxr$|j dij ddduS)z Helper function to determine if the result should be stored in the cache. Returns: bool: True if the result should be stored in the cache, False otherwise. Nr?zno-storeFT)rPr?supported_call_typesrvr#rO)r5r.r:s r+rz/LLMCachingHandler._should_store_result_in_cache+sy]]$ & M 22$> M&//0GMM4V4VV MGR(,,Z?tK  r*ctjJtjj0t|jtjjvryy)aO Helper function to determine if the call type is supported by the cache. call types are acompletion, aembedding, atext_completion, atranscription, arerank Defined on `litellm.types.utils.CallTypes` Returns: bool: True if the call type is supported by the cache, False otherwise. TF)rPr?rrvr#)r5r.s r+rQz2LLMCachingHandler._is_call_type_supported_by_cache;sB MM % 22>%../7==3U3UUr*processed_chunkc Kt||jtjj|j|j d}|1|j ||j|jd{yy7w)z Internal method to add the streaming response to the cache - If 'streaming_chunk' has a 'finish_reason' then assemble a litellm.ModelResponse object - Else append the chunk to self.async_streaming_chunks Trr0rIr/streaming_chunksrEN)rr.r:)rr0rVrWr/r3rr.r5rcomplete_streaming_responses r+ _add_streaming_response_to_cachez2LLMCachingHandler._add_streaming_response_to_cacheQs >"&&**,..!88   $ ' 2&&2"&"8"8**'   3 sA9B;B<Bct||jtjj|j|j d}||j ||jyy)zQ Sync internal method to add the streaming response to the cache FrN)rr:)rr0rVrWr/r4rrs r+%_sync_add_streaming_response_to_cachez7LLMCachingHandler._sync_add_streaming_response_to_cachemsn >"&&**,..!77   $ ' 2   2**   3r*rErc:|jdd||jdd|jdi|jdi|jdd|jdi|d }tj#tjjdi||d <nd|d <|j ||jd di||s|jd dn|jd d|jddt |d|jdd| y)a Helper function to update the LiteLLMLoggingObj environment variables. Args: logging_obj (LiteLLMLoggingObj): The logging object to update. model (str): The model being used. kwargs (Dict[str, Any]): The keyword arguments from the original function call. cached_result (Any): The cached result to log. is_async (bool): Whether the call is asynchronous or not. is_embedding (bool): Whether the call is for embeddings or not. custom_llm_provider (Optional[str]): The custom llm provider being used. Returns: None logger_fnNrBremetadata model_infoproxy_server_requeststream_response)rrrBrrrrrArusermessagesrtrCrGF) r7roptional_paramslitellm_paramsrtrCoriginal_responseadditional_argsrGrAr))rOrPr?r\update_environment_variablesrv) r5r8r7r:r rErrArs r+rYz9LLMCachingHandler._update_litellm_logging_obj_environments4 K6# :r2 :r2 **\26$*JJ/Et$L%zz*;R@#6   == $ ??I&I " 26N- .00FD))$ :r*ZZ,JJy$/!-0 ::h. 3 1 r*r2)FN)+r#r$r%rrrvrrVr6LiteLLMLoggingObjr r rrbrrr rxintrr}rr(r`rrrr[rSr rrrrrrZrrrrrQrrrYr)r*r+r-r-Js  #  S#X  %%  *+/A A $A ' A %% A  A S#XA uS#X'A  A V+/MCMC$MC' MC %% MC  MCS#XMCuS#X'MC MC^ Ad38n Ac A U3P_K_E`@a fnorfs kQ)12C)DkQH_56kQS#X kQ ' kQ %% kQkQ x)*D0 1kQZ E 5 U 2I#92I.2I%% 2I ## 2I  2Ih&%%  ##   855&*38n5  # ,m8]:#-1: &: : S#X :  :  : : &c]: r*r-r.r;.r<ctj|}t|jj }i}|r,t |D]\}}|t |ks||}|||< |Sr2)inspect signaturerU parameterskeysrr)r.r;r param_namesargs_to_kwargsrarg param_names r+rjrjsu !!"34Iy++0023KN #D/JE3s;''(/ -0z** r*r2)/r&rrVrrltypingrrrrrrr r r r pydanticr rPlitellm._loggingrrlitellm.caching.cachingrlitellm.types.cachingr(litellm.litellm_core_utils.logging_utilsrlitellm.types.rerankrlitellm.types.utilsrrrrrrr*litellm.litellm_core_utils.litellm_loggingrrrNrrr-rvrjr)r*r+rs    :+10W1 3Y 3s  s  p'+ 5c? # #s(^r*