
    hn                     (   d dl Z d dlZd dlmZmZ d dlmZmZmZmZm	Z	m
Z
mZmZ d dlmZ d dlmZ d dlZd dlmZmZ d dlmZ d dlmZ d d	lmZ d d
lmZmZmZ d dlmZm Z  erd dl!m"Z# d dl$m%Z& ee#ef   Z"e&Z%neZ"eZ% G d de      Z' G d de      Z(y)    N)datetime	timedelta)TYPE_CHECKINGAnyListLiteralOptionalTuple	TypedDictUnion)HTTPException)	BaseModel)	DualCacheModelResponseverbose_proxy_logger)CustomLogger)!_get_parent_otel_span_from_kwargs)CommonProxyErrorsCurrentItemRateLimitUserAPIKeyAuth)get_key_model_rpm_limitget_key_model_tpm_limit)Span)InternalUsageCachec                   n    e Zd ZU ee   ed<   ee   ed<   ee   ed<   ee   ed<   ee   ed<   ee   ed<   y)CacheObjectcurrent_global_requestsrequest_count_api_keyrequest_count_api_key_modelrequest_count_user_idrequest_count_team_idrequest_count_end_user_idN)__name__
__module____qualname__r	   dict__annotations__     h/var/www/Befach/backend/env/lib/python3.12/site-packages/litellm/proxy/hooks/parallel_request_limiter.pyr   r       s>    %d^+#D>)!)$/#D>)#D>)'~-r*   r   c                   H    e Zd ZdefdZd Zdededede	de
d	e
d
e
dee   de	ded   deeeef      defdZdefdZ	 d"dee	   defdZ	 d"dee	   dee	   dee	   dee	   dee	   dee	   dee   defdZdededede	fdZd Zd Zde	dedee   fd Zdedef fd!Z xZS )#!_PROXY_MaxParallelRequestsHandlerinternal_usage_cachec                     || _         y N)r.   )selfr.   s     r+   __init__z*_PROXY_MaxParallelRequestsHandler.__init__+   s
    $8!r*   c                     	 t        j                  |       t        j                  rt	        |       y y # t
        $ r Y y w xY wr0   )r   debuglitellmset_verboseprint	Exception)r1   print_statements     r+   print_verbosez/_PROXY_MaxParallelRequestsHandler.print_verbose.   s=    	 &&7""o& # 		s   04 	A A user_api_key_dictcachedata	call_typemax_parallel_requests	tpm_limit	rpm_limitcurrentr   rate_limit_type)keymodel_per_keyusercustomerteamvalues_to_update_in_cachereturnc                   K   t        j                  d|
 d|        |a|dk(  s
|dk(  s|dk(  r8| j                  t        j                  j
                   d|
 d| d| d| 	      d	dd	d
}|j                  |	|f       nt        |d         |k  r9|d   |k  r1|d   |k  r)|d   d	z   |d   |d   d	z   d
}|j                  |	|f       n`t        dd|
 dt        j                  j
                   d|d    d| d|d    d| d|d    d| dt        | j                               i      | j                  j                  |d|j                  d       d {    |S 7 w)NzCurrent Usage of z in this minute: r   z. Hit limit for z). Current limits: max_parallel_requests: z, tpm_limit: z, rpm_limit: additional_details   current_requestscurrent_tpmcurrent_rpmrP   rQ   rR     z1LiteLLM Rate Limit Handler for rate limit type = z. z. current rpm: z, rpm limit: z, current tpm: z, tpm limit: z!, current max_parallel_requests: z, max_parallel_requests: retry-afterstatus_codedetailheaders<   T)
cache_listttllitellm_parent_otel_span
local_only)r   inforaise_rate_limit_errorr   "max_parallel_request_limit_reachedvalueappendintr   strtime_to_next_minuter.   async_batch_set_cacheparent_otel_span)r1   r;   r<   r=   r>   r?   r@   rA   rB   r   rC   rI   new_vals                r+   check_key_in_limitsz5_PROXY_MaxParallelRequestsHandler.check_key_in_limits6   s     	!!00A'K	
 ?$)Y!^yA~11*;*^*^*d*d)eeu  wF  vG  Gp  qF  pG  GT  U^  T_  _l  mv  lw  (x 2   %&  G
 &,,.CW-MN*+,/DD&2&2 %,,>$?!$C&}5&}59G
 &,,.CW-MN  J?J[[]^o  _S  _S  _Y  _Y  ^Z  Zi  jq  r  j@  iA  AN  OX  NY  Yh  ip  q~  i  h@  @M  NW  MX  Xy  zA  BT  zU  yV  Vo  pE  oF  G&D,D,D,F(GH  ''==0%6%G%G	 > 
 	
 	
 	
s   EE EE c                     t        j                         }|t        d      z   j                  dd      }||z
  j	                         }|S )NrN   )minutesr   )secondmicrosecond)r   nowr   replacetotal_seconds)r1   rn   next_minuteseconds_to_next_minutes       r+   re   z5_PROXY_MaxParallelRequestsHandler.time_to_next_minuteo   sK    lln Yq11::!QR:S #."3!B!B!D%%r*   rM   c           	      n    d}||dz   |z   }t        dd| dt        | j                               i      )zX
        Raise an HTTPException with a 429 status code and a retry-after header
        z"Max parallel request limit reached rS   z#Max parallel request limit reached rT   rU   )r   rd   re   )r1   rM   error_messages      r+   r_   z8_PROXY_MaxParallelRequestsHandler.raise_rate_limit_error{   sQ     =))C/2DDM89K8LM"C(@(@(B$CD
 	
r*   r   r    r!   r"   r#   rg   c           	         K   ||||||g}| j                   j                  ||       d {   }	|	t        d d d d d d       S t        |	d   |	d   |	d   |	d   |	d   |	d         S 7 :w)	N)keysrg   )r   r   r    r!   r"   r#   r   rN               )r.   async_batch_get_cacher   )
r1   r   r   r    r!   r"   r#   rg   rw   resultss
             r+   get_all_cache_objectsz7_PROXY_MaxParallelRequestsHandler.get_all_cache_objects   s      $!'!!%
 11GG- H 
 

 ?(,&*,0&*&**.  $+AJ")!*(/
")!*")!*&-aj
 	

s   )A(A&;A(c                 ~  K   | j                  d       |j                  }|j                  }|t        j                  }|i }|j                  di       j                  dd       }t        |dt        j                        }|t        j                  }t        |dt        j                        }	|	t        j                  }	g }
d }|d}| j                  j                  |d|j                         d {   }|d}||k\  r| j                  d	| d
|       S | j                  j                  |dd|j                         d {    |j                  dd       }t        j                         j                  d      }t        j                         j                  d      }t        j                         j                  d      }| d| d| }| j                  |dnd || d| dnd ||| d| d| dnd |j                   |j                    d| dnd |j"                  |j"                   d| dnd |j$                  |j$                   d| dnd |j                         d {   }|/| d| d}| j'                  ||||||d   |||	d|
       d {    t)        |      t+        |      |j                  dd       }| d| d| d}t)        |      }t+        |      }d }d }|&|r|j                  |      }|r|j                  |      }| j'                  ||||t        j                  |d   ||xs t        j                  |xs t        j                  d|
       d {   }d }d }|r|||d   z
  }|||d   z
  }d| |d| |i}d|vri |d<   |d   j-                  |       |j                   }|y|j.                  }|j0                  }|t        j                  }|t        j                  }| d| d}| j'                  ||||t        j                  |d   |||d|
       d {    |j"                  }|y|j2                  } |j4                  }!| t        j                  } |!t        j                  }!| d| d}| j'                  ||||t        j                  |d    || |!d!|
       d {    |j$                  rt        |d"t        j                        }"t        |d#t        j                        }#|"t        j                  }"|#t        j                  }#|j$                   d| d}| j'                  ||||t        j                  ||d$   |"|#d%|
&       d {    t7        j8                  | j                  j;                  |
d'|j                  (             y 7 7 g7 D7 7 K7 7 7 Tw))Nz)Inside Max Parallel Request Pre-Call Hookmetadataglobal_max_parallel_requestsr@   rA   TrD   r]   r\   rN   zHit Global Limit: Limit=z, current: rL   rD   ra   r]   r\   model%Y-%m-%d%H%M-::::request_count)r   r   r    r!   r"   r#   rg   r   rD   )r;   r<   r=   r>   r?   rB   r   r@   rA   rC   rI   r    rE   rQ   rR   zlitellm-key-remaining-tokens-zlitellm-key-remaining-requests-r!   rF   r"   rH   end_user_tpm_limitend_user_rpm_limitr#   rG   )r;   r<   r=   r>   r?   r   rB   r@   rA   rC   rI   rY   rZ   r[   r\   )r:   api_keyr?   sysmaxsizegetgetattrr.   async_get_cacherg   r_   async_increment_cacher   rn   strftimer~   user_idteam_idend_user_idri   r   r   updateuser_tpm_limituser_rpm_limitteam_tpm_limitteam_rpm_limitasynciocreate_taskrf   )$r1   r;   r<   r=   r>   r   r?   r   r@   rA   rI   rh   _keyr   _modelcurrent_datecurrent_hourcurrent_minuteprecise_minutecache_objectsr   _tpm_limit_for_key_model_rpm_limit_for_key_modeltpm_limit_for_modelrpm_limit_for_model_remaining_tokens_remaining_requests_remaining_limits_datar   r   r   r   r   r   r   r   s$                                       r+   async_pre_call_hookz5_PROXY_MaxParallelRequestsHandler.async_pre_call_hook   sB     	FG#++ 1 G G ($'KK!<D'+xx
B'?'C'C*D(
$ -{CKKH	I-{CKKH	I
  	" #''31D,0,E,E,U,U):)K)K -V - '# '.*+'&*FF22)AB^A__j  lC  kD  (E 3  
 //EE#->-O-O	 F    '4(||~..z:||~..t4!006(><..9IJ+/+E+E 0; / & )2n-_= &6+= )2fXR'7G %,,8 %,,-R/?O %,,8 %,,-R/?O %00< %001N3C?S.??? ,F  ,
  &
B '.ir.1A$Q!**"3#&;%&=>&;## %*C +     $$56B&'89EXXgt,F)2fXR'7G " (??P'Q$'>?P'Q$"&"&!+*B*F*Fv*N'+*B*F*Fv*N' 44"3#&)kk%&CD&;-<-< /*C 5  G !%"&&2(;gm>T(T%&2*=@V*V' 0x8:K1&:<O&"
 %#%Z ##$:; $++.==N.==N%!$%!$'.ir.1A$Q!**"3#&)kk%&=>&;(( &*C +     $++.==N.==N%!$%!$'.ir.1A$Q!**"3#&)kk%&=>&;(( &*C +     ((!(!#7" ")!#7" ")%([["!)%([[" %001N3C?S "
 **"3#&)kk&;%&AB,, **C +    	%%;;4):)K)K < 	
 	G' &
HFV8Fs   C/V=1V'2AV=V*D%V=)V-*0V=V0CV=(V3)C	V=2V63BV=9V9:B,V=&V;'AV=*V=-V=0V=3V=6V=9V=;V=c                 	  K   ddl m} t        |      }	 | j                  d       |d   d   j	                  dd       }|d   d   d   }|d   d   j	                  d	d       }	|d   d   j	                  d
d       }
|d   d   j	                  dd       }|j	                  d      }|d   d   j	                  di       xs i }|)d}| j
                  j                  |dd|       d {    t        j                         j                  d      }t        j                         j                  d      }t        j                         j                  d      }| d| d| }d}t        |t              r|j                  j                  }g }|~| d| d}| j
                  j                  ||       d {   xs dddd}t        |d   dz
  d      |d   |z   |d   d}| j                  d| d|        |j!                  ||f        ||      }||d|v sd |v s|| d| d| d}| j
                  j                  ||       d {   xs dddd}t        |d   dz
  d      |d   |z   |d   d}| j                  d| d|        |j!                  ||f       |	d}t        |t              r|j                  j                  }|	 d| d}| j
                  j                  ||       d {   xs d|dd}t        |d   dz
  d      |d   |z   |d   d}| j                  d| d|        |j!                  ||f       |
d}t        |t              r|j                  j                  }|
 d| d}| j
                  j                  ||       d {   xs d|dd}t        |d   dz
  d      |d   |z   |d   d}| j                  d| d|        |j!                  ||f       |d}t        |t              r|j                  j                  }| d| d}| j
                  j                  ||       d {   xs d|dd}t        |d   dz
  d      |d   |z   |d   d}| j                  d| d|        |j!                  ||f       | j
                  j#                  |d!|"       d {    y 7 7 7 {7 7 17 7 # t$        $ r}| j                  |       Y d }~y d }~ww xY ww)#Nr   )#get_model_group_from_litellm_kwargskwargsz5INSIDE parallel request limiter ASYNC SUCCESS LOGGINGlitellm_paramsr   r   user_api_keyuser_api_key_user_iduser_api_key_team_iduser_api_key_model_max_budgetrF   user_api_key_metadataTr   r   r   r   r   r   r   rD   r\   rN   rO   rP   rQ   rR   zupdated_value in success call: z, precise_minute: model_rpm_limitmodel_tpm_limitrY   r   ))litellm.proxy.common_utils.callback_utilsr   r   r:   r   r.   r   r   rn   r   
isinstancer   usagetotal_tokensr   maxrb   rf   r8   )r1   r   response_obj
start_timeend_timer   r\   r   r   r   r   r   user_api_key_end_user_idr   r   r   r   r   r   r   rI   r   rB   rh   model_groupes                             r+   async_log_success_eventz9_PROXY_MaxParallelRequestsHandler.async_log_success_event  s    	
 7X7
 ]	"VW+12B+CJ+O+S+S.,( ""23J?OL#)*:#;J#G#K#K&$  $**:#;J#G#K#K&$  -33C,DZ,P,T,T/-) (.zz&'9$ '(4889PRTU  " ,75//EE#-E	 F    $<<>22:>L#<<>2248L%\\^44T:N ,~Q|nAn=MNNL,6+11>> )+%'#nB~&6oF & !% 9 9 I I--E !J !   )*#$#$  ),G4F,G!,KQ(O#*=#9L#H#*=#9 ""5gY>PQ_P`a *002G1QR
 >fEK(+%)>>(,AA4@ $nB{m2n5E_U & !% 9 9 I I--E !J !   )*#$#$  ),G4F,G!,KQ(O#*=#9L#H#*=#9 ""5gY>PQ_P`a *002G1QR
 $/ lM:#/#5#5#B#BL ,,B~.>oN & !% 9 9 I I--E !J !   )*#/#$  ),G4F,G!,KQ(O#*=#9L#H#*=#9 ""5gY>PQ_P`a *002G1QR
 $/ lM:#/#5#5#B#BL ,,B~.>oN & !% 9 9 I I--E !J !   )*#/#$  ),G4F,G!,KQ(O#*=#9L#H#*=#9 ""5gY>PQ_P`a *002G1QR
 (3 lM:#/#5#5#B#BL 00>2B/R & !% 9 9 I I--E !J !   )*#/#$  ),G4F,G!,KQ(O#*=#9L#H#*=#9 ""5gY>PQ_P`a *002G1QR++AA4)A B   q8JBBB(
  	"q!!	"s   S7CS %R=&CS 1S 2BS S	B'S 0S1B'S S	B'S  SA6S 7S8S <S7=S  S S S 	S S S 	S4S/*S7/S44S7c                 |  K   	 | j                  d       t        |      }|d   j                  di       xs i }|j                  dd       }|j                  dd       }| j                  d|        |y t        j                  j
                  t        |d         v ry |Od}	| j                  j                  |	d	|
       d {    | j                  j                  |	dd	|       d {    t        j                         j                  d      }
t        j                         j                  d      }t        j                         j                  d      }|
 d| d| }| d| d}| j                  j                  ||       d {   xs dddd}t        |d   dz
  d      |d   |d   d}| j                  d|        | j                  j                  ||d|       d {    y 7 ,7 7 j7 # t        $ r7}t!        j"                  dj%                  t        |                   Y d }~y d }~ww xY ww)Nz(Inside Max Parallel Request Failure Hookr   r   r   r   r   zuser_api_key: 	exceptionTr   r   r   r   r   r   r   r   r   r   rN   r   rO   rP   rQ   rR   zupdated_value in failure call: rY   )r[   r\   z;Inside Parallel Request Limiter: An exception occurred - {})r:   r   r   r   r`   ra   rd   r.   r   r   r   rn   r   r   async_set_cacher8   r   r   format)r1   r   r   r   r   r\   	_metadatar   r   r   r   r   r   r   r   rB   rh   r   s                     r+   async_log_failure_eventz9_PROXY_MaxParallelRequestsHandler.async_log_failure_event  s    Q	IJ 2@ % /044ZDJI+4==.,( %==>L~>?# !CCIIS{#N   0;9D"77GG $'+5M H    33II  #'1I	 J     (||~66zB'||~66t<!)!8!8!>$0><..AQ!R $nB~&6oF & !% 9 9 I I--E !J !   )*#$#$  ),G4F,G!,KQ(O#*=#9#*=#9 ""%DWI#NO//??)-E	 @   U&   	 **MTTF 	s   H<A0G9 4H<5(G9 H<%G9 G/&G9 *G2+BG9 
G5AG9 )G7*G9 .H</G9 2G9 5G9 7G9 9	H9-H4/H<4H99H<r   c                 &  K   ddl m} ddlm} ddlm} 	  |||| j                  j                  d|j                  d       d{   }|y|j                         S 7 # t        $ r&} |j                  dt        |             Y d}~yd}~ww xY ww)	a?  
        Helper to get the 'Internal User Object'

        It uses the `get_user_object` function from `litellm.proxy.auth.auth_checks`

        We need this because the UserApiKeyAuth object does not contain the rpm/tpm limits for a User AND there could be a perf impact by additionally reading the UserTable.
        r   r   )get_user_object)prisma_clientFN)r   r   user_api_key_cacheuser_id_upsertrg   proxy_logging_objz3Parallel Request Limiter: Error getting user object)litellm._loggingr   litellm.proxy.auth.auth_checksr   litellm.proxy.proxy_serverr   r.   
dual_cacherg   
model_dumpr8   r4   rd   )r1   r   r;   r   r   r   _user_id_rate_limitsr   s           r+   get_internal_user_objectz:_PROXY_MaxParallelRequestsHandler.get_internal_user_object  s      	:B<	)8+#'#<#<#G#G$!2!C!C"&* $  $+'2244$  	& &&Es1v 		sL   B/A AA BA BA 	B(B	B	BBc                   K   |j                   }t        j                         j                  d      }t        j                         j                  d      }t        j                         j                  d      }| d| d| }| d| d}	| j                  j                  |	|j                         d{   }
d}d}d}d}|
T|j                  |j                  |
d	   z
  }|j                  }|j                  |j                  |
d
   z
  }|j                  }t        |d      rt        |d      }nd}|t        |t              st        |t              r~t        |t              r|j                         }|j                  di       xs i }|||d<   |||d<   |||d<   |||d<   t!        |di |d|i       t"        | I  |||       d{   S yy7  7 	w)z;
        Retrieve the key's remaining rate limits.
        r   r   r   r   r   r   r   NrR   rQ   _hidden_paramsadditional_headerszx-ratelimit-remaining-requestszx-ratelimit-limit-requestszx-ratelimit-remaining-tokenszx-ratelimit-limit-tokens)r   r   rn   r   r.   r   rg   rA   r@   hasattrr   r   r   r'   r   r   setattrsuperasync_post_call_success_hook)r1   r=   r;   responser   r   r   r   r   r   rB   key_remaining_rpm_limitkey_rpm_limitkey_remaining_tpm_limitkey_tpm_limitr   _additional_headers	__class__s                    r+   r   z>_PROXY_MaxParallelRequestsHandler.async_post_call_success_hook   sJ     $++||~..z:||~..t4!006(><..9IJ#*)2n-=_ M ++;;%%6%G%G < 
 
 	 26'+15'+ **6%//'-2HH ( !2 ; ; **6%//'-2HH ( !2 ; ;8-.$X/?@N!N%~y1ZPT5U.)4!/!:!:!<"0"4"45I2"N"TRT&2 , $4 (DQ#$@A&2 , $2 (BO#$>? M>M#79LM ='  5 6V &3
hs%   B2G5G6DGGGGr0   )r$   r%   r&   r   r2   r:   r   r   r'   rd   rc   r	   r   r   r
   r   ri   floatre   r   r_   r   r   r~   r   r   r   r   r   __classcell__)r   s   @r+   r-   r-   )   s   9-? 97)7 7 	7
 7  #7 7 7 $7  #7 !!ST7 $(c3h#87 
7r
&U 
& 37
"*3-
	
. ,0(
!)#(
  (}(
 &.c]	(

  (}(
  (}(
 $,C=(
 #4.(
 
(
TI)I I 	I
 IVg"RRh"" *" 
$	"HDD-;D Dr*   r-   ))r   r   r   r   typingr   r   r   r   r	   r
   r   r   fastapir   pydanticr   r5   r   r   r   r   "litellm.integrations.custom_loggerr   'litellm.litellm_core_utils.core_helpersr   litellm.proxy._typesr   r   r   litellm.proxy.auth.auth_utilsr   r   opentelemetry.tracer   _Spanlitellm.proxy.utilsr   _InternalUsageCacher   r-   r)   r*   r+   <module>r      s{     
 ( W W W !   , 1 ; U X X
 1MD,D.) .{ {r*   