
    ht                        d dl Z d dlZd dlZd dlZd dlmZ d dlmZmZmZm	Z	m
Z
mZmZmZ d dlZd dlZd dlmZmZmZ d dlmZmZ d dlZd dlmZ d dlmZmZ d dlmZ d d	lm Z! d d
l"m#Z# d dl$m%Z%m&Z& d dl'm(Z( d dl)m*Z*m+Z+ d dl,m-Z- d dl.m/Z/ d dl0m1Z1 er	d dl2m3Z4 e4Z3neZ3d dl5m6Z6 dee7e8f   dee9   fdZ:ejv                  fdee7df   de7de<de9def
dZ= G d d      Z>y)    N)datetime)TYPE_CHECKINGAnyAsyncGeneratorCallableLiteralOptionalTupleUnion)HTTPExceptionRequeststatus)ResponseStreamingResponse)verbose_proxy_logger)(DD_TRACER_STREAMING_CHUNK_YIELD_RESOURCESTREAM_SSE_DATA_PREFIX)tracer)Logging)
safe_dumps)ProxyExceptionUserAPIKeyAuth)check_response_size_is_safe)get_logging_caching_headers3get_remaining_tokens_and_requests_from_request_data)route_request)ProxyLogging)Router)ProxyConfig)add_litellm_data_to_request
event_linereturnc                   K   t        | t              r| j                  d      n| } | j                  d      r| t	        d      d j                         }|r|dk(  ry	 t        j                  |      }t        |t              rd|v rt        |d   t              ro|d   j                  d      }d}t        |t              r|}nt        |t              r	 t        |      }|d|cxk  rd	k  r|S  |t        j                  d
|        yy# t        $ r t        j                  d|        Y Ow xY w# t        j                  t         j                  f$ r Y yw xY ww)zEParses an event line and returns an error code if present, else None.zutf-8data: Nz[DONE]errorcodez0Error code is a string but not a valid integer: d   iW  z+Error has invalid or non-convertible code: )
isinstancebytesdecode
startswithlenstriporjsonloadsdictgetintstr
ValueErrorr   warningJSONDecodeErrorjson)r!   json_strdataerror_code_raw
error_codes        c/var/www/Befach/backend/env/lib/python3.12/site-packages/litellm/proxy/common_request_processing.py_parse_event_data_for_errorr=   2   sj     '1U&C
'"  X&c(mo.4468x/!	<<)D4&tOtG}d3!%g!2!26!:,0
nc2!/J4%(%8
 )cZ.F3.F%% /G #.(00EnEUV ' & ,44N~N^_   &&(<(<= 		s[   AEA5D0 D	 D0 +E,D0 E	!D-*D0 ,D--D0 0#EEEE	generator
media_typeheadersdefault_status_codec                    
K   d
|}	 t        j                         r
  d{     j                          d{   

0	 t        
       d{   }||}t	        j
                  d|        dt        t        df   f
 fd	}	t         |	       |||      S 7 x7 b7 N# t        $ r"}t	        j
                  d|        Y d}~Vd}~ww xY w# t        $ r* dt        t        df   fd}t         |       |||      cY S t        $ rW}t	        j                  d|        dt        t        df   fd}t         |       ||t        j                        cY d}~S d}~ww xY ww)
z
    Creates a StreamingResponse by inspecting the first chunk for an error code.
    The entire original generator content is streamed, but the HTTP status code
    of the response is set based on the first chunk if it's a recognized error.
    Nz:Error detected in first stream chunk. Status code set to: z!Error parsing first chunk value: r"   c                    K   y wN rE       r<   	empty_genz,create_streaming_response.<locals>.empty_gen   s	     s   )r?   r@   status_codez,Error consuming first chunk from generator: c                 t   K   dt        j                  ddt        j                  di       d d y w)Nr$   r%   zError processing stream start)messager&   

zdata: [DONE]

)r7   dumpsr   HTTP_500_INTERNAL_SERVER_ERRORrE   rF   r<   error_gen_messagez4create_streaming_response.<locals>.error_gen_message   sV     4::w<[ek  fK  fK  1L  'M  N  O  OS  T  T$$s   68c                    K   't        j                  t              5   d d d        2 3 d {   } t        j                  t              5  |  d d d        1# 1 sw Y   <xY w7 8# 1 sw Y   HxY w6 y wrD   )r   tracer   )chunkfirst_chunk_valuer>   s    r<   combined_generatorz5create_streaming_response.<locals>.combined_generator   s{     (FG (''($ 	 	%FG  	( (	  %sM   A<A 
A<A:A,A:A<A.	A< A)%A<,A:.A7	3	A<)asyncioiscoroutine	__anext__r=   r   debug	ExceptionStopAsyncIterationr   r3   r   	exceptionr   rM   )r>   r?   r@   rA   final_status_codeerror_code_from_chunkerG   rN   rS   rR   s   `         @r<   create_streaming_responser^   `   s     (,+5
 y)'I #,"5"5"77(
T.I%/ )% )4(=%(..TUfTghLnS$Y&?  %	 y ( 8
)  T$**-Nqc+RSST  
	T	!: 	 !K!+	
 	
  
 	&&:1#>	

	%T	)B 	% !!==	
 	

s   E%C BC BC B# B! B# 3*E%C C !B# #	C,C	C 	CC 0E"E%E"AEE"E%E""E%c            %       B   e Zd ZdefdZedddddddddi dddedee   dee   dee   d	ee   d
ee   dee   dee	e
ef      dee   dee   dee   dee	e
eej                  f      defd       Z	 	 	 	 	 	 	 d2dedededededed   d
ee   dee   dee
   dee
   dee   dee   dee   deeef   fdZ	 	 	 	 	 	 	 	 	 d3ded ededed!   dededed"ed#ee   dee   dee   dee
   dee
   dee   dee   d
ee   d$ee   def$d%Z	 	 	 	 	 	 	 	 d4ded ededededed"ed#ee   dee   dee   dee
   dee
   dee   dee   d
ee   fd&Zd'edefd(Z	 d5ded$ee   defd)Z	 d6d*ededed
ee   fd+Z eded,   ded-   fd.       Z!ed/edefd0       Z"edededefd1       Z#y)7ProxyBaseLLMRequestProcessingr9   c                     || _         y rD   )r9   )selfr9   s     r<   __init__z&ProxyBaseLLMRequestProcessing.__init__   s	    	rF   N)call_idmodel_id	cache_keyapi_baseversionmodel_regionresponse_costhidden_params!fastest_response_batch_completionrequest_datatimeoutuser_api_key_dictrd   re   rf   rg   rh   ri   rj   rk   rl   rm   rn   r"   c                 V   h d}|xs i }||||r|j                  d      d   nd ||t        |      t        | j                        t        | j                        t        | j                        t        | j
                        t        |j                  dd             t        |j                  dd             |	t        |	      nd |t        |      nd d|j                         D ci c]  \  }}|t        |       c}}}|
r:t        |
      }|j                  |       t        |
      }|r|j                  |       	 |j                         D ci c]  \  }}||vr|t        |       c}}S c c}}w c c}}w # t        $ r$}t        j                  d|        i cY d }~S d }~ww xY w)N>   N None?r   _response_mslitellm_overhead_time_ms)x-litellm-call-idzx-litellm-model-idzx-litellm-cache-keyzx-litellm-model-api-basezx-litellm-versionzx-litellm-model-regionzx-litellm-response-costzx-litellm-key-tpm-limitzx-litellm-key-rpm-limitzx-litellm-key-max-budgetzx-litellm-key-spendzx-litellm-response-duration-mszx-litellm-overhead-duration-msz+x-litellm-fastest_response_batch_completionzx-litellm-timeoutzError setting custom headers: )splitr3   	tpm_limit	rpm_limit
max_budgetspendr1   itemsr   updater   rX   r   r%   )ro   rd   re   rf   rg   rh   ri   rj   rk   rl   rm   rn   kwargsexclude_valueskvr@   remaining_tokens_headerlogging_caching_headerskeyvaluer]   s                         r<   get_custom_headersz0ProxyBaseLLMRequestProcessing.get_custom_headers   s   " ,%+!("*#,*2s#A&!(&2'*='9'*+<+F+F'G'*+<+F+F'G(+,=,H,H(I#&'8'>'>#?.1!!.$7/ /2!!"<dC/
 5@ 56181DW$3
4 &,\\^4TQq#a&y45
8 CLQ $ NN23&A,&O#&67	 #*--/C. SZ  5
  	 &&)Gs'KLI	s6   &E/>E; E5+E; 5E; ;	F(F#F(#F(requestgeneral_settingsproxy_logging_objproxy_config
route_type)acompletion
aresponses
_arealtimeaget_responsesadelete_responsesacreate_batcharetrieve_batchafile_contentatext_completionacreate_fine_tuning_jobacancel_fine_tuning_jobalist_fine_tuning_jobsaretrieve_fine_tuning_jobalist_input_itemsaimage_editagenerate_contentagenerate_content_streamallm_passthrough_routeavector_store_searchavector_store_create
user_modeluser_temperatureuser_request_timeoutuser_max_tokensuser_api_basemodelc                 *  K   t        j                         }t        | j                  |||||       d {   | _        |j	                  dd       xs$ |xs  |xs | j                  j	                  dd       | j                  d<   |	r|	| j                  d<   |
r|
| j                  d<   |r|| j                  d<   |r|| j                  d<   t        | j                  d   t              rL| j                  d   t        j                  v r-t        j                  | j                  d      | j                  d<   |j                  j	                  dt        t        j                                     | j                  d	<   |j                  || j                  |
       d {   | _        t        j                  j                  d|t        j                  j                         |d| j                  \  }| _        || j                  d<   | j                  |fS 7 7 zw)N)r9   r   r   ro   rh   r   completion_modelr   temperaturerequest_timeout
max_tokensrg   rv   litellm_call_id)ro   r9   	call_type)original_function	rules_obj
start_timelitellm_logging_objrE   )r   nowr    r9   r1   r(   r3   litellmmodel_alias_mapr@   uuiduuid4pre_call_hookutilsfunction_setupRules)rb   r   r   ro   r   r   r   rh   r   r   r   r   r   r   r   logging_objs                   r<    common_processing_pre_call_logicz>ProxyBaseLLMRequestProcessing.common_processing_pre_call_logic   s    J \\^
5-/%
 
	   !3T: ,,, yy}}Wd+	 			' '7DIIm$+?DII'(&5DIIl#$1DIIj! tyy)3/		'"g&=&==!(!8!879K!LDIIg'.':':TZZ\!2(
		#$
 ,99/dii: : 
 
	 ")!=!= "
(mm))+!"
 ii	"
TY ,7		'(yy+%%o
P
s#   3HHE HHA8HHfastapi_response)r   r   r   r   r   r   r   r   r   r   r   r   r   select_data_generator
llm_routeris_streaming_requestc                   K   t        j                  dj                  t        j                  | j
                  dt                           | j                  ||||||||||||
|       d{   \  | _        }g }|j                  |j                  | j
                  |t        j                  |                   t        | j
                  ||	|       d{   }|j                  |       t        j                  | }| d{   }|d	   }t        |d
i       xs i }|j!                  dd      xs d}|j!                  dd      xs d}|j!                  dd      xs d}|j!                  dd      xs d}|j!                  dd      }|j!                  di       xs i }|	|	j#                  |      | j
                  d<   t        j$                  |j'                  | j
                  j!                  dd      d             | j)                  | j
                  |      s| j+                  |      rt        j,                  d!||j.                  |||||t        |dd      || j
                  |d|}|dk(  ru| j+                  |      r>t        j0                  |      r| d{   } n|} t3        | t4        j6                  |      S t3        |j9                         |j:                  |      S  |||| j
                        }!t=        |!d|       d{   S |j?                  | j
                  ||       d{   }t        |d
i       xs i }|j!                  di       xs i }|j@                  jC                  t        j,                  d!||j.                  |||||t        |dd      || j
                  |d|       tE        |        d{    |S 7 d7 7 7 :7 7 7 w)"zg
        Common request processing logic for both chat completions and responses API endpoints
        zRequest received by LiteLLM:
{}   )indentdefault)r   r   r   ro   rh   r   r   r   r   r   r   r   r   Nr   )r9   ro   r   )r9   r   r   r      _hidden_paramsre   rq   rf   rg   rj   rl   additional_headers)re   
deploymentr   success)r   r   )r9   r   allowed_model_region)ro   rd   re   rf   rg   rh   rj   ri   rl   rm   rk   r   contentrH   r@   )responsero   rm   ztext/event-stream)r>   r?   r@   )r9   ro   r   )r   rE   )#r   rW   formatr7   rL   r9   r3   r   appendduring_call_hookr`   _get_pre_call_typer   rT   gathergetattrr1   get_deploymentcreate_taskupdate_request_status_is_streaming_request_is_streaming_responser   r   rU   r   r   HTTP_200_OKaiter_bytesrH   r^   post_call_success_hookr@   r}   r   )"rb   r   r   ro   r   r   r   r   r   r   r   r   r   r   r   r   rh   r   r   tasksllm_callllm_responses	responsesr   rk   re   rf   rg   rj   rl   r   custom_headersr>   selected_data_generators"                                     r<   base_process_llm_requestz6ProxyBaseLLMRequestProcessing.base_process_llm_request\  sY    J 	"".55

499Q<	
 (,'L'L-//%!-!5+'! (M (
 "
	;  ..YY"37JJ) K  / 	
 '!!!	
 
 	X  
 ('	Q<*:B?E2 $$Z6<"!%%k48>B	 $$Z6<"%))/4@FB,9,=,=/-
) $1#4#45I2#N#TRT !&0&?&?&?&RDIIl#33 $		.? DY 4 	

 %%1E & 
((
 ;MM "3#33!#!+$%68NPRS2S!YY+ %N 55..x8**84*2N	$,	 - )$*$6$6 .  - ( 4 4 6$,$8$8 .  +@%&7!%+'
 752*   +AA.?( B 
 

 H.39r 	 +../CRHNB  '')<< "3#33!#!+$%68NPRS2S!YY+ %	
  *8<<<i"
:
 (^ %32
2 	=s   A*O",O-A+O"O,O"OFO"O A/O"O%O"5O6BO"	O 
O"O"O"O"O"O" O"c                 *  K   ddl m} | j                  |||d||||||	|
|||||       d {   }t        |t              r|S |j                          d {   }t        ||j                  |j                  |j                  d             S 7 a7 9w)Nr   )HttpPassThroughEndpointHelpersr   )r   r   ro   r   r   r   r   r   r   r   r   r   r   r   r   rh   )r@   r   r   )
;litellm.proxy.pass_through_endpoints.pass_through_endpointsr   r   r(   r   areadr   rH   get_response_headersr@   )rb   r   r   ro   r   r   r   r   r   r   r   r   r   r   r   rh   r   resultr   s                      r<   $base_passthrough_process_llm_requestzBProxyBaseLLMRequestProcessing.base_passthrough_process_llm_request  s     $	
 44-///!-%"7!-!5+'! 5 
 
( f/0M&**2GG# H 
 	
1
. 's!   +BB)BB8BBr   c                 ^    ddl }ddlm}m} |j	                  |      ryt        |||f      ryy)z
        Check if the response object is actually a streaming response by inspecting its type.

        This uses standard Python inspection to detect streaming/async iterator objects
        rather than relying on specific wrapper classes.
        r   N)r   AsyncIteratorTF)inspectcollections.abcr   r   
isasyncgenr(   )rb   r   r   r   r   s        r<   r   z4ProxyBaseLLMRequestProcessing._is_streaming_responseT  s4     	A h' h ?@rF   c                 &    |du ryd|v r|d   du ryy)z
        Check if the request is a streaming request.

        1. is_streaming_request is a dynamic param passed in
        2. if "stream" in data and data["stream"] is True
        TstreamFrE   )rb   r9   r   s      r<   r   z3ProxyBaseLLMRequestProcessing._is_streaming_requesth  s(      4'tX$ 6rF   r]   c                 h  K   t        j                  dt        |              |j                  ||| j                         d{    t        |dd      }t        j                  d||       t        |dd      }| j                  j                  dd      }t        j                  ||r|j                  nd|d	t        |d
d      | j                  |      }t        |di       xs i }	|	j                  |       t        |t              rSt        t        |dt        |            t        |dd      t        |dd      t        |dt        j                         |	      t        |       }
t        t        |d|
      t        |dd      t        |dd      t        |dd      t        |dd      |	      7 jw)zGRaises ProxyException (OpenAI API compatible) if an exception is raisedzLlitellm.proxy.proxy_server._handle_llm_api_exception(): Exception occured - ro   original_exceptionrm   Nlitellm_debug_inforq   zo[1;31mAn error occurred: %s %s

 Debug this by setting `--debug`, e.g. `litellm --model gpt-3.5-turbo --debug`rn   r   r   r   )ro   rd   rh   rj   ri   rm   rn   r@   detailtyperr   paramrH   )rJ   r   r   r&   r@   rJ   r&     )rJ   r   r   openai_coder&   r@   )r   rZ   r3   post_call_failure_hookr9   r   rW   r1   r`   r   r   r}   r(   r   r   r   HTTP_400_BAD_REQUEST)rb   r]   ro   r   rh   r   rn   _litellm_logging_objr   r@   	error_msgs              r<   _handle_llm_api_exceptionz7ProxyBaseLLMRequestProcessing._handle_llm_api_exceptionw  s     	&&Z[^_`[aZbc	
  66/  7 
 	
 	

 %Q(<bA"" C	
 y$
 =AIIMM!4=
 7II/8L$44RV !24JBO J 

 !Y+1r~&a' 8SV4Q/a&1Qv/J/JK  1vh	Ay)4FF+!Wf-640M3/
 	
S	
s   AF2F/E*F2)r   r   )
completionr   c                     | dk(  ry| dk(  ryy )Nr   r   r   r   rE   r   s    r<   r   z0ProxyBaseLLMRequestProcessing._get_pre_call_type  s     &<' (rF   rQ   c                 R    t        | t              rt        |       }t         | dS | S )z
        Helper function to format streaming chunks for Anthropic API format

        Args:
            chunk: A string or dictionary to be returned in SSE format

        Returns:
            str: A properly formatted SSE chunk string
        rK   )r(   r0   r   r   )rQ   	chunk_strs     r<   return_sse_chunkz.ProxyBaseLLMRequestProcessing.return_sse_chunk  s/     eT""5)I,-i[==LrF   c                  K   ddl m}m} t        j                  d       	 d}| 2 3 d{   }t        j                  dj                  |             |j                  ||||       d{   }t        |||f      rt        j                  |      }||z  }t        j                  |       7 7 H6 y# t        $ r}	t        j                  d	j                  t        |	                   |j                  ||	|
       d{  7   t        j                  d|	 d       t        |	t               r|	t#        j$                         }
t        |	       d|
 }t'        t)        |	d|      t)        |	dd      t)        |	dd      t)        |	dd            }t+        j,                  d|j/                         i      }t0         | d Y d}	~	yd}	~	ww xY ww)zm
        Anthropic /messages and Google /generateContent streaming data generator require SSE events
        r   )ModelResponseModelResponseStreamzinside generatorrq   Nz3async_data_generator: received streaming chunk - {})ro   r   r9   
str_so_far)response_objzIlitellm.proxy.proxy_server.async_data_generator(): Exception occured - {}r   z[1;31mAn error occurred: zP

 Debug this by setting `--debug`, e.g. `litellm --model gpt-3.5-turbo --debug`rK   rJ   r   rr   r   rH   r   )rJ   r   r   r&   r%   )litellm.types.utilsr  r  r   rW   r   async_post_call_streaming_hookr(   r   get_response_stringr`   r  rX   rZ   r3   r   r   	traceback
format_excr   r   r7   rL   to_dictr   )r   ro   rm   r   r  r  r  rQ   response_strr]   error_tracebackr   proxy_exceptionerror_returneds                 r<   async_sse_data_generatorz6ProxyBaseLLMRequestProcessing.async_sse_data_generator  s     	K""#560	BJ' L Le$**IPPQVW 0NN&7"%)	 O   em5H%IJ#*#>#>E#RL,.J 4DDUKK#L
  ($  	B **[bbF
 $::"3#$) ;   
 !&&/s  3E  F !]+"+"6"6"8"1vhd?*;<	,9i8Q/a&1Qs3	O "ZZ/2I2I2K(LMN+,^,<DAAA9	Bso   GB7 B5B1B5=B7 *B3+AB7 1B53B7 5B7 6G7
GAGDCGGGG)NNNNNNN)	NNNNNNNNF)NNNNNNNN)FrD   )$__name__
__module____qualname__r0   rc   staticmethodr   r	   r3   r   floatboolr2   httpxTimeoutr   r   r   r   r   r
   LiteLLMLoggingObjr   r   r   r   r   r   r   r   r   rX   r   r   r  r  rE   rF   r<   r`   r`      s   T   "&"&#'"&!%&*59(,<@')>B@)@ #@ 3-	@
 C=@ 3-@ #@ sm@  eSj 12@  ~@ ,4D>@ tn@ %sEMM 9:;@ 
@ @~ "&$(,004)-'+#G]&]& ]& *	]&
 (]& "]& $
]&: #;]&< SM=]&> #5/?]&@ 'uoA]&B "#C]&D  }E]&F }G]&H 
t&&	'I]&n (,#$(,004)-'+!%/4A # *	
 $
( ()* +, "-.  (/0 V$12 }34 SM56 #5/78 'uo9: "#;<  }=> #?@ 'tnAB 
CT (,#$(,004)-'+!%!5
5
 #5
 *	5

 (5
 5
 "5
  (5
 V$5
 }5
 SM5
 #5/5
 'uo5
 "#5
  }5
  #!5
ns t * BG08	( "&;
;
 *;
 (	;

 #;
z 78	*	+     " <B)<B <B (	<B <BrF   r`   )?rT   r7   r  r   r   typingr   r   r   r   r   r	   r
   r   r  r.   fastapir   r   r   fastapi.responsesr   r   r   litellm._loggingr   litellm.constantsr   r   %litellm.litellm_core_utils.dd_tracingr   *litellm.litellm_core_utils.litellm_loggingr   r  *litellm.litellm_core_utils.safe_json_dumpsr   litellm.proxy._typesr   r   litellm.proxy.auth.auth_utilsr   )litellm.proxy.common_utils.callback_utilsr   r   litellm.proxy.route_llm_requestr   litellm.proxy.utilsr   litellm.routerr   litellm.proxy.proxy_serverr   _ProxyConfig$litellm.proxy.litellm_pre_call_utilsr    r3   r)   r2   r=   r   r0   r^   r`   rE   rF   r<   <module>r.     s        	 	 	   2 2 9  1 9 S A ? E : , !FKK L+%U
2C +QT +d  &11	Sc4i(SS S 	S
 SlZ	B Z	BrF   