
    hf                         d dl Z d dlmZ d dlmZmZmZmZmZm	Z	 d dl
Z
d dlZd dlmZ d dlmZ d dlmZ d dlmZmZ d dlmZ d d	lmZmZmZmZmZ d
dlmZmZ ddl m!Z!  e!       Z"	  G d de      Z#y)    N)deepcopy)AnyCallableListOptionalUnioncast)verbose_logger)asyncify)
BaseAWSLLM)_get_httpx_clientget_async_httpx_client)AllMessageValues)CustomStreamWrapperEmbeddingResponseModelResponseUsage
get_secret   )AWSEventStreamDecoderSagemakerError   )SagemakerConfigc                   n   e Zd ZdefdZ	 ddededee   dededed	ee   fd
Z	di dddi fdede
dededededeeeej                   f      dedefdZ	 ddededefdZdee   dededee   dededee   dededefdZdee   dededee   dededededee   dedefdZi ddfdede
dededef
dZy)SagemakerLLMoptional_paramsc                 p   	 ddl m} |j                  dd       }|j                  dd       }|j                  dd       }|j                  dd       }|j                  dd       }|j                  d	d       }|j                  d
d       }	|j                  dd        |j                  dd       }
|j                  dd       }|Dt	        dd       }|t        |t              r|}t	        dd       }|t        |t              r|}|d}| j                  ||||||	||
|	      }||fS # t        $ r t        d      w xY w)Nr   )Credentials7Missing boto3 to call bedrock. Run 'pip install boto3'.aws_secret_access_keyaws_access_key_idaws_session_tokenaws_region_nameaws_role_nameaws_session_nameaws_profile_nameaws_bedrock_runtime_endpointaws_web_identity_tokenaws_sts_endpointAWS_REGION_NAME
AWS_REGION	us-west-2)	r!   r    r"   r#   r%   r&   r$   r(   r)   )botocore.credentialsr   ImportErrorpopr   
isinstancestrget_credentials)selfr   r   r    r!   r"   r#   r$   r%   r&   r(   r)   litellm_aws_region_namestandard_aws_region_namecredentialss                  e/var/www/Befach/backend/env/lib/python3.12/site-packages/litellm/llms/sagemaker/completion/handler.py_load_credentialszSagemakerLLM._load_credentials&   s   	Y8
 !0 3 34KT R+//0CTJ+//0CTJ)--.?F'++OTB*../A4H*../A4H*D	
 "1!4!45Mt!T*../A4H "&01BD&I#&2z'8 #:'1,'E$'3
(#9 #;&"-#'#7#7/"7/+--'#9- $8 
$
 O++_  	YWXX	Ys   D   D5Nmodeldatamessageslitellm_paramsr#   extra_headersc	                    	 ddl m}	 ddlm}
  |	|d|      }|j                  d      du r
d| d	| d
}n	d| d	| d}|j                  dd       }||}t        j                  |      j                  d      }t        j                  |||||      } |
d|||      }|j                  |       |d|v r|d   |j                  d<   |j                         }|S # t        $ r t	        d      w xY w)Nr   )	SigV4Auth)
AWSRequestr   	sagemakerstreamTzhttps://runtime.sagemaker.z.amazonaws.com/endpoints/z/invocations-response-streamz/invocationssagemaker_base_urlutf-8)headersr9   r;   r   r<   POST)methodurlr:   rE   Authorization)botocore.authr?   botocore.awsrequestr@   r.   getjsondumpsencodesagemaker_configvalidate_environmentadd_authrE   prepare)r3   r6   r9   r:   r;   r<   r   r#   r=   r?   r@   sigv4api_baserC   encoded_datarE   requestprepped_requests                     r7   _prepare_requestzSagemakerLLM._prepare_request]   s;   	Y/6 +{ODx(D03O3DD]^c]d  eA  BH3O3DD]^c]ddpqH,001EtL))Hzz$'..w7"77!+) 8 
 xlG
 	w%/]*J/<_/MGOOO,!//+A  	YWXX	Ys   C C.Fmodel_responseprint_verbosetimeoutacompletionrE   c                    | j                  |      \  }}t        |      }|j                  dd       }|j                  dd       }t        j
                  j                         }|j                         D ]  \  }}||vs|||<    |du rD|du r | j                  |||
||||||||||      }|S t        j                  |||||      }| j                  |||||||      }||j                  j                  d|i       t               }|j                  |j                   |j                  |j"                  |      }|j$                  d	k7  r.t'        |j$                  t)        |j+                               
      t-        d      }|j/                  |j/                  d            }t1        ||d|      }|j3                  |d|d|i       |S |du r| j5                  |||
||||||||||      S t        j                  |||||      }|||||||d}  | j                  d#i | }	 ||j                  j                  d|i       d}	t               }|j7                  g d||j                   |j                  d       	 |j                  |j                   |j                  |j"                  |	      }|j$                  d	k7  r!t'        |j$                  |j8                  
      	 t        jC                  |||||||||"	      S # t:        $ r'}!|j3                  g dt)        |!      d|i       |!d }!~!ww xY w# t:        $ r}!t=        j>                  dt)        |!             tA        |!di       j                  di       j                  dd      }"tA        |!di       j                  di       j                  dt)        |!            }#d |#v r|#d!z  }#t'        |"|#
      d }!~!ww xY w)$NrB   model_idT)r;   r9   custom_prompt_dicthf_model_namer   encodingrZ   logging_objr_   r#   r6   rE   r<   r9   r;   r   r<   rE   )r9   r:   r;   r   r<   r6   r#   $X-Amzn-SageMaker-Inference-Component)rH   rE   r:   rB      status_codemessage r9      
chunk_sizerA   completion_streamr9   custom_llm_providerrc   complete_input_dictinputapi_keyoriginal_responseadditional_args)r;   r9   r`   ra   rZ   rb   rc   r_   r   r6   r#   rE   r<   r9   r:   r   r<   r6   r#   r;        r@rr   rU   rE   rt   ru   rw   rH   rE   r:   r\   zSagemaker error %sresponseResponseMetadataHTTPStatusCode  ErrorMessage+Inference Component Name header is requiredJ
 pass in via `litellm.completion(..., model_id={InferenceComponentName})`	r9   raw_responserZ   rc   request_datar;   r   rb   r<    )"r8   r   r/   rL   litellmr   
get_configitemsasync_streamingrP   transform_requestrY   rE   updater   postrH   bodyrh   r   r1   readr   
iter_bytesr   	post_callasync_completionpre_calltext	Exceptionr
   errorgetattrtransform_response)$r3   r9   r;   rZ   r[   rb   rc   r   r<   r\   r`   ra   	logger_fnr]   rE   r6   r#   inference_paramsrB   r_   configkvr}   r:   prepared_requestsync_handlersync_responsedecoderrp   streaming_response_dataprepared_request_argserh   error_messages$                                       r7   
completionzSagemakerLLM.completion   s   $ (,'='=o'N$_#O4!%%h5"&&z48 ((335LLNDAq))&' #	 # T>d"//%'9"/$3%#1 +%$3 +##1 0   '99%$3#1# :  $(#8#8%$3#1 +$3 $9 $  ' %,,33?J  12 , 1 1(,,,44)..!	 !2 ! !,,3($1$=$= #M$6$6$8 9 
 0b9$+$6$6!,,,=%! &9&7(3 +	&" !!"4!6 =	 "  &%
 $((!#5+-!'! /' /- )  " !22+) 3 
 .,&. !
 1400I3HI9	Q# !((//;XF
 G,.L  +0 0 4 4/77! !  , 1 1(,,,44)..#	 !2 ! !,,3($1$=$= - 2 2  48  22&)#+) 3 

 
	
/  %%&)!f%:E$B	 &    	Q  !5s1v>:r*',%s+  :r*..w;??	3q6R  =M!nn [-PP	Qs9   &AL3 >A#L   	L0	"L++L00L3 3	O<BOOrU   c                 z  K   	 |$t        t        j                  j                        }|j	                  |||d       d {   }|j
                  dk7  r!t        |j
                  |j                        t        d      }|j                  |j                  d	            }|S 7 c# t        j                  $ r<}	|	j                  j
                  }
t        |
|	j                  j                        d }	~	wt        j                  $ r t        d
d      t        $ r}t        dt!        |            d }~ww xY ww)Nllm_providerT)rE   r:   rB   rf   rg   rj   rk   rl   rm   i  zTimeout error occurred.r   )r   r   LlmProviders	SAGEMAKERr   rh   r   r   r   aiter_bytesr   httpxHTTPStatusErrorr}   TimeoutExceptionr   r1   )r3   rU   rE   r:   rc   clientr}   r   rp   err
error_coder   s               r7   make_async_callzSagemakerLLM.make_async_calle  s.    &	B~/!(!5!5!?!? $[[	 )  H ##s*$ ( 4 4hmm  ,"5G ' 3 3$$$5! %$#6 $$ 	T11J ZARARSS%% 	U S:STT 	B S#a&AA	BsG   D;>B( B&A"B( %D;&B( (D8;7C22+D8D33D88D;r`   ra   r_   rc   c                   K   t         j                  ||i |ddi||       d {   }t        | j                        }|||||||d} |di | d {   }|
|j                  j                  d|
i       |j                  st        d      | j                  |j                  |j                  t        t        |j                        |       d {   }t        ||d|	      }|j                  g d
dd|i       |S 7 7 7 0w)NrB   Trd   rx   re   zPrepared request body is empty)rU   rE   r:   rc   rA   ro   rj   zfirst stream response receivedrr   rs   r   )rP   async_transform_requestr   rY   rE   r   r   
ValueErrorr   rH   r	   r1   r   r   )r3   r;   r9   r`   ra   r6   r#   r   rb   rZ   r_   rc   r<   rE   r:   asyncified_prepare_requestr   r   rp   r   s                       r7   r   zSagemakerLLM.async_streaming  sU      &==??$?) > 
 
 &.d.C.C%D".,&. !
 "<!T>S!TT$$++7B  $$=>>"&"6"6%))$,,c+001#	 #7 #
 
 1/ +#	
 	>2D9	 	 	
 "!_
" U
s4   $DD.DDA>DD-DDDc                   K   d}t        t        j                  j                        }t        j                  |||	||       d {   }t        | j                        }|||	||||d} |di | d {   }|
j                  g d||j                  |j                  d       	 ||j                  j                  d|i       	 |j                  |j                  |j                  |j                  |	       d {   }|j                  d
k7  r!t        |j                  |j                         	 t        j)                  ||||
|||	||	      S 7 7 7 X# t"        $ r*}|
j%                  |d   dt'        |      d|i       |d }~ww xY w# t"        $ r'}t'        |       }d|v r|dz  }t        d|      d }~ww xY ww)Nry   r   rd   rx   rj   rz   r{   re   r|   rf   rg   inputsrr   rs   r   r   r   r   r   )r   r   r   r   rP   r   r   rY   r   rH   rE   r   r   r   rh   r   r   r   r   r1   r   )r3   r;   r9   r`   ra   r6   r#   rb   rZ   r   rc   r_   rE   r<   r\   async_handlerr:   r   r   r   r}   r   r   s                          r7   r   zSagemakerLLM.async_completion  s*      . --77
 &==+) > 
 
 &.d.C.C%D".,&. !
 "<!T>S!TT'+,00+33 	 	
!	I# !((//;XF!.!3!3(,,,44)..#	 "4 "  ''3.($,$8$8(--  /$  22!)#+) 3 

 
	
A
& U(  %%x.&)!f%:D$A	 &    	I"1vhM<M!nn S-HH		Isx   AGE.G5E60G'F 6E =E>4E 2 GGE 	F!%FFF 	F>"F99F>>Grt   c           	         ddl }|j                  dd      }|j                  dd      }|j                  dd      }||j                  d|||      }n&t        d      xs |xs d	}|j                  d|
      }t	        |      }|j                  dd       t
        j                  j                         }|j                         D ]  \  }}||vs|||<    t        j                  d|i      j                  d      }d| d|d}|j                  |d||d       	 |j                  |d|d      }t        j&                  |d   j)                         j+                  d            }|j-                  |d|d |i!        |d"|        d#|vrt%        dd$      |d#   }t/        |t0              st%        d%d&|       g }t3        |      D ]  \  }}|j5                  d#||d'        d(|_        ||_        ||_        d}|D ]  } |t=        |j                  |             z  }! t?        |d)tA        |d|*             |S # t        $ ru}t        |di       j!                  di       j!                  dd      }t        |di       j!                  di       j!                  dt#        |            }t%        ||      d}~ww xY w)+zG
        Supports Huggingface Jumpstart embeddings like GPT-6B
        r   Nr    r!   r#   zsagemaker-runtime)service_namer!   r    region_namer*   r,   )r   r   rB   r   rD   zE
        response = client.invoke_endpoint(
            EndpointName=zA,
            ContentType="application/json",
            Body=f"z_",  # Use !r for safe representation
            CustomAttributes="accept_eula=true",
        )rj   )rr   request_strr{   zapplication/jsonzaccept_eula=true)EndpointNameContentTypeBodyCustomAttributesr}   r~   r   r   r   r   rg   r   utf8rr   rs   zraw model_response: 	embeddingzembedding not found in responsei  z"Response not in expected format - )objectindexr   listusage)prompt_tokenscompletion_tokenstotal_tokens)!boto3r/   r   r   r   r   r   r   r   rM   rN   rO   r   invoke_endpointr   r   rL   r1   r   loadsr   decoder   r0   r   	enumerateappendr   r:   r9   lensetattrr   )!r3   r9   rt   rZ   r[   rb   rc   r   r`   r<   r   r   r    r!   r#   r   r   r   r   r   r   r:   r   r}   r   rh   r   
embeddingsoutput_dataidxr   input_tokensr   s!                                    r7   r   zSagemakerLLM.embedding7  sQ   " 	 !0 3 34KT R+//0CTJ)--.?F( \\0"3&;+	 " F ,- " 
 \\0' " F $O4Xt, ((335LLNDAq))&' #	 # zz8U+,33G< !8 
	 	48U 	 	
	Q--".!3	 . H" ::hv.335<<VDE&2D9	 	 	
 	,XJ78h& )J  k*
*d+ <ZLI 
 '
3NC&9M 4
 !')$DC 566L  	*"#)	
 s  		Q:r*',%s+  :r*..w;??	3q6R  ![-PP		Qs   H? ?	J=A0J88J=)N)__name__
__module____qualname__dictr8   r1   r   r   r   rY   r   r   r   r   floatr   Timeoutboolr   r   r   r   r   r   r   r       r7   r   r   %   sm   5,5,@ )-. . 	.
 '(. . . .  ~.t :>!V
V
 V
 &	V

  V
 V
 V
 %u}} 456V
 V
 V
| .B.B .B 	.B`?"'(?" ?" !	?"
  }?" ?" &?" 3-?" ?" ?" ?"B_
'(_
 _
 !	_

  }_
 _
 &_
 _
 _
 3-_
 _
 _
T NN N *	N
  N Nr   r   )$rM   copyr   typingr   r   r   r   r   r	   r   r   litellm._loggingr
   #litellm.litellm_core_utils.asyncifyr   !litellm.llms.bedrock.base_aws_llmr   &litellm.llms.custom_httpx.http_handlerr   r   litellm.types.llms.openair   litellm.utilsr   r   r   r   r   common_utilsr   r   transformationr   rP   r   r   r   r7   <module>r      sY      = =   + 8 8 7  A +"$ `
: `
r   