
    h?                       d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlmZ d dlmZ d dlmZmZmZmZmZmZmZmZmZmZ d dlZd dlZd dlmZ d dlmZ d dlm Z  d dl!Z!d dl"Z!d dl#Z!d dl!m$Z$ d d	l%m&Z& d d
l'm(Z(m)Z)m*Z*m+Z+ d dl,m-Z- d dl.m/Z/ d dl0m1Z1 d dl2m3Z3 d dl4m5Z5 d dl6m7Z7 d dl8m9Z: d dl;m<Z< d dl=m>Z> d dl?m@Z@ d dlAmBZB d dlCmDZD d dlEmFZF d dlGmHZH d dlImJZJ d dlKmLZLmMZM d dlNmOZOmPZPmQZQ d dlRmSZS d dlTmUZUmVZV d dlWmXZX d dlYmZZZm[Z[m\Z\m]Z]m^Z^ d d l_m`Z`maZambZb d d!lcmdZd d d"lemfZg d d#lhmiZimjZj d d$lkmlZl d d%lmmnZn d d&lompZpmqZq d d'lrmsZsmtZt d d(lumvZvmwZwmxZxmyZy d d)lzm{Z{m|Z|m}Z}m~Z~mZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZ d d*lmZ d d+lmZmZ d d,lmZ d d,lmZ d d-lmZ d d.lmZmZmZmZmZmZmZmZmZmZ d/d0lmZ erd d1lmZ d d2lmZmZ eeef   ZneZeZeZ G d3 d4ejV                        Z G d5 d6      Zy)7    Ndefaultdict)	lru_cache)
TYPE_CHECKINGAnyCallableDictListLiteralOptionalTupleUnioncast)AsyncOpenAI)	BaseModel)overload)get_secret_str)verbose_router_logger)	DualCacheInMemoryCache
RedisCacheRedisClusterCache)DEFAULT_MAX_LRU_CACHE_SIZE)CustomLogger)run_async_function)!_get_parent_otel_span_from_kwargs)CredentialAccessor)tracer)Logging)RouterBudgetLimiting)LeastBusyLoggingHandler)LowestCostLoggingHandler)LowestLatencyLoggingHandler)LowestTPMLoggingHandler)LowestTPMLoggingHandler_v2)simple_shuffle)get_deployments_for_tag) add_fallback_headers_to_responseadd_retry_headers_to_response)"_get_router_metadata_variable_namereplace_model_in_jsonlshould_replace_model_in_jsonl)InitalizeCachedClient)get_dynamic_litellm_paramsis_clientside_credential)CooldownCache)DEFAULT_COOLDOWN_TIME_SECONDS_async_get_cooldown_deployments/_async_get_cooldown_deployments_with_debug_info_get_cooldown_deployments_set_cooldown_deployments)#_check_non_standard_fallback_formatget_fallback_model_grouprun_async_fallback)$ForwardClientSideHeadersByModelGroup)!get_num_retries_from_retry_policy)#async_raise_no_deployment_exceptionsend_llm_exception_alert)PromptCachingDeploymentCheck)ResponsesApiDeploymentCheck)0increment_deployment_failures_for_current_minute1increment_deployment_successes_for_current_minute)FlowItem	Scheduler)AllMessageValues	FileTypesOpenAIFileObjectOpenAIFilesPurpose)#CONFIGURABLE_CLIENTSIDE_AUTH_PARAMSVALID_LITELLM_ENVIRONMENTSAlertingConfigAllowedFailsPolicyAssistantsTypedDictCredentialLiteLLMParamsCustomPricingLiteLLMParamsCustomRoutingStrategyBase
DeploymentDeploymentTypedDictLiteLLM_ParamsMockRouterTestingParamsModelGroupInfoOptionalPreCallChecksRetryPolicyRouterCacheEnumRouterGeneralSettingsRouterModelGroupAliasItemRouterRateLimitErrorRouterRateLimitErrorBasicRoutingStrategy)ServiceTypes)GenericBudgetConfigTypeLiteLLMBatch)	ModelInfo)StandardLoggingPayload)
CustomStreamWrapperEmbeddingResponseModelResponseRulesfunction_setupget_llm_provider!get_non_default_completion_params
get_secretget_utc_datetimeis_region_allowed   )PatternMatchRouter)Span)
AutoRouterPreRoutingHookResponsec                       e Zd ZdZy)RoutingArgs<   N)__name__
__module____qualname__ttl     J/var/www/Befach/backend/env/lib/python3.12/site-packages/litellm/router.pyrq   rq      s    
Crx   rq   c            U       h   e Zd ZU g Zeed<   dZee   ed<   dZ	e
ed<   dZdZee   ed<   dZee   ed<   dZeeeeeef         ed	<   dddddddi dddddddddddd
dg g g i ddddi ddddddi dd e       df)deeee   eeeef      f      dee   dee   dee   dee
   dee   dee   dedeee      de
dee   dee
   dee
   dee
   dee   dee   dee   dee
   deded    d!eee      d"ed#ed$ed%eeeeeef   f      d&ed'ed(e
d)eeeef      d*eeef   d+ee
   d,ee    d-ee   d.ee   d/ed0   d1ee!   d2ed3ee"   d4ee#   d5ee   d6ed7dfTd8Z$d9 Z%d: Z&e'd;eeef   d7ee(e)f   fd<       Z*d=e(fd>Z+d/ee,ef   d2efd?Z-d@ Z.dA Z/dBee   fdCZ0d1ee!   fdDZ1dEefdFZ2dGedHeeeef      d7ee3e4f   fdIZ5dGedHeeeef      d7ee3e4f   fdJZ6e7dGedHee8   dKedL   d7e4fdM       Z9e7	 ddGedHee8   dKed   d7e3fdN       Z9e7	 ddGedHee8   dKeedL   ed   f   d7ee4e3f   fdO       Z9	 ddGedHee8   dKefdPZ9dGedHeeeef      d7ee3e4f   fdQZ:	 ddGedRedSee   d7dfdTZ;	 ddRedSee   d7dfdUZ<dEedRed7e=fdVZ>	 ddEedRedWee   d7dfdXZ?dEedRefdYZ@dRedZed7eeee
f      fd[ZAdRedZed7eeee
f      fd\ZBdRedZed7eeee
f      fd]ZCd^ee   dHeeeeef      eeeeef         f   fd_ZDdGedHeee8      fd`ZEe7dGedHeeeef      dKedL   d7e4fda       ZFe7	 ddGedHeeeef      dKed   d7e3fdb       ZF	 ddGedHeeeef      dKefdcZFe7	 ddGedHee8   dde
dKed   d7e3f
de       ZGe7dGedHee8   dde
dKedL   d7e4f
df       ZG	 ddGedHee8   dde
fdgZGdGedde
dhedieHedjf   dReeef   f
dkZIdGed7efdlZJdGedHee8   dReeef   fdmZKdnedGefdoZLdnedGefdpZMdnedGefdqZNdnedGefdrZOdsePdGefdtZQdsePdGefduZRdGedvedwefdxZSdGefdyZTdGefdzZU	 	 	 ddGedned{ee   d|ee   d}ee   f
d~ZV	 	 	 ddGedned{ee   d|ee   d}ee   f
dZWdGednefdZX	 	 	 ddedGed{ee   d|ee   d}ee   f
dZYdedGefdZZdGedhefdZ[dGedefdZ\dGedhefdZ]	 ddGedveeef   d}ee   d7e^fdZ_dveeef   dGefdZ`	 ddGedveeef   d}ee   d7e^fdZadveeef   dGefdZbdGed7ecfdZddGed7ecfdZedGed7effdZgdGed7effdZh	 ddGee   d7effdZidGefdZj	 ddhedee   fdZk	 ddheded   fdZl	 ddhedee   fdZmdhefdZn	 	 ddhedee   deeo   fdZp eqj                         d        Zs	 	 	 	 ddRedee   d"ee   d#ee   d$ee   f
dZt eqj                         d        ZudhefdZv	 ddRedee   fdZw	 	 	 	 	 ddexdee   dee   d#ee   d$ee   dee   fdZyd Zz	 dd"eeeee   f      dee   d7eee      fdZ{	 	 ddexde
de
dee   dee   d7ee
ef   fdZ|d Z}d7ee   fdZ~d7efdZdee   fdZdRedexd7efdZdedee   d7e
fdZd7efdZdGede3dRed7efdZdGedee   fdZdGedee   d7eHee   ee   f   fdZdEefdZ	 ddEedee   dee   fdZ	 	 ddGedee   dHeee8      dee   dee   dee   fdZdedefdZdedededed7ee=   f
dĄZded7efdńZdEe=fdƄZdEe=d7efdǄZdefdȄZdEe=d7e=fdɄZdEe=dedGefdʄZdEe=d7ee=   fd˄ZdEe=d7ee=   fd̄Zded7ee=   fd΄Zded7ee=   fdЄZded7ee   fdфZded7ee=   fdӄZe7	 ddEededdd7efdՄ       Ze7dEddeded7efdք       Z	 ddEee   dedee   d7efdׄZded7ee   fd؄Zded7ee   fdلZdeded7ee   fdۄZdeded7ee   fd݄Zded7ee   fdބZded7eHee
   ee
   f   fd߄Z ee      ded7ee   fd       Zded7eee
f   fdZ	 ddedee   d7efdZ	 ddee   ded7ee   fdZdeded7ee   fdZ	 ddedGedee   d7efdZ	 	 ddedee   dee   d7ee   fdZddee   d7ee   fdZ	 ddEedee   d7ee   fdZdee   d7efdZ	 ddee   d7ee   fdZ	 ddee   dee   d7eee      fdZ	 ddee   dee   d7eeee   f   fdZded7efdZd Zd ZddZ	 ddGededHeeeef      dee   fdZdGed7ee   fdZdGed7efdZ	 	 	 ddGedHeeeeef         dveeeef      dee   d7eHeeeef   f   f
dZ	 	 	 	 ddGededHeeeeef         dveeeef      dee   dee   d7eee   ef   fdZ	 	 	 ddGededHeeeeef         dveeeef      dee   f
dZ	 	 	 ddGededHeeeeef         dveeeef      dee   d7ee   fdZ	 	 	 	 ddGedHeeeeef         dveeeef      dee   dee   f
d Zdee   dee   d7ee   fdZ	 ddee   fdZ	 ddexdee   fdZĐdexfdZŐd ZƐdefd	ZȐd
 Zɐd Zy(  Routermodel_namesFcache_responsesi  default_cache_time_secondsNleastbusy_loggerlowesttpm_loggeroptional_callbacksINFOr   simple-shuffle
model_listassistants_config	redis_url
redis_host
redis_portredis_passwordcache_kwargscaching_groups
client_ttlpolling_intervaldefault_prioritynum_retriesmax_fallbackstimeoutstream_timeoutdefault_litellm_paramsdefault_max_parallel_requestsset_verbosedebug_level)DEBUGr   default_fallbacks	fallbackscontext_window_fallbackscontent_policy_fallbacksmodel_group_aliasenable_pre_call_checksenable_tag_filteringretry_afterretry_policymodel_group_retry_policyallowed_failsallowed_fails_policycooldown_timedisable_cooldownsrouting_strategy)r   
least-busyusage-based-routinglatency-based-routingcost-based-routingusage-based-routing-v2optional_pre_call_checksrouting_strategy_argsprovider_budget_configalerting_configrouter_general_settingsignore_invalid_deploymentsreturnc*                 ~   ddl m}* || _        |)| _        || _        || _        || _        dt        _        | j                  du rQ|dk(  r$t        j                  t        j                         n(|dk(  r#t        j                  t        j                         |(xs
 t               | _        || _        g | _        i | _        d}+d},i }-|
| _        ||K|Id}+|||-d	<   |||-d
<   |t)        |      |-d<   |||-d<   |-j+                  |       | j-                  |-      },|r7t        j.                   t        j0                  d'd|+i|-t        _        || _        t5        |,t7                     | _        t9        ||,      | _        || _        d| _        || _         g | _!        tE               | _#        i | _$        |\tK        jL                  |      }| jO                  |       | jP                  | _)        |D ]  }.d|.d   v sd| j$                  |.d   d   <   ! ng | _(        ||| _*        nt        jT                  | _*        |!xs tV        | _,        t[        | j.                  | jX                        | _.        |"| _/        t7               | _0        ||| _1        n;t        jb                  t        jb                  | _1        ntd        jf                  | _1        ||| _4        n;t        jh                  t        jh                  | _4        nt        jj                  | _4        |xs t        jl                  | _7        || _8        || _9        |#| _:        |xs t        jv                  }/| jy                  |/       |/| _;        |t        jz                  H|xs t        jz                  }/| jv                  | jv                  j}                  d|/i       n
d|/ig| _;        |xs t        j~                  | _?        |xs t        j                  }0| jy                  |0       |0| _@        t        t              | _C        t        t              | _D        t        t              | _E        g | _F        |xs i | _G        |xs i }t        j                  ||       | _I        || _J        | j                  j                  d|       | j                  j                  dd       | j                  j                  di       j+                  d|	i       i | _L        	 | j                  |#|%       d| _N        t        t        j                  t              r*t        j                  j                  | j                         n)t        j                  j                  | j                         t        t        j                  t              r*t        j                  j                  | j                         n| j                  gt        _U        t        t        j                  t              r*t        j                  j                  | j                         n| j                  gt        _X        t        t        j                  t              r*t        j                  j                  | j                         n| j                  gt        _[        t        j                  d| jt                   d| j
                   d| jv                   d| j                   d| j~                   d | j.                  j                   d!        |*       | _`        |%| _a        |&| _b        d| _c        t        j                  || j                  "      r|$|$j}                  d#       nd#g}$d| _f        |vt        |t              rt        d'i || _f        nt        |t              r|| _f        t        j                  d$j                  | j                  j                  d%                   || _l        d| _m        | vt        | t              rt        d'i | | _m        nt        | t              r| | _m        t        j                  d&j                  | j                  j                  d%                   |'| _o        |$| j                  |$       | j                  | j                          | j                          | j                          | j                          y)(ah  
        Initialize the Router class with the given parameters for caching, reliability, and routing strategy.

        Args:
            model_list (Optional[list]): List of models to be used. Defaults to None.
            redis_url (Optional[str]): URL of the Redis server. Defaults to None.
            redis_host (Optional[str]): Hostname of the Redis server. Defaults to None.
            redis_port (Optional[int]): Port of the Redis server. Defaults to None.
            redis_password (Optional[str]): Password of the Redis server. Defaults to None.
            cache_responses (Optional[bool]): Flag to enable caching of responses. Defaults to False.
            cache_kwargs (dict): Additional kwargs to pass to RedisCache. Defaults to {}.
            caching_groups (Optional[List[tuple]]): List of model groups for caching across model groups. Defaults to None.
            client_ttl (int): Time-to-live for cached clients in seconds. Defaults to 3600.
            polling_interval: (Optional[float]): frequency of polling queue. Only for '.scheduler_acompletion()'. Default is 3ms.
            default_priority: (Optional[int]): the default priority for a request. Only for '.scheduler_acompletion()'. Default is None.
            num_retries (Optional[int]): Number of retries for failed requests. Defaults to 2.
            timeout (Optional[float]): Timeout for requests. Defaults to None.
            default_litellm_params (dict): Default parameters for Router.chat.completion.create. Defaults to {}.
            set_verbose (bool): Flag to set verbose mode. Defaults to False.
            debug_level (Literal["DEBUG", "INFO"]): Debug level for logging. Defaults to "INFO".
            fallbacks (List): List of fallback options. Defaults to [].
            context_window_fallbacks (List): List of context window fallback options. Defaults to [].
            enable_pre_call_checks (boolean): Filter out deployments which are outside context window limits for a given prompt
            model_group_alias (Optional[dict]): Alias for model groups. Defaults to {}.
            retry_after (int): Minimum time to wait before retrying a failed request. Defaults to 0.
            allowed_fails (Optional[int]): Number of allowed fails before adding to cooldown. Defaults to None.
            cooldown_time (float): Time to cooldown a deployment after failure in seconds. Defaults to 1.
            routing_strategy (Literal["simple-shuffle", "least-busy", "usage-based-routing", "latency-based-routing", "cost-based-routing"]): Routing strategy. Defaults to "simple-shuffle".
            routing_strategy_args (dict): Additional args for latency-based routing. Defaults to {}.
            alerting_config (AlertingConfig): Slack alerting configuration. Defaults to None.
            provider_budget_config (ProviderBudgetConfig): Provider budget configuration. Use this to set llm_provider budget limits. example $100/day to OpenAI, $100/day to Azure, etc. Defaults to None.
            ignore_invalid_deployments (bool): Ignores invalid deployments, and continues with other deployments. Default is to raise an error.
        Returns:
            Router: An instance of the litellm.Router class.

        Example Usage:
        ```python
        from litellm import Router
        model_list = [
        {
            "model_name": "azure-gpt-3.5-turbo", # model alias
            "litellm_params": { # params for litellm completion/embedding call
                "model": "azure/<your-deployment-name-1>",
                "api_key": <your-api-key>,
                "api_version": <your-api-version>,
                "api_base": <your-api-base>
            },
        },
        {
            "model_name": "azure-gpt-3.5-turbo", # model alias
            "litellm_params": { # params for litellm completion/embedding call
                "model": "azure/<your-deployment-name-2>",
                "api_key": <your-api-key>,
                "api_version": <your-api-version>,
                "api_base": <your-api-base>
            },
        },
        {
            "model_name": "openai-gpt-3.5-turbo", # model alias
            "litellm_params": { # params for litellm completion/embedding call
                "model": "gpt-3.5-turbo",
                "api_key": <your-api-key>,
            },
        ]

        router = Router(model_list=model_list, fallbacks=[{"azure-gpt-3.5-turbo": "openai-gpt-3.5-turbo"}])
        ```
        r   )ServiceLoggingTr   r   localNredisurlhostportpasswordtype)redis_cachein_memory_cache)r   r   modellitellm_params)cachedefault_cooldown_time)fallback_param*)params
router_objr   max_retriesmetadatar   r   r   z)Intialized router with Routing strategy: z"

Routing enable_pre_call_checks: z

Routing fallbacks: z

Routing content fallbacks: z$

Routing context window fallbacks: z

Router Redis Caching=
)r   r   router_budget_limitingz+[32mRouter Custom Retry Policy Set:
{}[0mexclude_nonez3[32mRouter Custom Allowed Fails Policy Set:
{}[0mrw   )ulitellm._service_loggerr   r   r   r   r   r   litellmsuppress_debug_infor   setLevelloggingr   r   rW   r   r   deployment_namesdeployment_latency_mapr   strupdate_create_redis_cacher   Cacher}   r   r   rB   	schedulerr   default_deploymentr   provider_default_deployment_idsrl   pattern_routerauto_routerscopydeepcopyset_model_listr   healthy_deploymentsr   r1   r   r0   cooldown_cacher   failed_callsr   openaiDEFAULT_MAX_RETRIESr   ROUTER_MAX_FALLBACKSrequest_timeoutr   r   r   r   r   validate_fallbacksr   appendr   r   r   inttotal_calls
fail_callssuccess_callsprevious_modelsr   Chatchatr   
setdefaultdeployment_statsrouting_strategy_initaccess_groups
isinstance_async_success_callbacklistlogging_callback_manager"add_litellm_async_success_callbackdeployment_callback_on_successsuccess_callbackadd_litellm_success_callback#sync_deployment_callback_on_success_async_failure_callback"add_litellm_async_failure_callback$async_deployment_callback_on_failurefailure_callbackadd_litellm_failure_callbackdeployment_callback_on_failuredebugr   service_logger_objr   r   router_budget_loggerr    !should_init_router_budget_limiterr   dictrU   infoformat
model_dumpr   r   rJ   r   add_optional_pre_call_checks_initialize_alertinginitialize_assistants_endpointinitialize_router_endpointsapply_default_settings)1selfr   r   r   r   r   r   r}   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   
cache_typer   cache_configm
_fallbacks_content_policy_fallbackss1                                                    ry   __init__zRouter.__init__   s1   h 	;&*D'&&<#$8!&*#t#f$%..w||<'%..w}}=#>'<'> 	$ "3 	 ')#  	 ')$ Z%;
@V J$&/U#%'1V$%'*:V$)+9Z( -22<@K}}$ ' N: N N#2D #]_


 #-;
 !1"&-J*:<,0257!z2J
+-1__D$a 011PQD//2B0CG0LM  
  O $!.D!(!6!6D*K.K+**D4F4F
 "3O 	 "*D  ,&22D%99D$!.D"".!(!6!6D!(!=!=D9'"9"9,& 0 3'"3"3
z:#(G,E,E,Q*Gg.G.GJ~~)%%sJ&78#&
"3!4 %H(H(H 	%
 %H(H(H 	" 	/HI(A%(3)
 (3(
 +6+
  	 # 	
 "8!=2LL(>4P	 '=###..y'B##..}a@##..z2>EE~.	
 ')	 	""-"7 	# 	
 "g55t<,,OO33 ,,OO33 g..5,,II88 )-(P(P'QG$g55t<,,OO99
 99/G+ g..5,,II33 )-(K(K'LG$##78M8M7N O//3/J/J.K L""&..!1 2**.*G*G)H I1151N1N0O P$$(JJ$:$:#;2?	
 #1"2%:"&<#DH!AA!$:U:U
 (3(//0HI,D+E(37#,-$/$?,$?!L+6$0!!&&DKK%%00d0C % 	% CG!+.5,>,VAU,V)02DE,@)!&&LSS--88d8K :I#/--.FG+%%'++-((*##%rx   c                 ,    dg}| j                  |       y)z;
        Apply the default settings to the router.
        %forward_client_headers_by_model_groupN)r  )r  default_pre_call_checkss     ry   r  zRouter.apply_default_settingsj  s#     4:
 	))*ABrx   c                 $   t         j                  j                  t         j                  |        t         j                  j                  t         j                  |        t         j                  j                  t         j
                  |        t         j                  j                  t         j                  |        t         j                  j                  t         j                  |        t         j                  j                  t         j                  |        t         j                  j                  t         j                  |        | j                  B| j                  D ]2  }t         j                  j                  t         j                  |d       4 yy)z
        Pseudo-destructor to be invoked to clean up global data structures when router is no longer used.
        For now, unhook router's callbacks from all lists
        NF)require_self)r   r   #remove_callback_from_list_by_objectr   r   r   r   input_callbackservice_callback	callbacksr   )r  callbacks     ry   discardzRouter.discardu  s3   
 	((LL++T	
 	((LL$$d	
 	((LL++T	
 	((LL$$d	
 	((LL""D	
 	((LL$$d	
 	((LLt	

 "". 3300TT%%xe U  4 /rx   r  c                 P    | j                  d      rt        di | S t        di | S )za
        Initializes either a RedisCache or RedisClusterCache based on the cache_config.
        startup_nodesrw   )getr   r   )r  s    ry   r   zRouter._create_redis_cache  s.     O,$4|44---rx   r   c                 T    | j                   j                  || j                   _        yy)z
        Update the redis cache for the router, if none set.

        Allows proxy user to just do
        ```yaml
        litellm_settings:
            cache: true
        ```
        and caching to just work.
        N)r   r   )r  r   s     ry   _update_redis_cachezRouter._update_redis_cache  s%     ::!!)%*DJJ" *rx   c                 .   t        j                  d|        |t        j                  j                  k(  s|t        j                  k(  rt        | j                  | j                        | _        t        t        j                  t              r*t        j                  j                  | j                         n| j                  gt        _        t        t        j                  t              r*t        j                  j!                  | j                         y y |t        j"                  j                  k(  s|t        j"                  k(  rpt%        | j                  | j                  |      | _        t        t        j                  t              r*t        j                  j!                  | j&                         y y |t        j(                  j                  k(  s|t        j(                  k(  rpt+        | j                  | j                  |      | _        t        t        j                  t              r*t        j                  j!                  | j,                         y y |t        j.                  j                  k(  s|t        j.                  k(  rpt1        | j                  | j                  |      | _        t        t        j                  t              r*t        j                  j!                  | j2                         y y |t        j4                  j                  k(  s|t        j4                  k(  rpt7        | j                  | j                  i       | _        t        t        j                  t              r*t        j                  j!                  | j8                         y y y )NzRouting strategy: )router_cacher   )r(  r   routing_args)r   r  r[   
LEAST_BUSYvaluer!   r   r   r   r   r   r  r   r   r  r   add_litellm_callbackUSAGE_BASED_ROUTINGr$   r   USAGE_BASED_ROUTING_V2r%   lowesttpm_logger_v2LATENCY_BASEDr#   lowestlatency_logger
COST_BASEDr"   lowestcost_logger)r  r   r   s      ry   r   zRouter.routing_strategy_init  s    	""%78H7I#JK : : @ @@?#=#==$;!ZZDOO%D! '00$7&&--d.C.CD*.*?*?)@&'++T200EEdF[F[\ 3  C C I II?#F#FF$;!ZZ??2%D!
 '++T200EEdF[F[\ 3  F F L LL?#I#II'A!ZZ??2(D$
 '++T200EEdF^F^_ 3  = = C CC?#@#@@(C!ZZ??2)D%
 '++T200EEdF_F_` 3  : : @ @@?#=#==%=!ZZ??&D"
 '++T200EEdF\F\] 3 rx   c                 D   | j                  t        j                        | _        | j                  t        j                        | _        | j                  t        j                        | _        | j                  t        j
                        | _        | j                  t        j                        | _        | j                  t        j                        | _        | j                  t        j                        | _        | j                  t        j                        | _	        y N)
factory_functionr   acreate_assistantsadelete_assistantaget_assistantsacreate_threadaget_threada_add_messageaget_messagesarun_threadr  s    ry   r  z%Router.initialize_assistants_endpoint  s    "&"7"78R8R"S!%!6!6w7P7P!Q#44W5L5LM"33G4J4JK001D1DE!2273H3HI!2273H3HI001D1DErx   c                    | j                  t        j                  d      | _        | j                  t        j                  d      | _        | j                  t        j
                  d      | _        | j                  t        j                  d      | _        | j                  t        j                  d      | _        | j                  t        j                  d      | _        | j                  t        j                  d      | _	        | j                  t        j                  d	      | _
        | j                  t        j                  d
      | _        | j                  t        j                  d      | _        | j                  t        j                  d      | _        | j                  t        j                  d      | _        | j                  t        j                  d      | _        | j                  t        j                   d      | _        | j                  t        j"                  d      | _        | j                  t        j$                  d      | _        | j                  t        j&                  d      | _        ddlm}m}m}m} | j                  |d      | _        | j                  |d      | _        | j                  |d      | _        | j                  |d      | _        ddlm}m}m }m!} | j                  |d      | _        | j                  |d      | _         | j                  |d      | _        | j                  |d      | _!        y )N
moderation)	call_typeanthropic_messages
aresponsesafile_deleteafile_content	responsesaget_responsesadelete_responsesalist_input_items
_arealtimeacreate_fine_tuning_jobacancel_fine_tuning_jobalist_fine_tuning_jobsaretrieve_fine_tuning_jobalist_filesaimage_editallm_passthrough_router   )acreateasearchcreatesearchavector_store_searchavector_store_createvector_store_searchvector_store_create)agenerate_contentagenerate_content_streamgenerate_contentgenerate_content_streamr[  r]  r\  r^  )"r6  r   amoderationrC  aanthropic_messagesrD  rE  rF  rG  rH  rI  rJ  rK  rL  rM  rN  rO  
afile_listrQ  rR  litellm.vector_stores.mainrS  rT  rU  rV  rW  rX  rY  rZ  litellm.google_genair[  r\  r]  r^  )	r  rS  rT  rU  rV  r[  r\  r]  r^  s	            ry   r  z"Router.initialize_router_endpoints  s   00< 1 
 $(#8#8&&2F $9 $
  //, 0 
 !11  N 2 
 "22!!_ 3 
 ..w/@/@K.X"33"".> 4 
 "&!6!6%%1D "7 "
 "&!6!6%%1D "7 "
 //, 0 
 (,'<'<++7P (= (
$ (,'<'<++7P (= (
$ '+&;&;**6N '< '
# *.)>)>--9T *? *
& //- 0 
  00= 1 
 '+&;&;**6N '< '
# 	PO %)$9$95 %: %
! %)$9$95 %: %
! $(#8#83 $9 $
  $(#8#83 $9 $
 	
 	
 "&!6!6)< "7 "
 !% 5 5(: !6 !
 )-(=(=$0J )> )
% (,'<'<#/H (= (
$rx   r   c           	          |y|D ]J  }t        |t              st        d| d      t        |      dk7  s1t        d| dt        |       d       y)z3
        Validate the fallbacks parameter.
        NzItem 'z' is not a dictionary.rk   zDictionary 'z%' must have exactly one key, but has z keys.)r   r  
ValueErrorlen)r  r   fallback_dicts      ry   r   zRouter.validate_fallbacks\  sl     !+MmT2 6-8N!OPP=!Q& "=/1VWZ[hWiVjjpq 	 ,rx   c                    ||D ]  }d }|dk(  rt        | j                        }nQ|dk(  r-t        | j                  | j                  | j                        }n|dk(  rt               }n|dk(  r
t               }|u| j                  g | _        | j                  j                  |       t        j                  j                  |        y y )Nprompt_caching)r   r   )
dual_cacher   r   responses_api_deployment_checkr  )r=   r   r    r   r   r>   r9   r   r   r   r   r,  )r  r   pre_call_check	_callbacks       ry   r  z#Router.add_optional_pre_call_checksj  s     $/":48	!%55 <4:: NI#'?? 4#'::/3/J/J#'??!I
 $'GG ; =I#'NN D FI(..624/++229=44II)T% #; 0rx   
deploymentc                     	 t        j                  |      }|d   }d|v r|d   dd dz   |d<   |S # t        $ r(}t        j                  dt        |              |d}~ww xY w)z
        returns a copy of the deployment with the api key masked

        Only returns 2 characters of the api key and masks the rest with * (10 *).
        r   api_keyN   z
**********z+Error occurred while printing deployment - )r   r   	Exceptionr   r  r   )r  rn  _deployment_copyr   es        ry   print_deploymentzRouter.print_deployment  s}    
	#}}Z8#34D#ENN*,:9,Ebq,IH,Ty)## 	!''=c!fXF G		s   -0 	A!#AA!r   messagesc                     	 t        j                  d| d       ||d<   ||d<   | j                  |d<   | j                  ||        | j                  di |}|S # t
        $ r}|d}~ww xY w)	z
        Example usage:
        response = router.completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hey, how's it going?"}]
        zrouter.completion(model=z,..)r   rv  original_functionr   kwargsNrw   )r   r  _completion_update_kwargs_before_fallbacksfunction_with_fallbacksrr  )r  r   rv  rz  responsert  s         ry   
completionzRouter.completion  s    
	!''*B5'(NO#F7O!)F:*.*:*:F&'00uV0L3t33=f=HO 	G	s   AA 	A+$A&&A+c           	         d }	 | j                  |||j                  dd       |      }| j                  ||       |d   j                         }|d   }| j	                  ||      }|j                  dd       }||||j                  k7  rd }	n|}	|| j                         vr| j                  |       t        j                  di i ||| j                  |	d|}
t        j                  d	| d
       t        |
t              r.| j!                  ||
|      }|rt        j"                  d|d      |
S # t$        $ r,}t        j                  d	| dt'        |       d       |d }~ww xY w)Nspecific_deploymentr   rv  r  request_kwargsrn  rz  r   r   rp  rn  rv  cachingclientzlitellm.completion(model=)[32m 200 OK[0mr   r~  rz  Response output was blocked. messager   llm_provider)[31m Exception [0mrw   )get_available_deploymentpop_update_kwargs_with_deploymentr   _get_clientr$  rp  get_model_ids routing_strategy_pre_call_checksr   r  r}   r   r  r   rc   "_should_raise_content_policy_errorContentPolicyViolationErrorrr  r   )r  r   rv  rz  
model_namern  datapotential_model_clientdynamic_api_keymodel_clientr~  _should_raisert  s                ry   r{  zRouter._completion  s    
=	66!$*JJ/Dd$K%	 7 J //:f/U./446DgJ%)%5%5%f &6 &" %jjD9O+*6#'='E'EE#5 D..00555L))  (#33*	
 H "&&+J<7NO
 (M2 $ G G(6 !H ! !!== >#%'  O 	!&&+J<7KCPQF8SZ[ G		s   D2D7 7	E, 'E''E,streamTc                    K   y wr5  rw   r  r   rv  r  rz  s        ry   acompletionzRouter.acompletion        	   c                    K   y wr5  rw   r  s        ry   r  zRouter.acompletion  r  r  c                    K   y wr5  rw   r  s        ry   r  zRouter.acompletion  r  r  c                 &  K   	 ||d<   ||d<   ||d<   | j                   |d<   | j                  ||       |j                  d      xs | j                  }t	        j                         }| j                  |      }|r| j                  |||       d {   S |+t        |t              r | j                  di | d {   }n | j                  di | d {   }t	        j                         }	|	|z
  }
t        j                  | j                  j                  t        j                   |
d||	t#        |      	             |S 7 7 7 n# t$        $ r;}t        j                  t'        | |t)        j*                         |
             |d }~ww xY ww)Nr   rv  r  rx  ry  priority)r   rv  rz  r  servicedurationrB  
start_timeend_timeparent_otel_spanlitellm_router_instancer  error_traceback_stroriginal_exceptionrw   )_acompletionr|  r$  r   time_is_prompt_management_model_prompt_management_factoryr   r   schedule_acompletionasync_function_with_fallbacksasynciocreate_taskr  async_service_success_hookr\   ROUTERr   rr  r<   	traceback
format_exc)r  r   rv  r  rz  request_priorityr  r  r~  r  	_durationrt  s               ry   r  zRouter.acompletion  s    ,	#F7O!)F:%F8*.*;*;F&'00uV0L%zz*5N9N9NJ*.*J*J5*Q'*!<<%! =   
  +
;KS0Q!:!:!:!DV!DD!C!C!C!Mf!MMyy{H :-I''BB(//&+)%%Fv%N C 	 O- EM  		(,0#)(1(<(<(>'(	 G		sl   FBE
 EE
 F'E
 >E?E
 EA)E
 FE
 E
 E
 
	F6F		FFc                   K   d}i }	 t        j                  d| d|        t        |      }t        j                         }| j	                  |||j                  dd      |       d{   }|}t        j                         }	|	|z
  }
t        j                  | j                  j                  t        j                  |
d||	t        |                   | j                  ||       | j                  ||	       |d
   j                         }|d   }| j                  ||	      }| j                   |xx   dz  cc<   t#        j$                  di i ||| j&                  |d|}|j)                  dd      }| j+                  ||d      }|]t-        |t        j.                        rC|4 d{    	 | j1                  |||       d{    | d{   }ddd      d{    n&| j1                  |||       d{    | d{   }t-        t2              r.| j5                  |||      }|rt#        j6                  d|d      | j8                  |xx   dz  cc<   t        j:                  d| d       | j                  |||       |S 7 +7 7 7 7 # 1 d{  7  sw Y   xY w7 7 # t"        j<                  $ rf}|j)                  d
i       j)                  dd      }|j)                  d
i       j)                  dd      }|xj>                  d| d| z  c_        |d}~wt@        $ rE}t        j:                  d| dtC        |       d       || jD                  |xx   dz  cc<   |d}~ww xY ww) z
        - Get an available deployment
        - call it with a semaphore over the call
        - semaphore specific to it's rpm
        - in the semaphore,  make a check against it's local rpm before running
        NzInside _acompletion()- model: 
; kwargs: r  r  async_get_available_deploymentr  rn  r  r  r   r   rk   r  litellm_logging_objmax_parallel_requestsrn  rz  client_type)rn  logging_objr  r  r  r  r  zlitellm.acompletion(model=r  )rn  r~  r  r   r   z$

Deployment Info: request_timeout: z

timeout: r  r  rw   )#r   r  r   r  r  r  r  r  r  r  r\   r  _track_deployment_metricsr  r   _get_async_openai_model_clientr   r   r  r}   r$  r  r   	Semaphore&async_routing_strategy_pre_call_checksrc   r  r  r   r  Timeoutr  rr  r   r   )r  r   rv  rz  r  _timeout_debug_deployment_dictr  r  rn  r  r  r  r  	_responser  rpm_semaphorer~  r  rt   deployment_request_timeout_paramdeployment_timeout_params                        ry   r  zRouter._acompletion9  sB     
 	'|	!''0z&J  AHJ#BB!$*JJ/Dd$K%	  C   J .8*yy{H :-I''BB(//&>)%%Fv%N C 	 **%8H +  //:f/U./446DgJ>>% ? L Z(A-(++  (#33*	
 I 5;JJ%t5K !,,%3 - M
 (Zw00. ) 
/ 
/ EE#-$/)9 F   
 &/H
/ 
/ 
/ AA) +%5 B    "+? (M2 $ G G(6 !H ! !!== >#%'  z*a/*!&&,ZL8OP **%!!1 +  OMv
/

  /
/ 
/ 
/ 
/ +4  	/M/Q/Q "0c#T* - (F'I'I "(c)T" % IIABbAccn  pH  oI  J  JIG 	!&&,ZL8LSQRVHT[\ %
+q0+G	s   NA"J9 *J+EJ9 .J/J9 2J J	J JJ J9 %J&J9 J5	J9 J7BJ9 NJ9 J9 J J J9  J2&J)'J2.J9 7J9 9M>A!L--M>9A M99M>>Nrz  metadata_variable_namec                    |j                  d| j                        |d<   |j                  dt        t	        j
                                      d}| j                  |      r|}|j                  |i       j                  ||d       y)zm
        Adds/updates to kwargs:
        - num_retries
        - litellm_trace_id
        - metadata
        r   litellm_trace_idNr   )model_groupr   )r$  r   r   r   uuiduuid4_get_model_from_aliasr   )r  r   rz  r  r   s        ry   r|  z&Router._update_kwargs_before_fallbacks  s}     !'

=$:J:J K},c$**,.?@+/%%E%2 %0"5<<!8IJ	
rx   c                     | j                   j                         }|j                  di       xs i }|j                         D ]  \  }}|	|j	                  ||        |j	                  |i       j                  |       y)z
        Adds default litellm params to kwargs, if set.

        Handles inserting this as either "metadata" or "litellm_metadata" depending on the metadata_variable_name
        r   N)r   r   r  itemsr   r   )r  rz  r  defaultsmetadata_defaultskeyr+  s          ry   *_update_kwargs_with_default_litellm_paramsz1Router._update_kwargs_with_default_litellm_params  s|     ..335$LLR8>B #..*JC}c5) + 	0"5<<=NOrx   c                    |j                  di       j                         }|d   j                         }t        ||      }|j                  di       }t        t        |j                  d            }| j                  ||      }|j                  d      }	||d<   |	|d<   t        |t        di ||	      }
| j                  |

       |
S )z.
        Handle clientside credential
        
model_infor   )r   r  r   r  )r  r   idoriginal_model_idr  r   r  r  rw   )	r$  r   r.   r   r   _generate_model_idrO   rQ   upsert_deployment)r  rn  rz  r  r   dynamic_litellm_paramsr   r  	_model_idr  deployment_pydantic_objs              ry   _handle_clientside_credentialz$Router._handle_clientside_credential  s      ^^L"5::<
#$45::<!;)&"
 ::j"-3] ;<++#4J , 
	 'NN40$
4*;
&'",")C,BC!#

 	. 	 	
 '&rx   function_namec                    |j                  di       j                         }|d   d   }|d   j                  d      }|d   }t        |      rY| j                  ||      }|j                  j                         }|j                  j                  }|j                  j                  }t        |      }	|j                  |	i       j                  ||||d	       ||d<   | j                  ||d   
      |d<   | j                  ||	       y)z
        2 jobs:
        - Adds selected deployment, model_info and api_base to kwargs["metadata"] (used for logging)
        - Adds default litellm params to kwargs, if set.
        r  r   r   api_baser  )r  r  r  )rn  r  r  deployment_model_namerz  r  r   )rz  r  N)r$  r   r/   r  r  r
  r   r   r  r*   r   r   _get_timeoutr  )
r  rn  rz  r  r  deployment_litellm_model_namedeployment_api_baser  r  r  s
             ry   r  z%Router._update_kwargs_with_deployment  s7     ^^L"5::<
(23C(DW(M%()9:>>zJ *< 8#6:&*&H&H%f 'I '# 1;;FFHJ,C,R,R,X,X)"9"H"H"Q"Q!C'"
 	0"5<<;(/)>		
  *| --
+; < . 
y 	772H 	8 	
rx   c                     | j                  ||d      }|j                  dd      }||||j                  k7  rd}|S |}|S )a  
        Helper to get AsyncOpenAI or AsyncAzureOpenAI client that was created for the deployment

        The same OpenAI client is re-used to optimize latency / performance in production

        If dynamic api key is provided:
            Do not re-use the client. Pass model_client=None. The OpenAI/ AzureOpenAI client will be recreated in the handler for the llm provider
        asyncr  rp  N)r  r$  rp  )r  rn  rz  r  r  r  s         ry   r  z%Router._get_async_openai_model_client8  sh     "&!1!1!&g "2 "

 !**Y5'&2#9#A#AAL  2Lrx   r  c                     |j                  dd      xs> |j                  dd      xs* | j                  xs | j                  j                  dd      S )z=Helper to get stream timeout from kwargs or deployment paramsr   N)r$  r   r   )r  rz  r  s      ry   _get_stream_timeoutzRouter._get_stream_timeoutR  s_    
 JJ'. Gxx $G ""	G
 **../?F	
rx   c                     |j                  dd      xsf |j                  dd      xsR |j                  dd      xs> |j                  dd      xs* | j                  xs | j                  j                  dd      }|S )zAHelper to get non-stream timeout from kwargs or deployment paramsr   Nr   )r$  r   r   r  rz  r  r   s       ry   _get_non_stream_timeoutzRouter._get_non_stream_timeout_  s    
 JJy$' 	@zz+T2	@xx4	@
 xx!4	@ ||	@ **..y$? 	 rx   c                 ~    d}|j                  dd      r| j                  ||      }|| j                  ||      }|S )z6Helper to get timeout from kwargs or deployment paramsNr  Fr  )r$  r  r  r  s       ry   r  zRouter._get_timeoutq  sP    /3::h&..f4.HG?22D 3 G rx   modelsc                    K   dt         dt        t           f fd}dt         dt        t           dt        f fd}t	        |t
              rQt        d |D              r?g }|D ]  }|j                   |d||d|        t        j                  |  d{   }|S t	        |t
              rt        d	 |D              rg }t        |      D ]'  \  }	}
|D ]  }|j                   |d||	|
d
|        ) t        j                  |  d{   }t        t        |            D cg c]  }g  }}|D ]A  }t	        |t              r||d      j                  |d          .|d   j                  |       C |S yy7 7 qc c}w w)a6  
        Async Batch Completion. Used for 2 scenarios:
        1. Batch Process 1 request to N models on litellm.Router. Pass messages as List[Dict[str, str]] to use this
        2. Batch Process N requests to M models on litellm.Router. Pass messages as List[List[Dict[str, str]]] to use this

        Example Request for 1 request to N models:
        ```
            response = await router.abatch_completion(
                models=["gpt-3.5-turbo", "groq-llama"],
                messages=[
                    {"role": "user", "content": "is litellm becoming a better product ?"}
                ],
                max_tokens=15,
            )
        ```


        Example Request for N requests to M models:
        ```
            response = await router.abatch_completion(
                models=["gpt-3.5-turbo", "groq-llama"],
                messages=[
                    [{"role": "user", "content": "is litellm becoming a better product ?"}],
                    [{"role": "user", "content": "who is this"}],
                ],
            )
        ```
        r   rv  c                 |   K   	  j                   d| |d| d{   S 7 # t        $ r}|cY d}~S d}~ww xY wwzs
            Wrapper around self.async_completion that catches exceptions and returns them as a result
            r   rv  Nrw   r  rr  r   rv  rz  rt  r  s       ry   _async_completion_no_exceptionszARouter.abatch_completion.<locals>._async_completion_no_exceptions  D     -T--WEHWPVWWWW 0   <$ "$ <$ 	949<9<idxc                    K   	  j                   d| |d| d{   |fS 7 # t        $ r}||fcY d}~S d}~ww xY wwr  r  )r   rv  r	  rz  rt  r  s        ry   *_async_completion_no_exceptions_return_idxzLRouter.abatch_completion.<locals>._async_completion_no_exceptions_return_idx  sS     *$**TTVTT T  #vs4   A & $& A & 	=8=A =A c              3   <   K   | ]  }t        |t                y wr5  )r   r  .0r  s     ry   	<genexpr>z+Router.abatch_completion.<locals>.<genexpr>  s     -TajD.A-T   r  Nc              3   <   K   | ]  }t        |t                y wr5  )r   r   r  s     ry   r  z+Router.abatch_completion.<locals>.<genexpr>  s     /V
1d0C/Vr  )r   r	  rv  rk   r   rw   )r   r
   rC   r   r   r   allr   r  gather	enumeraterangerf  tuple)r  r  rv  rz  r  r  _tasksr   r~  r	  r  rG  _final_responsess   `             ry   abatch_completionzRouter.abatch_completion|  s    H				"&'7"8				+,	 	& h%#-T8-T*TF=gET\g`fgh   %^^V44HO$'C/VX/V,VF )( 3W#EMMB "'S7FL $ !4 &nnf55I<A#h-<P/Qq/QO/Q%h.#HQK077D#A&--h7	 &
 #"# -W' 5 6/Qs8   BFE;A4FE=F&	E?/AF=F?Fc           	          K   dt         dt        t           f fd}g }|D ]  }|j                   |d||d|        t	        j
                  |  d{   }|S 7 w)a  
        Async Batch Completion - Batch Process multiple Messages to one model_group on litellm.Router

        Use this for sending multiple requests to 1 model

        Args:
            model (List[str]): model group
            messages (List[List[Dict[str, str]]]): list of messages. Each element in the list is one request
            **kwargs: additional kwargs
        Usage:
            response = await self.abatch_completion_one_model_multiple_requests(
                model="gpt-3.5-turbo",
                messages=[
                    [{"role": "user", "content": "hello"}, {"role": "user", "content": "tell me something funny"}],
                    [{"role": "user", "content": "hello good mornign"}],
                ]
            )
        r   rv  c                 |   K   	  j                   d| |d| d{   S 7 # t        $ r}|cY d}~S d}~ww xY wwr  r  r  s       ry   r  z]Router.abatch_completion_one_model_multiple_requests.<locals>._async_completion_no_exceptions  r  r  r  Nrw   )r   r
   rC   r   r  r  )r  r   rv  rz  r  r  message_requestr~  s   `       ry   -abatch_completion_one_model_multiple_requestsz4Router.abatch_completion_one_model_multiple_requests  sv     ,				"&'7"8		 'OMM/ /=C  ( !00 1s   AAAAc                    K   y wr5  rw   r  s        ry   "abatch_completion_fastest_responsez)Router.abatch_completion_fastest_response  r  r  c                    K   y wr5  rw   r  s        ry   r   z)Router.abatch_completion_fastest_response  r  r  c                    K   |j                  d      D cg c]  }|j                          }}dt        dt        t        t        t        f      dt
        dt        dt        t        t        t        f   f
 fd}g dt        j                  ffd	}|D ]2  }t        j                   |d|||d
|      }	j                  |	       4 r_t        j                  t        j                          d{   \  }
|
D ]'  } ||       d{   }|d|j"                  d<   |c S  r_t        d      c c}w 7 F7 0w)z
        model - List of comma-separated model names. E.g. model="gpt-4, gpt-3.5-turbo"

        Returns fastest response from list of model names. OpenAI-compatible endpoint.
        ,r   rv  r  rz  r   c                    K   	  j                   d| ||d| d{   S 7 # t        j                  $ r& t        j                  dj                  |               t        $ r}|cY d}~S d}~ww xY ww)zn
            Wrapper around self.acompletion that catches exceptions and returns them as a result
            r   rv  r  Nz4Received 'task.cancel'. Cancelling call w/ model={}.rw   )r  r  CancelledErrorr   r  r	  rr  )r   rv  r  rz  rt  r  s        ry   r  zRRouter.abatch_completion_fastest_response.<locals>._async_completion_no_exceptions%  sw     -T--fEHU[f_effff)) %++JQQRWX  s>   A4% #% A4% A A1%A,&A1'A4,A11A4taskc                   K   	 |  d {   }t        |t        t        f      r@t        j                  d       D ]  }|j                           |	 j                  |        S 	 	 j                  |        y 7 n# t        $ r Y S w xY w# t        $ r Y /w xY w# t        $ r Y y w xY w# 	 j                  |        w # t        $ r Y w w xY wxY ww)Nz=Received successful response. Cancelling other LLM API calls.)	r   rc   ra   r   r  cancelremoveKeyErrorrr  )r'  resulttpending_taskss      ry   check_responsezARouter.abatch_completion_fastest_response.<locals>.check_response7  s     #f}6I&JK)//W +
 +!!((. L!((. $         !((. s   CB
 A9AB
 A;$C%B( 'B 8C9B
 ;	BCBC
	BB( BB( 	B%"C$B%%C(C*B<;C<	CCCCCr%  )return_whenNT!fastest_response_batch_completionzAll tasks failedrw   )splitstripr   r
   r	   boolr   r   rc   ra   rr  r  Taskr  r   waitFIRST_COMPLETED_hidden_params)r  r   rv  r  rz  r  r  r  r/  r'  donecompleted_taskr,  r.  s   `            @ry   r   z)Router.abatch_completion_fastest_response  sS     &+[[%56!'')66		"&tCH~"6	@D	PS	="5y@A	  	w|| 	, E&&/ (6EKD
   &  (/7+B+B) #D- #'-n==%QUF))*MN!M #'	  *++ 7h# >s:   D?D6CD?4D;5D?D=D?D?+D?=D?r  c                    K   y wr5  rw   r  r   rv  r  r  rz  s         ry   r  zRouter.schedule_acompletionh  r  r  c                    K   y wr5  rw   r<  s         ry   r  zRouter.schedule_acompletionn  r  r  c                   K   t        |      }t        t        j                               }t	        ||d      }| j
                  j                  |       d {    t        j                         | j                  z   }	t        j                         }
| j
                  j                  }d}|
|	k  r| j                  ||       d {   \  }}| j
                  j                  |j                  |j                  |       d {   }|rn7t        j                  |       d {    t        j                         }
|
|	k  r|r]	  | j                   d|||d| d {   }|j"                  j%                  di        |j"                  d   j'                  d	d
i       |S t-        j.                  d|d      7 `7 7 7 7 c# t(        $ r}t+        |d|       |d }~ww xY ww)Nzgpt-3.5-turbor  
request_idr  requestFr   r  r  r  health_deploymentsr%  additional_headers%x-litellm-request-prioritization-usedTr  %Request timed out while polling queuer   r  rw   )r   r   r  r  rA   r   add_requestr  r   r   _async_get_healthy_deploymentspollr@  r  r  sleepr  r8  r   r   rr  setattrr   r  )r  r   rv  r  r  rz  r  _request_iditemr  	curr_timepoll_intervalmake_request_healthy_deploymentsr  r  rt  s                    ry   r  zRouter.schedule_acompletionv  s     =VD$**,'"&
 nn(((666 99;-IIK	77(",0,O,O.> -P - '# ! "&!4!4????#7 "5 " L
 mmM222 IIK	 (" "2$"2"2 #(6#EK# 	 ((334H"M(()=>EE<dC ! 
 //?% K 	7' 3
  :x0s   AG#F8A,G#F;<G#F=G#"F?#G#G#G GAG G#;G#=G#?G#G 	G GG  G#rx  args.c                   K   t        |      }t        t        j                               }t	        |||      }| j
                  j                  |       d {    t        j                         | j                  z   }	t        j                         }
| j
                  j                  }d}|
|	k  r| j                  ||       d {   \  }}| j
                  j                  |j                  |j                  |       d {   }|rn7t        j                  |       d {    t        j                         }
|
|	k  r|ri	  ||i | d {   }t!        |j"                  t$              r<|j"                  j'                  di        |j"                  d   j)                  ddi       |S t/        j0                  d
|d      7 l7 7 7 7 ~# t*        $ r}t-        |d	|       |d }~ww xY ww)Nr?  rA  FrC  rD  rF  rG  Tr  rH  r   r  )r   r   r  r  rA   r   rI  r  r   r   rJ  rK  r@  r  r  rL  r   r8  r  r   r   rr  rM  r   r  )r  r   r  rx  rT  rz  r  rN  rO  r  rP  rQ  rR  rS  r  r  rt  s                    ry   _schedule_factoryzRouter._schedule_factory  s     =VD$**,'"
 nn(((666 99;-IIK	77(",0,O,O.> -P - '# ! "&!4!4????#7 "5 " L
 mmM222 IIK	 (" 
"3T"DV"DD	i66=,,778LbQ,,-ABII@$G ! 
 //?% I 	7' 3
 E  :x0s   AG0GA,G0G<G0G
G0"G#G0G0G GAG +G0G0
G0G0G 	G-G((G--G0c                     | j                  |      }|yt        |      dk7  ry|d   d   j                  dd       }|yd|v r'|j                  d      d   }|t        j
                  v ryy)	Nr  Frk   r   r   r   /T)get_model_listrf  r$  r2  r   )_known_custom_logger_compatible_callbacks)r  r   r   litellm_modelsplit_litellm_models        ry   r  z"Router._is_prompt_management_model  s    ((E(:
z?a"1&67;;GTJ -"/"5"5c":1"="g&W&WWrx   c                 n  K   |j                  dd       }|$t        di dt               t               d|\  }}t	        t
        |      }| j                  |dddg|j                  dd             }| j                  ||	       |d
   j                         }|j                  dd       }|j                  d      xs |d
   j                  dd       }|j                  d      xs |d
   j                  dd       }	|j                  dd       xs |d
   j                  dd       }
|t        |t              st        d| dt        |             |	*t        |	t              st        d|	 dt        |	             |j                  ||t!        |      ||	|
      \  }}}i |||}||d<   ||d<   ||d<   ||d<   |	|d<   |
|d<   | j#                  |      }|t%        |      dk(  r.|j                  d       t'        j(                  di | d {   S  | j*                  di | d {   S 7 7 w)Nr  r  )rx  	rules_objr  userpromptrolecontentr  r   rv  r  r  r   r   	prompt_idprompt_variablesprompt_labelz*Prompt ID is not set or not a string. Got=z, type=z2Prompt variables is set but not a dictionary. Got=)rz  )r   rv  non_default_paramsrf  rg  rh  rv  rX  r   rx  rw   )r$  re   rd   ri   r   LiteLLMLoggingr  r  r  r   r   r   re  r   r  get_chat_completion_promptrg   rZ  rf  r   r  r  )r  r   rv  rz  litellm_logging_objectprompt_management_deploymentr  r\  rf  rg  rh  optional_params_model_lists                ry   r  z!Router._prompt_management_factory  s     "(,A4!H!)-; .)6!&"2"4 	.*"F "&n6L!M'+'D'D%(;< &

+@$ G (E (
$ 	++3F 	, 	
 ,,<=BBD$/JJ{+ !/K0

#k4
  	 "::
 
)*:;??
 	
 zz.$7 $;W<

#nd
# 	 Jy#$><YKwtT]N_`  '
;KT0RDEUDVV]^bcs^t]uv  #==@O-% > 
		
 7D6F6o6w%z(>$%'{%5!"!-~))U);#k"2a"7JJ*+ ,,6v6667T77A&AAA 7As$   HH5H1H5,H3-H53H5ra  c                    	 ||d<   ||d<   | j                   |d<   |j                  d| j                        |d<   |j                  di       j	                  d|i        | j
                  di |}|S # t        $ r}|d }~ww xY w)Nr   ra  rx  r   r   r  rw   )_image_generationr$  r   r   r   r}  rr  r  ra  r   rz  r~  rt  s         ry   image_generationzRouter.image_generationR  s    
	#F7O%F8*.*@*@F&'$*JJ}d>N>N$OF=!j"-44mU5KL3t33=f=HO 	G	   A.A1 1	B:A<<Bc           	         d}	 t        j                  d| d|        | j                  |dddg|j                  dd             }| j	                  ||	       |d
   j                         }| j                  ||	      }| j                  |xx   dz  cc<   | j                  |       t        j                  di i ||| j                  |d|}| j                  |xx   dz  cc<   t        j                  d| d       |S # t        $ rE}	t        j                  d| dt        |	       d       || j                   |xx   dz  cc<   |	d }	~	ww xY w)Nr  #Inside _image_generation()- model: r  r`  ra  rb  r  re  r  r   rk   r  ra  r  r  zlitellm.image_generation(model=r  r  r  rw   )r   r  r  r  r  r   r  r   r  r   rs  r}   r   r  rr  r   r   )
r  ra  r   rz  r  rn  r  r  r~  rt  s
             ry   rq  zRouter._image_generation_  s   
*	!''5eWJvhO 66#)h?@$*JJ/Dd$K 7 J
 //:f/U./446D>>% ? L
 Z(A-( 11Z1H// $#33*	
 H z*a/*!&&1*=TU O 	!&&1*=QRUVWRXQYY`a %
+q0+G	s   C?D 	EA EEc           	      l  K   	 ||d<   ||d<   | j                   |d<   |j                  d| j                        |d<   | j                  ||        | j                  di | d {   }|S 7 # t
        $ r;}t        j                  t        | |t        j                         |             |d }~ww xY ww)Nr   ra  rx  r   ry  r  rw   )_aimage_generationr$  r   r|  r  rr  r  r  r<   r  r  rr  s         ry   aimage_generationzRouter.aimage_generation  s     	#F7O%F8*.*A*AF&'$*JJ}d>N>N$OF=!00uV0L?T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		s;   B4A A- $A+%A- *B4+A- -	B166B,,B11B4c           	      6  K   |}	 t        j                  d| d|        t        |      }| j                  |dddg|j	                  dd       |       d {   }| j                  ||       |d	   j                         }|d
   }| j                  ||      }| j                  |xx   dz  cc<   t        j                  di i ||| j                  |d|}	| j                  ||d      }
|
\t        |
t        j                        rB|
4 d {    	 | j!                  ||       d {    |	 d {   }	d d d       d {    n%| j!                  ||       d {    |	 d {   }	| j"                  |xx   dz  cc<   t        j$                  d| d       |	S 7 L7 7 {7 s7 e# 1 d {  7  sw Y   OxY w7 `7 X# t&        $ rE}t        j$                  d| dt)        |       d       || j*                  |xx   dz  cc<   |d }~ww xY ww)Nrv  r  r`  ra  rb  r  r  r  r   r   rk   rw  r  r  r  z litellm.aimage_generation(model=r  r  r  rw   )r   r  r   r  r  r  r   r  r   r   rz  r}   r  r   r  r  r  r   r  rr  r   r   )r  ra  r   rz  r  r  rn  r  r  r~  r  rt  s               ry   ry  zRouter._aimage_generation  s    
D	!''5eWJvhO  AH#BB#)h?@$*JJ/Dd$K%	  C   J //:f/U./446DgJ>>% ? L
 Z(A-(00 $#33*	
 H !,,%3 - M (Zw00. ) . . EE#-@P F    &.~H. . . AA)<L B    "*>z*a/*!&&2:,>UV OqJ.
  .. . . . *  	!&&2:,>RSVWXSYRZZab %
+q0+G	s   HAG F$B8G F'G F/-F).	F/7F+8F/<G F-G #G$	G -G.5G #H$G 'G )F/+F/-G /G5F86G=G G 	HA HHHfilec           	      .  K   	 ||d<   ||d<   | j                   |d<   | j                  ||        | j                  di | d{   }|S 7 # t        $ r;}t	        j
                  t        | |t        j                         |             |d}~ww xY ww)a  
        Example Usage:

        ```
        from litellm import Router
        client = Router(model_list = [
            {
                "model_name": "whisper",
                "litellm_params": {
                    "model": "whisper-1",
                },
            },
        ])

        audio_file = open("speech.mp3", "rb")
        transcript = await client.atranscription(
        model="whisper",
        file=audio_file
        )

        ```
        r   r|  rx  ry  Nr  rw   )	_atranscriptionr|  r  rr  r  r  r<   r  r  )r  r|  r   rz  r~  rt  s         ry   atranscriptionzRouter.atranscription  s     .	#F7O!F6N*.*>*>F&'00uV0L?T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		;   BAA AA BA 	B6BBBc           	      ,  K   |}	 t        j                  d| d|        t        |      }| j                  |dddg|j	                  dd       |       d {   }| j                  ||       |d	   j                         }| j                  ||      }| j                  |xx   d
z  cc<   t        j                  di i ||| j                  |d|}	| j                  ||d      }
|
\t        |
t        j                        rB|
4 d {    	 | j!                  ||       d {    |	 d {   }	d d d       d {    n%| j!                  ||       d {    |	 d {   }	| j"                  |xx   d
z  cc<   t        j$                  d| d       |	S 7 G7 7 {7 s7 e# 1 d {  7  sw Y   OxY w7 `7 X# t&        $ rE}t        j$                  d| dt)        |       d       || j*                  |xx   d
z  cc<   |d }~ww xY ww)Nz!Inside _atranscription()- model: r  r`  ra  rb  r  r  r  r   rk   )r|  r  r  r  r  r  zlitellm.atranscription(model=r  r  r  rw   )r   r  r   r  r  r  r   r  r   r   r  r}   r  r   r  r  r  r   r  rr  r   r   )r  r|  r   rz  r  r  rn  r  r  r~  r  rt  s               ry   r~  zRouter._atranscription  s    
B	!''3E7*VHM  AH#BB#)h?@$*JJ/Dd$K%	  C   J //:f/U./446D>>% ? L
 Z(A-(--  #33*	
 H !,,%3 - M (Zw00. ) . . EE#-@P F    &.~H. . . AA)<L B    "*>z*a/*!&&/
|;RS OmF.
  .. . . . *  	!&&/
|;OPSTUPVxW^_ %
+q0+G	s   HAG FB3G F"G F*(F$)	F*2F&3F*7G F(G F?	G (G)5G HG "G $F*&F*(G *F<0F31F<8G G 	HA HHHinputvoicec           	        K   	 ||d<   ||d<   | j                  |dddg|j                  dd      |       d{   }| j                  ||	       |d
   j                         }|d    | j                  j                         D ])  \  }}||vr|||<   |dk(  s||   j                  |       + | j                  ||d      }	|j                  dd      }
|
|	|
|	j                  k7  rd}n|	}t        j                  di i |d|i| d{   }|S 7 7 # t        $ r;}t        j                  t        | |t!        j"                         |             |d}~ww xY ww)a  
        Example Usage:

        ```
        from litellm import Router
        client = Router(model_list = [
            {
                "model_name": "tts",
                "litellm_params": {
                    "model": "tts-1",
                },
            },
        ])

        async with client.aspeech(
            model="tts",
            voice="alloy",
            input="the quick brown fox jumped over the lazy dogs",
            api_base=None,
            api_key=None,
            organization=None,
            project=None,
            max_retries=1,
            timeout=600,
            client=None,
            optional_params={},
        ) as response:
            response.stream_to_file(speech_file_path)

        ```
        r  r  r`  ra  rb  r  Nr  ry  r   r   r   r  r  rp  r  r  rw   )r  r  r|  r   r   r  r   r  r$  rp  r   aspeechrr  r  r  r<   r  r  )r  r   r  r  rz  rn  r  kvr  r  r  r~  rt  s                 ry   r  zRouter.aspeechZ  s    @4	#F7O#F7O#BB#)h?@$*JJ/Dd$K%	  C   J 00uV0L./446DM3399;1VO !F1I*_1I$$Q' < &*%5%5%f' &6 &" %jjD9O+*6#'='E'EE#5$__ l  H OM>  		(,0#)(1(<(<(>'(	 G		sR   E 6D DAD A4D DD E D D 	E"6EEE c           	      6  K   	 ||d<   t         |d<   | j                  |d<   | j                  ||        | j                  di | d {   }|S 7 # t        $ r;}t        j                  t        | |t        j                         |             |d }~ww xY wwNr   r  rx  ry  r  rw   )
r  _arerankr|  r  rr  r  r  r<   r  r  r  r   rz  r~  rt  s        ry   arerankzRouter.arerank  s     	#F7O#F7O*.--F&'00uV0L?T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		s;   BAA 	A
A BA 	B6BBBc           	        K   d }	 t        j                  d| d|        | j                  ||j                  dd       |       d {   }| j	                  ||       |d   j                         }|d   }| j                  ||      }| j                  |xx   dz  cc<   t        j                  di i || j                  |d	| d {   }| j                  |xx   dz  cc<   t        j                  d
| d       |S 7 7 8# t        $ rE}t        j                  d
| dt        |       d       || j                  |xx   dz  cc<   |d }~ww xY ww)NzInside _rerank()- model: r  r  )r   r  r  r  r   r   rk   )r  r  zlitellm.arerank(model=r  r  r  rw   )r   r  r  r  r  r   r  r   r   r  r}   r   r  rr  r   r   )	r  r   rz  r  rn  r  r  r~  rt  s	            ry   r  zRouter._arerank  s    
'	!''+E7*VHE  $BB$*JJ/Dd$K%  C   J
 //:f/U./446DgJ>>% ? L Z(A-($__ #33* 	 H z*a/*!&&(4KL O9  	!&&(4HQPWX %
+q0+G	sO   EAD D	BD D5D ED D 	EA EEEis_retryis_fallbackis_asyncc                    d|dg}	 ||d<   ||d<   |j                  d| j                        |d<   |j                  di       j                  d|i       | j	                  |||j                  dd       	      }|d
   j                         }	| j                  j                         D ])  \  }
}|
|vr|||
<   |
dk(  s||
   j                  |       + t        j                  di i |	|| j                  d|S # t        $ r}|d }~ww xY w)Nr`  rb  r   ra  r   r   r  r  re  r   )ra  r  rw   )r$  r   r   r   r  r  r   r   r  r   text_completionr}   rr  )r  r   ra  r  r  r  rz  rv  rn  r  r  r  rt  s                ry   r  zRouter.text_completion  s1    $78	#F7O%F8$*JJ}d>N>N$OF=!j"-44mU5KL 66!$*JJ/Dd$K 7 J ./446D3399;1VO !F1I*_1I$$Q' < **s-r-rSWSgSg-rkq-rss 	G	s   B2C9 ;=C9 9	D	DD	c           	        K   |j                  dd       9| j                  ||j                  d      | j                  ||f|       d {   S 	 ||d<   ||d<   | j                  |d<   | j                  ||        | j                  di | d {   }|S 7 M7 # t        $ r;}t        j                  t        | |t        j                         |             |d }~ww xY ww)	Nr  )r   r  rx  rT  rz  r   ra  rx  ry  r  rw   )r$  rV  r  atext_completion_atext_completionr|  r  rr  r  r  r<   r  r  )	r  r   ra  r  r  r  rz  r~  rt  s	            ry   r  zRouter.atext_completion	  s      ::j$'3//J/"&"7"7V_ 0   	#F7O%F8*.*@*@F&'00uV0L?T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		sI   AC"	B
C"AB BB C"B 	C$6CCC"c           	      2  K   	 t        j                  d| d|        t        |      }| j                  |d|dg|j	                  dd       |       d {   }| j                  ||       |d   j                         }|d	   }| j                  ||      }| j                  |xx   d
z  cc<   t        j                  di i ||| j                  |d|}	| j                  ||d      }
|
\t        |
t        j                        rB|
4 d {    	 | j!                  ||       d {    |	 d {   }	d d d       d {    n%| j!                  ||       d {    |	 d {   }	| j"                  |xx   d
z  cc<   t        j$                  d| d       |	S 7 L7 7 {7 s7 e# 1 d {  7  sw Y   OxY w7 `7 X# t&        $ rE}t        j$                  d| dt)        |       d       || j*                  |xx   d
z  cc<   |d }~ww xY ww)N#Inside _atext_completion()- model: r  r`  rb  r  r  r  r   r   rk   rw  r  r  r  zlitellm.atext_completion(model=r  r  r  rw   )r   r  r   r  r  r  r   r  r   r   r  r}   r  r   r  r  r  r   r  rr  r   r   )r  r   ra  rz  r  rn  r  r  r  r~  r  rt  s               ry   r  zRouter._atext_completion:	  s    C	!''5eWJvhO  AH#BB#)f=>$*JJ/Dd$K%	  C   J //:f/U./446DgJ>>% ? L Z(A-(// $#33*	
 H !,,%3 - M (Zw00. ) . . EE#-@P F    &.~H. . . AA)<L B    "*>z*a/*!&&1*=TU OoH.
  .. . . . *  	!&&1%8LSQRVHT[\  &!+&G	   HAG F"B8G F%G F-+F',	F-5F)6F-:G F+G !G"	G +G,5G !H"G %G 'F-)F-+G -F?3F64F?;G G 	HA HHH
adapter_idc           	        K   	 ||d<   ||d<   | j                   |d<   |j                  d| j                        |d<   |j                  di       j	                  d|i        | j
                  di | d {   }|S 7 # t        $ r;}t        j                  t        | |t        j                         |             |d }~ww xY ww)	Nr   r  rx  r   r   r  r  rw   )_aadapter_completionr$  r   r   r   r  rr  r  r  r<   r  r  )	r  r  r   r  r  r  rz  r~  rt  s	            ry   aadapter_completionzRouter.aadapter_completion	  s     	#F7O#-F< *.*C*CF&'$*JJ}d>N>N$OF=!j"-44mU5KL?T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		s;   CA0A= 4A;5A= :C;A= =	C6B<<CCc           	      2  K   	 t        j                  d| d|        t        |      }| j                  |dddg|j	                  dd       |       d {   }| j                  ||       |d	   j                         }|d
   }| j                  ||      }| j                  |xx   dz  cc<   t        j                  di i ||| j                  |d|}	| j                  ||d      }
|
\t        |
t        j                        rB|
4 d {    	 | j!                  ||       d {    |	 d {   }	d d d       d {    n%| j!                  ||       d {    |	 d {   }	| j"                  |xx   dz  cc<   t        j$                  d| d       |	S 7 L7 7 {7 s7 e# 1 d {  7  sw Y   OxY w7 `7 X# t&        $ rE}t        j$                  d| dt)        |       d       || j*                  |xx   dz  cc<   |d }~ww xY ww)Nz&Inside _aadapter_completion()- model: r  r`  zdefault textrb  r  r  r  r   r   rk   )r  r  r  r  r  r  z"litellm.aadapter_completion(model=r  r  r  rw   )r   r  r   r  r  r  r   r  r   r   r  r}   r  r   r  r  r  r   r  rr  r   r   )r  r  r   rz  r  rn  r  r  r  r~  r  rt  s               ry   r  zRouter._aadapter_completion	  s    C	!''8z&R  AH#BB#)nEF$*JJ/Dd$K%	  C   J //:f/U./446DgJ>>% ? L Z(A-(22 ",#33*	
 H !,,%3 - M (Zw00. ) . . EE#-@P F    &.~H. . . AA)<L B    "*>z*a/*!&&4ZL@WX OoH.
  .. . . . *  	!&&4UG;OPSTUPVxW^_  &!+&G	r  c           	      f  K   	 ||d<   ||d<   | j                   |d<   | j                  ||d       t        j                  d| d|         | j                  d
i | d{   }|S 7 # t
        $ r;}t        j                  t        | |t        j                         |	             |d}~ww xY ww)
        Helper function to make a generic LLM API call through the router, this allows you to use retries/fallbacks with litellm router
        r   original_generic_functionrx  litellm_metadatar   rz  r  z3Inside ageneric_api_call_with_fallbacks() - model: r  Nr  rw   )(_ageneric_api_call_with_fallbacks_helperr|  r   r  r  rr  r  r  r<   r  r  )r  r   rx  rz  r~  rt  s         ry   !_ageneric_api_call_with_fallbacksz(Router._ageneric_api_call_with_fallbacks	  s     	#F7O2CF./*.*W*WF&'00FCU 1  "''EeWJW]V^_ @T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		s;   B1AA* !A("A* 'B1(A* *	B.36B))B..B1r  c           	      V  K   |j                  dd      }d}	 t        |      }	 | j                  |||j                  dd      |j                  dd             d{   }| j                  |||	       |d
   j                         }	|	d   }
| j                  |
xx   dz  cc<    |di i |	d| j                  i|}| j                  ||d      }|\t        |t        j                        rB|4 d{    	 | j                  ||       d{    | d{   }ddd      d{    n%| j                  ||       d{    | d{   }| j                  |
xx   dz  cc<   t        j                   d|
 d       |S 7 +# t        $ r!}|r |dd|i| d{  7  cY d}~S |d}~ww xY w7 7 7 7 # 1 d{  7  sw Y   |xY w7 7 # t        $ rE}t        j                   d| dt#        |       d       || j$                  |xx   dz  cc<   |d}~ww xY ww)r  passthrough_on_no_deploymentFr  rv  Nr  )r   r  rv  r  r   rn  rz  r  r   rk   r  r  r  r  z'ageneric_api_call_with_fallbacks(model=r  r  r  rw   )r  r   r  r$  rr  r  r   r   r}   r  r   r  r  r  r   r   r  r   r   )r  r   r  rz  r  r  r  rn  rt  r  r  r~  r  s                ry   r  z/Router._ageneric_api_call_with_fallbacks_helper
  s     (.zz2PRW'X$;D	@H
#'#F#F#)#ZZ
D9(.

3H$(O	 $G $ 
 //%fM 0  ./446DgJZ(A-( 1 t33 H !,,%3 - M (Zw00. ) . . EE#-@P F    &.~H. . . AA)<L B    "*>z*a/*!&&9*E\] Os  /!:!Q!Q&!QQQQ>.
  .. . . . *  	!&&9%@TUXYZU[T\\cd  &!+&G	s  H)G 8F
 FF
 !BG 4F75G 8F?F9	F?F;F?G *F=+G G	G G5G H)F
 
	F4F/"F%#F/'F4(G ,H)-F//F44G 9F?;F?=G ?GGGG G 	H&!A H!!H&&H)c           
         |j                   }	 t        j                  d| d| d|        | j                  ||j	                  dd      |j                  dd            }| j                  ||d	       |d
   j                         }|d   }| j                  |xx   dz  cc<   | j                  |       	 t        |d         \  }}	}} |di i ||	| j                  d|}
| j                  |xx   dz  cc<   t        j                  | d| d       |
S # t        $ r d}	Y \w xY w# t        $ rG}t        j                  | d| dt        |       d       || j                   |xx   dz  cc<   |d}~ww xY w)a  
        Make a generic LLM API call through the router, this allows you to use retries/fallbacks with litellm router
        Args:
            model: The model to use
            original_function: The handler function to call (e.g., litellm.completion)
            **kwargs: Additional arguments to pass to the handler function
        Returns:
            The response from the handler function
        z&Inside _generic_api_call() - handler: z	, model: r  rv  Nr  re  generic_api_callr  r   r   rk   r  r  )custom_llm_providerr  z(model=r  r  r  rw   )rs   r   r  r  r$  r  r  r   r   r  rf   rr  r}   r   r  r   r   )r  r   rx  rz  handler_namern  r  r  r  r  r~  rt  s               ry    _generic_api_call_with_fallbacksz'Router._generic_api_call_with_fallbacksQ
  s    )11.	!''8iPUwV`ag`hi 66J5$*JJ/Dd$K 7 J
 //%fDV 0  ./446DgJZ(A-( 11Z1H+/?d7m/T,&1 ) +>#33 	H z*a/*!&&.
|3JK O!  +&*#+"  	!&&.w.B3q6('R  &!+&G	s>   B'D* 6D 
AD* D'$D* &D''D* *	E:3AE55E:c                    	 ||d<   ||d<   | j                   |d<   |j                  d| j                        |d<   |j                  di       j	                  d|i        | j
                  di |}|S # t        $ r}|d }~ww xY w)Nr   r  rx  r   r   r  rw   )
_embeddingr$  r   r   r   r}  rr  r  r   r  r  rz  r~  rt  s          ry   	embeddingzRouter.embedding
  s    		#F7O#F7O*.//F&'$*JJ}d>N>N$OF=!j"-44mU5KL3t33=f=HO 	G	rt  c           	         d }	 t        j                  d| d|        | j                  |||j                  dd             }| j	                  ||       |d   j                         }|d   }| j                  ||d	      }|j                  d
d       }||||j                  k7  rd }	n|}	| j                  |xx   dz  cc<   | j                  |       t        j                  di i ||| j                  |	d|}
| j                  |xx   dz  cc<   t        j                  d| d       |
S # t         $ rE}t        j                  d| dt#        |       d       || j$                  |xx   dz  cc<   |d }~ww xY w)NzInside embedding()- model: r  r  )r   r  r  r  r   r   syncr  rp  rk   r  r  r  r  zlitellm.embedding(model=r  r  r  rw   )r   r  r  r  r  r   r  r$  rp  r   r  r   r  r}   r   r  rr  r   r   )r  r  r   rz  r  rn  r  r  r  r  r~  rt  s               ry   r  zRouter._embedding
  s   
4	!''-eWJvhG 66$*JJ/Dd$K 7 J
 //:f/U./446DgJ%)%5%5%f& &6 &" %jjD9O+*6#'='E'EE#5Z(A-( 11Z1H(( "#33*	
 H z*a/*!&&*:,6MN O 	!&&*:,6J3q6(RYZ %
+q0+G	s   D+D0 0	E>9A E99E>c           	      .  K   	 ||d<   ||d<   | j                   |d<   | j                  ||        | j                  di | d {   }|S 7 # t        $ r;}t	        j
                  t        | |t        j                         |             |d }~ww xY wwr  )	_aembeddingr|  r  rr  r  r  r<   r  r  r  s          ry   
aembeddingzRouter.aembedding
  s     	#F7O#F7O*.*:*:F&'00uV0L?T??I&IIHO J 		(,0#)(1(<(<(>'(	 G		r  c           	      .  K   d }	 t        j                  d| d|        t        |      }| j                  |||j	                  dd       |       d {   }| j                  ||       |d   j                         }|d   }| j                  ||      }| j                  |xx   dz  cc<   t        j                  di i ||| j                  |d	|}	| j                  ||d
      }
|
\t        |
t        j                        rB|
4 d {    	 | j!                  ||       d {    |	 d {   }	d d d       d {    n%| j!                  ||       d {    |	 d {   }	| j"                  |xx   dz  cc<   t        j$                  d| d       |	S 7 L7 7 {7 s7 e# 1 d {  7  sw Y   OxY w7 `7 X# t&        $ rE}t        j$                  d| dt)        |       d       || j*                  |xx   dz  cc<   |d }~ww xY ww)NzInside _aembedding()- model: r  r  )r   r  r  r  r  r   r   rk   r  r  r  r  zlitellm.aembedding(model=r  r  r  rw   )r   r  r   r  r  r  r   r  r   r   r  r}   r  r   r  r  r  r   r  rr  r   r   )r  r  r   rz  r  r  rn  r  r  r~  r  rt  s               ry   r  zRouter._aembedding
  s    
B	!''/wjI  AH#BB$*JJ/Dd$K%	  C   J //:f/U./446DgJ>>% ? L
 Z(A-()) "#33*	
 H !,,%3 - M (Zw00. ) . . EE#-@P F    &.~H. . . AA)<L B    "*>z*a/*!&&+J<7NO OmF.
  .. . . . *  	!&&+J<7KCPQF8SZ[ %
+q0+G	s   HAG F B8G F#G F+)F%*	F+3F'4F+8G F)G G  	G )G*5G H G #G %F+'F+)G +F=1F42F=9G G 	HA HHHc           	      b  K   	 ||d<   | j                   |d<   |j                  d| j                        |d<   | j                  ||        | j                  di | d {   }|S 7 # t
        $ r;}t        j                  t        | |t        j                         |             |d }~ww xY ww)Nr   rx  r   ry  r  rw   )_acreate_filer$  r   r|  r  rr  r  r  r<   r  r  r  s        ry   acreate_filezRouter.acreate_file8  s     
	#F7O*.*<*<F&'$*JJ}d>N>N$OF=!00uV0L?T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		s;   B/AA( A& A( %B/&A( (	B,16B''B,,B/c                    K   	 ddl m} t        j                  d| d        t	               j                  |dddgj                  dd       	       d {   }d
t        dt        f fd}g }t        |t              r|j                   ||             n|D ]  }|j                   ||              t        j                  |  d {   }t        |      dk(  rt        d       |||      }	t        t        |d         }
|	|
j                   d<   |
S 7 7 M# t        $ rH}t        j"                  d| d dt%        |       d       | j&                  |xx   dz  cc<   |d }~ww xY ww)Nr   )add_model_file_id_mappingsr  r  r`  files-api-fake-textrb  r  r   rv  r  r  r  rn  r   c           	      &  K   t        j                        }j                  | |d       | d   j                         }|d   }j                  | |      }j                  |xx   dz  cc<   t        |d         \  }}}}t        t        t           j                  d            }t        t        t           j                  d	            }	|	r|st        d
      t        |      }
|
rt        |	|      }	|	|d	<   t        j                  di i ||j                   |d|}j#                  | |d      }|\t%        |t&        j(                        rB|4 d {    	 j+                  |        d {    | d {   }d d d       d {    n%j+                  |        d {    | d {   }j,                  |xx   dz  cc<   t/        j0                  d| d       |S 7 7 x7 p7 b# 1 d {  7  sw Y   LxY w7 ]7 Uw)Nr  r  r   r   r  rk   r  purposer|  z2file and file_purpose are required for create_file)r  )file_contentnew_model_namer  r  r  r  r  r  litellm.acreate_file(model=r  rw   )r   r   r  r  r   rf   r   r   rF   r$  rD   rr  r,   r+   r   r  r}   r  r   r  r  r  r   r   r  )rn  kwargs_copyr  r  r  stripped_modelr  r  r  r|  replace_model_in_jsonl_boolr~  r  rz  r  r  s                ry   create_file_for_deploymentz8Router._acreate_file.<locals>.create_file_for_deploymentd  s    "mmF333)&"0 4 
 ""2388:!']
#BB)&  C     ,1, =Mw-=9 3Q
 x(:;VZZ	=RSHY/F1CD7#L  /L#/+ /1%)'5D
 +/K'"// /B#'#7#7".	
 & !% 0 0)& 7 !1 ! !,!7#4#42  - 2 2 #II'1DT J    *2>2 2 2 EE#-@P F    &.~H"":.!3.%**1*=TU  )2
 $22 2 2 2  .s   EHG0H!G89G2:	G8G4G8HG6H/H0	H9H:7H2G84G86H8H
>H?H
HHNo healthy deployments found.)r   rG  model_file_id_mappingr  , r  r  rk   )!litellm.router_utils.common_utilsr  r   r  r   async_get_healthy_deploymentsr  r  rE   r   r   r  r  rf  rr  r   r8  	exceptionr   r   )r  r   rz  r  r   r  tasksrn  rG  r  returned_responsert  r  s   ` `         @ry   r  zRouter._acreate_fileP  s    
{	T!''5eWJvhO  AH(,(J(J#)6KLM$*JJ/Dd$K%!1 )K ) #O T O FV O b E-t478KLM"5JLL!;J!GH #6 &nne44I9~" ?@@$>$79%! !%%5y| D% ,,-DE %$[#B 5  	!++-eWBvh>RSVWXSYRZZab  &!+&G	sP   E:AD&  D"!A5D& D$A
D& !E:"D& $D& &	E7/AE22E77E:c           	      |  K   	 ||d<   | j                   |d<   |j                  d| j                        |d<   t        d      }| j	                  |||        | j
                  di | d {   }|S 7 # t        $ r;}t        j                  t        | |t        j                         |             |d }~ww xY ww)	Nr   rx  r   _acreate_batchr  r  r  rw   )r  r$  r   r*   r|  r  rr  r  r  r<   r  r  )r  r   rz  r  r~  rt  s         ry   acreate_batchzRouter.acreate_batch  s     
	#F7O*.*=*=F&'$*JJ}d>N>N$OF=!%G.&" 00'= 1 
 @T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		s;   B<A(A5 ,A3-A5 2B<3A5 5	B9>6B44B99B<c                 b  K   	 t        j                  d| d|        t        |      }| j                  |dddg|j	                  dd       |       d {   }|d   j                         }|d	   }| j                  ||d
       | j                  ||      }| j                  |xx   dz  cc<   t        |d	         \  }}	}}t        j                  di i ||	| j                  |d|}
| j                  ||d      }|\t        |t        j                         rB|4 d {    	 | j#                  ||       d {    |
 d {   }
d d d       d {    n%| j#                  ||       d {    |
 d {   }
| j$                  |xx   dz  cc<   t        j&                  d| d       |
S 7 a7 7 {7 s7 e# 1 d {  7  sw Y   OxY w7 `7 X# t(        $ rH}t        j*                  d| d| dt-        |       d       || j.                  |xx   dz  cc<   |d }~ww xY ww)Nz Inside _acreate_batch()- model: r  r`  r  rb  r  r  r   r   r  r  r  rk   r  r  r  r  r  zlitellm.acreate_batch(model=r  zlitellm._acreate_batch(model=r  r  r  rw   )r   r  r   r  r  r   r  r  r   rf   r   r  r}   r  r   r  r  r  r   r  rr  r  r   r   )r  r   rz  r  rn  r  r  r  r  r  r~  r  rt  s                ry   r  zRouter._acreate_batch  s    
I	!''25'F8L  AH#BB#)6KLM$*JJ/Dd$K%	  C   J ./446DgJ//%fDT 0   >>% ? L Z(A-( ,<$w-+P(A"Aq,, +>#33*	
 H !,,%3 - M (Zw00. ) . . EE#-@P F    &.~H. . . AA)<L B    "*>z*a/*!&&.zl:QR O{R.
  .. . . . *  	!++/wb@TUXYZU[T\\cd  &!+&G	s   H/AG F7CG $F:%G (G F<	G
F>GG G G 6G7	G  G5G 6H/7G :G <G>G G GG	GG G 	H,$AH''H,,H/c                    	K   	 t              }|3 j                  |dddgj                  dd      |       d{   }n j                         }|t	        d      g 	dt
        f	 fd	}|^t        |t              rNt        |      d
kD  r@t        j                  |D cg c]  } |t        t
        |             c}ddi d{   }n<|/t        |t              r |t        t
        |             d{   }nt	        d      |=t        |t              r|S t        |t              r|D ]  }t        |t              s|c S  	r	d
   t	        dj                  	            7 /c c}w 7 7 z# t        $ r;}t        j                  t!         t#        j$                         |             |d}~ww xY ww)z
        Iterate through all models in a model group to check for batch

        Future Improvement - cache the result.
        Nr`  zretrieve-api-fake-textrb  r  r  Router not yet initialized.r  c                 @  K   	 | d   j                  d      }| d   j                         }|j                  d      }|t        d|        |st        |      \  }}}}t        j                        }
j                  t        t        |       |d       |j                  dd        |j                  dd        t        j                  d	i i |d|i| d {   S 7 # t        $ r0}dd l} |j                          	j                  |       Y d }~y d }~ww xY ww)
Nr   r   r  z2Model not found in litellm_params for deployment: r  aretrieve_batchr  r   rw   )r$  r   rr  rf   r   r  r   r  r  r   r  r  	print_excr   )r  r   r  r  r  
new_kwargsrt  r  rz  receieved_exceptionsr  s           ry   try_retrieve_batchz2Router.aretrieve_batch.<locals>.try_retrieve_batch^  sJ    " &'78<<WEE%&67<<>D*.((3H*I'}'PQ[P\] 
 /7G"'84.1 "&v!6J77#'j#9)&7 8 
 NN#8$?HH2D9!(!8!8 ""13F )"    !  $'I'')(//2 sA   DCC" C C" D C" "	D+&DDDDr   return_exceptionsTr  z7Unable to find batch in any model. Received errors - {}r  )r   r  r  rZ  rr  rP   r   r   rf  r  r  r   r  r^   r	  r  r<   r  r  )
r  r   rz  r  filtered_model_listr  resultsr,  rt  r  s
   ` `      @ry   r  zRouter.aretrieve_batchA  s    j	@H  <<'-:RST(.

3H$(O#)%5 =   $ '+&9&9&;#"* =>>#% # 5H # N $/2D9+,q0 ' &9! +40CU+KL!
 '+!  %0Z#T6 !3,.AB!    ?@@ "g|4"N.")%fl;#)M #*
 $*1-- IPP( ot2  		(,0#)(1(<(<(>'(	 G		s}   G:E< E0A%E< 'E3
E< E80E< ?E: #E< #G$%E< 
E< G#E< 3E< :E< <	G 6F;;G  Gc                   K   | j                  |      }|t        d      dt        ffd}t        j                  |D cg c]
  } ||       c}  d{   }dg dddd}|D ]g  }||d	   t        |d	      rt        |d	      |d	<   t        |d
      |d
<   |d   j                  |j                         t        |dd      du scd|d<   i |S c c}w 7 w)zQ
        Return all the batches across all deployments of a model group.
        rX  Nr  r   c                 x   K   	 t        j                  di i | d    d {   S 7 # t        $ r Y y w xY ww)Nr   rw   )r   alist_batchesrr  ry  s    ry   r  z0Router.alist_batches.<locals>.try_retrieve_batch  sT     $22 ;/0;F;     s(   :+ )+ :+ 	7:7:r   F)objectr  first_idlast_idhas_morer  r  r  r  T)	rZ  rr  rP   r  r  hasattrgetattrextendr  )r  r   rz  r  r  r  final_resultsr,  s     `     ry   r  zRouter.alist_batches  s     #11U1C&9::	,? 	  5HIE 'I
 

 
 F! ,49T07
0KM*-+269+Ei(f%,,V[[9 6:u5=04M*-  / J
s*   ?CCCCC-AC
Cr  c                   K   | j                  |j                  dd      |       |j                  d      ru| j                  |d         r`| j                  |d   |       d {   }|d   d   |d<   |d   j	                         }| j                  ||       |j                  |        |di | d {   S 7 V7 w)	Nr   r  ry  rX  )r   r  r   r  rw   )r|  r$  rZ  r  r   r  r   )r  rx  r  rz  rn  r  s         ry   )_pass_through_moderation_endpoint_factoryz0Router._pass_through_moderation_endpoint_factory  s      	,,**Wb) 	- 	
 ::g4#6#6&/#6#R#BBWo%  C   J ))9:7CF7O./446D//% 0  MM$&0000 1s%   A#C %B<&AC 7B>8C >C rB  )
assistantsrA  rC  rD  rG  rH  rI  rE  rF  rK  rL  rM  rN  rO  rP  rQ  rR  rJ  r[  r]  r\  r^  rW  rX  rY  rZ  c                      dv r&	 	 ddt         t           dt         t           f fd}|S 	 	 ddt         t           dt         t           f fd}|S )z
        Creates appropriate wrapper functions for different API call types.

        Returns:
            - A synchronous function for synchronous call types
            - An asynchronous function for asynchronous call types
        )rG  r]  r^  rY  rZ  r  r  c                 ,     j                   ddi|S )Nrx  rw   )r  )r  r  rz  rx  r  s      ry   sync_wrapperz-Router.factory_function.<locals>.sync_wrapper4  s*    
 =t<< &7;A rx   c                   K   dk(  r j                   d| |d| d {   S dk(  r j                  ddi| d {   S dv r j                  ddi| d {   S dk(  r j                  ddd| d {   S d	v r j                  ddi| d {   S d
v r j                  d| d| d {   S dv r j                  d| |d| d {   S y 7 7 7 7 n7 P7 17 w)Nr  )rx  r  r  rA  rx  )rC  rD  rK  rL  rM  rN  rO  rP  rQ  r[  r\  rR  T)rx  r  )rH  rI  rJ  )rW  rX  )rx  r  )rE  rF  rw   ))_pass_through_assistants_endpoint_factoryr  r  _init_responses_api_endpoints _init_vector_store_api_endpoints)r  r  rz  rB  rx  r  s      ry   async_wrapperz.Router.factory_function.<locals>.async_wrapper@  s    
 L(KTKK &7(;! 	   l*KTKK &7;A      DTCC &7   66CTCC &715   
   
 @T?? &7      CTBB &7(;   
 ??CTCC &7(;! 	   @g 
s{   C<C. C<C0C<#C2$!C<C4C<%C6& C<C8!C<(C:)C<0C<2C<4C<6C<8C<:C<NN)r   r   r   )r  rx  rB  r  r  s   ```  ry   r6  zRouter.factory_function  sn    P  
 
 6:(,%-c]    26$(?	!)#?	SM?	B rx   c                 D   K   |r	d|vr||d<    |di | d{   S 7 w)zJ
        Initialize the Vector Store API endpoints on the router.
        r  Nrw   rw   )r  rx  r  rz  s       ry   r  z'Router._init_vector_store_api_endpoints  s2      #8#F,?F()&00000s     c                    K   ddl m} |j                  |j                  d            }|||d<    | j                  dd|i| d{   S 7 w)z
        Initialize the Responses API endpoints on the router.

        GET, DELETE Responses API Requests encode the model_id in the response_id, this function decodes the response_id and sets the model to the model_id.
        r   )ResponsesAPIRequestUtilsresponse_idNr   rx  rw   )litellm.responses.utilsr  get_model_id_from_response_idr$  r  )r  rx  rz  r  model_ids        ry   r  z$Router._init_responses_api_endpoints  sh      	E+IIJJ}%
 &F7O;T;; 
/

 
 	
 
s   AAAAr  c                    K   |E| j                   .| j                   d   }|j                  | j                   d          nt        d       |d||d| d{   S 7 w)z@Internal helper function to pass through the assistants endpointNr  r   z'custom_llm_provider' must be set. Either via:
 `Router(assistants_config={'custom_llm_provider': ..})` 
or
 `router.arun_thread(custom_llm_provider=..)`)r  r  rw   )r   r   rr  )r  rx  r  r  rz  s        ry   r  z0Router._pass_through_assistants_endpoint_factory  s      &%%1&*&<&<=R&S#d445EFG s  ' 
 3F
FL
 
 	
 
s   AAAAc                 	  K   |j                  d      }|j                  dd      }|j                  d| j                        }|j                  d| j                        }|j                  d| j                        }|j                  dd      }	 | j                  |||||	       | | j                  |i |d|i d{   }	n | j                  |i | d{   }	t        j                  d
|	        t        |	d      }	|	S 7 F7 -# t        $ r}
t        j                  dt        j                                 |
}d}|j                  d      }d}|du s||
| |d|}d|vr| j                  |d<   d|vrd|d<   	 t        j                  d       t        |      }|r/|j!                  ||d       t#        |i | d{  7  }	|	cY d}
~
S t%        |
t&        j(                        r|F| j+                  ||      }|||j!                  ||d       t#        |i | d{  7  }	|	cY d}
~
S dj-                  |||      }t        j                  dj-                  |             |
xj.                  dj-                  |      z  c_        nt%        |
t&        j0                        r|F| j+                  ||      }|||j!                  ||d       t#        |i | d{  7  }	|	cY d}
~
S dj-                  |||      }t        j                  dj-                  |             |
xj.                  dj-                  |      z  c_        ||t        j                  d|        t3        |t5        t6        |            \  }}|
|||   d   }|Dt        j                  d| d |        t9        |d!      r|xj.                  d| d | z  c_        ||j!                  ||d       t#        |i | d{  7  }	|	cY d}
~
S n# t        $ ru}t;        |      }t        j<                  d"j-                  t7        |      t        j                         t?        | |#       d{  7               t7        |      }Y d}~nd}~ww xY wt9        |d!      rW|xj.                  d$j-                  ||      z  c_        tA        |      dkD  r$|xj.                  d%j-                  |      z  c_        |d}
~
ww xY ww)&z
        Try calling the function_with_retries
        If it fails after num_retries, fall back to another model group
        r   disable_fallbacksFr   r   r   mock_timeoutN)rz  r  r   r   r   zAsync Response: r   )r~  attempted_fallbacks	Tracebackr  T)litellm_routerr  r   fallback_depthzTrying to fallback b/w models)r   )fallback_model_grouporiginal_model_group)r   r  zmodel={}. context_window_fallbacks={}. fallbacks={}.

Set 'context_window_fallback' - https://docs.litellm.ai/docs/routing#fallbackszGot 'ContextWindowExceededError'. No context_window_fallback set. Defaulting                             to fallbacks, if available.{})msgz
{}zmodel={}. content_policy_fallback={}. fallbacks={}.

Set 'content_policy_fallback' - https://docs.litellm.ai/docs/routing#fallbackszGot 'ContentPolicyViolationError'. No content_policy_fallback set. Defaulting                             to fallbacks, if available.{}zinside model fallbacks: r   z7No fallback model group found for original model_group=z. Fallbacks=r  zlitellm.router.py::async_function_with_fallbacks() - Error occurred while trying to do fallbacks - {}
{}

Debug Information:
Cooldown Deployments={}r  r  z<. Received Model Group={}
Available Model Group Fallbacks={}z
Error doing the fallback: {})!r$  r  r   r   r   _handle_mock_testing_fallbacksasync_function_with_retriesr   r  r(   rr  r  r  r   r  r6   r   r8   r   r   ContextWindowExceededError(_get_fallback_model_group_from_fallbacksr	  r  r  r7   r   r   r  r   errorr3   rf  )r  rT  rz  r  r   r   r   r   r  r~  rt  r  r  r  fallback_failure_exception_strinput_kwargsis_non_standard_fallback_formaterror_messagegeneric_fallback_idxnew_exceptionr  s                        ry   r  z$Router.async_function_with_fallbacks  s     &,ZZ%8,2JJ7JE,R$*JJ{DNN$K	39::&(E(E4
  4:::&(E(E4
  zz.$7H	%//'#)A)A 0  '!A!A!A"#"2>"  "B!A!A4!R6!RR!''*:8*(EF7!$%H O S  s	%!'')I4H4H4J3K(LM!"#' 28**W2E -/* D(,@,H #'&8 L l2040B0B_-|312-.OD%**+JK 3V'3/ 3 ''4=4H &8&&&    H
 $Oa!C!CD/; II*B,7 J  - 07"44$++8L8L *<!*** $ $  ( )q  )x  )x')A9) .22!;;A6 -< 		V]]=%AA	7#F#FG/; II*B,7 J  - 07"44$++8L8L *<!*** $ $  ( )p  )w  )w')A9) .22!;;A6 -< 		V]]=%AA	([-D)//2J9+0VW 1"+$(k$:,, -40</89M/Ns/S,+3-22UVaUbbnoxnyz ##5yA.66<st  tA  AM  NW  MX  ;Y  Y600 ''4H4H &8&&&    H
 $O D#DV#L %++ o  v  vM*!,,.M48-=  	 25]1C.D )95"**.m.t.t(/ * 56:&..8??:. %$gs	%s   B
S91D	 >D?D	 D*D	 S9D	 D	 	
S6A,S1 APGPS6S9AP'H*(P.S6/S94B2P&K)'P-S6.S93D
P=P >PS6S9
S1	R	AR&Q)'R?S1R		A(S11S66S9r  c                    t        j                  |      }|j                  ,|j                  du rt        j                  |dd| d|       |j
                  ,|j
                  du rt        j                  |dd| d|       |j                  -|j                  du rt        j                  |dd| d|       yy)	a  
        Helper function to raise a litellm Error for mock testing purposes.

        Raises:
            litellm.InternalServerError: when `mock_testing_fallbacks=True` passed in request params
            litellm.ContextWindowExceededError: when `mock_testing_context_fallbacks=True` passed in request params
            litellm.ContentPolicyViolationError: when `mock_testing_content_policy_fallbacks=True` passed in request params
        NTr  #This is a mock exception for model=z#, to trigger a fallback. Fallbacks=)r   r  r  zF, to trigger a fallback.                     Context_Window_Fallbacks=zF, to trigger a fallback.                     Context_Policy_Fallbacks=)	rR   from_kwargsmock_testing_fallbacksr   InternalServerErrormock_testing_context_fallbacksr  %mock_testing_content_policy_fallbacksr  )r  rz  r  r   r   r   mock_testing_paramss          ry   r
  z%Router._handle_mock_testing_fallbacks  s     6AA&I66B#::dB--!=k]Jmnwmxy   >>J#BBdJ44!=k] K..F-GI   EEQ#IITQ55!=k] K..F-GI  R Rrx   c           
        K   t        j                  d       |j                  d      }|j                  d| j                        }t	        |      }|j                  d| j
                        }|j                  d| j                        }|j                  d      }|j                  d      }	|j                  d|j                  d	            xs i }
d
|
v rFt        |
d
   t              r3| j                  |
d
         }||
j                  dt        |      i       t        j                  d| d|	        	 | j                  ||        | j                  |g|i | d {   }t        |dd       }|S 7 # t         $ rO}d }|}t#        |dd       }|t        |t$              r|}		 | j'                  |j                  d      xs d|       d {  7  \  }}| j)                  ||||||       | j*                  | j,                  &| j/                  ||j                  d            }||}	|	dkD  r| j1                  ||      }n t        j2                  d|	        | j5                  ||	|	||      }t7        j8                  |       d {  7   t;        |	      D ]  }	  | j                  |g|i | d {  7  }t=        j>                  |      r| d {  7  }t        ||dz   |	      }|c cY d }~S # t         $ r}| j1                  ||      }|	|z
  }|j                  d      }| | j'                  ||       d {  7  \  }}ng }| j5                  |||	||      }t7        j8                  |       d {  7   Y d }~d }~ww xY w tA        |      tB        jD                  v rtG        |d|	       tG        |d|       |d }~ww xY ww)Nz#Inside async function with retries.rx  r   r   r   r   r   r  r   r  rX  model_group_sizez/async function w/ retries: original_function - z, num_retries - )r  rz  r   )r~  attempted_retriesr   r  rC  )r  r   all_deploymentsr   regular_fallbacksr   )r  r  )rz  rt  z#Retrying request with num_retries: )rt  remaining_retriesr   r   r   rk   r   )$r   r  r  r   r   r   r   r$  r   r   rZ  r   rf  %_handle_mock_testing_rate_limit_error	make_callr)   rr  r  r   rJ  should_retry_this_errorr   r   r:   	log_retryr  _time_to_sleep_before_retryr  rL  r  inspectiscoroutinefunctionr   r   LITELLM_EXCEPTION_TYPESrM  )r  rT  rz  rx  r   r  r   r   r  r   	_metadatar   r~  rt  current_attemptr  deployment_num_retriesrS  _all_deployments_retry_policy_retriesr   r"  _modelr  _timeouts                            ry   r  z"Router.async_function_with_retries  sN    ##$IJ"JJ':;JJ{DNN;	<VD#)::&(E(E$
  $*::&(E(E$
  &,ZZ%8jj/ !**%7J9OPVTV	I%*Y}5Ms*S,,	-8P,QJ%  "4c*o!FG##=>O=PP`al`mn	
r	%66' 7  ,T^^,=OOOOH4!QDH O	 P
  h	%"O!"%,Qt%D"%1j&7 5 99jj)/R!1 :   $  (($8 0)A"+)A )  !!-00< )-(N(N0fjj>Q )O )% )4"7KQv9KL!&&5k]C ::$"-'$8 0 ; K --,,,#(#5%2%3T^^4E%W%WPV%WWWH22  *2>><!)*9A*=$/ H
 $O  2!^^6Q^?F(3o(E%,2JJw,?F) #'"E"E"(-= #F #  0 02,#??,*;$/,@(8  @  H "--111-2! $6P &'7+J+JJ*M;G*M?K$$Qh	%s   D2O5,E8 !E6"E8 5O6E8 8
OAOGB=OJO$K==K >K=K K=5O7O8O=	NANM

9NN
NON;OOOc                    K   |j                  d      } ||i |}t        j                  |      st        j                  |      r
| d{   }| j	                  ||       d{   }|S 7 !7 w)z^
        Handler for making a call to the .completion()/.embeddings()/etc. functions.
        r   N)r~  r  )r$  r(  r)  isawaitableset_response_headers)r  rx  rT  rz  r  r~  s         ry   r$  zRouter.make_callT  sx      jj)$d5f5&&x0G4G4G4Q%~H22; 3 
 
  &
s$   A	A1A-A1&A/'A1/A1c                 @   |j                  dd      }| j                  |      }d}|:t        |      dk(  r,t        t        t
           |d   d   j                  d            }|:|du r5t        j                  d	|        t        j                  |d
d| d|      yy)z
        Helper function to raise a mock litellm.RateLimitError error for testing purposes.

        Raises:
            litellm.RateLimitError error when `mock_testing_rate_limit_error=True` passed in request params
        mock_testing_rate_limit_errorNrX  rk   r   r   r   TzTlitellm.router.py::_mock_rate_limit_error() - Raising mock RateLimitError for model=r  r  z , to trigger a rate limit error.)r   r  r  r   )r  rZ  rf  r   r   r   r$  r   r  r   RateLimitError)r  rz  r  r6  available_modelsr   s         ry   r#  z,Router._handle_mock_testing_rate_limit_errorc  s     9?

+T9
%  ..+.F%)'C0@,AQ,F/23CDHHWK
 *5-5!&&fgrfst ((!=k]Jjk'	  6 6rx   r  r   r   r!  c                    d}|t        |t              rt        |      }d}|t        |t              rt        |      }t        |t        j                        r||t        |t        j
                        r||t        |t        j                        r|t        |t        j                        r|dk  r|t        |      dkD  r|t        |t        j                        r	 |dk  r||dk  r|y)au  
        1. raise an exception for ContextWindowExceededError if context_window_fallbacks is not None
        2. raise an exception for ContentPolicyViolationError if content_policy_fallbacks is not None

        2. raise an exception for RateLimitError if
            - there are no fallbacks
            - there are no healthy deployments in the same model group
        r   rk   T)
r   r   rf  r   r  r  NotFoundErrorr   r7  AuthenticationError)	r  r  r   r   r   r   r!  _num_healthy_deployments_num_all_deploymentss	            ry   r%  zRouter.should_retry_this_error  s    " $% *z:Mt/T'*+>'?$ &:ot+L#&#7  ug@@A(4K ugAAB(4KeW223KeV223(A-%1)*Q.eV778
 %) $q(Krx   c                 4    t        | j                  g|i |S )z~
        Sync wrapper for async_function_with_fallbacks

        Wrapped to reduce code duplication and prevent bugs.
        )r   r  )r  rT  rz  s      ry   r}  zRouter.function_with_fallbacks  s     "$"D"DVtVvVVrx   c                 l    |yd}|D ])  }t        |j                               d   |k(  s#||   } |S  |S )a@  
        Returns the list of fallback models to use for a given model group

        If no fallback model group is found, returns None

        Example:
            fallbacks = [{"gpt-3.5-turbo": ["gpt-4"]}, {"gpt-4o": ["gpt-3.5-turbo"]}]
            model_group = "gpt-3.5-turbo"
            returns: ["gpt-4"]
        Nr   )r   keys)r  r   r  r  rO  s        ry   r  z/Router._get_fallback_model_group_from_fallbacks  sQ     48DDIIK #{2'+K'8$##	  $#rx   rt  r"  c                    |t        |      dk(  rn!|t        |t              rt        |      dkD  ryd}t        |d      r,t        |j                  d      r|j                  j
                  }t        |d      r|j                  }|%t        j                  |||| j                        }|S t        j                  ||| j                        }|S )	z
        Calculate back-off, then retry

        It should instantly retry only when:
            1. there are healthy deployments in the same model group
            2. there are fallbacks for the completion call
        Nrk   r   r~  headerslitellm_response_headers)r"  r   response_headersmin_timeout)r"  r   rE  )
rf  r   r   r  r~  rB  rC  r   _calculate_retry_afterr   )r  rt  r"  r   r   r   rD  r   s           ry   r'  z"Router._time_to_sleep_before_retry  s    " &3+?1+D+.5'(1,481j!gajj)&D zz11101 99'44"3'!1 ,,	G  44"3' ,,G rx   c                   K   	 |j                  dd      }|t        d      |d   j                  d      y|d   d   j                  dd      }|j                  dd      }|j                  dd      }||yt        |t              rt	        |      }t        |      }	|j                  d	d
      }
t               }|j                  d      }t        j                  j                  j                  |||      }| j                  j                  ||
|	t        j                  j                         d{    t        j                   j                  j                  |||      }| j                  j                  |d|	t        j                  j                         d{    t#        | |       |S 7 ~7 # t$        $ r7}t'        j(                  dj                  t	        |                   Y d}~yd}~ww xY ww)zG
        Track remaining tpm/rpm quota for model in model_list
        standard_logging_objectNzstandard_logging_object is Noner   r   rn  r  r  total_tokensr   %H-%Mr  current_minuter   r  r+  r  rv   rk   r  deployment_idzOlitellm.router.Router::deployment_callback_on_success(): Exception occured - {})r$  re  r   r   r   r   ri   strftimerV   TPMr+  r	  r   async_increment_cacherq   rv   RPMr@   rr  r   r  )r  rz  completion_responser  r  rH  deployment_namer  r  r  rI  dtrL  tpm_keyrpm_keyrt  s                   ry   r   z%Router.deployment_callback_on_success  s    F	HN

)4I# '. !BCC&'++J7?"()9"::"F"J"J $# 699-N,00TB&"*C(RB#DV#L &=&A&A.RS&T
 &'!#" *--33::. ;  jj66&%5#--	 7    *--33::. ;  jj66%5#--	 7    B,0"$
 /  	!++ahhF
 	sk   G83F5 G8A F5 8G89B9F5 2F13A*F5 F3F5 0G81F5 3F5 5	G5>-G0+G80G55G8c                    d}|d   j                  d      nc|d   d   j                  dd      }|d   j                  di       xs i }|j                  dd      }||yt        |t              rt        |      }|t	        | |      }|S y)z
        Tracks the number of successes for a deployment in the current minute (using in-memory cache)

        Returns:
        - key: str - The key used to increment the cache
        - None: if no key is found
        Nr   r   r  r  r  rN  )r$  r   r   r   r@   )	r  rz  rT  r  r  r  r  r  r  s	            ry   r   z*Router.sync_deployment_callback_on_successn  s     "#''
3; !12:>BB=RVWK 0155lBGM2Jd+B"bjB$W>C(, C Jrx   c                    t        j                  d       	 |j                  dd      }t        |dd      }|j                  di       j                  di       }t        j
                  j                  j                  |      }|j                  di       j                  d	d      }	d}
| t        j                  j                  |
      }
|	|	dk\  r|	}n|
|
dk\  r|
}n| j                  }t        |t              r1|j                  dd      }t        | |       t        | ||||      }|S t        j                  d       y# t        $ r}|d}~ww xY w)a`  
        2 jobs:
        - Tracks the number of failures for a deployment in the current minute (using in-memory cache)
        - Puts the deployment in cooldown if it exceeds the allowed fails / minute

        Returns:
        - True if the deployment should be put in cooldown
        - False if the deployment should not be put in cooldown
        z1Router: Entering 'deployment_callback_on_failure'r  Nstatus_coder  r   r  )r  r   )rD  r   r  rN  r  exception_statusr  rn  time_to_cooldownzWRouter: Exiting 'deployment_callback_on_failure' without cooldown. No model_info found.F)r   r  r$  r  r   litellm_core_utilsexception_mapping_utils_get_response_headersutils&_get_retry_after_from_exception_headerr   r   r  r?   r5   rr  )r  rz  rT  r  r  r  r]  _model_infoexception_headersdeployment_cooldownheader_cooldown_time_to_cooldownrO  r,  rt  s                  ry   r  z%Router.deployment_callback_on_failure  ss     	##$WX6	

;5I&y-D **%5r:>>|RPK ' : : R R h h#, !i !
 #)**-=r"B"F"F# #O ,")--"V"V%6 #W # #.3F!3K$7! ,A1E$3!$($6$6!+t, +d ;@,0"/ 3,0%5'0,%6 %++m  	G	s   DD? )D? ?	EE

ErT  c                 (  K   |d   d   j                  dd      }|d   d   j                  dd      }|d   j                  di       xs i }|j                  dd      }||yt        |t              rt        |      }t	        |      }	t               }
|
j                  d      }t        j                  j                  j                  |||	      }| j                  j                  |d
|	t        j                  j                         d{    y7 w)z3
        Update RPM usage for a deployment
        r   r   rn  Nr  r  r  rJ  rK  rk   rM  )r$  r   r   r   r   ri   rP  rV   rS  r+  r	  r   rR  rq   rv   )r  rz  rT  r  r  rU  r  r  r  r  rV  rL  rX  s                ry   r   z+Router.async_deployment_callback_on_failure  s     !!12:>BB$
 -.z:>>}dS,-11,CIr
^^D$'"*C RB<VD

 "%%++22. 3 
 jj..-%%	 / 
 	
 	
s   DD
DDc                    	 d|v rdnd}t        |      j                  t        |      d}|j                         D ]W  \  }}||ddfvr|||<   ||k(  st	        |t
              s*i ||<   ||   j                         D ]  \  }}|dk7  s|||   |<    Y t        | j                        dkD  r| j                  j                  d       | j                  j                  |       | j                  ||   d<   |S # t        $ r}|d	}~ww xY w)
z
        When a retry or fallback happens, log the details of the just failed model call - similar to Sentry breadcrumbing
        r  r   )exception_typeexception_stringrv  rx  r      r   N)r   rs   r   r  r   r  rf  r   r  r   rr  )	r  rz  rt  _metadata_varprevious_modelr  r  
metadata_k
metadata_vs	            ry   r&  zRouter.log_retry  s+   	&8F&B"
 
 #'q'"2"2$'FN 	 ]J8KLL()N1%-'Jq$,?46N=1282G2M2M2O.
J%)::<FN1-j9 3P  4''(1,$$((+  ''77;7K7KF=!"34M 	G	s+   AC7 C7 %#C7 	A-C7 7	D DDrO  r  c                     |}| j                   j                  ||d      }|#d}| j                   j                  ||dd       |S |dz  }| j                   j                  ||d       |S )zf
        Update deployment rpm for that minute

        Returns:
        - int: request count
        T)r  r  
local_onlyrk   rr   )r  r+  rs  rv   )r  r+  rs  )r   	get_cache	set_cache)r  rO  r  rX  request_counts        ry   _update_usagezRouter._update_usage!  s      

,,*:t - 
  MJJ  =Tr !   QMJJ  =T !  rx   c                 n    | j                   y| j                   D ]  }t        |t              sd|v s y y)NFr   T)r   r   r  )r  fallbacks     ry   _has_default_fallbackszRouter._has_default_fallbacks<  s6    >>!H(D)(? ' rx   r~  c                    |j                   r5t        |j                         dkD  r|j                   d   j                  dk7  ry|j                  d| j                        }|2d}|D ](  }t        |j                               d   |k(  s#||   } n |y| j                         ryt        j                  dj                  ||             y)z
        Determines if a content policy error should be raised.

        Only raised if a fallback is available.

        Else, original response is returned.
        r   content_filterFr   NTzyContent Policy Error occurred. No available fallbacks. Returning original response. model={}, content_policy_fallbacks={})choicesrf  finish_reasonr$  r   r   r@  rz  r   r  r	  )r  r   r~  rz  r   r  rO  s          ry   r  z)Router._should_raise_content_policy_errorE  s     H$4$4 5 9"004DD#)::&(E(E$
 
 $/#' 0		$Q'50+/;( 1
 $/((*"" H  O  O/	

 rx   c                     g }	 | j                  |      \  }}t        |t              rg S 	 t	        | |      }g }|D ]  }|d   d   |v r|j                  |         ||fS # t        $ r Y Aw xY w)Nr  r	  r  r  )#_common_checks_available_deploymentr   r  rr  r4   r   r  r   r  r.  r  unhealthy_deploymentsr   rn  s           ry   _get_healthy_deploymentszRouter._get_healthy_deploymentsk  s    !#	"&"J"J #K #A *D1	 2
 !:$(;K!
 %'*J,'-1FF#**:6	 + #$444  		s   &A" "	A.-A.c                   K   g }	 | j                  |      \  }}t        |t              rg |fS 	 t	        | |       d{   }g }|D ]  }|d   d   |v r|j                  |         ||fS # t        $ r Y Iw xY w7 <w)z
        Returns Tuple of:
        - Tuple[List[Dict], List[Dict]]:
            1. healthy_deployments: list of healthy deployments
            2. all_deployments: list of all deployments
        r  r	  Nr  r  )r  r   r  rr  r2   r   r  s           ry   rJ  z%Router._async_get_healthy_deployments  s      "$	"&"J"J #K #A *D1+++ 2
 'F$(;K'
 !
 %'*J,'-1FF#**:6	 +
 #$444  		!
s3   A?(A. A? A=-A?.	A:7A?9A::A?c                 r    t         j                  D ]$  }t        |t              s|j	                  |       & y)a  
        Mimics 'async_routing_strategy_pre_call_checks'

        Ensures consistent update rpm implementation for 'usage-based-routing-v2'

        Returns:
        - None

        Raises:
        - Rate Limit Exception - If the deployment is over it's tpm/rpm limits
        N)r   r  r   r   rl  )r  rn  rm  s      ry   r  z'Router.routing_strategy_pre_call_checks  s,     !**I)\2((4 +rx   r  c           
      |  K   t         j                  D ].  }t        |t              s	 |j	                  ||       d{    0 y7 # t         j
                  $ r}|t        j                  |j                  |t        j                         t        j                                      t        j                  |j                  |t        j                         f      j                          t!        | |j"                  ||d   d   | j$                         |d}~wt&        $ r}|t        j                  |j                  |t        j                         t        j                                      t        j                  |j                  |t        j                         f      j                          |d}~ww xY ww)O  
        For usage-based-routing-v2, enables running rpm checks before the call is made, inside the semaphore.

        -> makes the calls concurrency-safe, when rpm limits are set for a deployment

        Returns:
        - None

        Raises:
        - Rate Limit Exception - If the deployment is over it's tpm/rpm limits
        Nr  traceback_exceptionr  targetrT  r  r  r\  )r   r  r   r   async_pre_call_checkr7  r  r  async_failure_handlerr  r  r  	threadingThreadfailure_handlerstartr5   r[  r   rr  )r  rn  r  r  rm  rt  s         ry   r  z-Router.async_routing_strategy_pre_call_checks  s}    " !**I)\2(#88EUVVV + W-- ".++'==*+4=4H4H4J)- >  "((#.#>#>"#Y%9%9%;!<  %'-04)*+,#-l#;D#A)-);); G  ".++'==*+4=4H4H4J)- >  "((#.#>#>"#Y%9%9%;!<  %'GsF   $F<AAAF<AF9B<DF9"BF44F99F<r  c           
        K   |}t         j                  D ]2  }t        |t              s	 |j	                  |||||       d{   }4 |S 7 # t
        $ r}	|t        j                  |j                  |	t        j                         t        j                                      t        j                  |j                  |	t        j                         f      j                          |	d}	~	ww xY ww)r  r   r   rv  r  r  Nr  r  )r   r  r   r   async_filter_deploymentsrr  r  r  r  r  r  r  r  r  r  r  )
r  r   r   rv  r  r  r  returned_healthy_deploymentsrm  rt  s
             ry   !async_callback_filter_deploymentsz(Router.async_callback_filter_deployments  s     ( (;$ **I)\2'@@"'0L%-+9-= A   1 +8 ,+1 ! ".++'==*+4=4H4H4J)- >  "((#.#>#>"#Y%9%9%;!<  %'Gs;   &C1AAAC1A	C.BC))C..C1r   c                    |}|j                         D ]  \  }}t        |t              r||z  }n7t        |t              r|t	        j
                  |      z  }n|t        |      z  }t        |t              r||z  }it        |t              r|t	        j
                  |      z  }|t        |      z  } t        j                  |j                               }|j                         S )z
        Helper function to consistently generate the same id for a deployment

        - create a string from all the litellm params
        - hash
        - use hash as id
        )
r  r   r   r  jsondumpshashlibsha256encode	hexdigest)r  r  r   
concat_strr  r  hash_objects          ry   r  zRouter._generate_model_id   s     !
"((*DAq!S!a
At$djjm+
c!f$
!S!a
At$djjm+
c!f$
 + nnZ%6%6%89$$&&rx   deployment_info_model_name_litellm_paramsrd  c                    	 t        di |}t        di ||||d}t        j                  j	                         D ]0  }|j
                  j                  |      |j
                  |   ||<   2 |j                  j                  }|t        j                  ||i       |j
                  j                  }|j
                  j                  |j
                  j                  dz   |z   }t        j                  ||i       | j                  |      dur3t        j                  d|j                    d|j                  d	           y| j#                  |      }|j%                  d
      }	| j&                  j)                  |	       |S # t*        $ r1}
| j,                  rt        j.                  d|
 d       Y d}
~
y|
d}
~
ww xY w)a^  
        Create a deployment object and add it to the model list

        If the deployment is not active for the current environment, it is ignored

        Returns:
        - Deployment: The deployment object
        - None: If the deployment is not active for the current environment (if 'supported_environments' is set in litellm_params)
        r  N)
model_costrY  r  TzIgnoring deployment z% as it is not active for environment supported_environmentsr   zError creating deployment: 1, ignoring and continuing with other deployments.rw   )rQ   rO   rM   model_fieldsr@  r   r$  r  r  r   register_modelr   r  $deployment_is_active_for_environmentr   warningr  _add_deploymentto_jsonr   r   rr  r   r  )r  r  r  r  rd  r   rn  fieldr  r   rt  s              ry   _create_deploymentzRouter._create_deployment<  s    ;	-;-No-NN# !&-&	J 4@@EEG,,007C)3)B)B5)IK& H
 ",,//H#&& +  %3399K((<<H--AACG+U  "" 99Z9P &--*:+@+@*AAfgqg|g|  ~V  hW  gX  Y ---DJ&&D&9EOO""5) 	..%//1!4ef 	s,   AF C*F A F 	G%F<:F<<Gc                 <    |j                   j                  d      ryy)z
        Check if the deployment is an auto-router deployment.

        Returns True if the litellm_params model starts with "auto_router/"
        zauto_router/TF)r   
startswith)r  r   s     ry   _is_auto_router_deploymentz!Router._is_auto_router_deployment  s     **>:rx   c                    ddl m} |j                  j                  }|j                  j                  }||t        d      |j                  j                  }|t        d      |j                  j                  }|t        d       ||j                  |||||       }|j                  | j                  v rt        d|j                   d	      || j                  |j                  <   y)
z
        Initialize the auto-router deployment.

        This will initialize the auto-router and add it to the auto-routers dictionary.
        r   )rn   Nzzauto_router_config_path or auto_router_config is required for auto-router deployments. Please set it in the litellm_paramszfauto_router_default_model is required for auto-router deployments. Please set it in the litellm_paramszhauto_router_embedding_model is required for auto-router deployments. Please set it in the litellm_params)r  auto_router_config_pathauto_router_configdefault_modelembedding_modelr  zAuto-router deployment z3 already exists. Please use a different model name.)
/litellm.router_strategy.auto_router.auto_routerrn   r   r  r  re  auto_router_default_modelauto_router_embedding_modelr  r   )r  rn  rn   r  r  r  r  autor_routers           ry   init_auto_router_deploymentz"Router.init_auto_router_deployment  s    	O1;1J1J1b1b,6,E,E,X,X"*/A/I  Z  [  ['1'@'@'Z'Z   F  G  G)3)B)B)^)^"  H  I  I#-!,,$;1'+$($
   D$5$556z7L7L6M  NA  B  C  C3?*//0rx   c           	      P   |j                   d|j                   vs|j                   d   yt        d      }|t        d      |t        vrt        dt         d|       |j                   d   D ]"  }|t        vst        dt         d| d	|        ||j                   d   v ryy
)a  
        Function to check if a llm deployment is active for a given environment. Allows using the same config.yaml across multople environments

        Requires `LITELLM_ENVIRONMENT` to be set in .env. Valid values for environment:
            - development
            - staging
            - production

        Raises:
        - ValueError: If LITELLM_ENVIRONMENT is not set in .env or not one of the valid values
        - ValueError: If supported_environments is not set in model_info or not one of the valid values
        r  TLITELLM_ENVIRONMENT)secret_namezPSet 'supported_environments' for model but not 'LITELLM_ENVIRONMENT' set in .envz#LITELLM_ENVIRONMENT must be one of z. but set as: z&supported_environments must be one of z for deployment: F)r  r   re  rH   )r  rn  litellm_environment_envs       ry   r  z+Router.deployment_is_active_for_environment  s     !!)'z/D/DD$$%=>F,9NO&b  &@@56P5QQ_`s_tu  ))*BCD55 <=W<XXfgkfll}  I  ~J  K  D *"7"78P"QQrx   c                    t        j                  |      }g | _        |D ]
  }|j                  d      }|j                  d      }t	        |t
              rI|j                         D ]6  \  }}t	        |t              s|j                  d      s)t        |      ||<   8 |j                  di       }d|vr| j                  ||      }	|	|d<   |j                  dd       8t	        |d   t              r%|d   D ]  }
|
|d<   | j                  ||||        | j                  ||||        t        j                  d| j!                                 |D cg c]  }|d   	 c}| _        y c c}w )	Nr  r   os.environ/r  r  organization)r  r  r  rd  z
Initialized Model List )r   r   r   r  r   r  r  r   r  rh   r  r$  r   r  r   r  get_model_namesr|   )r  r   original_model_listr   r  r  r  r  rd  _idorgr  s               ry   r   zRouter.set_model_list  s   "mmJ7 )E))L1K#ii(89O/40+113DAq!!S)all=.I-7]* 4 !&		, ;K ;&--k?K$'D!"">48D/J +>:C69ON3++(-$/(7$/	 ,  ; ''$) +$3 +	 ( 9 )F 	##'(<(<(>'?@	
 6@@AlO@@s   E,c                    dd l }t        j                  |j                  j                  |j                  j                  dd             \  }}}}|t        j                  vrt        d|       | j                  j                  |j                  j                         |j                  j                  (t        |dd       t        |d      |j                  _
        |j                  j                  (t        |dd       t        |d      |j                  _        d|j                  v r{| j                  j                  |j                  |j!                  d	             |j"                  j$                  r/| j&                  j                  |j"                  j$                         |j                  j                  d
g       xs g }|D ]i  }|j                  di       }	dD ]P  }
|
|	v s|	|
   j)                  d      s|	|
   j+                  dd      }|j,                  j                  |d      |	|
<   R k | j/                  |||j                  j                         | j1                  |j                        r| j3                  |       |S )Nr   r  r   r  zUnsupported provider - rpmtpmr   Tr   dataSources
parameters)endpointr  r  r  )rn  r  r   )r   r  )osr   rf   r   r   r$  provider_listrr  r   r   r  r  r  r  r   add_patternr  r  r  r   r  replaceenviron'_initialize_deployment_for_pass_throughr  r  )r  rn  r  r0  r  r  r  data_sourcesdata_sourcer   	param_keyenv_names               ry   r  zRouter._add_deployment  s\    $$++11 * 9 9 = =%t!
	
 g&;&;;56I5JKLL 	$$Z%>%>%D%DE
 %%))1
E40<,3J,FJ%%) %%))1
E40<,3J,FJ%%) *''' ++%%z'9'9t'9'L $$''44;;J<Q<Q<T<TU "0044]BGM2'K __\26F0	&6)+<+G+G+V%i088KH(*

x(DF9%	 1 ( 	44! 3++11 	5 	
 ***:S:S*T,,
,Crx   c                    |j                   j                  du rSddlm} |j                   j                  *t        j                  |j                   j                        }ni }|dk(  r|j                  d      xs |j                   j                  }|j                  d      xs |j                   j                  }|j                  d      xs |j                   j                  }||t        d	      |j                  |||
       nf|j                  d      xs |j                   j                  }	|j                  d      xs |j                   j                  }
|j                  ||	|
       	 y)a*  
        Optional: Initialize deployment for pass-through endpoints if `deployment.litellm_params.use_in_pass_through` is True

        Each provider uses diff .env vars for pass-through endpoints, this helper uses the deployment credentials to set the .env vars for pass-through endpoints
        Tr   )passthrough_endpoint_routerN	vertex_aivertex_projectvertex_locationvertex_credentialsz]vertex_project, and vertex_location must be set in litellm_params for pass-through endpoints.)
project_idlocationr  r  rp  )r  r  rp  )r   use_in_pass_through>litellm.proxy.pass_through_endpoints.llm_passthrough_endpointsr  litellm_credential_namer   get_credential_valuesr$  r  r  r  re  add_vertex_credentialsr  rp  set_pass_through_credentials)r  rn  r  r   r  credential_valuesr  r  r  r  rp  s              ry   r  z.Router._initialize_deployment_for_pass_through^  s    $$88D@ ((@@L$6$L$L--EE%! %'!"k1%))*:; @!00?? 
 &))*;< A!00@@  
 &))*>? D!00CC #
 ")_-D$w  ,BB-,'9 C  &))*5 :!0099 
 &)))4 9!0088  ,HH(;%# I 
 rx   c                    |j                   j                  | j                         v ry|j                  d      }| j	                  |       | j
                  j                  |       | j                  j                  |j                         |S )z
        Parameters:
        - deployment: Deployment - the deployment to be added to the Router

        Returns:
        - The added deployment
        - OR None (if deployment already exists)
        NTr   r  )	r  r  r  r  r  r   r   r|   r  )r  rn  _deployments      ry   add_deploymentzRouter.add_deployment  s~       ##t'9'9';; !((d(;
3 	{+
 5 56rx   c                    	 |j                   j                  xs d}| j                  |      }|x|j                  |j                  k(  ryd}t	        | j
                        D ]'  \  }}|d   d   |j                   j                  k(  s&|}) || j
                  j                  |       | j                  |       |S # t        $ r1}| j                  rt        j                  d| d       Y d}~y|d}~ww xY w)	z
        Add or update deployment
        Parameters:
        - deployment: Deployment - the deployment to be added to the Router

        Returns:
        - The added/updated deployment
        r  r  Nr  r  r  zError upserting deployment: r  )r  r  get_deploymentr   r  r   r  r  rr  r   r   r  )r  rn  _deployment_model_id_deployment_on_routerremoval_idxr	  r   rt  s           ry   r  zRouter.upsert_deployment  s
   !	#-#8#8#;#;#Ar :>:M:M- ;N ;! %0,,0E0T0TT .2"+DOO"<JC\*40J4I4I4L4LL&) #= *OO''4 :6 	..%++21#5fg 	s*   AB< 
<B< 4B< <	C6%C1/C11C6r  c                     d}t        | j                        D ]  \  }}|d   d   |k(  s|} 	 || j                  j                  |      }|S y# t        $ r Y yw xY w)z
        Parameters:
        - id: str - the id of the deployment to be deleted

        Returns:
        - The deleted deployment
        - OR None (if deleted deployment not found)
        Nr  r  )r  r   r  rr  )r  r  deployment_idxr	  r  rO  s         ry   delete_deploymentzRouter.delete_deployment  so     0FCt$*!$ 1	)**>: 		s   A 	AAr  c                     | j                   D ]n  }d|v sd|d   v s||d   d   k(  st        |t              rt        di |c S t        |t              r|c S t	        dj                  t        |                   y)l
        Returns -> Deployment or None

        Raise Exception -> if model found in invalid format
        r  r  zModel invalid format - {}Nrw   r   r   r  rO   rr  r	  r   )r  r  r   s      ry   r  zRouter.get_deployment  s     __Eu$|1D)Du\2488!%.)2E22#E:6$'(C(J(J4PU;(WXX % rx   c                     | j                  |      }|yt        di |j                  j                  d      j                  d      S )zE
        Returns -> dict of credentials for a given model id
        r  NTr   rw   )r  rL   r   r
  )r  r  rn  s      ry   get_deployment_credentialsz!Router.get_deployment_credentials  sU     ((((;
& 
''222E

*$*
'	(rx   model_group_namec                     | j                   D ]^  }|d   |k(  st        |t              rt        di |c S t        |t              r|c S t	        dj                  t        |                   y)r  r  zModel Name invalid - {}Nrw   r  )r  r  r   s      ry   "get_deployment_by_model_group_namez)Router.get_deployment_by_model_group_name  sg     __E\"&66eT*%...z2 L#$=$D$DT%[$QRR % rx   received_model_namec                      y r5  rw   r  rn  r  r  s       ry   get_router_model_infozRouter.get_router_model_info#       	rx   c                      y r5  rw   r  s       ry   r  zRouter.get_router_model_info)  r   rx   c                 
   |&| j                  |      }||j                  d      }|t        d      |j                  di       j                  dd      }|"|j                  di       j                  dd      }|}t	        j
                  |j                  di       j                  d	d
      t        di |j                  di             \  }}}	}	|dk(  r|t        j                  d       n|dk7  r|}| j                  j                  |      }
d|v rp|
n|
D ]i  }	 |j                  di       j                  d      |j                  di       j                  d      k(  r#|j                  di       j                  d	      } nk |j                  dj                  |            sdj                  ||      }n|}t	        j                  |      }|j                  di       }|j                  |       |S # t        $ r Y w xY w)a  
        For a given model id, return the model info (max tokens, input cost, output cost, etc.).

        Augment litellm info with additional params set in `model_info`.

        For azure models, ignore the `model:`. Only set max tokens, cost values if base_model is set.

        Returns
        - ModelInfo - If found -> typed dict with max tokens, input cost, etc.

        Raises:
        - ValueError -> If model is not mapped yet
        Nr  Tr   zDeployment not foundr  
base_modelr   r   r  r   r   azurezCould not identify azure model. Set azure 'base_model' for accurate max tokens, cost tracking, etc.- https://docs.litellm.ai/docs/proxy/cost_tracking#spend-tracking-for-azure-openai-modelsr   r  z{}/z{}/{}r  rw   )r  r
  re  r$  r   rf   rQ   r   r  r   routerr  r  r	  get_model_infor   )r  rn  r  r  r  r  r   r0  r  r  potential_modelspotential_modelmodel_info_namer  user_model_infos                  ry   r  zRouter.get_router_model_info/  s*   & >--r-:K&(333F
344  ^^L"599,M
#(8"=AA,PTUJ -4,D,D..!126::7BG)QJNN;KR,PQ-
)#Q ')j.@!'' O !G+E#22889LMe| 0 <'7O	*..|R@DD '^^L"=AA$GH %4$7$78H"$M$Q$Q '%E "H (8 -@ AB%nn-@%HO#O++/B
 %..r:/*! % s   A$G66	HHc                 ^    | j                   D ]  }d|v sd|d   v s||d   d   k(  s|c S  y)z
        For a given model id, return the model info

        Returns
        - dict: the model in list with 'model_name', 'litellm_params', Optional['model_info']
        - None: could not find deployment in list
        r  r  Nr   )r  r  r   s      ry   r  zRouter.get_model_info|  sB     __Eu$|1D)D|,T22 L % rx   c                 Z    | j                  |      }|y|d   }| j                  |      S )zT
        Return list of all models in the same model group as that model id
        )r  Nr  rX  )r  rZ  )r  r  r  r  s       ry   get_model_groupzRouter.get_model_group  s>    
 ((B(/
-
""j"99rx   r  c           	      B   ddl m} d}d}d}	 t        j                  j	                  |      }	 t        j                  |      }|7|5t        t         |t        t        |      j                         |            }|S ||}|S # t
        $ r Y aw xY w# t
        $ r Y Yw xY w)z
        For a given model id, return the model info

        1. Check if model_id is in model info
        2. If not, check if litellm model name is in model info
        3. If not, return None
        r   )_update_dictionaryNr  )litellm.utilsr  r   r  r$  rr  r  r   r_   r  r   )r  r  r  r  r  custom_model_infolitellm_model_name_model_infos          ry   get_deployment_model_infoz Router.get_deployment_model_info  s     	5*.
,0=A%	 ' 2 2 6 6x @	,3,B,B,T) (-J-V"<=BBD%J  +66J'  		
  		s"   B B 	BB	BBuser_facing_model_group_namec                 	   d}d}d}d}| j                  |      }|y|D ]/  }d}	d|v r|d   |k(  rd}	n!d|v r| j                  j                  |      d}	|	s9t        di |d   }
|
j                  }d}||j                  dd      }|"|j                  di       j                  dd      }|"|j                  di       j                  dd      }d}||j                  d	d      }|"|j                  di       j                  d	d      }|"|j                  di       j                  d	d      }	 |j                  di       j                  d
d      }|| j                  ||
j                        }nd}d\  }}	 t        j                  |
j                  |
j                        \  }}}}|0t        j$                  ||      }|g }t'        |ddddd|d|d
      }|t)        di ||gd|}nG||j*                  vr|j*                  j-                  |       |j                  dd      -|d   (|j.                  |d   |j.                  kD  r
|d   |_        |j                  dd      -|d   (|j0                  |d   |j0                  kD  r
|d   |_        |j                  dd      (|j2                  |d   |j2                  kD  r
|d   |_        |j                  dd      (|j4                  |d   |j4                  kD  r
|d   |_        |j                  dd      	 |d   du rd|_        |j                  dd      |d   du rd|_        |j                  dd      |d   du rd|_        |j                  dd      |d   du rd|_        |j                  dd      |d   du rd|_        |j                  dd      |d   du rd|_         |j                  dd      |d   
|d   |_!        |j                  dd      ||j                  d      }|j                  d	d      ||j                  d	      }|	|d}||z  }|'|d}||z  }2 ||||_"        |||_#        |||_        |S # t        $ r d}Y w xY w# t        j                  j                  $ r8}t        j                  dj!                  t#        |                   Y d}~*d}~ww xY w)z
        For a given model group name, return the combined model info

        Returns:
        - ModelGroupInfo if able to construct a model group
        - None if error constructing model group info
        NrX  Fr  Tr   r  r  r  r  )r  r  )r  r  r  z.litellm.router.py::get_model_group_info() - {}r   r   )
r  
max_tokensmax_input_tokensmax_output_tokensinput_cost_per_tokenoutput_cost_per_tokenlitellm_providermodesupported_openai_paramssupports_system_messages)r  	providersr  r  r  r  "supports_parallel_function_callingsupports_visionsupports_function_callingsupports_web_searchsupports_url_contextsupports_reasoningr  rw   )$rZ  r   r  rQ   #configurable_clientside_auth_paramsr$  r  r   rr  r   rf   r  
exceptionsBadRequestErrorr   r  r	  r   get_supported_openai_paramsModelMapInforS   r!  r   r  r  r  r  r"  r#  r$  r%  r&  r'  r  r  r  )r  r  r  model_group_info	total_tpm	total_rpmr(  r   r   is_matchr   _deployment_tpm_deployment_rpmr  r  r\  r  r  rt  r  s                       ry   _set_model_group_infozRouter._set_model_group_info  s-    6:#'	#'	SW+((K(@
EH%%*=*L%''--k:F+Fe4D.EFN BB 0 .2O&"'))E4"8&"')),<b"A"E"EeT"R&"'))L""="A"A%"N .2O&"'))E4"8&"')),<b"A"E"EeT"R&"'))L""="A"A%"N	" 99\26::4F'!%!?!?!)n6J6J "@ "J "&J
 +1'M<4;4L4L(..(6(J(J51|Q !*1*M*M'\+' +2.0+)##%)&*)**+%1,C-1
  '#1 $'C&2^ %$   '7'A'AA$..55lCNN#5t<H"#56B(99A%&89*;;< 9CCU8V$5NN#6=I"#67C(::B%&9:*<<= :DDW9X$6>>"8$?K$99A!"89&;;< =G.=$9 >>"94@L$::B!"9:&<<= >H/>$: NN#GN "#GHDPJN$GNN#4d;G"#45=7;$4NN#>EQ"#>?4GAE$>NN#8$?K"#89TA;?$8NN#94@L"#9:dB<@$9 NN#7>J"#78D@:>$7NN#<dCO"#<=I?I1@$< >>%.:?V&0nnU&;O>>%.:?V&0nnU&;O*$ !I_,	*$ !I_,	w  x '$'0 $$'0 $ 3>7 !D  w  "!
" %%55 %++DKKCPQFS s+   0AQ2:0R2R RS!-SSc                     || j                   v rT| j                   |   }t        |t              r|}nt        |t              r|d   du ry|d   }ny| j	                  ||      S | j	                  ||      S )z
        For a given model group name, return the combined model info

        Returns:
        - ModelGroupInfo if able to construct a model group
        - None if error constructing model group info or hidden model group
        hiddenTNr   )r  r  )r   r   r   r  r3  )r  r  rO  _router_model_groups       ry   get_model_group_infozRouter.get_model_group_info  s     $000))+6D$$&*#D$'>T)*.w-'--/-8 .   ))#+ * 
 	
rx   c                   K   t               }|j                  d      }g }g }| j                  |      }|y|D ]  }|j                  di       j                  d      }|d   j                  d      }	||	=|j	                  t
        j                  j                  j                  ||	|	             |j	                  t
        j                  j                  j                  ||	|	              ||z   }
| j                  j                  |

       d{   }|y|dt        |       }|t        |      d }d}|!|D ]  }t        |t              s|d}||z  } d}|!|D ]  }t        |t              s|d}||z  } ||fS 7 qw)z
        Returns current tpm/rpm usage for model group

        Parameters:
        - model_group: str - the received model name from the user (can be a wildcard route).

        Returns:
        - usage: Tuple[tpm, rpm]
        rJ  rX  Nr  r  r  r   r   )r  r   rL  )r@  r   )ri   rP  rZ  r$  r   rV   rQ  r+  r	  rS  r   async_batch_get_cacherf  r   r   )r  r  rV  rL  tpm_keysrpm_keysr   r   r  r\  combined_tpm_rpm_keyscombined_tpm_rpm_valuestpm_usage_listrpm_usage_list	tpm_usager-  	rpm_usages                    ry   get_model_group_usagezRouter.get_model_group_usage  s     
 ! ((K(@
E %		, ; ? ? EB+01A+B+F+F,M z]2OO##))00'#1 1  OO##))00'#1 1   * !)8 3(,

(H(H& )I )
 #
 #*)@3x=)Q)@X)Q $(	%#a% ($%	NI	 $ $(	%#a% ($%	NI	 $
 )##3#
s   DFF<F$F5F)maxsizec                 $    | j                  |      S )z
        Cached version of get_model_group_info, uses @lru_cache wrapper

        This is a speed optimization, since set_response_headers makes a call to get_model_group_info on every request
        )r7  )r  r  s     ry   _cached_get_model_group_infoz#Router._cached_get_model_group_info  s     ((55rx   c                 <  K   | j                  |      }||j                  |j                  }nd }||j                  |j                  }nd }||i S | j                  |       d {   \  }}i }|||xs dz
  |d<   ||d<   |||xs dz
  |d<   ||d<   |S 7 1w)Nr   x-ratelimit-remaining-tokenszx-ratelimit-limit-tokensx-ratelimit-remaining-requestszx-ratelimit-limit-requests)rE  r  r  rB  )r  r  r-  	tpm_limit	rpm_limitcurrent_tpmcurrent_rpmreturned_dicts           ry   get_remaining_model_group_usagez&Router.get_remaining_model_group_usage  s     <<[I',<,@,@,L(,,II',<,@,@,L(,,II!2I)-)C)CK)P#P [ <E q=M89 9BM45 >G q?M:; ;DM67 $Qs   A&B(B)2Bc                 ~  K   t        |t              rt        |d      rt        |j                  t              r|j                  j                  di        ||j                  d   d<   |j                  d   }d|vr?d|vr;|9| j                  |       d{   }|j                         D ]  \  }}|	|||<    |S 7 &w)a  
        Add the most accurate rate limit headers for a given model response.

        ## TODO: add model group rate limit headers
        # - if healthy_deployments > 1, return model group rate limit headers
        # - else return the model's rate limit headers
        r8  rF  zx-litellm-model-grouprG  rH  N)r   r   r  r8  r  r   rN  r  )r  r~  r  rF  remaining_usageheaderr+  s          ry   r4  zRouter.set_response_headers.  s      x+"23822D9##../CRH  ##$89' "*!8!89M!N /6HH4<NN+(,(L(L) # &5%:%:%<MFE(5:*62 &= #s   BB=B;B=2
B=exclude_team_modelsc                     g }| j                   D ]^  }d|v sd|d   v s|d   d   }|r|d   j                  d      r/||d   |k(  r|j                  |       K|N|j                  |       ` |S )z\
        if 'model_name' is none, returns all.

        Returns list of model id's.
        r  r  team_idr  )r   r$  r   )r  r  rR  idsr   r  s         ry   r  zRouter.get_model_idsR  s     __Eu$|1D)D<(.&5+>+B+B9+M)eL.AZ.OJJrN'JJrN % 
rx   team_model_namerT  c                     | j                   D ]<  }|d   j                  d      }|d   j                  d      }||k(  s1||k(  s7|d   c S  y)z
        Map a team model name to a team-specific model name.

        Returns:
        - team_model_name: str - the team-specific model name
        - None: if no team-specific model name is found
        r  rT  team_public_model_namer  N)r   r$  )r  rV  rT  r   model_team_idmodel_team_public_model_names         ry   map_team_modelzRouter.map_team_modelf  sb     __E!,/33I>M+0+>+B+B(,( (0OC\** % rx   c                 |    |/|d   j                  d      |k(  r||d   j                  d      k(  ry|	|d   |k(  ryy)zU
        Get the team-specific model name if team_id matches the deployment.
        r  rT  rX  Tr  Fr$  )r  r  r   rT  s       ry   should_include_deploymentz Router.should_include_deploymentz  sV     l#''	2g=eL1556NOO#l(;z(Irx   model_aliasc                     g }| j                   D ]V  }| j                  |||      s|,t        j                  |      }||d<   |j	                  |       F|j	                  |       X |S )ze
        Return all deployments of a model name

        Used for accurate 'get_model_list'.
        )r  r   rT  r  )r   r^  r   r   r   )r  r  r_  rT  returned_modelsr   alias_models          ry   _get_all_deploymentszRouter._get_all_deployments  su     68__E--%UG .  *"&--"6K0;K-#**;7#**51 % rx   c                    | j                         xs g }g }|D ]m  }|j                  d      }| j                  |      r(| j                  ||      }|s;|j	                  |       M|j	                  |j                  dd             o |S )u  
        Returns all possible model names for the router, including models defined via model_group_alias.

        If a team_id is provided, only deployments configured with that team_id (i.e. team‐specific models)
        will yield their team public name.
        r  )rn  rT  r  r  )rZ  r$  _is_team_specific_model_get_team_specific_modelr   )r  rT  deploymentsr|   rn  r  rV  s          ry   r  zRouter.get_model_names  s     ))+1r%J#5J++J7"&"?"?)7 #@ # #&&7"":>>,#CD & rx   c                 ~    |j                  d      xs i }|y||j                  d      k(  r|j                  d      S y)a  
        Get the team-specific model name if team_id matches the deployment.

        Args:
            deployment: DeploymentTypedDict - The model deployment
            team_id: Optional[str] - If passed, will return router models set with a `team_id` matching the passed `team_id`.

        Returns:
            str: The `team_public_model_name` if team_id matches
            None: If team_id doesn't match or no team info exists
        r  NrT  rX  r]  )r  rn  rT  r  s       ry   rf  zRouter._get_team_specific_model  sD     &0^^L%A%GR
jnnY//>>":;;rx   r  c                 >    t        |xr |j                  d            S )z
        Check if model info contains team-specific configuration.

        Args:
            model_info: Model information dictionary

        Returns:
            bool: True if model has team-specific configuration
        rT  )r4  r$  )r  r  s     ry   re  zRouter._is_team_specific_model  s     J<:>>)#<==rx   c                    g }| j                   j                         D ]l  \  }}|||k7  rt        |t              r|}n*t        |t              rt        di |}|d   du rD|d   }nK|j                  | j                  ||             n |S )z
        Helper function to get model list from model alias.

        Used by `.get_model_list` to get model list from model alias.
        r5  Tr   )r  r_  rw   )r   r  r   r   r  rX   r  rc  )r  r  ra  r_  model_value_router_model_name_model_values          ry   get_model_list_from_model_aliasz&Router.get_model_list_from_model_alias  s     68(,(>(>(D(D(F$K%+*C+s+*5"K.8G;G)T1)5g)>&""))1{ *  )G( rx   c                    t        | d      rg }|"|j                  | j                  ||             t        | d      r!|j                  | j                  |             t	        |      dk(  rG| j
                  j                  |      }|*|(|D ]#  }t        di |}||d<   |j                  |       % ||| j                  z  }|S |S y)	z
        Includes router model_group_alias'es as well

        if team_id specified, returns matching team-specific models
        r   N)r  rT  r   rX  r   r  rw   )
r  r  rc  rn  rf  r   r  rP   r   r   )r  r  rT  ra  potential_wildcard_modelsr  deployment_typed_dicts          ry   rZ  zRouter.get_model_list  s     4&9;O%&&--W-U t01&&88J8O ?#q(,0,?,?,E,Ej,Q)).G.S60C0Ha0H->H-l;'../DE 7
 !4??2&&""rx   model_access_groupc                 4   ddl m}  |t              }| j                  |      }|rq|D ]l  }|j	                  d      }|s|j	                  dg       xs g D ]=  }| ||k(  s|d   }||   j                  |       %|d   }||   j                  |       ? n |S )a  
        If model_name is provided, only return access groups for that model.

        Parameters:
        - model_name: Optional[str] - the received model name from the user (can be a wildcard route). If set, will only return access groups for that model.
        - model_access_group: Optional[str] - the received model access group from the user. If set, will only return models for that access group.
        r   r   rX  r  r   r  )collectionsr   r   rZ  r$  r   )	r  r  rr  r   r   r   r  rd  groups	            ry   get_model_access_groupszRouter.get_model_access_groups  s     	,#D)((J(?
eeL1!,"!E!K!K-9$(::-.|_
 -e 4 ; ;J G)*<J)%077
C "L   rx   c                     | j                  |      }t        |      dk(  ry|j                  |g       }|D ]   }| j                  j	                  |        y y)zG
        Return True if model access group is a wildcard route
        )rr  r   FrA  T)rv  rf  r$  r   r  )r  rr  r   r  r   s        ry   )_is_model_access_group_for_wildcard_routez0Router._is_model_access_group_for_wildcard_route=  sm     441 5 
 }"""#5r:E""(((7C 
 rx   c                     t        |       }i }g d}|D ]K  }||v r||   ||<   |dk(  s| j                  dk(  s%| j                  j                  j	                         ||<   M |S )a  
        Get router settings method, returns a dictionary of the settings and their values.
        For example get the set values for routing_strategy_args, routing_strategy, allowed_fails, cooldown_time, num_retries, timeout, max_retries, retry_after
        )r   r   r   r   r   r   r   r   r   r   r   r   r   r   )varsr   r1  r)  r  )r  	_all_vars_settings_to_returnvars_to_includevars        ry   get_settingszRouter.get_settingsT  s}    
 J	 
 #Ci+4S>#C(..))-DD+/+D+D+Q+Q+V+V+X#C( # #"rx   c                    g d}g d}| j                         }|D ]  }||v rg||v rt        ||         }t        | ||       '|dk(  r1|d   ||   k7  r&| j                  ||   |j	                  di              t        | |||          nt        j                  dj                  |              t        j                  d| j                                 y)	z-
        Update the router settings.
        )r   r   r   r   r   r   r   r   r   r   r   )r   r   r   r   r   r   r   r   zSetting {} is not allowedzUpdated Router settings: N)r  r   rM  r   r$  r   r  r	  )r  rz  _allowed_settings_int_settings_existing_router_settingsr~  _casted_values          ry   update_settingszRouter.update_settingst  s    


 %)$5$5$7!C''-'$'s$4MD#}5 1156HIVTW[X22-3C[28** 73 3  D#vc{3%++,G,N,Ns,ST' ( 	##&?@Q@Q@S?T$UVrx   c                 |   |d   d   }t        |      }|dk(  rhdj                  |      }| j                  j                  |d|      }|5t	        j
                  | |       | j                  j                  |d|      }|S |dk(  r]|j                  d	      du r%| d
}| j                  j                  |d|      }|S | d}| j                  j                  |d|      }|S |j                  d	      du r$| d}| j                  j                  ||      }|S | d}| j                  j                  ||      }|S )a  
        Returns the appropriate client based on the given deployment, kwargs, and client_type.

        Parameters:
            deployment (dict): The deployment dictionary containing the clients.
            kwargs (dict): The keyword arguments passed to the function.
            client_type (str): The type of client to return.

        Returns:
            The appropriate client based on the given client_type and kwargs.
        r  r  r  z{}_max_parallel_requests_clientTr  rs  r  )r  r   r  r  _stream_async_client_async_client_stream_client)r  r  _client)r   r	  r   rt  r-    set_max_parallel_requests_clientr$  )r  rn  rz  r  r  r  	cache_keyr  s           ry   r  zRouter._get_client  s    l+D1+LV+T119@@JIZZ))$AQ * F ~%FF,0
 --!dEU .  MG#zz(#t+'j(<=	--!dEU .  'j6	--!dEU .  zz(#t+'j7	--!4D .  'j0	--!4D .  rx   c           	      	   t        j                  d|        t        j                  |      }g }	 t	        j
                  |      }d}	d}
d}t        |      }t               }|j                  d      }| d| }| j                  j                  |d	|
      xs i }t        |      D ]  \  }}	 |j!                  di       j!                  dd      }|"|j!                  di       j!                  dd      }| j#                  ||      }|xs" |j!                  di       j!                  dd      }t%        |t&              rZ|j!                  dd      Ht%        |d   t(              r5||d   kD  r-|j+                  |       d	}	|
dj                  ||d   |      z  }
|j!                  di       }|j!                  di       j!                  dd      }| j                  j                  |d	|
      xs d}t%        |t&              r| j.                  dk7  rv|j!                  |d      ||<   t1        |||         }t%        |t&              rB|j!                  dd      0t%        |d   t(              r|d   |k  r|j+                  |       d	}|M|j!                  d      <|j!                  d      }|)t3        t5        d!i ||      s|j+                  |       +|/t        j6                  du sCt	        j8                  |t5        d!i |      \  }}}}t	        j:                  ||      }|t        j<                  j?                  |      }dg}|jA                         D ]8  \  }} ||vs||v st        j                  d|        |j+                  |       :  tC        |      tC        |      k(  r<	 |d	u rtE        |      |	d	u r't	        jF                  dj                  |
      |d       tC        |      dkD  r!tI        |      D ]  }|jK                  |        tC        |      dkD  rt        j<                  jM                  |      }|S # t        $ r9}t        j                  dj                  t        |                   |cY d}~S d}~ww xY w# t        $ r8}t        j,                  dj                  t        |                   Y d}~5d}~ww xY w)"a  
        Filter out model in model group, if:

        - model context window < message length. For azure openai models, requires 'base_model' is set. - https://docs.litellm.ai/docs/proxy/cost_tracking#spend-tracking-for-azure-openai-models
        - filter models above rpm limits
        - if region given, filter out models not in that region / unknown region
        - [TODO] function call and model doesn't support function calling
        z2Starting Pre-call checks for deployments in model=)rv  zllitellm.router.py::_pre_call_checks: failed to count tokens. Returning initial list of deployments. Got - {}NFr  rJ  z:rpm:Tr  r  r  r   )rn  r  r   r  z%Model={}, Max Input Tokens={}, Got={}zAn error occurs - {}r  r   r   r  allowed_model_region)r   r  r  r  )passed_paramsresponse_formatz1INVALID MODEL INDEX @ REQUEST KWARG FILTERING, k=r  z~litellm._pre_call_checks: Context Window exceeded for given call. No models have context window large enough for this call.
{}r  rw   )'r   r  r   r   r   token_counterrr  r  r	  r   r   ri   rP  r   rt  r  r$  r  r   r  r   r   r  r   maxrj   rQ   drop_paramsrf   r+  rb  get_non_default_paramsr  rf  rZ   r  reversedr  _get_order_filtered_deployments)!r  r   r   rv  r  _returned_deploymentsinvalid_model_indicesinput_tokensrt  _context_window_error_potential_error_str_rate_limit_errorr  rV  rL  rX  model_group_cacher	  rn  r  r  r  r  current_request_cache_localcurrent_requestr  r  r  r  ri  special_paramsr  r  s!                                    ry   _pre_call_checkszRouter._pre_call_checks  se     	##@H	
 !%.A B "	)"00(CL !&!!<^L W-G5 01JJ  ?O !   	 	  ))>?OCW'^^L"=AA,PTU
%!+0@"!E!I!I$d"J "77)u 8 
 # jnn5Er&J&N&NT'
 z40"'94@L #:.@#A3G(:6H+II-44S904-,CJJ %z2D'E|,
 ! )nn-=rBO!~~lB7;;D"EH 

$$ TDT %   	 ( ,d3))-EE.?.C.CHa.P!(+"%/1B81L#
 5'++E48D #?5#93?+E2oE-44S9,0)  *"&&'=>J'5'9'9:P'Q$'3,'5'H'H-A .44S9  )g.A.AU.J3:3K3K0Q0Q40*Aq +2*M*M5H+' +2 *1)M)M&4 *N *& '8%8N 2 8 8 :1$;;^@S177"STUSV W 288= !;M  @\ $%-B)CC !D(/  '$.88 ^  e  e,  !#  $%) 56%))#. 7 $%)$+MM$Q$Q%%! %$G  	)!''~  F  FF
 )(	)l  W%//0F0M0McRSf0UVVWs6   Q /C(R	R.R	R	R	S-SSc                 t    || j                   vry| j                   |   }t        |t              r|}|S |d   }|S )z
        Get the model from the alias.

        Returns:
        - str, the litellm model name
        - None, if model is not in model group alias
        Nr   )r   r   r   )r  r   _items      ry   r  zRouter._get_model_from_alias  sK     ...&&u-eS!E  'NErx   c                 X    | j                   D cg c]  }|d   d   |k(  s| c}S c c}w )z6
        Get the deployment by litellm model.
        r   r   r  )r  r   r  s      ry    _get_deployment_by_litellm_modelz'Router._get_deployment_by_litellm_model  s.      ??Taa0@.A'.Je.STTTs   ''r  c                    |du r|| j                  |      fS || j                         v rY| j                  |      }|*|j                  j                  }||j                  d      fS t        d| d| j                         | j                  |      }||}|| j                  vrY| j                  j                  |      }|r||fS | j                  +t        j                  | j                        }	||	d   d<   ||	fS | j                  |	      }
t        |
      d
k(  r| j                  |      }
t!        j"                  d|
        t        |
      d
k(  rU| j%                  |	      d| dj'                  |      }nd| dj'                  |      }t)        j*                  ||d      t(        j,                  r%|t(        j,                  v rt(        j,                  |   }||
fS )aF  
        Common checks for 'get_available_deployment' across sync + async call.

        If 'healthy_deployments' returned is None, this means the user chose a specific deployment

        Returns
        - str, the litellm model name
        - List, if multiple models chosen
        - Dict, if specific model chosen
        Tr  r  r   zBLiteLLM Router: Trying to call specific deployment, but Model ID :z6 does not exist in                     Model ID List: r   r   rX  r   zinitial list of deployments: zYou passed in model=z+. There is no 'model_name' with this stringz1. There are no healthy deployments for this modelr  r  )r  r  r  r   r   r
  re  r  r|   r   get_deployments_by_patternr   r   r   rc  rf  r   r  rZ  r	  r   r*  model_alias_map)r  r   rv  r  r  rn  deployment_model_model_from_aliaspattern_deploymentsupdated_deploymentr   r  s               ry   r  z*Router._common_checks_available_deployment  sW   $ $&$??e?LLLd((**,,e,<J%#-#<#<#B#B ')>)>D)>)QQQTUZT[ \$$($6$6#79 
 !66U6C(%E((("&"5"5"P"P #Q # #111 &&2%)]]++&" AF"#34W=000 #7757I"#q("&"G"Ge"G"T##+,?+@A	
 "#q(""e"4<07bcjj 17hipp ))  ""u0G0G'G++E )))rx   c                   K   ddl m} | j                  ||||      \  }} |||      }t        |t              r|S t        | |       d{   }	t        j                  d|	        t        j                  d|	        | j                  ||		      }| j                  |||t        t        t           |      nd||
       d{   }| j                  r,|*| j                  |t        t        t           |      ||      }t!        | |||       d{   }t#        |      dk(  rt%        | ||       d{   }
|
|S 7 7 }7 07 w)z
        Get the healthy deployments for a model.

        Returns:
        - List[Dict], if multiple models chosen
        *OR*
        - Dict, if specific model chosen
        r   )filter_team_based_modelsr   rv  r  r  )r   r  r	  Nzasync cooldown deployments: zcooldown_deployments: r   cooldown_deploymentsr  r   r   rv  r  )llm_router_instancer   r  r   r  r   r  )r  r  r  r   r  r2   r   r  _filter_cooldown_deploymentsr  r   r
   rC   r   r  r	   r'   rf  r;   )r  r   r  rv  r  r  r  r  r   r  r  s              ry   r  z$Router.async_get_healthy_deployments	  s    " 	O%)%M%M 3	 &N &
"" 7 3)

 )40&&%D$(;K&
  
 	##*+?*@A	
 	##&<=Q<R$ST"?? 3!5 @ 

 %)$J$J 3:B:NT*+X6TX)- %K %
 
 &&8+?"&"7"7$(d5H$I!-	 #8 # %< $) 3	%
 
 "#q(A(,!1 I
 O""[ 

$
sJ   AEEA9EEAEE#E:E	;	EEE	Ec           
        K   | j                   dk7  rR| j                   dk7  rC| j                   dk7  r4| j                   dk7  r%| j                   dk7  r| j                  |||||      S 	 t        |      }| j                  |||||       d{   }||j                  }|j
                  }| j                  ||||||	       d{   }t        |t              r|S t        j                         }	| j                   dk(  r4| j                  (| j                  j                  ||||
       d{   }
n| j                   dk(  r4| j                  (| j                  j                  ||||
       d{   }
n| j                   dk(  r5| j                  )| j                  j                  |||||       d{   }
n`| j                   dk(  rt        | ||      S | j                   dk(  r2| j                  &| j                  j                  ||       d{   }
nd}
|
t!        | ||       d{   }|t#        j$                  d| d| j'                  |
       d|        t        j                         }||	z
  }t)        j*                  | j,                  j/                  t0        j2                  |d||	|             |
S 7 27 7 7 W7 7 7 # t4        $ r}t7        j8                         }|j|j;                  dd      }|Vt=        j>                  |j@                  ||f      jC                          t)        j*                  |jE                  ||             |d}~ww xY ww)z
        Async implementation of 'get_available_deployments'.

        Allows all cache calls to be made async => 10x perf impact (8rps -> 100 rps).
        r   r   r   r   r   )r   rv  r  r  r  r   r  rv  r  r  N)r   r  rv  r  r  r  r  r   rv  r  )r  r   rv  r  r  r  r   r   r  r   r  $get_available_deployment for model: , Selected deployment:  for model: z2<routing_strategy>.async_get_available_deployments)r  r  rB  r  r  r  r  r  )#r   r  r   async_pre_routing_hookr   rv  r  r   r  r  r/  async_get_available_deploymentsr3  r1  r&   r   r;   r   r  ru  r  r  r  r  r\   r  rr  r  r  r$  r  r  r  r  r  )r  r   r  rv  r  r  r  pre_routing_hook_responser   r  rn  r  r  r  rt  r  r  s                    ry   r  z%Router.async_get_available_deployment\  s     !!%==%%)99%%)==%%)@@%%500!$7- 1  A	@P /3.I.I-!$7 /J / )% )41774==
 )-(J(J-!$7!1 )K ) # -t4**J%%)AA,,8 22RR$),?!)#	 S    %%)==**6 00PP$),?!)#	 Q    %%)@@--9 33SS$),?!)#'5 T    &&*::%(,(;  %%5))5 //OO$),? P    "
!"E,0%5# 	
  !&&6ug=TUYUjUjkuUvTw  xD  EJ  DK  L yy{H :-I''BB(//&R%5)% C 	 S)#"&0  	"+"6"6"8),001FM*$$*::!45 eg''#99!=PQ G!	s   A"M!%$K 	J;
8K J>K M!AK *K+AK -K.AK 1K2!K M!;K K
K *K+BK :M!;K >K K K K 
K K 	MBMMM!c                    K   || j                   v r+| j                   |   j                  |||||       d{   S y7 w)z
        This hook is called before the routing decision is made.

        Used for the litellm auto-router to modify the request before the routing decision is made.
        r  N)r   r  )r  r   r  rv  r  r  s         ry   r  zRouter.async_pre_routing_hook  sX       D%%%**51HH-!$7 I    s   5?=?c                    | j                  ||||      \  }}t        |t              r|S t        |      }t	        | |      }| j                  ||      }| j                  r|| j                  ||||      }t        |      dk(  rU| j                  |      }	| j                  j                  |	|      }
t	        | |      }t        ||
| j                  |	      | j                  d
k(  r*| j                  | j                  j                  ||      }n| j                  dk(  rt!        | ||      S | j                  dk(  r+| j"                  | j"                  j                  |||      }nx| j                  dk(  r,| j$                   | j$                  j                  ||||      }n=| j                  dk(  r,| j&                   | j&                  j                  ||||      }nd}|nt)        j*                  d| d       | j                  |      }	| j                  j                  |	|      }
t	        | |      }t        ||
| j                  |	      t)        j*                  d| d| j-                  |       d|        |S )zB
        Returns the deployment based on routing strategy
        r  r	  r  Nr  r   rX  )	model_idsr  )r   r   r   cooldown_listr   r  r   r  r   )r  r   r  r   r  r   r  z, No deployment availabler  r  )r  r   r  r   r4   r  r   r  rf  r  r   get_min_cooldownrY   r   r   get_available_deploymentsr&   r1  r   r/  r   r  ru  )r  r   rv  r  r  r  r   r  r  r  _cooldown_time_cooldown_listrn  s                ry   r  zRouter.get_available_deployment  s8    &*%M%M 3	 &N &
"" )40&&+L,
  9$(;K 
 #?? 3!5 @ 
 &&8+?"&"7"7$7!-	 #8 # "#q(**e*<I!00AA#6F B N 7(,?ON ','+'B'B,	    L0T5J5J5V..HH!7J I J ""&66 "$($7  !!%<<))522LL!$7- M J !!%::%%1..HH!$7!	 I J !!%==((411KK!$7!	 L J J!&&6ug=VW **e*<I!00AA#6F B N 7(,?ON ','+'B'B,	  	""25'9PQUQfQfgqQrPss  AF  @G  H	
 rx   r  c                     g }t        j                  d|        |D ]   }|d   d   }||v s|j                  |       " |D ]  }|j                  |        |S )a  
        Filters out the deployments currently cooling down from the list of healthy deployments

        Args:
            healthy_deployments: List of healthy deployments
            cooldown_deployments: List of model_ids cooling down. cooldown_deployments is a list of model_id's cooling down, cooldown_deployments = ["16700539-b3cd-42f4-b426-6a12a1bb706a", "16700539-b3cd-42f4-b426-7899"]

        Returns:
            List of healthy deployments
        zcooldown deployments: r  r  )r   r  r   r*  )r  r   r  deployments_to_removern  rO  s         ry   r  z#Router._filter_cooldown_deployments  sr     !###&<=Q<R$ST-J&|4T:M 44%,,Z8 . 0J&&z2 0""rx   c                     	 |j                  di       j                  dd      }||| j                  ||       yyy# t        $ r+}t        j                  dt        |              Y d}~yd}~ww xY w)z7
        Tracks successful requests rpm usage.
        r  r  Nz$Error in _track_deployment_metrics: )r$  rw  rr  r   r  r   )r  rn  r  r~  r  rt  s         ry   r  z Router._track_deployment_metrics  s    		Y!~~lB7;;D$GH'&& "2 (    	Y!''*NsSTvh(WXX	Ys   8= 	A1!A,,A1r  c                 H    t        ||| j                  | j                        S )N)r  r  r   r   )"_get_num_retries_from_retry_policyr   r   )r  r  r  s      ry   r:   z(Router.get_num_retries_from_retry_policy  s)     2#%)%B%B**	
 	
rx   c                    | j                   }|yt        |t        j                        r|j                  |j                  S t        |t        j
                        r|j                  |j                  S t        |t        j                        r|j                  |j                  S t        |t        j                        r|j                  |j                  S t        |t        j                        r|j                  |j                  S yy)a  
        BadRequestErrorRetries: Optional[int] = None
        AuthenticationErrorRetries: Optional[int] = None
        TimeoutErrorRetries: Optional[int] = None
        RateLimitErrorRetries: Optional[int] = None
        ContentPolicyViolationErrorRetries: Optional[int] = None
        N)r   r   r   r*  BadRequestErrorAllowedFailsr;  AuthenticationErrorAllowedFailsr  TimeoutErrorAllowedFailsr7  RateLimitErrorAllowedFailsr  'ContentPolicyViolationErrorAllowedFails)r  r  r   s      ry   get_allowed_fails_from_policyz$Router.get_allowed_fails_from_policy  s     >B=V=V' y'"9"9:$@@L'CCCy'"="=>$DDP'GGGy'//2$==I'@@@y'"8"89$??K'BBBy'"E"EF$LLX'OOO Y Grx   c                 J   ddl m} | j                  y | j                  } ||j                  dg|j                        }|| _        t        j                  j                  |       t        j                  j                  |j                         t        j                  d       y )Nr   )SlackAlertingslack)alerting_thresholdalertingdefault_webhook_urlz2[94m
Initialized Alerting for litellm.Router[0m
)1litellm.integrations.SlackAlerting.slack_alertingr  r   r  webhook_urlslack_alerting_loggerr   r   r,  r   !response_taking_too_long_callbackr   r  )r  r  router_alerting_config_slack_alerting_loggers       ry   r  zRouter._initialize_alerting  s    S'151E1E!.5HHY 6 B B"
 &<"((==>TU((EE"DD	
 	""H	
rx   CustomRoutingStrategyc                 `    t        | d|j                         t        | d|j                         y)a  
        Sets get_available_deployment and async_get_available_deployment on an instanced of litellm.Router

        Use this to set your custom routing strategy

        Args:
            CustomRoutingStrategy: litellm.router.CustomRoutingStrategyBase
        r  r  N)rM  r  r  )r  r  s     ry   set_custom_routing_strategyz"Router.set_custom_routing_strategy  s4     	&!::	

 	,!@@	
rx   c                 N    d t         _        | j                  j                          y r5  )r   r   flush_cacher?  s    ry   r  zRouter.flush_cache"  s    

 rx   c                     g t         _        g t         _        g t         _        g t         _        d | _        | j                          y r5  )r   r   r   r   r   r   r  r?  s    ry   resetzRouter.reset&  s6    #% *,'#% *,' rx   )F)r   r5  )FFF)T)r  r  )NNNN)NNNNN)NF)NNF)NNFN)rs   rt   ru   r|   r
   __annotations__r}   r   r4  r~   r   tenacityr   r!   r   r$   r   r   r   r   r   rW   rP   r	   r   rK   r  r  floatr   rX   rU   rJ   rT   r]   rI   r  r  r!  staticmethodr   r   r   r&  r[   r   r  r  r   r  ru  rc   ra   r  r{  r   rC   r  r  r|  r  rO   r  r  r  r  r  r  r  r  r   r  r   rV  r  r  rs  rq  rz  ry  rD   r  r~  r  r  r  r  r  r  r  r  r  r  r  rb   r  r  r  r  rE   r  r  r^   r  r  r  r  r  r6  r  r  r   r  r   wrapr  r
  r  r$  r#  rr  r%  r}  r  r'  r   r   r  r   r&  rm   rw  rz  r  r  rJ  r  rj  r  r  r  r  rQ   r  r  r  r   r   r  r  r  r  r  r  r  r  r,  r  r  r  r_   r  rS   r3  r7  rB  r   r   rE  rN  r4  r  r[  r^  rc  r  rf  re  rn  rZ  rv  rx  r  r  r  r  r  r  r  r  r  ro   r  r  r  r  r:   r  r  rN   r  r  r  rw   rx   ry   r{   r{      s   K&+OXd^+&11H:>h67>:>h67>MQeL(C,G&H!IJQ ;?#'$($((,*/ ,0*.%) #'*. 7;!06 )+)+ ',%*     ,0 DH&(DH48 "#+0[g&$*+T$sCx.-AAB
g& $$78g& C=g& SMg& SMg& !g& "$g& g& !K
g&" #g&& #5/'g&( #3-)g&, c]-g&.  
/g&4 %5g&6 !7g&8 !)!
9g&> (0}?g&@ Ag&B _-Cg&D $I
Eg&J Kg&L #'Mg&N #'Og&P $eC!::;;<
Qg&V !%Wg&X #Yg&Z [g&\ +t#$
]g&b #'#
cg&h  
ig&n '
og&t  
ug&z $D>{g&| "&
}g&L #++@"AMg&N  $Og&P !))@ AQg&R ".1Sg&T "*!"
Ug&Z %)[g&\ 
]g&R	 D 	.38n	.	z,,	-	. 	.+ +? %os&: ;?TX?B	F]
~$ U(01F(GU24 ($(c3h$8	}11	2&AA$(c3h$8A	}11	2AJ $()9$:DKDM	 
 UZ$()9$:DKEN	 
 kp$()9$:DI'RV-Y`afYgJgDh	"M1	2  	33 '(3 	3jII$(c3h$8I	}11	2I^ 1;	

 
 !)	

 

, EOPP4<SMP	P*''(,'	'B (,	+
+
 +
  }	+

 
+
Z t 4

"&
	%s
#	$
"&	%s
#	$$	4 	t 	uczAR8S 	Z#S	Z# T#s(^,d4S#X3G.HHIZ#x++$(.>)?$@+^ $(c3h$8BI$-	  SX$(c3h$8BI%.	  	K,K, tCH~&K, 	K,b di$()9$:FISZ[`Sa	 
 $()9$:FISZ[_S`	  << '(< 	<|;; ; $	;
 CHo; S#X;z  $MBMB '(MB S#X	MB^s 3 , ,C ,\c # *Fs F3 FP( (3 (TD) DC DLT3 Ts T3 Tl3 *)C )^ $)&+#(## # 4.	#
 d^# 4.#R $)&+#(## # 4.	#
 d^# 4.#JDS D# DT $)&+#(  4.	
 d^ 4.:DS D DL-5>MM5=M^;;-5;B $)	 S$Y 4.	 
$6c4i 0 6 6x $(	 S$Y 4.	 
2DuS$Y'7 D DN 
	0@@ 
	@D 
	>NN 
	Nd  $t}t 
	tl//l .21#1 &c]1p =}#} #
}D .21#1 &c]1
#
2 .2(,	
#
 &c]
 %	
, V[[]Y% Y%| &*$(3737-- c]- D>	-
 #+4.- #+4.-^ V[[]J% J%X   :>!!)1#!L /3*.3737,0@@ &d^@ "$	@
 #+4.@ #+4.@ $D>@DW &*$S$s)^,-$ c]$ 
$s)		$< /3*.// / 	/
 &d^/ "$/ 
sEz	/fPd! 
#!FG 
GR 
+3C= 
D# # #t #J 4<TN	6 $$$1$;?$	$L5c 5Xd^ 5.55,4TN5	tDz4:%	&5<54 5( 15	;; #4.; n-	;F *.041,1, "$Z1, 4 012	1,
 #4.1, !1, n-1,f'c '4 '8KK K 	K
 K 
*	KZ D @j @@&z &d &P+A +AZT* T Tl:$:;>:GJ:x 8L 0*J *8J;O *XC HZ,@ .s x
/C "	(3 	(8D> 	( #	*	$ EI58>B	 
 58>A	  !	KTNK !K SM	K
 
KZ $ 
:# 
:(4. 
:%%),%	)	%N\ \ >A\ 	.	!\ |
 
8P 
>G$G$	x}hsm+	,G$R 1266	.	!6 36 cSVh B ;?""*23-"	"J MR"3-EI	c(c C HSM * FJ&*5=c]	& &*!%	 c] #	
 
!	"2x} S	 0 IM-8@	#*
>(4. 
>T 
> +/"3-	!	"@ JN#"3-#9A##	$*+	,#L UY"3-DLSM	c49n	<"%	.#@0Wd4v *.}%}% "}% tCH~&	}%
 !}%~3 8C= &Uc Ud U 48,0.3S*S* 4S#X/0S* c4i()	S*
 &d^S* 
sE$*%%	&S*r 48,0.3+/Q#Q# Q# 4S#X/0	Q#
 c4i()Q# &d^Q# #4.Q# 
tDz4	 Q#n 48,0.3\\ \ 4S#X/0	\
 c4i()\ &d^\D 48,0.3  4S#X/0	
 c4i() &d^ 
(	)B 48,0.3)-{{ 4S#X/0{ c4i()	{
 &d^{ !{z##':#EI#Y#	d#8 FJY,4TNY$ BF
"
19#
&Py &PP
0
%>
.!rx   r{   )r  r   enumr  r(  r  r   r  r  r  r  rt  r   	functoolsr   typingr   r   r   r	   r
   r   r   r   r   r   httpxr   r   pydanticr   typing_extensionsr   r   litellm.litellm_core_utils2litellm.litellm_core_utils.exception_mapping_utilsr   litellm._loggingr   litellm.caching.cachingr   r   r   r   litellm.constantsr   "litellm.integrations.custom_loggerr   #litellm.litellm_core_utils.asyncifyr   'litellm.litellm_core_utils.core_helpersr   .litellm.litellm_core_utils.credential_accessorr   %litellm.litellm_core_utils.dd_tracingr   *litellm.litellm_core_utils.litellm_loggingr   rj  &litellm.router_strategy.budget_limiterr    "litellm.router_strategy.least_busyr!   #litellm.router_strategy.lowest_costr"   &litellm.router_strategy.lowest_latencyr#   &litellm.router_strategy.lowest_tpm_rpmr$   )litellm.router_strategy.lowest_tpm_rpm_v2r%   &litellm.router_strategy.simple_shuffler&   )litellm.router_strategy.tag_based_routingr'   /litellm.router_utils.add_retry_fallback_headersr(   r)    litellm.router_utils.batch_utilsr*   r+   r,   /litellm.router_utils.client_initalization_utilsr-   2litellm.router_utils.clientside_credential_handlerr.   r/   #litellm.router_utils.cooldown_cacher0   &litellm.router_utils.cooldown_handlersr1   r2   r3   r4   r5   ,litellm.router_utils.fallback_event_handlersr6   r7   r8   >litellm.router_utils.forward_clientside_headers_by_model_groupr9   *litellm.router_utils.get_retry_from_policyr:   r  !litellm.router_utils.handle_errorr;   r<   Dlitellm.router_utils.pre_call_checks.prompt_caching_deployment_checkr=   Clitellm.router_utils.pre_call_checks.responses_api_deployment_checkr>   >litellm.router_utils.router_callbacks.track_deployment_metricsr?   r@   litellm.schedulerrA   rB   litellm.types.llms.openairC   rD   rE   rF   litellm.types.routerrG   rH   rI   rJ   rK   rL   rM   rN   rO   rP   rQ   rR   rS   rT   rU   rV   rW   rX   rY   rZ   r[   litellm.types.servicesr\   litellm.types.utilsr]   r^   r_   r,  r`   r  ra   rb   rc   rd   re   rf   rg   rh   ri   rj   &router_utils.pattern_match_deploymentsrl   opentelemetry.tracerm   _Spanr  rn   ro   Enumrq   r{   rw   rx   ry   <module>r     s              #        &  ! 9 " 2  9 ; B U M 8 P G F H N J P A M 
 R >  
 2      . 0 E ) 9 6   G1
 DDJ $)) ui uirx   