h-: dZddlZddlZddlmZmZmZmZmZddl Z ddl m Z ddl m Z mZmZddlmZddlmZmZerdd lmZdd lmZeZeeefZneZeZ dd ed ed eeefdeedef dZ dd edeed eeefdedee def dZ!d eded eeefdedef dZ" dd eded eeefdeedee def dZ#d edeedeefdZ$d edeedee%fdZ&d edeedeefdZ'd edededefdZ(d edefdZ)d eeefdefdZ*y) a Router cooldown handlers - _set_cooldown_deployments: puts a deployment in the cooldown list - get_cooldown_deployments: returns the list of deployments in the cooldown list - async_get_cooldown_deployments: ASYNC: returns the list of deployments in the cooldown list N) TYPE_CHECKINGAnyListOptionalUnion)verbose_router_logger)DEFAULT_COOLDOWN_TIME_SECONDS!DEFAULT_FAILURE_THRESHOLD_PERCENT+SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD)router_cooldown_event_callback)*get_deployment_failures_for_current_minute+get_deployment_successes_for_current_minute)Span)Routerlitellm_router_instancemodel_idexception_status exception_strreturnc dg}| |D]}||vsyt|tr t|}|dk\r|dkr|dk(ry|dk(ry|dk(ry|d k(ryyy#t$rYywxYw) a> A function to determine if a cooldown is required based on the exception status. Parameters: model_id (str) The id of the model in the model list exception_status (Union[str, int]): The status of the exception. Returns: bool: True if a cooldown is required, False otherwise. APIConnectionErrorFiTiii) isinstancestrint Exception)rrrrignored_stringsignored_strings b/var/www/Befach/backend/env/lib/python3.12/site-packages/litellm/router_utils/cooldown_handlers.py_is_cooldown_requiredr"'s %/0  %"1!]2 #2 & ,"#34  s "'7#'=3&!S(!S(!S( s,AA+AAA A AA deploymentoriginal_exceptiontime_to_cooldownc||j|tjdy|.tj|ddrtjdy|j rtjdy|tjd yt |||t| stjd y||jvrtjd yy )a Helper that decides if cooldown logic should be run Returns False if cooldown logic should not be run Does not run cooldown logic when: - router.disable_cooldowns is True - deployment is None - _is_cooldown_required() returns False - deployment is in litellm_router_instance.provider_default_deployment_ids - exception_status is not one that should be immediately retried (e.g. 401) idzSShould Not Run Cooldown Logic: deployment id is none or model group can't be found.Fg& .>)ababs_tolz@Should Not Run Cooldown Logic: time_to_cooldown is effectively 0z8Should Not Run Cooldown Logic: disable_cooldowns is Truez1Should Not Run Cooldown Logic: deployment is None)rrrrzCShould Not Run Cooldown Logic: _is_cooldown_required returned FalsezOShould Not Run Cooldown Logic: deployment is in provider_default_deployment_idsT) get_model_grouprdebugmathisclosedisable_cooldownsr"rprovider_default_deployment_idsrr#rr$r%s r!_should_run_cooldown_logicr4_s&  " 2 2j 2 A I## a  #  ) ##$fg00## F ##$WX 7),-   ## Q ,LLL## ]  c|j|}d}|t|dk(rd}|jt|durt ||}t ||}||z}d} |dkDr|||zz } t jd || ||t|} | d k(r|sy| d k(r |tk\ry| tkDr|sytjt| duryyt||| S)a_ Helper that decides if a deployment should be put in cooldown Returns True if the deployment should be put in cooldown Returns False if the deployment should not be put in cooldown Deployment is put in cooldown when: - v2 logic (Current): cooldown if: - got a 429 error from LLM API - if %fails/%(successes + fails) > ALLOWED_FAILURE_RATE_PER_MINUTE - got 401 Auth error, 404 NotFounder - checked by litellm._should_retry() - v1 logic (Legacy): if allowed fails or allowed fail policy set, coolsdown if num fails in this minute > allowed fails r'Fr Tr)r deployment_idr)rzYpercent fails for deployment = %s, percent fails = %s, num successes = %s, num fails = %srg?) status_code)rr#r$)r-lenallowed_fails_policy_is_allowed_fails_set_on_routerrrrr.cast_exception_status_to_intr r litellm _should_retry-should_cooldown_based_on_allowed_fails_policy) rr#rr$ model_group is_single_deployment_model_groupnum_successes_this_minutenum_fails_this_minutetotal_requests_this_minute percent_failsexception_status_ints r!_should_cooldown_deploymentrHsM2*99Z9HK',$3{#3q#8+/(44< +$;    %P$;:% !!K$;:! &?AV%V" % )1),AAM ## g   % !   <s(557I%44OO-P  #%#  -  1 $ q)5 17F&Gr!u&G#&G"<_RO#%#  -  1 $ q)5 17F&Gr!u&G#&G &&'Hs' A7c|j|xs |j}|jxst}|jj |xsd}|dz}||kDry|jj |||y)z Check if fails are within the allowed limit and update the number of fails. Returns: - True if fails exceed the allowed limit (should cooldown) - False if fails are within the allowed limit (should not cooldown) ) exception)keyrr T)rfvaluettlF)get_allowed_fails_from_policy allowed_failsrJr failed_calls get_cache set_cache)rr#r$rjrJ current_fails updated_failss r!r@r@s ==( >  1 # 0 0  --N1N,88BBzBRWVWM!A%M}$,,66-] 7  r5cZ|jy|jtjk7ryy)z Check if Router.allowed_fails is set or is Non-default Value Returns: - True if Router.allowed_fails is set or is Non-default Value - False if Router.allowed_fails is None or is Default Value FT)rjr>r7s r!r<r<s-,,4,,0E0EE r5ct|tr t|}|S|S#t$rt j d|dd}Y|SwxYw)Nz'Unable to cast exception status to int z. Defaulting to status=500.r)rrrrrr.)rs r!r=r=sh"C( #"#34     # ! ' '9:J9KKfg  #    #s !$A A )N)NN)+__doc__rMr/typingrrrrrr>litellm._loggingrlitellm.constantsr r r 'litellm.router_utils.cooldown_callbacksr )router_callbacks.track_deployment_metricsrropentelemetry.tracer_Spanlitellm.routerr_Router LitellmRouterrrboolr"floatr4rHrOr^rYr`rcr@r<r=r5r!rs_ <<2 S 10M  DM D$( 5*55CHo5C= 5  5z)- A*A ACHoA A uo A  AHY*YYCHoY Y  Y@!%(, <*<<CHo< < uo <  <~'*'tn' #Y'8*tn %[$'*'>Ftn' #Y': *    F* " 5c? s r5