hp,ddlZddlZddlZddlZddlZddlZddlZddlmZ ddl m Z m Z m Z mZddlZddlZddlmZe rddlmZne Zej*j-ej.dZej2ddZedk(reddlmZdZGd d eZd efd Z Gd dZ!ejDejFddddejFddddejFddddejFdddejFdd d!ejFd"d#dd$ejFd%dd&ejFd'dd$ejFd(dd)ejFd*d+e$d,-ejFd.d/d+e$d0d12ejFd3d/d+e$d4d52ejFd6d/d+e$d78ejFd9de%d:;ejFdde&d?;ejFd@d+dABejFdCd+dDBejFdEdFddGejFdHde%dI;ejFdJd+e$dK;ejFdLdedM;ejFdNdOd/d+e$dP8ejFdQd+dRSejFdTd+dUSejFdVd/d+dWXejFdYd/d+dZXejFd[d\e&d];ejFd^d/d+d_XejFd`d/d+daXejFdbdedcddeejFdfdedgdheejFdidedj;ejFdkd+d/dlmejFdnd+d/domejFdpd+d/dqmejFdrde&dsdteduZ'e(dvk(re'yy)wN) TYPE_CHECKINGAnyOptionalUnion) load_dotenv)FastAPIzlitellm.secrets LITELLM_MODEDEV)EnumceZdZdZdZy)LiteLLMDatabaseConnectionPool <N)__name__ __module__ __qualname__database_connection_pool_limit database_connection_pool_timeoutS/var/www/Befach/backend/env/lib/python3.12/site-packages/litellm/proxy/proxy_cli.pyr r !s%'"')$rr returncdddlm}|jd||jd|tj|}tj|j }|j |tj|d}tj|j|}|S)Nr)verbose_proxy_loggerzurl: zparams: T)doseq)query) litellm._loggingrdebugurlparseparse_qsrupdate urlencode urlunparse_replace)urlparamsr parsed_url parsed_query encoded_query modified_urls rappend_query_paramsr+&s5se}-&23""3'J$$Z%5%56L&&|4@M&&z':':':'OPL rceZdZedZedZededededee effdZ e ddeded e ed e ed e f d Z e dd ededededede ef dZededed edededef dZedZedZedZy)ProxyInitializationHelperscttjjd}tjd|dy)Nlitellmz LiteLLM: Current Version =  ) importlibmetadataversionclickecho) pkg_versions r_echo_litellm_versionz0ProxyInitializationHelpers._echo_litellm_version4s.((00;  2;-rBCrctdtjd|d|d}ttj|jdy)Nz) LiteLLM: Health Testing models in confighttp://:z/health)r%)indent)printhttpxgetjsondumps)hostportresponses r_run_health_checkz,ProxyInitializationHelpers._run_health_check9sA :;9974&$w!?@ djj34rrBrCmodeltestcZ|xsd}tjd|ddl}d|d|}t|tr|}n t d|j d|}|jjj|d d d gd }tjd|td||jjj|d d d gd} | D]} tjd| td|jj|d } t| y)Nz gpt-3.5-turbozE LiteLLM: Making a test ChatCompletions request to your proxy. Model=rr9r:zInvalid test valuez My API Key)api_keybase_urluserz*this is a test request, write a short poem)rolecontent)rFmessages max_tokensz LiteLLM: response from proxy zN LiteLLM: Making a test ChatCompletions + streaming r equest to proxy. Model=T)rFrOstreamz'LiteLLM: streaming response from proxy z$ making completion request to proxy)rFprompt) r4r5openai isinstancestr ValueErrorOpenAIchat completionscreater=) rBrCrFrG request_modelrSapi_baseclientrDstream_responsechunkcompletion_responses r_run_test_chat_completionz4ProxyInitializationHelpers._run_test_chat_completion?s^0  TUbTc d  TF!D6* dC H12 2|hG;;**11#K 2    4XJ?@ ]^k]l m !++1188#K 9  %E JJ@H I% 56$0077(T8  !"rN log_configkeepalive_timeoutrcddl}d||d}|td|||d<n|jrtdd|d<|||d<|S) z8 Get the arguments for `uvicorn` worker rNzlitellm.proxy.proxy_server:app)apprBrCzUsing log_config: rbz,Using json logs. Setting log_config to None.timeout_keep_alive)r/r= json_logs)rBrCrbrcr/ uvicorn_argss r_get_default_unvicorn_init_argsz:ProxyInitializationHelpers._get_default_unvicorn_init_argsusn 4  ! &zl3 4)3L &    @ A)-L &  (1BL- .rressl_certfile_pathssl_keyfile_pathcipherscddl}ddlm}ddlm}t d|d|d|} |d|g| _|+|)t d|d |d || _|| _||| _ |j||| y) z5 Initialize litellm with `hypercorn` rN)serve)Config)LiteLLM Proxy: Starting server on r:z using Hypercorn /LiteLLM Proxy: Using SSL with certfile:  and keyfile:  ) asynciohypercorn.asynciornhypercorn.configror=bindcertfilekeyfilerlrun) rerBrCrjrkrlrtrnroconfigs r_init_hypercorn_serverz1ProxyInitializationHelpers._init_hypercorn_servers ++ :4&$G` a q'(  (-=-I DEVDWWefvewxAB 0FO-FN"!(  E#v&'r num_workersc Dtjdk(rnddl}Gfddjjj }t d|dd|d |d|d d d d dd}||t d|d|d||d<||d<|||jy)z- Run litellm with `gunicorn` ntrNc0eZdZdfd ZdZdZxZS)NProxyInitializationHelpers._run_gunicorn_server..StandaloneApplicationc|xsi|_||_t| dd}|dz}t t dt d|dt dt ddy) Nz curl --location 'http://0.0.0.0:z/chat/completions' \ai --header 'Content-Type: application/json' \ --data ' { "model": "gpt-3.5-turbo", "messages": [ { "role": "user", "content": "what llm are you" } ] }' zLiteLLM: Test your local proxy with: "litellm --test" This runs an openai.ChatCompletion request to your proxy [In a new terminal tab] z8LiteLLM: Curl Command Test for your local proxy z  z;Docs: https://docs.litellm.ai/docs/simple_proxy z5See all Router/Swagger docs on http://0.0.0.0:)options applicationsuper__init__r=)selfrer _endpoint_str curl_command __class__rCs rrzWProxyInitializationHelpers._run_gunicorn_server..StandaloneApplication.__init__s&}" #&  "7tfycc}}wN)cfgritemssettingssetlower)rkeyvaluer{s r load_configzZProxyInitializationHelpers._run_gunicorn_server..StandaloneApplication.load_configs88'+/,,*<*<*>&C$(("3"338IU F  F"(,,.JCxx+ SYY[%8#1s'B'c|jSr)r)rs rloadzSProxyInitializationHelpers._run_gunicorn_server..StandaloneApplication.loads'''rr)rrrrrr __classcell__)rrCs@rStandaloneApplicationrs$ L 9 (rrrpr:z with z workers zuvicorn.workers.UvicornWorkerT-iXz+%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s)rwworkers worker_classpreload accesslogtimeoutaccess_log_formatrqrrrsrxry)rer)osnamegunicorn.app.baserebaseBaseApplicationr=rz) rBrCrer}rjrkgunicornrgunicorn_optionss ` r_run_gunicorn_serverz/ProxyInitializationHelpers._run_gunicorn_servers 77d?  $7 (HLL$5$5$E$E7 (r :4&$vk]Zk l fAdV$";!N   (-=-I DEVDWWefvewxAB ,= Z (*: Y '#/?@DDFrc ddg}ttjd5}tj|||dddy#1swYyxYw#t $r}t d|dYd}~yd}~wwxYw)Nollamarnw)stdoutstderrzh LiteLLM Warning: proxy started with `ollama` model `ollama serve` failed with Exceptionz-. Ensure you run `ollama serve` )openrdevnull subprocessPopen Exceptionr=)commandres r_run_ollama_servez,ProxyInitializationHelpers._run_ollama_serves )Gbjj#& J'  I J J J  iijhkl    s3AAAA A A A0A++A0cddl}|j|j|j5}|jd|fdk(cdddS#1swYyxYw)Nr localhost)socketAF_INET SOCK_STREAM connect_ex)rCrss r_is_port_in_usez*ProxyInitializationHelpers._is_port_in_use"sJ ]]6>>6+=+= > :!<<d 349 : : :s A  Ac*tjdvryy)zBHelper function to determine the event loop type based on platform)win32cygwincliNuvloop)sysplatformrrr_get_loop_typez)ProxyInitializationHelpers._get_loop_type)s <<5 5r)NNr)rrr staticmethodr7rErUintrboolrardictrirr|rrrrrrrr-r-3sDD55 3#3#3#3#D#I 3#3#j%)+/ SM$C=   4"& (  ( ( ( (  ( # ( (D^G^G^G^G ^G  ^G  ^G^G@  :: rr-z--hostz0.0.0.0z!Host for the server to listen on.HOST)defaulthelpenvvarz--portzPort to bind the server to.PORTz --num_workerszONumber of uvicorn / gunicorn workers to spin up. By default, 1 uvicorn is used. NUM_WORKERSz --api_basez API base URL.)rrz --api_versionz2024-07-01-previewz$For azure - pass in the api version.z--modelz-mz)The model name to pass to litellm expectsz--aliaszThe alias for the model - use this to give a litellm model name (e.g. "huggingface/codellama/CodeLlama-7b-Instruct-hf") a more user-friendly name ("codellama")z --add_keyz --headerszheaders for the API callz--saveTzSave the model-specific config)is_flagtyperz--debugFzTo debug the inputDEBUG)rrrrrz--detailed_debugzTo view detailed debug logsDETAILED_DEBUGz --use_queuez)To use celery workers for async endpoints)rrrrz --temperaturezSet temperature for the model)rrrz --max_tokenszSet max tokens for the modelz--request_timeoutz+Set timeout in seconds for completion callsz --drop_paramszDrop any unmapped params)rrz--add_function_to_promptz5If function passed but unsupported, pass it as promptz--configz-cz]Path to the proxy configuration file (e.g. config.yaml). Usage `litellm --config config.yaml`z --max_budgetz`Set max budget for API calls - works for hosted models like OpenAI, TogetherAI, Anthropic, etc.`z --telemetryzZHelps us know if people are using this feature. Turn this off by doing `--telemetry False`z --log_configz&Path to the logging configuration filez --versionz-vzPrint LiteLLM versionz--healthz:Make a chat/completions request to all llms in config.yaml) flag_valuerz--testz4proxy chat completions url to make a test request toz --test_asyncz9Calls async endpoints /queue/requests and /queue/response)rrrz--iam_token_db_authz!Connects to RDS DB with IAM tokenz--num_requestsrz-Number of requests to hit async endpoint withz--run_gunicornzTStarts proxy via gunicorn, instead of uvicorn (better for managing multiple workers)z--run_hypercornz@Starts proxy via hypercorn, instead of uvicorn (supports HTTP/2)z--ssl_keyfile_pathz^Path to the SSL keyfile. Use this when you want to provide SSL certificate when starting proxySSL_KEYFILE_PATH)rrrrz--ssl_certfile_pathz_Path to the SSL certfile. Use this when you want to provide SSL certificate when starting proxySSL_CERTFILE_PATHz --ciphersz!Ciphers to use for the SSL setup.z--use_prisma_migratezHUse prisma migrate instead of prisma db push for database schema updates)rrrz--localzfor local debuggingz--skip_server_startupzASkip starting the server after setup (useful for migrations only)z--keepalive_timeoutzSSet the uvicorn keepalive timeout in seconds (uvicorn timeout_keep_alive parameter)KEEPALIVE_TIMEOUTc% nt}%|r ddlm}&m}'m}(m})n ddlm}&m}'m}(m})|durtjy|rd|vr|tj|durtj||y|durtj||||y|rtj|}|)dYid |d |d |d |d | d| d| d| d| d|d|d|d|d|d|d|d| ddl}+d},d}-i}.|rddlm}/t)j*d}0t)j*d}1t)j*d }2t)j*d!}3t)j*d"}4|/|0|1|2#}5d$|2d%|5d&|0d%|1d'|3 }6|4r|6d(|4z }6|6t(j,d)<d*t(j,d+<dd,lm}7t)j*d-dJt)j*d-d*k(r2|7}8|8j3D]\}9}:|:t(j,|9<| ddl};|'}<|;j7||>&d1|>vr"|>d1durddl}?d|?_|?jA|=j;d2i}.|.i}.|.r#|.j;d3d}@|d dd?l3m4}Lt)j*d)d.|,|-d@}MLd)d=}Btk|B|M}N|Nt(j,d)<t)j*d>d9|,|-d@}Mt)j*d>}Btk|B|M}N|Nt(j,d><tmj6dAgdBd}OOrDddDl8m9}PddEl:m;}Qm<}R|R|.j;dFdCur PdGnQj{|"Hn t}dI|dJk(r+tj|rtjdKdL}ddl}?| dur|?jddMlCm}(|#r t}dNytj|||!|$O}S|dCurV|dCurR||t}dP|dQ|dR|SdS<||SdT<tj}T|TrTSdU<|+j6dYiSdV|iy|durtj|||(|||Wy|durtj|(||||| Xyy#t $r}*t d|*dd}*~*wt$r&}*dt|*vr|*ddlm}&m}'m}(m})Yd}*~*Fd}*~*wwxYw#t"$r tdwxYw#t"$r td.wxYw#tn$rdC}OYwxYw)ZNr)KeyManagementSettings ProxyConfigresave_worker_configrzMissing dependency z$. Run `pip install 'litellm[proxy]'`zlitellm[proxy]TrrFaliasr\ api_versionrdetailed_debug temperaturerPrequest_timeout max_budget telemetry drop_paramsadd_function_to_promptheaderssaver{ use_queuezLuvicorn, gunicorn needs to be imported. Run - `pip install 'litellm[proxy]'`dr)generate_iam_auth_token DATABASE_HOST DATABASE_PORT DATABASE_USER DATABASE_NAMEDATABASE_SCHEMA)db_hostdb_portdb_userz postgresql://r:@/z?schema= DATABASE_URLTrueIAM_TOKEN_DB_AUTH)decrypt_env_var USE_AWS_KMSz?yaml needs to be imported. Run - `pip install 'litellm[proxy]'`)config_file_pathlitellm_settingsrggeneral_settingskey_management_systemkey_management_settings database_urlDATABASE_USERNAMEDATABASE_PASSWORDrrz os.environ/z../..)get_secret_str) default_value DIRECT_URL) get_secret)connection_limit pool_timeoutprisma)capture_outputF)check_prisma_schema_diff) PrismaManagershould_update_prisma_schemadisable_prisma_schema_update)db_url) use_migratezYUnable to connect to DB. DATABASE_URL found in environment, but prisma package not found.rii)rez>LiteLLM: Setup complete. Skipping server startup as requested.)rBrCrbrcrqrrrs ssl_keyfile ssl_certfileloopr)rBrCrer}rjrk)rerBrCrjrkrlr)Hlocals proxy_serverrrrerModuleNotFoundError ImportErrorrUr-r7rrErar@loadsuvicornr litellm.proxy.auth.rds_iam_tokenrrgetenvenviron*litellm.secret_managers.aws_secret_managerrrrtrz get_configr?r/rg _turn_on_jsoninitialize_secret_manager_key_management_settingsurllibparse quote_plusr rrr startswithgetcwdrpathinsertabspathrchdirrTlitellm.secret_managers.mainrr+rFileNotFoundError litellm.proxy.db.check_migrationr litellm.proxy.db.prisma_clientr r setup_databaser=rrandomrandint_turn_on_debuglitellm.proxy.proxy_serverrirrr|)UrBrCr\rrFradd_keyrrrrrrPrrrr{rrrGlocalr} test_asynciam_token_db_auth num_requestsrhealthr3 run_gunicorn run_hypercornrkrjrlrbuse_prisma_migrateskip_server_startuprcargsrrrerrrdb_connection_pool_limitdb_connection_timeoutrrrrrdb_name db_schematoken_db_urlr new_env_varkvrt proxy_config_configrr/rrr database_hostdatabase_usernamedatabase_password database_namedatabase_username_encdatabase_password_encdatabase_name_enc original_dirrrr&r*is_prisma_runnabler r r rh loop_typesU r run_serverrS0s| 8D      ,$"88: U"x'7"446 ~"44T4@ t|"<* IInd + 7yyt,8 +C99^T2>-E(=F$.nD#QL#6|V#LL1=BJJ~.99\40<-E(=F$&99\#:L#6|V#LL/;BJJ|,z$?%)""U 0(,,-KL -D9!00=O0Po 4<6FFtL>>$.D T !  " " $ 3  P  1QQ!/ R 5 ]e%; ,1A1MHIZH[[ijzi{|EF/? ]+/@ ^,2AACI'0 V$ GKK  # T ! & ; ;'"3!1 < d " & = ="3!1 > #O # %%aS(LM  3q6) ^ ^  l !U `% +%*" +sN \(%]5^ B3^%( ]21] ]2 ]--]25^  ^"% ^43^4__main__))r1r@rr/rr urllib.parser!r"rtypingrrrrr4r>dotenvrfastapirr&appendr%config_filenamer litellm_modeenumr rr rUr+r-roptionrfloatrrSrrrrr_st  66 G  #ryy/ 5M  *D*  {{z  i&IRXh+HQWX  Z   lD?  /  tT(S  k $Ok4.HI h46VW         &     4 T4SDs1O  6  ot2LM   @   h     k    e    1      E   ?    D     ,    8     _     K    i    j    ,     S  iu;PQ    L    ^  f R              N XJ  @ YnfR  zLr