hAczUddlZddlZddlZddlZddlZddlZddlZddlZddlZddl m Z ddl m Z ddlm Z ddl mZmZmZmZmZddlmZddlmZdd lmZmZmZdd lmZmZmZmZdd lm Z m!Z!dd l"m#Z#m$Z$m%Z%dd l&m'Z'ddl(m)Z)ddl*m+Z+erddl,m-Z-ej\e/Z0dZ1dZ2gdZ3ee4e5d<dZ6dZ7ddddddddddde8dee8e fde8dee8dee9d eeee8e8fd!eeee8e8fd"ee4d#e9d$e4fd%Z:Gd&d'ejvZd*e>ddde8de8de8f d+Z?d*e>d,eeeN)datetime)Path)Lock) TYPE_CHECKINGListOptionalTupleUnion)quote) constants)CommitOperationAdd UploadInfo_fetch_upload_modes)LocalUploadFileMetadataLocalUploadFilePathsget_local_upload_pathsread_upload_metadata)DEFAULT_REVISION REPO_TYPES)DEFAULT_IGNORE_PATTERNSfilter_repo_objectstqdm) _format_size)is_xet_available) sha_fileobj)HfApi d) 2Kr}iiXiCOMMIT_SIZE_SCALET<)revisionprivateallow_patternsignore_patterns num_workers print_reportprint_report_everyapirrepo_id folder_path repo_typer)r*r+r,r-r.r/c j| td|tvrtdt|t}tj j j stdd|g}nt|tr|g}|tz }|'tjxsd} t| dz d}|j|||d } tjd | | j }t#xr|j%|||d j&} | rt(nt*}t-fd j/dD||}|Dcgc]}t1|}}tjdt3|dt5|dDcgc]}|t7|j8f}}t;||}t=|Dcgc]#}t?j@tB|||||d%}}|D]}|jE| rtGd|jIztKjJ} tKjLdtKjJ|z | k\r/| rtO|jItKjJ}|jQrtSjdn|D]}|jUtj|jItSjdycc}wcc}wcc}w)zUpload a large folder to the Hub in the most resilient way possible. See [`HfApi.upload_large_folder`] for the full documentation. NzFor large uploads, `repo_type` is explicitly required. Please set it to `model`, `dataset` or `space`. If you are using the CLI, pass it as `--repo-type=model`.z"Invalid repo type, must be one of zProvided path: 'z' is not a directoryr T)r1r3r*exist_okzRepo created: xetEnabled)r1r3r)expandc3~K|]4}|js|jj6ywN)is_file relative_toas_posix).0pathr2s `/var/www/Befach/backend/env/lib/python3.12/site-packages/huggingface_hub/_upload_large_folder.py z/upload_large_folder_internal..rs1idZ^ZfZfZh  + & / / 1is=$=z**/*)r+r,zFound z candidate files to uploadzRecovering from metadata files)desc)statusr0r1r3r))targetkwargsz zIs done: exiting main loopzUpload is complete!)+ ValueErrorrrr expanduserresolveis_dir isinstancestrros cpu_countmax create_repologgerinfor1r repo_info xet_enabledUPLOAD_BATCH_SIZE_XETUPLOAD_BATCH_SIZE_LFSrglobrlenrr path_in_repoLargeUploadStatusrange threadingThread _worker_jobstartprintcurrent_reporttimesleep_print_overwriteis_doneloggingjoin)r0r1r2r3r)r*r+r,r-r.r/nb_coresrepo_urlis_xet_enabledupload_batch_sizefiltered_paths_listrelpath paths_listpathsitemsrC_threadsthreadlast_report_tss ` r@upload_large_folder_internalrt5s$& I   "=j\JKK#{#..088:K    +K=8LMNN OS )*+..O<<>&Q(Q,* w)W_cdH KK. +,G   MM  +2@-EZ.ik>N>Nv>Vi%' Obb7(g>bJb KK&Z))CDE *+KL   $[%2D2DEF E  u&7 8F{#    "&$   G   fv,,../YY[N  1 99; '+= = !6!6!89!YY[N >>  LL5 6    KK%%'( LL&']c  sL&#L+(L0ceZdZejZejZejZejZejZ y) WorkerJobN) __name__ __module__ __qualname__enumautoSHA256GET_UPLOAD_MODE PREUPLOAD_LFSCOMMITWAITr@rvrvsC TYY[FdiikODIIKM TYY[F 499;DrrvcdeZdZdZddeedefdZdefdZde ded e dd fd Z de fd Z de fd Zy )rYzBContains information, queues and tasks for a large upload process.rorjc||_tj|_tj|_tj|_tj|_t|_d|_ d|_ d|_ ||_ d|_ d|_d|_t!j"|_d|_t|_|jD]}|\}}|j*|jj-|0|j.|jj-|X|j.dk(r(|j0s|j j-||j2s|j j-|t4j7d|j8dy)Nrr lfszSkipping file z! (already uploaded and committed))roqueueQueue queue_sha256queue_get_upload_modequeue_preupload_lfs queue_commitrlocknb_workers_sha256nb_workers_get_upload_modenb_workers_preupload_lfsrjnb_workers_commitnb_workers_waitinglast_commit_attemptrnow _started_at _chunk_idx _chunk_locksha256put upload_mode is_uploaded is_committedrPdebugrX)selfrorjitemrnmetadatas r@__init__zLargeUploadStatus.__init__s\ 7<{{}@E ">Ckkm 7<{{}F &'/0'-.%&7&''(48 #<<> !%JJD"OE8&!!%%d+%%-**..t4%%.x7K7K((,,T2**!!%%d+ ~e.@.@-AAbcdrreturncl|j5t|jcdddS#1swYyxYwr:)rr&rrs r@ target_chunkzLargeUploadStatus.target_chunks,    6$T__5 6 6 6s*3successnb_itemsdurationNc |j5|s/tjd|d|xjdzc_nI|t|jk\r3|dkr.tj d|d|xjdz c_t dt|jttdz |_dddy#1swYyxYw)NzFailed to commit z9 files at once. Will retry with less files in next batch.r (zSuccessfully committed z. at once. Increasing the limit for next batch.r) rrPwarningrr&rQrNminrW)rrrrs r@ update_chunkzLargeUploadStatus.update_chunks    W!28*O:PST:T%UVDO W W Ws B0CCc d}d}d}d}d}d}d}d}d} d} d} |j5|jD]\} } | jr| dz } | | jz } | dz } | j|dz }|| jz }| j dk(r|dz }| j |dz }| j r|dz }|| jz }| js|dz }|| jz }t| }tj}|jd}||jz }t|jdd}d}|d|d |d z }|d z }|d z }|d |d| d t|d|d z }|d|d|d t|d|d z }|dkDr |d|dz }|d|d| d t|d|d z }|d| dz }|dz }|d|jdz }|d|j dz }|d|j"dz }|d|j$dz }|d|j&dz }|dz }|cdddS#1swYyxYw)z >G *4+J+J*K3O OG )F)F(GsK KG d&<&<%=SA AG 4#:#:";2> >G x G_/ / / sB*H4EH44H=c~|j5td|jDcdddS#1swYyxYw)Nc3VK|]!\}}|jxs |j#ywr:rrr>rprs r@rAz,LargeUploadStatus.is_done..,s(e;1hx,,F0F0FFe'))rallrors r@rdzLargeUploadStatus.is_done*s7 YY feZ^ZdZdee f f fs3<)r )rwrxry__doc__r JOB_ITEM_TintrrboolfloatrrKr`rdrrr@rYrYsnL ed:. e3 eD6c6 WD WC W5 WT W==~ffrrYrCcP d}t|}|y|\}}|tjk(rX|d} t||jj ||j5|xjdzc_dddnT|tjk(r t!||||||D]}|\} } | j"r| j$dk(r|j&j |@| j$d k(r|j(j |k|jj ||j5|xj*dzc_dddnz|tj,k(r` t/||||||D]}|j(j | |j5|xj0dzc_dddn|tj2k(rt5j4} d} t7|||||t5j4| z }|j9| t;|||j5t5j4|_|xj>dzc_dddnV|tj@k(rCt5jBtD|j5|xjFdzc_#ddd#t $rt$rR} tjd| tj|jj |Yd} ~ d} ~ wwxYw#1swYrxYw#t $rt$r7} tjd| tjYd} ~ d} ~ wwxYw#1swYxYw#t $rt$rY} tjd | tj|D]}|j&j |Yd} ~ fd} ~ wwxYw#1swYExYw#t $rt$r[} tjd | tj|D]}|j(j |d } Yd} ~ }d} ~ wwxYw#1swYxYw#1swYxYw) a Main process for a worker. The worker will perform tasks based on the priority list until all files are uploaded and committed. If no tasks are available, the worker will wait for 10 seconds before checking again. If a task fails for any reason, the item(s) are put back in the queue for another worker to pick up. Read `upload_large_folder` docstring for more information on how tasks are prioritized. TNrzFailed to compute sha256: r )r0r1r3r)zFailed to get upload mode: rregularzFailed to preupload LFS: zFailed to commit: F)$_determine_next_jobrvr|_compute_sha256rrKeyboardInterrupt ExceptionrPerror traceback format_excrrrr}_get_upload_moderrrrrr~_preupload_lfsrrra_commitrrWrrrrbWAITING_TIME_IF_NO_TASKSr)rCr0r1r3r)next_jobjobrorerprstart_tsrrs r@r]r]/s AE'v.    U )"" "8D .%,,006 .((A-( . .I-- - ' CI`hi" 8))''50..2248))Y6''++D10044T: 711Q61 7 7I++ + 9u#w)^fg!D''++D1" 5//14/ 5 5I$$ $yy{HG 39W_`yy{X-H   UX > .-1YY[*((A-( . .INN " JJ/ 0 /))Q.) /u %  . 9!=>$$&##''-- .  . . %  ' :1#>?$$&& '( 7 7%  9 8<=$$&!D..2248" 9  5 5%   1!56$$&!D''++D1"   . .  / /s&K"L?M <N02N#/P9P/R8RL<*AL77L<?M N,NNN #P5AP  PPR 0ARR RR%rc.|j5|jdk(r|jjdkDr|jt j |jz dkDrg|xjdz c_t jdtjt|j|jfcdddS|jdk(r|jjdk\rg|xjdz c_t jdtjt|j|jfcdddS|jjtk\re|xjdz c_t jdtdtjt|jtfcdddS|j j|j"k\rr|j$dk(rc|xj$dz c_t jd tj&t|j |j"fcdddS|j(jdkDrg|j*dk(rX|xj*dz c_t jd tj,t/|j(fcdddS|jjdkDrl|jdk(r]|xjdz c_t jd tjt|jtfcdddS|j j|j"k\r|j$dk(st0j2sc|xj$dz c_t jd tj&t|j |j"fcdddS|j(jdkDrX|xj*dz c_t jd tj,t/|j(fcdddS|jjdkDr]|xjdz c_t jdtjt|jtfcdddS|j jdkDrc|xj$dz c_t jd tj&t|j |j"fcdddS|jdk(r|jjdkDr|jt j |jz dkDrg|xjdz c_t jdtjt|j|jfcdddS|jdk(r|jjdkDr|j(jdk(r|jjdk(r|j jdk(r|j*dk(r|jdk(rv|j$dk(rg|xjdz c_t jdtjt|j|jfcdddSt5d|j6Drt j9d dddy|xj:dz c_t jdt<dtj>gfcdddS#1swYyxYw)Nri,r z;Job: commit (more than 5 minutes since last commit attempt)zJob: commit (>100 files ready)zJob: get upload mode (>z files ready)z5Job: preupload LFS (no other worker preuploading LFS)z.Job: sha256 (no other worker computing sha256)z:Job: get upload mode (no other worker getting upload mode)zJob: preupload LFSz Job: sha256zJob: get upload moder(z-Job: commit (1 min since last commit attempt)z Job: commitc3VK|]!\}}|jxs |j#ywr:rrs r@rAz&_determine_next_job..s(a[Q&&@(*@*@@arz.All files have been processed! Exiting worker.zNo task available, waiting... (zs)) rrrqsizerrarPrrvr_get_nrrMAX_NB_FILES_FETCH_UPLOAD_MODErr}rrjrr~rrr|_get_oner HF_HUB_ENABLE_HF_TRANSFERrrorQrrr)rCs r@rrs i(  $ $ )##))+a/**6 f8886A  $ $ ) $ LLV W$$fV-@-@&BUBUBW&XYi(i( % % *v/B/B/H/H/Jc/Q  $ $ ) $ LL9 :$$fV-@-@&BUBUBW&XY!i(i(& ) ) / / 15S S  - - 2 - LL23Q2RR_` a--vf6R6RTr/st-i(i(2 ' ' - - /63K3K KPVPoPostPt  + +q 0 + LLP Q++VF4N4NPVPhPh-ij9i(i(> & & (1 ,1I1IQ1N  $ $ ) $ LLI J$$hv/B/B&CDEi(i(J ) ) / / 1A 5&:[:[_`:`  - - 2 - LLU V--vf6R6RTr/stQi(i(X ' ' - - /63K3K K  + +q 0 8[8[  + +q 0 + LL- .++VF4N4NPVPhPh-ijci(i(h & & (1 ,  $ $ ) $ LL '$$hv/B/B&CDoi(i(t ) ) / / 1A 5  - - 2 - LL/ 0--vf6R6RTr/st{i(i(@ ' ' - - /! 3  + +q 0 + LL- .++VF4N4NPVPhPh-ijGi(i(N  $ $ )##))+a/**6 f8886A  $ $ ) $ LLH I$$fV-@-@&BUBUBW&XY]i(i(f  $ $ )##))+a/##))+q0,,2249**002a7((A-11Q6//14  $ $ ) $ LL '$$fV-@-@&BUBUBW&XY}i(i(BaTZT`T`a a KKH IGi(i(N  % % * % LL:;S:TTVW XNNB'Si(i(i(sfB:` B ` $A<` *B` A:` A?` B` 9A+` .A0` (A6` (B9` +D` 2` >A`  `rc|\}}|jB|jjd5}t|j |_ddd|j |y#1swYxYw)z1Compute sha256 of a file and save it in metadata.Nrb)r file_pathopenrhexsave)rrnrfs r@rrs^OE8 __ ! !$ ' 31)!n002HO 3 MM% 3 3s A&&A/roc `|Dcgc] }t|}}t||||jt|d|jt ||D]N\}}|\}} |j | _|j| _ |j| _ | j|Pycc}w)zmGet upload mode for each file and update metadata. Also receive info if the file should be ignored. )safe) additionsr3r1headersr)endpointN) _build_hacky_operationr_build_hf_headersr rzip _upload_moder_should_ignorer _remote_oid remote_oidr) ror0r1r3r)rradditionrnrs r@rrs ;@@$'-@I@%%'xb) eY/hx'44!)!8!8&22 e 0AsB+c|Dcgc] }t|}}|j|||||D]\}}d|_|j|ycc}w)z(Preupload LFS files and update metadata.)r1r3r)rTN)rpreupload_lfs_filesrr ror0r1r3r)rrrnrs r@rr-sd:?@$'-@I@ !x# e!AsAc|Dcgc] }t|}}|j||||d|D]\}}d|_|j|ycc}w)zCommit files to the repo.z(Add files using upload-large-folder tool)r1r3r) operationscommit_messageTN)r create_commitrrrs r@rr<sg:?@$'-@I@A !x $ e!AsAceZdZddZy)HackyCommitOperationAddNcnt|jtrt|j|_yyr:)rJpath_or_fileobjrrKrs r@ __post_init__z%HackyCommitOperationAdd.__post_init__Qs* d**D 1#&t';';#AIEIIK >> >sAreportc$|dz }tjjtfd|j D}t |D]@}t jjdt jjdBt jj|t jjdt|j dz zt jjy)zPrint a report, overwriting the previous lines. Since tqdm in using `sys.stderr` to (re-)write progress bars, we need to use `sys.stdout` to print the report. Note: works well only if no other process is writing to `sys.stdout`! rc3@K|]}t|zdzyw)r N)rW)r>lineterminal_widths r@rAz#_print_overwrite..~sSt3t9.2Ssz zrN) shutilget_terminal_sizecolumnssum splitlinesrZsysstdoutwriterWflush)rnb_linesrprs @r@rcrcqs dNF--/77NSv?P?P?RSSH8_ $ " JJVJJSNS1B1B1DR1H-IIJKJJr)JrzrerLrrr#r[rarrpathlibrrtypingrrrr r urllib.parser rr _commit_apirrr _local_folderrrrrrrutilsrrrutils._cache_managerrutils._runtimer utils.sharhf_apir getLoggerrwrPrrr&r__annotations__rTrUrKrrtEnumrvrrYr]rrrrrrrrrrcrrr@r5s    >>LLvv3EE.,"   8 $!$O49O#"6:7;!% o( o( o(sDy!o(  o( sm o(d^o(U49c>23o(eDIsN34o(#o(o(o(n ')@@ A ufufpj/ j/ j/j/ j/  j/Zj( 1j(huYPTU_P`E`?a6bj(d*D,7SUXdglp, $z*  3 SV be jn  4 # ' C C [^ cg (=0=  0G &-$z2B?+??Z8H?STr