
    ⾙h                    2   d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
mZ d dlmZ ddlmZmZ ddlmZ ddlmZmZmZmZmZ ddlmZmZmZ dd	lmZ dd
lmZ ddl m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z' d dlm(Z( d dl)m*Z*m+Z+ ddlm,Z,m-Z-m.Z. dZ/ej`                  d    dk(  Z1 ejd                         dk(  r	ddl3m4Z5 eZ6nd dlm7Z5 ejl                  Z6	 ejp                  Z8ej`                  dk\  rejt                  Z;nejv                  Z;d Z<dZ=dZ>d Z?dZ@dZAdZBdZCd ZDdZEdZF eGedd      ZHdZI eGedd       ZDdZJdZK ej                         ZMej                  ZNd ZOd ZPd  ZQd! ZRdBd"ZS G d# d$e;      ZT G d% d&eU      ZV G d' d(eU      ZWd) ZX G d* d+      ZY G d, d-e      ZZ G d. d/eZ      Z[ G d0 d1eZ      Z\ G d2 d3eZ      Z] G d4 d5eZ      Z^ G d6 d7      Z_ G d8 d9      Z` G d: d;e`      Za G d< d=      Zb G d> d?eb      Zc G d@ dAe_      Zdy# e9$ r dZ8Y 4w xY w)C    N)deque)partial   )	cpu_countget_context)util)TERM_SIGNALhuman_statuspickle_loadsreset_signalsrestart_state)	get_errnomem_rsssend_offset)ExceptionInfo)DummyProcess)CoroStopRestartFreqExceededSoftTimeLimitExceeded
TerminatedTimeLimitExceededTimeoutErrorWorkerLostError	monotonicQueueEmpty)FinalizedebugwarningzEchild process exiting after exceeding memory limit ({0}KiB / {1}KiB)
   Windows)kill_processtree)killg    _B)r"   r"            SIGUSR1g      $@EX_OKi,  皙?c                     	 | j                   }|t        t         | j                               S |S # t        $ r d }Y /w xY wN)r   AttributeErrorr   fileno)
connectionnatives     I/var/www/Befach/backend/env/lib/python3.12/site-packages/billiard/pool.py_get_send_offsetr3   x   sJ    '' ~{J$5$5$788M	  s   0 >>c                 $    t        t        |        S r-   )listmapargss    r2   mapstarr9      s    T
    c                 L    t        t        j                  | d   | d               S )Nr   r   )r5   	itertoolsstarmapr7   s    r2   starmapstarr>      s"    	!!$q'47344r:   c                 T     t        j                         j                  | g|i | y r-   )r   
get_loggererror)msgr8   kwargss      r2   rA   rA      s#    DOOC1$1&1r:   c                 T    | t        j                         ur| j                  |       y y r-   )	threadingcurrent_threadstop)threadtimeouts     r2   stop_if_not_currentrJ      s$    Y--//G 0r:   c                   P    e Zd ZdZd ZerddZd Zd Zd Z	yddZd	 Zd
 Zd Z	y)LaxBoundedSemaphorez^Semaphore that checks that # release is <= # acquires,
    but ignores if # releases >= value.c                 N    | xj                   dz  c_         | j                          y Nr   )_initial_valueacquireselfs    r2   shrinkzLaxBoundedSemaphore.shrink   s    q r:   Nc                 >    t         j                  | |       || _        y r-   
_Semaphore__init__rO   rR   valueverboses      r2   rW   zLaxBoundedSemaphore.__init__   s    e,"'Dr:   c                     | j                   5  | xj                  dz  c_        | xj                  dz  c_        | j                   j                          d d d        y # 1 sw Y   y xY wrN   )_condrO   _valuenotifyrQ   s    r2   growzLaxBoundedSemaphore.grow   sM     $##q(#q 

!!#$ $ $s   AAA$c                     | j                   }|5  | j                  | j                  k  r%| xj                  dz  c_        |j                          d d d        y # 1 sw Y   y xY wrN   )r\   r]   rO   
notify_allrR   conds     r2   releasezLaxBoundedSemaphore.release   sM    ::D &;;!4!44KK1$KOO%& & &   ?AA c                     | j                   | j                  k  r0t        j                  |        | j                   | j                  k  r/y y r-   )r]   rO   rV   rd   rQ   s    r2   clearzLaxBoundedSemaphore.clear   s6    ++ 3 33""4( ++ 3 33r:   c                 @    t         j                  | ||       || _        y r-   rU   rX   s      r2   rW   zLaxBoundedSemaphore.__init__   s    eW5"'Dr:   c                     | j                   }|5  | xj                  dz  c_        | xj                  dz  c_        |j                          d d d        y # 1 sw Y   y xY wrN   )_Semaphore__condrO   _Semaphore__valuer^   rb   s     r2   r_   zLaxBoundedSemaphore.grow   sP    ((D ##q(#&&!+&  s   ;AAc                     | j                   }|5  | j                  | j                  k  r%| xj                  dz  c_        |j                          d d d        y # 1 sw Y   y xY wrN   )rj   rk   rO   	notifyAllrb   s     r2   rd   zLaxBoundedSemaphore.release   sT    ((D %))D,?,??**a/*NN$% % %re   c                     | j                   | j                  k  r0t        j                  |        | j                   | j                  k  r/y y r-   )rk   rO   rV   rd   rQ   s    r2   rg   zLaxBoundedSemaphore.clear   s:    ((4+>+>>""4( ((4+>+>>r:   r   N)
__name__
__module____qualname____doc__rS   PY3rW   r_   rd   rg    r:   r2   rL   rL      s9    + 	(	$	&	)
	(		%	)r:   rL   c                   .     e Zd ZdZ fdZd Zd Z xZS )MaybeEncodingErrorzVWraps possible unpickleable errors, so they can be
    safely sent through the socket.c                     t        |      | _        t        |      | _        t        |   | j                  | j                         y r-   )reprexcrY   superrW   )rR   rz   rY   	__class__s      r2   rW   zMaybeEncodingError.__init__   s0    9%[
4::.r:   c                 N    d| j                   j                  dt        |       dS )N<z: >)r|   rp   strrQ   s    r2   __repr__zMaybeEncodingError.__repr__   s    !^^44c$i@@r:   c                 <    d| j                   d| j                  dS )NzError sending result: 'z'. Reason: 'z'.)rY   rz   rQ   s    r2   __str__zMaybeEncodingError.__str__   s    JJ" 	"r:   )rp   rq   rr   rs   rW   r   r   __classcell__r|   s   @r2   rw   rw      s    '/
A"r:   rw   c                       e Zd ZdZy)WorkersJoinedzAll workers have terminated.N)rp   rq   rr   rs   ru   r:   r2   r   r      s    &r:   r   c                     t               r-   )r   )signumframes     r2   soft_timeout_sighandlerr      s    

!!r:   c                   r    e Zd Z	 	 	 	 ddZd Zd Zd ZddZd Zd Z	e
edfd	Zd
 Zd Zd ZefdZd Zy)WorkerNc                    |t        |      t        k(  r|dkD  sJ || _        || _        || _        || _        || _        || _        |	| _        |||c| _	        | _
        | _        |
| _        || _        | j                  |        y Nr   )typeintinitializerinitargsmaxtasksmax_memory_per_child	_shutdownon_exitsigprotectioninqoutqsynqwrap_exceptionon_ready_countercontribute_to_object)rR   r   r   r   r   r   r   sentinelr   r   r   r   r   s                r2   rW   zWorker.__init__   s     DNc$9hlKK&  $8!!*),dD&$)TY, 0!!$'r:   c                    | j                   | j                  | j                  c|_         |_        |_        | j                   j                  j	                         |_        | j                  j                  j	                         |_        | j                  rw| j                  j                  j	                         |_        | j                  j                  j	                         |_	        t        | j                  j                        |_        nd x|_        x|_	        |_        | j                   j                  j                  |_        | j                  j                  j                  |_        t        | j                   j                        |_        |S r-   )r   r   r   _writerr/   inqW_fd_readeroutqR_fdsynqR_fdsynqW_fdr3   send_syn_offset_send_syn_offsetsend
_quick_putrecv
_quick_getsend_job_offset)rR   objs     r2   r   zWorker.contribute_to_object   s
   &*hh		499#38hh&&--/yy((//19999,,335CL99,,335CL"24993D3D"ECAEECLE3<#*>))..**//.txx/?/?@
r:   c                 &   | j                   | j                  | j                  | j                  | j                  | j
                  | j                  | j                  | j                  | j                  | j                  | j                  | j                  ffS r-   )r|   r   r   r   r   r   r   r   r   r   r   r   r   rQ   s    r2   
__reduce__zWorker.__reduce__  sn    ~~HHdiiD,<,<MM4==$..$,, 3 3T5N5N!!	 
 
 	
r:   c                    t         j                  d gdfd	}|t         _        t        j                         }| j	                          | j                          | j                  |       	 t        j                  | j                  |             | j                  |d   d        y # t        $ r/}t        d| |d       | j                  |d   |       Y d }~Jd }~ww xY w# | j                  |d   d        w xY w)Nc                     | d<    |       S r   ru   )status_exit	_exitcodes    r2   exitzWorker.__call__.<locals>.exit  s    !IaL= r:   pidzPool process %r error: %rr   exc_infor   r-   )sysr   osgetpid_make_child_methods
after_forkon_loop_startworkloop	ExceptionrA   _do_exit)rR   r   r   rz   r   r   s       @@r2   __call__zWorker.__call__  s    F		! iik  "s#	3HHT]]s]+,
 MM#y|T2	  	2-tS1EMM#y|S11	2 MM#y|T2s*   /%B+ +	C#4%CC& C##C& &C>c                    ||rt         nt        }| j                  | j                  ||       t        j                  dk7  rO	 | j
                  j                  t        ||ff       t        j                  d       t        j                  |       y t        j                  |       y # t        j                  |       w xY w)Nwin32r   )
EX_FAILUREr*   r   r   platformr   putDEATHtimesleepr   r   )rR   r   exitcoderz   s       r2   r   zWorker._do_exit+  s    %(zeH<<#LLh'<<7"#		usHo67

1"HHX "s   8B' 'B>c                      y r-   ru   rR   r   s     r2   r   zWorker.on_loop_start;      r:   c                     |S r-   ru   )rR   results     r2   prepare_resultzWorker.prepare_result>  s    r:   c                     |xs t        j                         } j                  j                  } j                  } j
                  } j                  } j                  xs d} j                  }	 j                  }
 j                   fd}d}	 ||r||k  r |
       }|r|\  }}|t        k(  sJ |\  }}}}} |t        || |       ||ff       r ||      }|sO	 d |	 ||i |      f}	  |t        ||||ff       |dz  }|dkD  r[t'               }|dk  rt)        d       |dkD  r<||kD  r7t+        t,        j/                  ||             t0         j3                  |       S ||r||k  r |d|       |r$||k(  rt0        nt4        	  j3                  |       S t6         j3                  |       S # t        $ r dt               f}Y w xY w# t        $ ra}t!        j"                         \  }}}	 t%        ||d         }t        t$        ||f      } |t        ||d|f|ff       ~n# ~w xY wY d }~7d }~ww xY w#  j3                  |       w xY w)	Nr   c                     d}	 |dkD  r1t        d| j                  j                  j                         d               }|r|\  }}|t        k(  ry|t
        k(  sJ y|dz  }`)Nr   r   <   z(!!!WAIT FOR ACK TIMEOUT: job:%r fd:%r!!!r   FT)rA   r   r   r/   NACKACK)jidireqtype_r8   _wait_for_synrR   s        r2   wait_for_synz%Worker.workloop.<locals>.wait_for_synM  sr    Ar6Dtyy00779AG#o"%KE4}$ C<'<Q r:   TFr   z'worker unable to determine memory usage)	completedzworker exiting after %d tasks)r   r   r   r   r   r   r   r   r   wait_for_jobr   TASKr   r   r   READYr   r   rw   r   rA   r!   MAXMEM_USED_FMTformat
EX_RECYCLE_ensure_messages_consumedr   r*   )rR   r    nowr   r   r   r   r   r   r   r   r   r   r   r   args_jobr   funr8   rC   confirmr   rz   _tbwrappedeinfoused_kbr   s   `                            @r2   r   zWorker.workloopA  s    RYY[iimm,,====#88=A,,(())	 	/	@"xI4H"n#&LE5 D=(=05-CCvsAsuc8<=>$".s"3&$:"&sD7KF7K(L!M$US!VW$=>? NI+a/"))"a<!"KL"Q;75I+I#O$:$: ')=%? @#- **Y*?] "xI4HJ 19=%.(%:z
J **Y*?  **Y*?C % :"'!9: % 	$#&<<>1b$&8fQi&HG$1 2GR3 %E  a%(I JK "	$: **Y*?s   AI "F< 3G AI *I -I 5I $I <GI GI 	I H<95H0.H<0H33H<6I <II Ic                     | j                   syt        t              D ]B  }| j                   j                  |k\  rt	        d|        yt        j                  t               D t        d       y)zr Returns true if all messages sent out have been received and
        consumed within a reasonable amount of time Fz*ensured messages consumed after %d retriesTz<could not ensure all messages were consumed prior to exiting)	r   range)GUARANTEE_MESSAGE_CONSUMPTION_RETRY_LIMITrY   r    r   r   ,GUARANTEE_MESSAGE_CONSUMPTION_RETRY_INTERVALr!   )rR   r   retrys      r2   r   z Worker._ensure_messages_consumed  sb     $$DEE$$**i7BEJJJCD	 F
 	  	r:   c                 .   t        | j                  d      r$| j                  j                  j                          t        | j                  d      r$| j                  j
                  j                          | j                   | j                  | j                    t        | j                         t        t        j                  t        t               	 t        j                  t        j                  t        j                         y # t        $ r Y y w xY w)Nr   r   )full)hasattrr   r   closer   r   r   r   r   r   SIG_SOFT_TIMEOUTsignalr   SIGINTSIG_IGNr.   rQ   s    r2   r   zWorker.after_fork  s    488Y'HH""$499i(II##%'Ddmm, 	4--. 'MM*,CD	MM&--8 		s   2D 	DDc                     |j                   t        |d      rO|j                  j                  t        |d      r%|j                  r|j                  t
        ffd	}|S fd}|S fd}|S )Nr   get_payloadc                 "    d |              fS NTru   )rI   loadsr   s     r2   _recvz'Worker._make_recv_method.<locals>._recv  s    {}!555r:   c                 (     |       r	d        fS yNTFNru   )rI   _pollgets    r2   r  z'Worker._make_recv_method.<locals>._recv  s    W~#SU{*&r:   c                 N    	 d |       fS # t         j                  $ r Y yw xY wNTrI   r  r   )rI   r  s    r2   r  z'Worker._make_recv_method.<locals>._recv  s.    'W!555{{ '&'s   
 $$)r  r   r   pollr   r   )rR   connr  r  r  r   s      @@@r2   _make_recv_methodzWorker._make_recv_method  si    hh4#LL%%Et]+0@0@"..)5 6 ' '
 r:   c                     | j                  | j                        | _        | j                  r!| j                  | j                        | _        y d | _        y r-   )_make_protected_receiver   r   r   r   )rR   r  s     r2   r   zWorker._make_child_methods  sD     88B $		 "99$))D/3 	r:   c                     | j                  |      | j                  r| j                  j                  nd t        ffd	}|S )Nc                 `   r        r | d       t        t              	  d      \  }}|sy 	 | | d       t        t              |S # t        t        f$ rQ}t	        |      t
        j                  k(  rY d }~y  | dt        |      j                         t        t              d }~ww xY w)Nzworker got sentinel -- exiting      ?zworker got %s -- exiting)

SystemExitr*   EOFErrorIOErrorr   errnoEINTRr   rp   r   )r    readyr   rz   _receiveshould_shutdowns       r2   receivez/Worker._make_protected_receive.<locals>.receive  s    ?#467 ''-%c]
s  {67 ,,J g& -S>U[[00$s)2D2DE ,,	-s   A B-B(=+B((B-)r  r   is_setr    )rR   r  r  r  r  s      @@r2   r  zWorker._make_protected_receive  s9    ))$/37>>$..//t 	$ r:   )
NNru   NNNTTNNr-   )rp   rq   rr   rW   r   r   r   r   r   r   r    r   r   r   r   r  r   r   r  ru   r:   r2   r   r      s`    HJ7;48=A("
3,  #	t K@Z ,. )5 5
r:   r   c                   D     e Zd Zd Zd Z fdZd ZddZd Zd Z	 xZ
S )	
PoolThreadc                 `    t        j                  |        t        | _        d| _        d| _        y NFT)r   rW   RUN_state_was_starteddaemon)rR   r8   rC   s      r2   rW   zPoolThread.__init__  s&    d#!r:   c                    	 | j                         S # t        $ rb}t        dt        |       j                  |d       t        t        j                         t               t        j                          Y d }~y d }~wt        $ rA}t        dt        |       j                  |d       t        j                  d       Y d }~y d }~ww xY w)NzThread %r crashed: %rr   r   )bodyr   rA   r   rp   _killr   r   r	   r   r   r   r   rR   rz   s     r2   runzPoolThread.run  s    
	99;" 	)4:+>+>"))+{+HHJJ 	)4:+>+>HHQKK	s!    	CAA88C7C  Cc                 :    d| _         t        t        |   |i | y r  )r$  r{   r  start)rR   r8   rC   r|   s      r2   r,  zPoolThread.start  s     j$%t6v6r:   c                      y r-   ru   rQ   s    r2   on_stop_not_startedzPoolThread.on_stop_not_started  r   r:   c                 `    | j                   r| j                  |       y | j                          y r-   )r$  joinr.  rR   rI   s     r2   rG   zPoolThread.stop
  s&    IIg  "r:   c                     t         | _        y r-   )	TERMINATEr#  rQ   s    r2   	terminatezPoolThread.terminate  s	    r:   c                     t         | _        y r-   )CLOSEr#  rQ   s    r2   r   zPoolThread.close  s	    r:   r-   )rp   rq   rr   rW   r*  r,  r.  rG   r4  r   r   r   s   @r2   r  r    s&    7# r:   r  c                   $     e Zd Z fdZd Z xZS )
Supervisorc                 0    || _         t        | 	          y r-   )poolr{   rW   )rR   r:  r|   s     r2   rW   zSupervisor.__init__  s    	r:   c                    t        d       t        j                  d       | j                  }	 |j                  }t	        d|j
                  z  d      |_        t        d      D ]O  }| j                  t        k(  s|j                  t        k(  s+|j                          t        j                  d       Q ||_        | j                  t        k(  r_|j                  t        k(  rL|j                          t        j                  d       | j                  t        k(  r|j                  t        k(  rLt        d       y # t        $ r" |j                          |j                           w xY w)Nzworker handler startingg?
   r   r+   zworker handler exiting)r    r   r   r:  r   
_processesr   r#  r"  _maintain_poolr   r   r0  )rR   r:  
prev_stater   s       r2   r'  zSupervisor.body  s   '(

3yy	 ++J!.rDOO/CQ!GD2Y;;#%$++*<'')JJsO  ",D++$);##%

3 ++$); 	&'	 # 	JJLIIK	s   AD: :D: BD: :+E%)rp   rq   rr   rW   r'  r   r   s   @r2   r8  r8    s    (r:   r8  c                   0     e Zd Z fdZd Zd Zd Z xZS )TaskHandlerc                 h    || _         || _        || _        || _        || _        t
        |           y r-   )	taskqueuer   outqueuer:  cacher{   rW   )rR   rC  r   rD  r:  rE  r|   s         r2   rW   zTaskHandler.__init__>  s1    " 	
r:   c           	         | j                   }| j                  }| j                  }t        |j                  d       D ]Y  \  }}d }d}	 t        |      D ]'  \  }}| j                  rt        d        n$	  ||       ) |rt        d        ||dz          Y n t        d	       | j                          y # t        $ r t        d       Y  6t        $ r; |d d \  }}		 ||   j                  |	dt               f       n# t        $ r Y nw xY wY w xY w# t        $ rX |r|d d nd\  }}	||v r"||   j                  |	dz   dt               f       |r t        j                  d        ||dz          Y 5w xY w)
Nz'task handler found thread._state != RUNzcould not put task on queuer&   Fzdoing set_length()r   )r   r   ztask handler got sentinel)rE  rC  r   iterr  	enumerater#  r    r  r   _setr   KeyErrorr   tell_others)
rR   rE  rC  r   taskseq
set_lengthtaskr   r   inds
             r2   r'  zTaskHandler.bodyF  sx   

NN	hh#'	t#<GZDA&(1GAt{{GH
!D	  2  "23"1q5)1 $=B -.3 # ;<$ !#'8S!!#JOOC%1IJ' ! !	!  &'+48S%<#JOOC!Ge]_-EFJJ34q1u%&sg   *D1B39D3DD
DC;:D;	D	DD	D
DDDAE10E1c                    | j                   }| j                  }| j                  }	 t        d       |j                  d        t        d       |D ]
  } |d         	 t        d       y # t        $ r t        d       Y "w xY w)Nz/task handler sending sentinel to result handlerz(task handler sending sentinel to workersz/task handler got IOError when sending sentinelsztask handler exiting)rD  r   r:  r    r  )rR   rD  r   r:  ps        r2   rL  zTaskHandler.tell_othersp  s{    ==hhyy
	ECDLL <=D	 
 	$%  	ECD	Es   6A) )B ?B c                 $    | j                          y r-   )rL  rQ   s    r2   r.  zTaskHandler.on_stop_not_started  s    r:   )rp   rq   rr   rW   r'  rL  r.  r   r   s   @r2   rA  rA  <  s    (T&&r:   rA  c                   H     e Zd Z fdZd Zd Zd Zd Zd Zd Z	d Z
 xZS )	TimeoutHandlerc                 h    || _         || _        || _        || _        d | _        t
        |           y r-   )	processesrE  t_softt_hard_itr{   rW   )rR   rW  rE  rX  rY  r|   s        r2   rW   zTimeoutHandler.__init__  s1    "
r:   c                 T    t        fdt        | j                        D        d      S )Nc              3   J   K   | ]  \  }}|j                   k(  r||f  y wr-   r   .0r   procr   s      r2   	<genexpr>z1TimeoutHandler._process_by_pid.<locals>.<genexpr>  ,      
!!Txx3 1I
    #NN)nextrI  rW  r   s    `r2   _process_by_pidzTimeoutHandler._process_by_pid  s+     
%.t~~%>
  	r:   c                    t        d|       | j                  |j                        \  }}|sy |j                  d       	 t	        |j                  t
               y # t        $ r'}t        |      t        j                  k7  r Y d }~y d }~ww xY w)Nzsoft time limit exceeded for %rTsoft)
r    re  _worker_pidhandle_timeoutr(  r   OSErrorr   r  ESRCH)rR   r   process_indexrz   s        r2   on_soft_timeoutzTimeoutHandler.on_soft_timeout  sz    /5..s? 	%	#//#34 	~, -	s   A 	B%BBc                 \   |j                         ry t        d|       	 t        |j                        # t        $ r) |j	                  |j
                  dt               f       Y nw xY w| j                  |j                        \  }}|j                  d       |r| j                  |       y y )Nzhard time limit exceeded for %rFrg  )r  r    r   _timeoutrJ  _jobr   re  ri  rj  _trywaitkill)rR   r   rm  rn  s       r2   on_hard_timeoutzTimeoutHandler.on_hard_timeout  s    99;/5	#CLL11  	9HHSXX}78	9 ..s? 	&g& s   4 /A&%A&c                 :   t        d|j                         	 t        j                  |j                        |j                  k(  rXt        d|j                         t        j
                  t        j                  |j                        t        j                         n|j                          |j                  j                  d      ry t        d|j                         	 t        j                  |j                        |j                  k(  rXt        d|j                         t        j
                  t        j                  |j                        t        j                         y t        |j                  t               y # t        $ r Y w xY w# t        $ r Y y w xY w)Nztimeout: sending TERM to %szIworker %s is a group leader. It is safe to kill (SIGTERM) the whole groupr+   r  z/timeout: TERM timed-out, now sending KILL to %szIworker %s is a group leader. It is safe to kill (SIGKILL) the whole group)r    _namer   getpgidr   killpgr   SIGTERMr4  _popenwaitrk  SIGKILLr(  rR   workers     r2   rs  zTimeoutHandler._trywaitkill  s   +V\\:
	zz&**%3acicmcmn		"**VZZ0&..A  " }}!!#!.?N	zz&**%3acicmcmn		"**VZZ0&..Afjj'*  		  		s+   BE?  BF $F ?	F
F	FFc              #   F  K   | j                   | j                  }}t               }| j                  }| j                  }d }| j
                  t        k(  rt        j                  | j                        |rt        fd|D              }j                         D ]k  \  }}|j                  }	|j                  }
|
|}
|j                  }||} ||	|      r	 ||       D||vsI ||	|
      sS ||       |j                  |       m d  | j
                  t        k(  ry y w)Nc                 0    | r|syt               | |z   k\  ryy r!  r   )r,  rI   s     r2   
_timed_outz2TimeoutHandler.handle_timeouts.<locals>._timed_out  s     {ego- .r:   c              3   ,   K   | ]  }|v s|  y wr-   ru   )r^  krE  s     r2   r`  z1TimeoutHandler.handle_timeouts.<locals>.<genexpr>  s     ;!U
A;   	)rY  rX  setro  rt  r#  r"  copyrE  items_time_accepted_soft_timeoutrq  add)rR   rY  rX  dirtyro  rt  r  r   r   ack_timesoft_timeouthard_timeoutrE  s               @r2   handle_timeoutszTimeoutHandler.handle_timeouts  s    dkk....	 kkS  IIdjj)E ;u;;++-3--"00'#)L"||'#)Lh5#C(e^
8\(J#C(IIaL ( / kkS s   CD!"	D!,2D!D!c                     | j                   t        k(  r@	 | j                         D ]  }t        j                  d        	 | j                   t        k(  r@t        d       y # t
        $ r Y w xY w)Nr  ztimeout handler exiting)r#  r"  r  r   r   r   r    rR   r   s     r2   r'  zTimeoutHandler.body  s^    kkS --/AJJsO 0 kkS  	'(  s   *A   	A,+A,c                     | j                   | j                         | _         	 t        | j                          y # t        $ r
 d | _         Y y w xY wr-   )rZ  r  rd  StopIterationrR   r8   s     r2   handle_eventzTimeoutHandler.handle_event  sB    88++-DH	N 	DH	s   9 AA)rp   rq   rr   rW   re  ro  rt  rs  r  r'  r  r   r   s   @r2   rU  rU    s,    '*.$L)r:   rU  c                   L     e Zd Z	 d fd	Zd Zd Zd	dZd
dZd ZddZ	 xZ
S )ResultHandlerc                     || _         || _        || _        || _        || _        || _        || _        d | _        d| _        || _	        |	| _
        |
| _        | j                          t        | 9          y )NF)rD  r  rE  r  join_exited_workersputlockr   rZ  _shutdown_completecheck_timeoutson_job_readyon_ready_counters_make_methodsr{   rW   )rR   rD  r  rE  r  r  r  r   r  r  r  r|   s              r2   rW   zResultHandler.__init__  sv     !
	#6 *"',(!2r:   c                 (    | j                  d       y )NT)r  )finish_at_shutdownrQ   s    r2   r.  z!ResultHandler.on_stop_not_started  s    5r:   c                     	  j                    j                   j                   j                  fd} fd}d }t        |t
        |t        |ix	 _        	fd}| _        y )Nc                 n    d_         	 |    j                  ||||       y # t        t        f$ r Y y w xY wr   )R_ackrK  r.   )r   r   time_acceptedr   r   rE  r   s        r2   on_ackz+ResultHandler._make_methods.<locals>.on_ack(  s<    MOc
=#x@n- s   " 44c                     | |||       	 |    }
j                  rpt        t        |j	                               d       }|rK|
j                  v r=
j                  |   }|j                         5  |xj                  dz  c_        d d d        |j                         s		j                          	 |j                  ||       y # t         $ r Y y w xY w# 1 sw Y   NxY w# t         $ r Y y w xY wrN   )
rK  r  rd  rH  worker_pidsget_lockrY   r  rd   rJ  )r   r   r   r   item
worker_pidr   rE  r  r  rR   s          r2   on_readyz-ResultHandler._make_methods.<locals>.on_ready0  s    'S!S'2Sz %%!$t'7'7'9":DA
*0F0F"F'+'='=j'I$)224 4(..!3.4 ::<&OO%		!S!  4 4  s/   C 3C4C" 	CCC"	C.-C.c                     	 t        j                  | t               y # t        $ r'}t	        |      t
        j                  k7  r Y d }~y d }~ww xY wr-   )r   r%   r	   rk  r   r  rl  )r   r   rz   s      r2   on_deathz-ResultHandler._make_methods.<locals>.on_deathG  s=    [) S>U[[0 1s    	AAAc                 Z    | \  }}	  |   |  y # t         $ r t        d||       Y y w xY w)NzUnknown job state: %s (args=%s))rK  r    )rO  stater8   state_handlerss      r2   on_state_changez4ResultHandler._make_methods.<locals>.on_state_changeR  s?    KE4F%u%t, F7EFs    **)	rE  r  r   r  r   r   r   r  r  )
rR   r  r  r  r  rE  r  r  r   r  s
   `    @@@@@r2   r  zResultHandler._make_methods"  sj    

,,**((		.	 %0
 	
,	F  /r:   c              #   t  K   | j                   }| j                  }	 	  ||      \  }}| j                  r*| j                  t        k(  sJ t	        d       t               |r%|t	        d       t                ||       |dk7  ry y d  o# t        t        f$ r}t	        d|       t               d }~ww xY ww)N result handler got %r -- exitingz,result handler found thread._state=TERMINATEzresult handler got sentinelr   )r  r  r  r  r    r   r#  r3  )rR   rI   r  r  r  rO  rz   s          r2   _process_resultzResultHandler._process_resultZ  s     yy..!"7mt
 {{{{i///DEj <78"*$%a<+  X& !8#>j !s(   B8B A#B8B5B00B55B8c                     | j                   t        k(  r9| j                  | j                  d      | _        	 t	        | j                         y y # t
        t        f$ r
 d | _        Y y w xY wr   )r#  r"  rZ  r  rd  r  r   )rR   r/   eventss      r2   r  zResultHandler.handle_eventu  s\    ;;#xx//2 TXX	 
 "8,   s   A A'&A'c                    t        d       	 | j                  t        k(  r,	 | j                  d      D ]  } 	 | j                  t        k(  r,| j                          y # t        $ r Y w xY w# | j                          w xY w)Nzresult handler startingr  )r    r#  r"  r  r   r  r  s     r2   r'  zResultHandler.body~  sx    '(	&++$!11#6 7 ++$ ##%    ##%s-   A, A A, 	A)&A, (A))A, ,A>c           
         d| _         | j                  }| j                  }| j                  }| j                  }| j
                  }| j                  }| j                  }d }	|r`| j                  t        k7  rM| |        	  |d      \  }
}|
r|t        d       : ||       	  |d       |r| j                  t        k7  rMt#        |d
      r?t        d       	 t%        d      D ]%  }|j&                  j	                         s n	 |        ' t        dt)        |      | j                         y # t        t        f$ r}t        d|       Y d }~y d }~ww xY w# t        $ rL t               }|	s|}	n:||	z
  dkD  rt        d       Y t        dt        t!        ||	z
  dz
  d	                   Y w xY w# t        t        f$ r Y w xY w)NTr  r  z&result handler ignoring extra sentinel)shutdowng      @z!result handler exiting: timed outz6result handler: all workers terminated, timeout in %ssr   r   z"ensuring that outqueue is not fullr<  z7result handler exiting: len(cache)=%s, thread._state=%s)r  r  rD  rE  r  r  r  r  r#  r3  r  r  r    r   r   absminr   r   r   len)rR   r  r  rD  rE  r  r  r  r  time_terminater  rO  rz   r   r   s                  r2   r  z ResultHandler.finish_at_shutdown  s   "&hh==

yy"66,,..y0) "3it
 <BC%C#T2! y0: 8Y'67rA#++002E # 	G%j$++	'K X& 8#> ! 
Ck%%(N^+c1AB +c#"6"<a@AC
C, X& sB   =D, !	E 3F, ,E;EE+F)%F)(F),F>=F>r-   )r  rc  F)rp   rq   rr   rW   r.  r  r  r  r'  r  r   r   s   @r2   r  r  
  s.     BF$66/p6 
&6'r:   r  c                   "   e Zd ZdZdZeZeZeZeZe	Z	e
Z
	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d=dZd Zd Zd Zd	 Zd
 Zd Zd Zd Zd Zd Zd Zd Zd>dZd Zd Zd Zd Zd Zd Zd Zd Z d Z!d?dZ"d?dZ#d Z$d Z%d  Z&d! Z'd" Z(d# Z)d$ Z*d% Z+d& Z,di fd'Z-d@d(Z.	 	 dAd)Z/d@d*Z0dBd+Z1	 	 dBd,Z2di ddddddddddfd-Z3d. Z4d@d/Z5	 	 dAd0Z6	 	 dAd1Z7e8d2        Z9d3 Z:d4 Z;d5 Z<e8d6        Z=d7 Z>d8 Z?e8d9        Z@eAd:        ZBeAd;        ZCeDd<        ZEy)CPoolzS
    Class which supports an async version of applying functions to arguments.
    TNru   c                 p   |xs
 t               | _        || _        | j                          t	               | _        i | _        t        | _        || _	        || _
        || _        || _        || _        || _        || _        |xs t         | _        |
| _        || _        || _        || _        || _        i | _        || _        t3        |xs | j                  d uxs | j                  d u      | _        |r&t6         t9        j:                  t=        d             d }|| j?                         n|| _         |xs tC        | j@                  dz        | _"        tG        ||	xs d      | _#        |tI        |      stK        d      |tI        |      stK        d      | j                  jL                  | _'        g | _(        i | _)        i | _*        || _+        |xs tY        | j@                        | _-        t]        | j@                        D ]  }| j_                  |        | ja                  |       | _1        |r| jb                  je                          | jg                  | j
                  | jh                  | jj                  | jP                  | j                        | _6        |r| jl                  je                          d | _7        | j4                  r| jq                  | jP                  | j                  | j                  | j                        | _9        tu               | _;        d| _<        | j{                          |s1| jr                  j|                  | _7        nd | _9        d| _<        d | _;        | j                         | _@        | j                  j|                  | _A        |r| j                  je                          t        | | j                  | j
                  | j                  | jj                  | jP                  | jb                  | jl                  | j                  | j                  | jr                  | j                         f
d      | _F        y )	NWSoft timeouts are not supported: on this platform: It does not have the SIGUSR1 signal.d   r   zinitializer must be a callablez on_process_exit must be callableF   )r8   exitpriority)Gr   _ctxsynack_setup_queuesr   
_taskqueue_cacher"  r#  rI   r  _maxtasksperchild_max_memory_per_child_initializer	_initargs_on_process_exitLOST_WORKER_TIMEOUTlost_worker_timeouton_process_upon_process_downon_timeout_seton_timeout_cancelthreadsreadersallow_restartboolenable_timeoutsr   warningswarnUserWarningr   r=  roundmax_restartsr   callable	TypeErrorProcess_Process_pool	_poolctrl_on_ready_countersputlocksrL   _putlockr   _create_worker_processr8  _worker_handlerr,  rA  r   	_outqueue_task_handlerr  rU  _timeout_handlerLock_timeout_handler_mutex_timeout_handler_started_start_timeout_handlerr  create_result_handler_result_handlerhandle_result_eventr   _terminate_pool_inqueue_help_stuff_finish_args
_terminate)rR   rW  r   r   maxtasksperchildrI   r  r  r  max_restart_freqr  r  r  r  r  	semaphorer  r  r  on_process_exitcontextr   r  rC   r   s                            r2   rW   zPool.__init__  s   $ ,{}	'(!1%9"'! /#6#M:M *.,!2*# *LL$*T) 
 ,4MM+I   L.7.?$..*Y(HE$//C2G,H*<9I9NQO"8K+@<==&x/H>??		))
"$ !I%8%It'A''* (  $t4  &&(!--doo.2oo.2nn.2jj.2kk	;
 $$&" $($7$7

DKK!!4<<%D! +/&D',1D)'') &*&;&;&H&H#$(D!,1D)*.D'  $99;#'#7#7#D#D   &&("$&&//4==$..**d22D4F4F&&''..0	2
 
r:   c                 &     | j                   |i |S r-   )r  )rR   r8   kwdss      r2   r  zPool.ProcessI  s    t}}d+d++r:   c                 D    |j                  | j                  |            S )N)target)r   r  r}  s     r2   WorkerProcesszPool.WorkerProcessL  s    **4<<v<+FGGr:   c                     | j                   | j                  | j                  | j                  | j                  | j
                  | j                  | j                  | j                  | j                  f	d| j                  i|S )Nr  )r  r  r   r  _poll_result_join_exited_workersr  r   r  r  r  )rR   extra_kwargss     r2   r  zPool.create_result_handlerO  su    !t!!NNDOOT[[t88MM4--t/B/B	
 261H1H	

 
 	
r:   c                      y r-   ru   )rR   r   r   r   r   s        r2   r  zPool.on_job_readyX  r   r:   c                 H    | j                   | j                  | j                  fS r-   )r  r  r  rQ   s    r2   r  zPool._help_stuff_finish_args[  s    }}d00$**<<r:   c                 6    	 t               S # t        $ r Y yw xY wrN   )r   NotImplementedErrorrQ   s    r2   r   zPool.cpu_count^  s!    	;" 		s   	 	c                 4     | j                   j                  | S r-   )r  r  r  s     r2   r  zPool.handle_result_eventd  s    0t##00$77r:   c                      y r-   ru   )rR   r~  queuess      r2   _process_register_queueszPool._process_register_queuesg  r   r:   c                 T    t        fdt        | j                        D        d      S )Nc              3   J   K   | ]  \  }}|j                   k(  r||f  y wr-   r   r]  s      r2   r`  z'Pool._process_by_pid.<locals>.<genexpr>k  ra  rb  rc  )rd  rI  r  r   s    `r2   re  zPool._process_by_pidj  s+     
%.tzz%:
  	r:   c                 4    | j                   | j                  d fS r-   )r  r  rQ   s    r2   get_process_queueszPool.get_process_queuesp  s    }}dnnd22r:   c                    | j                   r| j                  j                         nd }| j                         \  }}}| j                  j	                  d      }| j                  | j                  |||| j                  | j                  | j                  || j                  | j                  | j                  | j                  |            }| j                  j                  |       | j!                  ||||f       |j"                  j%                  dd      |_        d|_        ||_        |j+                          || j,                  |j.                  <   || j0                  |j.                  <   | j2                  r| j3                  |       |S )Nr   )r   r   r   r   r  
PoolWorkerT)r  r  Eventr  Valuer  r   r  r  r  r  r  _wrap_exceptionr  r  appendr  namereplacer%  indexr,  r  r   r  r  )rR   r   r   r   r   r   r   ws           r2   r  zPool._create_worker_processs  sC   (,(:(:499??$113T499??3/t{{tT..""Hd.C.C ,,//!%!;!;-  + 	 
 	 	

!%%a#tT):;	<8		 (quu)9&q!r:   c                      y r-   ru   r}  s     r2   process_flush_queueszPool.process_flush_queues  r   r:   c           	         d}t        | j                  j                               D cg c]   }|j                         s|j                  r|" c}D ]D  }|xs
 t               }|j                  \  }}||z
  |j                  kD  s3| j                  ||       F |rt        | j                        s
t               i i c}t        t        t        | j                                    D ]  }| j                  |   }|j                  }	|j                  }
|
|	0t        d|       |
|j!                          t        d|       ||j"                  <   |	||j"                  <   |	t$        t&        fvr:t)        |dd      s-t+        d|j,                  |j"                  t/        |	      d       | j1                  |       | j                  |= | j2                  |j"                  = | j4                  |j"                  =  r| j                  D cg c]  }|j"                   c}t        | j                  j                               D ]  }t7        fd	|j9                         D        d      }|r~| j;                  ||       |j                         rM|j=                  |      xs d}	j=                  |      }|rt)        |d
d      r|j?                  |	       | jA                  |||	       |jB                  }|jD                  }|r-|jG                         s| j;                  ||j"                         |s|jG                         r| j;                  ||j"                         " j                         D ]3  }| jH                  s|s| jK                  |       | jI                  |       5 t        |j                               S g S c c}w c c}w )zCleanup after any worker processes which have exited due to
        reaching their specified lifetime. Returns True if any workers were
        cleaned up.
        Nz!Supervisor: cleaning up worker %dzSupervisor: worked %d joined_controlled_terminationFz Process %r pid:%r exited with %rr   r   c              3   2   K   | ]  }|v s|vr|  y wr-   ru   )r^  r   all_pidscleaneds     r2   r`  z,Pool._join_exited_workers.<locals>.<genexpr>  s&      ?Sw#X*=  ?s   _job_terminated)&r5   r  valuesr  _worker_lostr   _lost_worker_timeoutmark_as_worker_lostr  r  r   reversedr   r   rz  r    r0  r   r*   r   getattrrA   r  r
   r   r  r  rd  r  on_job_process_downr  _set_terminatedon_job_process_lost	_write_to_scheduled_for	_is_aliver  _process_cleanup_queues)rR   r  r   r   	lost_timelost_ret	exitcodesr   r~  r   popenr  acked_by_goner_  write_to	sched_forr$  r%  s                   @@r2   r  zPool._join_exited_workers  s1   
 
 $((:(:(<#= =C99;3+;+;  =C$C"%"2"2IxY!9!99((h7= C

O/!%DJJ01AZZ]FHMME} 491=$KKM4a8&,

#(0	&**%E:#66#F,EuM:VZZh1G!"
 ))&1JJqMNN6::.++FJJ7- 2. '+zz2!2HDKK..01 $?COO$5 ?! !,,S-@99;#,==#?#D1&{{=9GD2CU$K//9 44 #]H
  #}}H # 2 2I(:(:(<00hllC"9+>+>+@00immD7 2: "..*''#44V<((0	 +
 	((*++	O=F 3s   %OOc                      y r-   ru   )rR   r   r~  s      r2   on_partial_readzPool.on_partial_read  r   r:   c                      y r-   ru   r}  s     r2   r3  zPool._process_cleanup_queues  r   r:   c                      y r-   ru   )rR   r   pid_gones      r2   r-  zPool.on_job_process_down  r   r:   c                 &    t               |f|_        y r-   )r   r(  )rR   r   r   r   s       r2   r/  zPool.on_job_process_lost  s    %K2r:   c                     	 t        dj                  t        |      |j                              # t         $ r |j	                  d dt               f       Y y w xY w)Nz(Worker exited prematurely: {0} Job: {1}.F)r   r   r
   rr  rJ  r   )rR   r   r   s      r2   r*  zPool.mark_as_worker_lost  sV    	!:AA *CHH6   	5HHTE=?34	5s   .0 %AAc                     | S r-   ru   rQ   s    r2   	__enter__zPool.__enter__      r:   c                 "    | j                         S r-   )r4  )rR   r   s     r2   __exit__zPool.__exit__  s    ~~r:   c                      y r-   ru   rR   ns     r2   on_growzPool.on_grow   r   r:   c                      y r-   ru   rH  s     r2   	on_shrinkzPool.on_shrink  r   r:   c                 &   t        | j                               D ]k  \  }}| xj                  dz  c_        | j                  r| j                  j	                          |j                          | j                  d       ||dz
  k\  sk y  t        d      )Nr   z&Can't shrink pool. All processes busy!)rI  _iterinactiver=  r  rS   terminate_controlledrL  
ValueError)rR   rI  r   r~  s       r2   rS   zPool.shrink  sw    "4#5#5#78IAvOOq O}}$$&'')NN1AEz 9 EFFr:   c                     t        |      D ]>  }| xj                  dz  c_        | j                  s%| j                  j                          @ | j	                  |       y rN   )r   r=  r  r_   rJ  )rR   rI  r   s      r2   r_   z	Pool.grow  sB    qAOOq O}}""$  	Qr:   c              #   Z   K   | j                   D ]  }| j                  |      r|  y wr-   )r  _worker_activer}  s     r2   rN  zPool._iterinactive  s'     jjF&&v. !s   !++c                 ~    | j                   j                         D ]   }|j                  |j                         v s  y y)NTF)r  r'  r   r  )rR   r~  r   s      r2   rS  zPool._worker_active  s4    ;;%%'CzzS__.. ( r:   c                    t        | j                  t        | j                        z
        D ]o  }| j                  t
        k7  r y	 |r+||   t        t        fvr| j                  j                          | j                  | j                                t        d       q y# t        $ r | j                  j                          Y Rw xY w)zBring the number of pool processes up to the specified number,
        for use after reaping workers which have exited.
        Nzadded worker)r   r=  r  r  r#  r"  r*   r   r   step
IndexErrorr  _avail_indexr    )rR   r6  r   s      r2   _repopulate_poolzPool._repopulate_pool$  s     tTZZ89A{{c!*1eZ5H!H&&++- ''(9(9(;<.! :  *""'')*s   -B#CCc                     t        | j                        | j                  k  sJ t        d | j                  D              t	        fdt        | j                        D              S )Nc              3   4   K   | ]  }|j                     y wr-   )r  )r^  rR  s     r2   r`  z$Pool._avail_index.<locals>.<genexpr>5  s     2!agg2s   c              3   ,   K   | ]  }|vs|  y wr-   ru   )r^  r   indicess     r2   r`  z$Pool._avail_index.<locals>.<genexpr>6  s     J!'9IAJr  )r  r  r=  r  rd  r   )rR   r]  s    @r2   rX  zPool._avail_index3  sK    4::0002tzz22JuT__5JJJr:   c                 $    | j                          S r-   )r  rQ   s    r2   did_start_okzPool.did_start_ok8  s    ,,...r:   c                     | j                         }| j                  |       t        t        |            D ])  }| j                  | j                  j                          + y)zF"Clean up any exited workers and start replacements for them.
        N)r  rY  r   r  r  rd   )rR   joinedr   s      r2   r>  zPool._maintain_pool;  sM     **,f%s6{#A}}(%%' $r:   c                 F   | j                   j                  t        k(  r&| j                  t        k(  r	 | j                          y y y # t        $ r" | j                          | j                           t        $ r)}t        |      t        j                  k(  rt        | d }~ww xY wr-   )r  r#  r"  r>  r   r   r0  rk  r   r  ENOMEMMemoryErrorr)  s     r2   maintain_poolzPool.maintain_poolD  s    &&#-$++2D	##% 3E- ' 

		 S>U\\1%3.s   A 2B 7$BB c                 .     j                   j                          _         j                   j                          _         j                  j                  j
                   _         j                  j                  j                   _	         fd}| _
        y )Nc                 t    j                   j                  j                  |       rdj                         fS yr  )r  r   r  r   rI   rR   s    r2   r  z(Pool._setup_queues.<locals>._poll_resultW  s0    ~~%%**73T__...r:   )r  SimpleQueuer  r  r   r   r   r   r   r   r  rR   r  s   ` r2   r  zPool._setup_queuesQ  sh    		--/..0--//44..0055	 )r:   c                     | j                   rP| j                  C| j                  5  | j                  s!d| _        | j                  j	                          d d d        y y y # 1 sw Y   y xY wr  )r  r  r  r  r,  rQ   s    r2   r  zPool._start_timeout_handler]  sc     <<D11=,, 24448D1))//12 2 ><2 2s   .AA'c                 l    | j                   t        k(  r!| j                  |||      j                         S y)z8
        Equivalent of `func(*args, **kwargs)`.
        N)r#  r"  apply_asyncr  )rR   funcr8   r  s       r2   applyz
Pool.applyf  s3     ;;###D$599;; r:   c                 v    | j                   t        k(  r&| j                  ||t        |      j	                         S y)z
        Like `map()` method but the elements of the `iterable` are expected to
        be iterables as well and will be unpacked as arguments. Hence
        `func` and (a, b) becomes func(a, b).
        N)r#  r"  
_map_asyncr>   r  rR   rn  iterable	chunksizes       r2   r=   zPool.starmapm  s7     ;;#??4#.	;;>35A r:   c                 ^    | j                   t        k(  r| j                  ||t        |||      S y)z=
        Asynchronous version of `starmap()` method.
        N)r#  r"  rq  r>   rR   rn  rs  rt  callbackerror_callbacks         r2   starmap_asynczPool.starmap_asyncw  s3    
 ;;#??4;	#+^= = r:   c                 l    | j                   t        k(  r!| j                  |||      j                         S y)zx
        Apply `func` to each element in `iterable`, collecting the results
        in a list that is returned.
        N)r#  r"  	map_asyncr  rr  s       r2   r6   zPool.map  s1    
 ;;#>>$)<@@BB r:   c                    | j                   t        k7  ry|xs | j                  }|dk(  rSt        | j                  |      | j
                  j                  fdt        |      D        j                  f       S |dkD  sJ t        j                  ||      }t        | j                  |      | j
                  j                  fdt        |      D        j                  f       d D        S )zP
        Equivalent of `map()` -- can be MUCH slower than `Pool.map()`.
        Nr   r  c              3   T   K   | ]  \  }}t         j                  ||fi ff ! y wr-   r   rr  r^  r   xrn  r   s      r2   r`  zPool.imap.<locals>.<genexpr>  3      2Q atR89 2   %(c              3   \   K   | ]#  \  }}t         j                  |t        |fi ff % y wr-   r   rr  r9   r^  r   r  r   s      r2   r`  zPool.imap.<locals>.<genexpr>  3      6Q a1$;< 6   ),c              3   .   K   | ]  }|D ]  }|   y wr-   ru   r^  chunkr  s      r2   r`  zPool.imap.<locals>.<genexpr>       ?U?D?D?   )r#  r"  r  IMapIteratorr  r  r   rI  _set_lengthr  
_get_tasksrR   rn  rs  rt  r  task_batchesr   s    `    @r2   imapz	Pool.imap  s     ;;#1MT5M5M>!$++6IKFOO2&x02""! 
 Mq= =??49EL!$++6IKFOO6&|46""! 
 @f??r:   c                    | j                   t        k7  ry|xs | j                  }|dk(  rSt        | j                  |      | j
                  j                  fdt        |      D        j                  f       S |dkD  sJ t        j                  ||      }t        | j                  |      | j
                  j                  fdt        |      D        j                  f       d D        S )zL
        Like `imap()` method but ordering of results is arbitrary.
        Nr   r}  c              3   T   K   | ]  \  }}t         j                  ||fi ff ! y wr-   r  r  s      r2   r`  z&Pool.imap_unordered.<locals>.<genexpr>  r  r  c              3   \   K   | ]#  \  }}t         j                  |t        |fi ff % y wr-   r  r  s      r2   r`  z&Pool.imap_unordered.<locals>.<genexpr>  r  r  c              3   .   K   | ]  }|D ]  }|   y wr-   ru   r  s      r2   r`  z&Pool.imap_unordered.<locals>.<genexpr>  r  r  )r#  r"  r  IMapUnorderedIteratorr  r  r   rI  r  r  r  r  s    `    @r2   imap_unorderedzPool.imap_unordered  s    
 ;;#1MT5M5M>*1DF OO2&x02""! 
 Mq= =??49EL*1DF OO6&|46""! 
 @f??r:   c                 
   | j                   t        k7  ry|	xs | j                  }	|
xs | j                  }
|xs | j                  }|	r&t
         t        j                  t        d             d}	| j                   t        k(  r|| j                  n|}|r&| j                  | j                  j                          t        | j                  |||||	|
|| j                  | j                  || j                   r| j"                  nd|      }|
s|	r| j%                          | j&                  r5| j(                  j+                  t,        |j.                  d|||ffgdf       |S | j1                  t,        |j.                  d|||ff       |S y)a  
        Asynchronous equivalent of `apply()` method.

        Callback is called when the functions return value is ready.
        The accept callback is called when the job is accepted to be executed.

        Simplified the flow is like this:

            >>> def apply_async(func, args, kwds, callback, accept_callback):
            ...     if accept_callback:
            ...         accept_callback()
            ...     retval = func(*args, **kwds)
            ...     if callback:
            ...         callback(retval)

        Nr  )r  r  callbacks_propagatesend_ackcorrelation_id)r#  r"  r  rI   r  r   r  r  r  r  r  rP   ApplyResultr  r  r  r  r  r  r  r  r   r   rr  r   )rR   rn  r8   r  rw  rx  accept_callbacktimeout_callbackwaitforslotr  rI   r  r  r  r   s                  r2   rm  zPool.apply_async  st   , ;;##8t'8'8)T\\1MT5M5M,4MM+I   L;;#+6+>$--KKt}}8%%' X8Hg7J#22"&"8"8$7*.++4-F ,++-||##tfkk4$($.6 '7 &89=%? @ M T4t'L MNM+ r:   c                      y r-   ru   )rR   responser   r   fds        r2   r  zPool.send_ack  r   r:   c                     | j                  |      \  }}|$	 t        ||xs t               d|_        d|_        y y # t
        $ r'}t        |      t        j                  k7  r Y d }~y d }~ww xY wr  )	re  r(  r	   r"  r&  rk  r   r  rl  )rR   r   sigr_  r   rz   s         r2   terminate_jobzPool.terminate_job  sq    &&s+a,c3-+.
 04,'+$   S>U[[0 1s   < 	A,A''A,c                 6    | j                  ||t        |||      S )z<
        Asynchronous equivalent of `map()` method.
        )rq  r9   rv  s         r2   r{  zPool.map_async
  s"    
 (GY.
 	
r:   c                   	 | j                   t        k7  ryt        |d      st        |      }|5t	        t        |      t        | j                        dz        \  }}|r|dz  }t        |      dk(  rd}t        j                  |||      }t        | j                  |t        |      ||      	| j                  j                  	fdt        |      D        df       	S )zY
        Helper function to implement map, starmap and their async counterparts.
        N__len__r'   r   r   rx  c              3   T   K   | ]  \  }}t         j                  ||fi ff ! y wr-   r  )r^  r   r  mapperr   s      r2   r`  z"Pool._map_async.<locals>.<genexpr>'  s5      C"&!Q  $fkk1fqdB%GH Cr  )r#  r"  r   r5   divmodr  r  r  r  	MapResultr  r  r   rI  )
rR   rn  rs  r  rt  rw  rx  extrar  r   s
      `     @r2   rq  zPool._map_async  s    
 ;;#x+H~H%c(mS_q5HIIuQ	x=AItXyA4;;	3x=(*8: C*3L*ACDHJ 	Kr:   c              #   t   K   t        |      }	 t        t        j                  ||            }|sy | |f )wr-   )rH  tupler<   islice)rn  itsizer  s       r2   r  zPool._get_tasks+  s=     "Xi&&r401A)O	 s   68c                     t        d      )Nz:pool objects cannot be passed between processes or pickled)r  rQ   s    r2   r   zPool.__reduce__4  s    !H
 	
r:   c                 8   t        d       | j                  t        k(  r|t        | _        | j                  r| j                  j                          | j                  j                          | j                  j                  d        t        | j                         y y )Nzclosing pool)r    r#  r"  r6  r  rg   r  r   r  r   rJ   rQ   s    r2   r   z
Pool.close9  sk    n;;#DK}}##%  &&(OO% 4 45 r:   c                     t        d       t        | _        | j                  j	                          | j                          y )Nzterminating pool)r    r3  r#  r  r4  r  rQ   s    r2   r4  zPool.terminateC  s.     !&&(r:   c                     t        |        y r-   )rJ   )task_handlers    r2   _stop_task_handlerzPool._stop_task_handlerI  s
    L)r:   c                    | j                   t        t        fv sJ t        d       t	        | j
                         t        d       | j                  | j                         t        d       t	        | j                         t        d       t        | j                        D ]F  \  }}t        d|dz   t        | j                        |       |j                  7|j                          H t        d       y )Nzjoining worker handlerjoining task handlerjoining result handlerzresult handler joinedzjoining worker %s/%s (%r)r   zpool join complete)r#  r6  r3  r    rJ   r  r  r  r  rI  r  r  rz  r0  )rR   r   rR  s      r2   r0  z	Pool.joinM  s    {{ui0000&'D001$% 2 23&'D001%&djj)DAq-q1uc$**oqIxx# * 	"#r:   c                 b    | j                   j                         D ]  }|j                           y r-   )r  r'  r  )rR   es     r2   restartzPool.restart\  s"    &&(AEEG )r:   c                 \   t        d       | j                  j                          |j                         rw| j                  j                         r\| j                  j                          t        j                  d       |j                         r| j                  j                         rZy y y y )Nz7removing tasks from inqueue until task handler finishedr   )	r    _rlockrP   is_aliver   r  r   r   r   )inqueuer  r  s      r2   _help_stuff_finishzPool._help_stuff_finish`  sy     	GH ##%'//*>*>*@OO  "JJqM ##%'//*>*>*@%*@%r:   c                 &    |j                  d        y r-   )r   )clsrD  r:  s      r2   _set_result_sentinelzPool._set_result_sentineli  s    Tr:   c                    t        d       |j                          |j                          |j                  d        t        d        | j                  |
  |j                          | j	                  ||       |	|	j                          |rBt        |d   d      r3t        d       |D ]#  }|j                         s|j                          % t        d       | j                  |       t        d       |j                          |	 t        d       |	j                  t               |rpt        |d   d      rat        d	       |D ]F  }|j                         st        d
|j                         |j                  7|j                          H t        d       |r|j                          |r|j                          y y )Nzfinalizing poolz&helping task handler/workers to finishr   r4  zterminating workersr  r  zjoining timeout handlerzjoining pool workerszcleaning up worker %dzpool workers joined)r    r4  r   r  r  r   r2  r  rG   TIMEOUT_MAXr  r   rz  r0  r   )r  rC  r  rD  r:  worker_handlerr  result_handlerrE  timeout_handlerhelp_stuff_finish_argsrR  s               r2   r  zPool._terminate_poolm  sr    	   " d67 67  "  40&%%' GDG[1'(;;=KKM  	$%|,&'&+,  -GDG[1()::<11559xx+  '(MMONN r:   c                 h    | j                   D cg c]  }|j                  j                   c}S c c}w r-   )r  rz  r   )rR   r  s     r2   process_sentinelszPool.process_sentinels  s$    +/::6a!!666s   /)NNru   NNNNNr   NNNNTNFFFNNNFr  )r   r-   )NNNro   )Frp   rq   rr   rs   r  r   r8  rA  rU  r  r   rW   r  r  r  r  r  r   r  r  re  r  r  r   r  r<  r3  r-  r/  r*  rC  rF  rJ  rL  rS   r_   rN  rS  rY  rX  r_  r>  re  r  r  ro  r=   ry  r6   r  r  rm  r  r  r{  rq  staticmethodr  r   r   r4  r  r0  r  r  classmethodr  r  propertyr  ru   r:   r2   r  r    s'    OFJK#N!M1BDCG%)56#!% $#'$!%&*!&!x
t,H
=834Qf3	 
G
"K
/(
)2  " <A 7;48=C@8 89+/@> &(b!$%)t!%t(*#'6p
, 3704
 <@150  

6 * *$     4 4l 7 7r:   r  c                       e Zd ZdZdZdZdddddedddddfdZd Zd Z	d Z
d Zd Zd	 Zd
 ZddZd ZddZddZd ZddZd Zd Zy)r  Nru   c                    || _         t               | _        t        j                         | _        t        t              | _        || _	        || _
        || _        || _        || _        || _        || _        || _        |	| _        |
| _        |xs d| _        || _        d| _        d| _        d | _        d | _        d | _        | || j                  <   y )Nru   F)r  r  _mutexrE   r  _eventrd  job_counterrr  r  	_callback_accept_callback_error_callback_timeout_callbackrq  r  r)  _on_timeout_set_on_timeout_cancel_callbacks_propagate	_send_ack	_accepted
_cancelledri  r  _terminated)rR   rE  rw  r  r  rx  r  rI   r  r  r  r  r  r  s                 r2   rW   zApplyResult.__init__  s     -foo'%	! /-!1)$7!-"3$7$=2!!"diir:   c                     dj                  | j                  j                  | j                  | j                  | j                               S )Nz&<{name}: {id} ack:{ack} ready:{ready}>)r  idackr  )r   r|   rp   rr  r  r  rQ   s    r2   r   zApplyResult.__repr__  s<    7>>((yydnnDJJL ? 
 	
r:   c                 6    | j                   j                         S r-   )r  r  rQ   s    r2   r  zApplyResult.ready  s    {{!!##r:   c                     | j                   S r-   )r  rQ   s    r2   acceptedzApplyResult.accepted  s    ~~r:   c                 >    | j                         sJ | j                  S r-   )r  _successrQ   s    r2   
successfulzApplyResult.successful  s    zz||}}r:   c                     d| _         y)zOnly works if synack is used.TN)r  rQ   s    r2   _cancelzApplyResult._cancel  s	    r:   c                 P    | j                   j                  | j                  d        y r-   )r  poprr  rQ   s    r2   discardzApplyResult.discard  s    		4(r:   c                     || _         y r-   )r  rR   r   s     r2   r4  zApplyResult.terminate  s
    !r:   c                 z    	 t        |xs d       # t         $ r | j                  d dt               f       Y y w xY wNr   F)r   rJ  r   r  s     r2   r.  zApplyResult._set_terminated  s<    	6v{^,, 	6IIdUMO45	6s    %::c                 8    | j                   r| j                   gS g S r-   ri  rQ   s    r2   r  zApplyResult.worker_pids  s    %)%5%5  !=2=r:   c                 :    | j                   j                  |       y r-   )r  r{  r1  s     r2   r{  zApplyResult.wait  s    !r:   c                     | j                  |       | j                         st        | j                  r| j                  S | j                  j
                  r-   )r{  r  r   r  r]   	exceptionr1  s     r2   r  zApplyResult.get  s>    		'zz|==;;++'''r:   c                     |r
	  ||i | y y # | j                   $ r  t        $ r}t        d|d       Y d }~y d }~ww xY w)Nz"Pool callback raised exception: %rr   r   )r  r   rA   )rR   r   r8   rC   rz   s        r2   safe_apply_callbackzApplyResult.safe_apply_callback  sT    "T$V$  ,,  ":C " ""s    >9>c                     | j                   7| j                  | j                   ||r| j                  n| j                         y y )N)rh  rI   )r  r  r  rq  )rR   rh  s     r2   rj  zApplyResult.handle_timeout	  sB    !!-$$&&T.2** %  .r:   c                 F   | j                   5  | j                  r| j                  |        |\  | _        | _        | j                  j                          | j                  r&| j                  j                  | j                  d        | j                  r2| j                  r&| j                  | j                  | j                         | j                  >| j                  r2| j                  s&| j                  | j                  | j                         d d d        y # 1 sw Y   y xY wr-   )r  r  r  r]   r  r  r  r  r  rr  r  r  r  rR   r   r   s      r2   rJ  zApplyResult._set  s    [[ 	7&&''-),&DM4;KKOO~~ 		40 ~~$--((NNDKK1'(((((($++7#	7 	7 	7s   DDD c           	         | j                   5  | j                  rJ| j                  r>d| _        |r+| j                  t        || j
                  |      cd d d        S 	 d d d        y d| _        || _        || _        | j                         r&| j                  j                  | j
                  d        | j                  r'| j                  | | j                  | j                         t        }| j                  rH	 | j                  ||       | j                  r)|r'| j                  ||| j
                  |      cd d d        S | j                  r |r| j                  ||| j
                  |       d d d        y # | j                   $ r t        } t"        $ r	 t        }Y w xY w# | j                  r.|r+| j                  ||| j
                  |      c cY cd d d        S w w xY w# 1 sw Y   y xY wr  )r  r  r  r  r   rr  r  ri  r  r  r  r  r  rq  r   r  _propagate_errorsr   )rR   r   r  r   r   r  s         r2   r  zApplyResult._ack%  s   [[ 	C4>>!%>>$TYYI		C 	C
 	C 	C "DN"/D"Dzz|		40##$$T4+=+=t}}MH$$))#}= ~~(#~~$c499h 5	C 	C: ~~(xdiiB=	C 	C$ -- #H  $#H$ ~~(#~~$c499h  5	C 	C2 +3~3	C 	Cs[   AGG$BG5E1+G<,G1$FFFF/G
GGGG#r-   r  )rp   rq   rr   r(  r0  r1  r  rW   r   r  r  r  r  r  r4  r.  r  r{  r  r  rj  rJ  r  ru   r:   r2   r  r    s    LIN8<"&t$3F $%'$ $ <
$)"6>"("7*Cr:   r  c                   *    e Zd Zd Zd Zd Zd Zd Zy)r  c                 R   t         j                  | |||       d| _        || _        d g|z  | _        dg|z  | _        d g|z  | _        d g|z  | _        || _        |dk  r/d| _	        | j                  j                          || j                  = y ||z  t        ||z        z   | _	        y )Nr  TFr   )r  rW   r  _lengthr]   r  ri  r  
_chunksize_number_leftr  r  rr  r  )rR   rE  rt  lengthrw  rx  s         r2   rW   zMapResult.__init__M  s    %. 	 	
 fvo6) 6F?#fvo#> !DKKOOdii  &) 3d6I;M6N NDr:   c                    |\  }}|r|| j                   || j                  z  |dz   | j                  z   | xj                  dz  c_        | j                  dk(  rt| j                  r| j                  | j                          | j                  r&| j
                  j                  | j                  d        | j                  j                          y y d| _
        || _         | j                  r| j                  | j                          | j                  r&| j
                  j                  | j                  d        | j                  j                          y )Nr   r   F)r]   r  r  r  r  r  r  rr  r  r  r  r  )rR   r   success_resultsuccessr   s        r2   rJ  zMapResult._set_  s    (IODKKDOO+QUdoo,EF"  A%>>NN4;;/>>KKOODIIt4! & "DM DK##$$T[[1~~		40KKOOr:   c                 X   || j                   z  }t        |dz   | j                   z  | j                        }t        ||      D ]/  }d| j                  |<   || j
                  |<   || j                  |<   1 | j                         r'| j                  j                  | j                  d        y y Nr   T)r  r  r  r   r  ri  r  r  r  r  rr  )rR   r   r  r   r8   r,  rG   js           r2   r  zMapResult._acks  s    DOO#AET__,dll;ud#A $DNN1"%DQ%2D" $ ::<KKOODIIt, r:   c                 ,    t        | j                        S r-   )allr  rQ   s    r2   r  zMapResult.accepted}  s    4>>""r:   c                 F    | j                   D cg c]  }|s|	 c}S c c}w r-   r  r   s     r2   r  zMapResult.worker_pids  s    #//73777s   N)rp   rq   rr   rW   rJ  r  r  r  ru   r:   r2   r  r  K  s    O$(-#8r:   r  c                   J    e Zd ZdZefdZd Zd
dZeZd Z	d Z
d Zd Zd	 Zy)r  Nc                 "   t        j                  t        j                               | _        t	        t
              | _        || _        t               | _	        d| _
        d | _        d| _        i | _        g | _        || _        | || j                  <   y r  )rE   	Conditionr  r\   rd  r  rr  r  r   _itemsrn  r  _ready	_unsorted_worker_pidsr)  )rR   rE  r  s      r2   rW   zIMapIterator.__init__  sq    (()9:
%	g$7!diir:   c                     | S r-   ru   rQ   s    r2   __iter__zIMapIterator.__iter__  rD  r:   c                    | j                   5  	 | j                  j                         }d d d        \  }}|r|S t        |      # t        $ r | j                  | j
                  k(  rd| _        t        | j                   j                  |       	 | j                  j                         }n9# t        $ r- | j                  | j
                  k(  rd| _        t        t        w xY wY w xY w# 1 sw Y   xY wr  )r\   r  popleftrW  rn  r  r  r  r{  r   r   )rR   rI   r  r  rY   s        r2   rd  zIMapIterator.next  s    ZZ 	''{{**,	'  L!  ';;$,,."&DK''

(';;..0D! '{{dll2&*++&&	' '	' 	'sA   C*AA
C'B+*C'+6C!!C'$C*&C''C**C3c                    | j                   5  | j                  |k(  r| j                  j                  |       | xj                  dz  c_        | j                  | j                  v rn| j                  j                  | j                        }| j                  j                  |       | xj                  dz  c_        | j                  | j                  v rn| j                   j                          n|| j                  |<   | j                  | j                  k(  rd| _        | j                  | j                  = d d d        y # 1 sw Y   y xY wr  )r\   rn  r  r  r  r  r^   r  r  r  rr  r   s      r2   rJ  zIMapIterator._set  s    ZZ 	+{{a""3'q kkT^^3..,,T[[9CKK&&s+KK1$K kkT^^3 

!!#$'q!{{dll*"KK		*	+ 	+ 	+s   CD=A!D==Ec                     | j                   5  || _        | j                  | j                  k(  r8d| _        | j                   j	                          | j
                  | j                  = d d d        y # 1 sw Y   y xY wr  )r\   r  rn  r  r^   r  rr  )rR   r  s     r2   r  zIMapIterator._set_length  s\    ZZ 	+!DL{{dll*"

!!#KK		*	+ 	+ 	+s   AA//A8c                 :    | j                   j                  |       y r-   )r  r  )rR   r   r  r   r8   s        r2   r  zIMapIterator._ack  s      %r:   c                     | j                   S r-   )r  rQ   s    r2   r  zIMapIterator.ready  s    {{r:   c                     | j                   S r-   )r  rQ   s    r2   r  zIMapIterator.worker_pids  s       r:   r-   )rp   rq   rr   r(  r  rW   r  rd  __next__rJ  r  r  r  r  ru   r:   r2   r  r    s9    L2E  , H+"+&!r:   r  c                       e Zd Zd Zy)r  c                 H   | j                   5  | j                  j                  |       | xj                  dz  c_        | j                   j	                          | j                  | j
                  k(  rd| _        | j                  | j                  = d d d        y # 1 sw Y   y xY wr  )	r\   r  r  rn  r^   r  r  r  rr  r   s      r2   rJ  zIMapUnorderedIterator._set  su    ZZ 	+KKs#KK1KJJ{{dll*"KK		*	+ 	+ 	+s   BBB!N)rp   rq   rr   rJ  ru   r:   r2   r  r    s    +r:   r  c                   :    e Zd ZddlmZ eZddZd Zed        Z	y)
ThreadPoolr   )r  Nc                 4    t         j                  | |||       y r-   )r  rW   )rR   rW  r   r   s       r2   rW   zThreadPool.__init__  s    dI{H=r:   c                      t                _        t                _         j                  j                   _         j                  j
                   _         fd}| _        y )Nc                 L    	 dj                  |       fS # t        $ r Y yw xY wr
  )r   r   rh  s    r2   r  z.ThreadPool._setup_queues.<locals>._poll_result  s0    #T__W_=== #"#s    	##)r   r  r  r   r   r  r   r  rj  s   ` r2   r  zThreadPool._setup_queues  sF    --++..,,	#
 )r:   c                     | j                   5  | j                  j                          | j                  j                  d gt	        |      z         | j                   j                          d d d        y # 1 sw Y   y xY wr-   )	not_emptyqueuerg   extendr  ra   )r  r  r:  s      r2   r  zThreadPool._help_stuff_finish  sa      	+MM!MM  $#d)!34((*	+ 	+ 	+s   AA33A<)NNru   )
rp   rq   rr   dummyr  r   rW   r  r  r  ru   r:   r2   r&  r&    s(    .G>) + +r:   r&  r-   )er  r  r<   r   r   r   r   rE   r   r  collectionsr   	functoolsr    r   r   r   commonr	   r
   r   r   r   compatr   r   r   r   r   r.  r   
exceptionsr   r   r   r   r   r   r   r   r,  r   r   r   r    r!   r   version_infort   system_winr$   r(  r|  r%   r  r.   	SemaphorerV   r"  r6  r3  r   r   r   r   r   r*   r   r   r,  r   r  r   r   countr  r  r3   r9   r>   rA   rJ   rL   r   rw   r   r   r   r  r8  rA  rU  r  r  r  r  r  r  r&  ru   r:   r2   <module>r:     sf      	   
      $   4 3        * * 
qQ8??	! 0G nnG''K
 v$$J %%J 		 		
 	


 69d3   GQ,/ )/2 ,ioo~~52
4)* 4)v" ""'I '"{ {F& &R"( "(JH* HV@Z @Fv'J v'rc7 c7TWC WC|68 68zK! K!d	+L 	+ + +O=  Ks   H HH