File: //lib64/python3.8/multiprocessing/__pycache__/pool.cpython-38.opt-2.pyc
U
e5d~ @ sd d dgZ ddlZddlZddlZddlZddlZddlZddlZddlZddlm Z ddl
m Z ddl
m Z m
Z
ddlmZ d Zd
Zd Zd Ze Zd
d Zdd ZG dd deZG dd dZdd ZG dd deZ d+dd Z d d Z G d d d e Z G d!d d e!Z"G d"d# d#e!Z#e#Z$G d$d% d%e#Z%G d&d' d'e!Z&G d(d) d)e&Z'G d*d de"Z(dS ),Pool
ThreadPool N)Empty )util) get_context TimeoutError)waitINITRUNCLOSE TERMINATEc C s t t| S N)listmapargs r ,/usr/lib64/python3.8/multiprocessing/pool.pymapstar/ s r c C s t t| d | d S )Nr r )r itertoolsstarmapr r r r starmapstar2 s r c @ s e Zd Zdd Zdd ZdS )RemoteTracebackc C s
|| _ d S r tb)selfr r r r __init__: s zRemoteTraceback.__init__c C s | j S r r r r r r __str__< s zRemoteTraceback.__str__N)__name__
__module__ __qualname__r r r r r r r 9 s r c @ s e Zd Zdd Zdd ZdS )ExceptionWithTracebackc C s0 t t|||}d|}|| _d| | _d S )N z
"""
%s""") tracebackformat_exceptiontypejoinexcr )r r) r r r r r @ s
z ExceptionWithTraceback.__init__c C s t | j| jffS r ) rebuild_excr) r r r r r
__reduce__E s z!ExceptionWithTraceback.__reduce__N)r r! r" r r+ r r r r r# ? s r# c C s t || _| S r )r __cause__)r) r r r r r* H s
r* c s, e Zd Z fddZdd Zdd Z ZS )MaybeEncodingErrorc s. t || _t || _tt| | j| j d S r )reprr) valuesuperr- r )r r) r/ __class__r r r T s
zMaybeEncodingError.__init__c C s d| j | jf S )Nz(Error sending result: '%s'. Reason: '%s')r/ r) r r r r r Y s zMaybeEncodingError.__str__c C s d| j j| f S )Nz<%s: %s>)r2 r r r r r __repr__] s zMaybeEncodingError.__repr__)r r! r" r r r3
__classcell__r r r1 r r- P s r- r Fc
C s |d k r(t |tr|dks(td||j}| j}t| drR| j |j |d k rb|| d}|d ks~|r||k rz
| } W n( t
t fk
r t
d Y qY nX | d krt
d q| \}
} } }
}zd| |
|f}W nH tk
r0 } z(|r| tk rt||j}d|f}W 5 d }~X Y nX z||
| |f W nR tk
r } z2t||d }t
d | ||
| d|ff W 5 d }~X Y nX d } }
} } }
}|d7 }qft
d
| d S ) Nr zMaxtasks {!r} is not valid_writerr z)worker got EOFError or OSError -- exitingz worker got sentinel -- exitingTFz0Possible encoding error while sending result: %sz worker exiting after %d tasks)
isinstanceintAssertionErrorformatputgethasattrr5 close_readerEOFErrorOSErrorr debug Exception_helper_reraises_exceptionr#
__traceback__r- )inqueueoutqueue initializerinitargsZmaxtaskswrap_exceptionr: r; Z completedtaskjobifuncr kwdsresultewrappedr r r workera sN
$
rR c C s | d S r r )Zexr r r rC s rC c s. e Zd Zdd fdd
Z fddZ ZS )
_PoolCacheNnotifierc s || _ t j|| d S r )rU r0 r )r rU r rN r1 r r r s z_PoolCache.__init__c s t | | s | jd d S r )r0 __delitem__rU r: )r itemr1 r r rV s z_PoolCache.__delitem__)r r! r" r rV r4 r r r1 r rS s rS c @ s e Zd ZdZedd ZdKddZeje fdd Z
d
d Z d d
Z edd Z
edd Zdd Zedd Zedd Zdd Zdd Zdi fd d ZdLd d ZdMd d!ZdNd"d#Zd$d% ZdOd'd(ZdPd)d*Zdi ddfd+d,ZdQd-d.Z dRd/d0Z edSd1d2Z e d3d4 Z ed5d6 Z!ed7d8 Z"ed9d: Z#d;d< Z$d=d> Z%d?d@ Z&dAdB Z'edCdD Z(e dEdF Z)dGdH Z*dIdJ Z+dS )Tr Tc O s | j ||S r Process)ctxr rN r r r rY s z Pool.ProcessNr c C s g | _ t| _|pt | _| t | _| j | _ t
| j d| _ || _ || _
|| _|d krjt phd}|dk rztd|d k rt|std|| _z | W nH tk
r | j D ]}|jd kr| q| j D ] }| q Y nX | }tj t j | j | j| j| j | j| j | j | j!| j
| j| j | j"|| j fd| _#d| j#_$t%| j#_| j#& tj t j'| j| j(| j!| j | j fd| _)d| j)_$t%| j)_| j)& tj t j*| j!| j+| j fd| _,d| j,_$t%| j,_| j,& t-j.| | j/| j| j | j!| j | j | j#| j)| j,| j f dd| _0t%| _d S ) NrT r z&Number of processes must be at least 1z initializer must be a callabletargetr T )r Z exitpriority)1_poolr
_stater _ctx
_setup_queuesqueue SimpleQueue
_taskqueue_change_notifierrS _cache_maxtasksperchild _initializer _initargsos cpu_count
ValueErrorcallable TypeError
_processes_repopulate_poolrB exitcode terminater( _get_sentinels threadingZThreadr _handle_workersrY _inqueue _outqueue_wrap_exception_worker_handlerdaemonr start
_handle_tasks
_quick_put
_task_handler_handle_results
_quick_get_result_handlerr ZFinalize_terminate_pool
_terminate)r processesrG rH maxtasksperchildcontextp sentinelsr r r r s
z
Pool.__init__c C s> | j |kr:|d| t| d t| dd d k r:| jd d S )Nz&unclosed running multiprocessing pool )sourcere )r_ ResourceWarninggetattrre r: )r Z_warnr r r r __del__ s
z Pool.__del__c C s0 | j }d|j d|j d| j dt| j d S )N<.z state=z pool_size=>)r2 r! r" r_ lenr^ )r clsr r r r3 s z
Pool.__repr__c C s | j jg}| jjg}||S r )rw r> re )r Ztask_queue_sentinelsZself_notifier_sentinelsr r r rs s
zPool._get_sentinelsc C s dd | D S )Nc S s g | ]}t |d r|jqS )sentinel)r< r ).0rR r r r
<listcomp> s
z.Pool._get_worker_sentinels.<locals>.<listcomp>r Zworkersr r r _get_worker_sentinels s zPool._get_worker_sentinelsc C sP d}t tt| D ]6}| | }|jd k rtd| | d}| |= q|S )NFcleaning up worker %dT)reversedranger rq r rA r( )poolZcleanedrL rR r r r _join_exited_workers s
zPool._join_exited_workersc C s0 | | j| j| j| j| j| j| j| j| j | j
S r ) _repopulate_pool_staticr` rY ro r^ rv rw rh ri rg rx r r r r rp . s zPool._repopulate_poolc
C sf t |t| D ]P}
|| t|||||| fd} | jdd| _d| _| || t d qd S )Nr[ rY Z
PoolWorkerTz added worker)
r r rR namereplacerz r{ appendr rA ) rZ rY r r rE rF rG rH r rI rL wr r r r 7 s
z Pool._repopulate_pool_staticc
C s* t |r&t | |||||||||
d S r )r r r )
rZ rY r r rE rF rG rH r rI r r r _maintain_poolJ s
zPool._maintain_poolc C s4 | j | _| j | _| jjj| _| jjj| _ d S r )
r` rc rv rw r5 sendr} r> recvr r r r r ra V s zPool._setup_queuesc C s | j tkrtdd S )NzPool not running)r_ r rl r r r r _check_running\ s
zPool._check_runningc C s | ||| S r ) apply_asyncr; )r rM r rN r r r apply` s z
Pool.applyc C s | ||t| S r )
_map_asyncr r; r rM iterable chunksizer r r r g s zPool.mapc C s | ||t| S r )r r r; r r r r r n s z Pool.starmapc C s | ||t|||S r )r r r rM r r callbackerror_callbackr r r
starmap_asyncv s zPool.starmap_asyncc
c sj z,d}t |D ]\}}||||fi fV qW n8 tk
rd } z||d t|fi fV W 5 d }~X Y nX d S )Nr ) enumeraterB rC )r Z
result_jobrM r rL xrP r r r _guarded_task_generation~ s z Pool._guarded_task_generationr c C s | |dkr:t| }| j| |j|||jf |S |dk rPtd|t
|||}t| }| j| |jt ||jf dd |D S d S )Nr z Chunksize must be 1+, not {0:n}c s s | ]}|D ]
}|V q
qd S r r r chunkrW r r r <genexpr> s z Pool.imap.<locals>.<genexpr>) r IMapIteratorrd r: r _job _set_lengthrl r9 r
_get_tasksr r rM r r rO task_batchesr r r imap s4 z Pool.imapc C s | |dkr:t| }| j| |j|||jf |S |dk rPtd|t
|||}t| }| j| |jt ||jf dd |D S d S )Nr z Chunksize must be 1+, not {0!r}c s s | ]}|D ]
}|V q
qd S r r r r r r r s z&Pool.imap_unordered.<locals>.<genexpr>) r IMapUnorderedIteratorrd r: r r r rl r9 r r r r r r r imap_unordered s0 zPool.imap_unorderedc C s6 | t| ||}| j|jd|||fgd f |S Nr )r ApplyResultrd r: r )r rM r rN r r rO r r r r s zPool.apply_asyncc C s | ||t|||S r )r r r r r r map_async s zPool.map_asyncc
C s | t|dst|}|d krJtt|t| jd \}}|rJ|d7 }t|dkrZd}t|||}t| |t|||d} | j
| | j ||d f | S )N__len__ r r r )
r r< r divmodr r^ r r MapResultrd r: r r )
r rM r Zmapperr r r Zextrar rO r r r r s,
zPool._map_asyncc C s" t | |d | s | q d S )N)timeout)r emptyr; )r change_notifierr r r r _wait_for_updates s zPool._wait_for_updatesc C sp t }|jtks |rX|jtkrX| ||||||| |
| |
| ||
}| || q|d t
d d S )Nzworker handler exiting) rt current_threadr_ r r
r r r r: r rA )r cache taskqueuerZ rY r r rE rF rG rH r rI r r threadZcurrent_sentinelsr r r ru s
zPool._handle_workersc
C sp t }t| jd D ]\}}d }z|D ]}|jtkrBtd qz || W q& tk
r }
zB|d d \} } z|| | d|
f W n t
k
r Y nX W 5 d }
~
X Y q&X q&|rtd |r|d nd} || d W qW
q
W 5 d } }} X qtd z6td | d td |D ]} |d q.W n t k
r` td
Y nX td d S ) Nz'task handler found thread._state != RUN Fzdoing set_length()r r ztask handler got sentinelz/task handler sending sentinel to result handlerz(task handler sending sentinel to workersz/task handler got OSError when sending sentinelsztask handler exiting)
rt r iterr; r_ r r rA rB _setKeyErrorr: r@ )
r r: rF r r r ZtaskseqZ
set_lengthrJ rK rP idxr r r r r| sB
zPool._handle_tasksc C s t }z
| }W n$ ttfk
r6 td Y d S X |jtkrNtd q|d krbtd q|\}}}z|| || W n t k
r Y nX d } }}q|r@|jt
kr@z
| }W n$ ttfk
r td Y d S X |d k rtd q|\}}}z|| || W n t k
r0 Y nX d } }}qt | drtd z,t dD ] }| j
sv q| q`W n ttfk
r Y nX tdt||j d S ) Nz.result handler got EOFError/OSError -- exitingz,result handler found thread._state=TERMINATEzresult handler got sentinelz&result handler ignoring extra sentinelr> z"ensuring that outqueue is not full
z7result handler exiting: len(cache)=%s, thread._state=%s)rt r r@ r? r rA r_ r r r r
r< r r> pollr )rF r; r r rJ rK rL objr r r r : s\
zPool._handle_resultsc c s0 t |}tt||}|s d S | |fV qd S r )r tupler islice)rM itsizer r r r r v s
zPool._get_tasksc C s t dd S )Nz:pool objects cannot be passed between processes or pickled)NotImplementedErrorr r r r r+ s zPool.__reduce__c C s2 t d | jtkr.t| _t| j_| jd d S )Nz closing pool)r rA r_ r r ry re r: r r r r r= s
z
Pool.closec C s t d t| _| d S )Nzterminating pool)r rA r
r_ r r r r r rr s
zPool.terminatec C sj t d | jtkr tdn| jttfkr4td| j | j | j
| j D ] }| qXd S )Nz joining poolzPool is still runningzIn unknown state) r rA r_ r rl r r
ry r( r~ r r^ )r r r r r r( s
z Pool.joinc C s@ t d | j | r<| j r<| j t d qd S )Nz7removing tasks from inqueue until task handler finishedr )
r rA Z_rlockacquireis_aliver> r r timesleep)rE task_handlerr r r r _help_stuff_finish s
zPool._help_stuff_finishc
C sX t d t|_|d t|_t d | ||t| | sXt| dkrXtdt|_|d |d t d t
|k r| |rt |d drt d |D ]}
|
j
d kr|
qt d t
|k r| t d t
|k r| |rTt |d drTt d
|D ](}
|
r*t d |
j |
q*d S ) Nzfinalizing poolz&helping task handler/workers to finishr z.Cannot have cache with result_hander not alivezjoining worker handlerrr zterminating workerszjoining task handlerzjoining result handlerzjoining pool workersr )r rA r
r_ r: r r r r8 rt r r( r< rq rr pid) r r rE rF r r Zworker_handlerr Zresult_handlerr r r r r r sB
zPool._terminate_poolc C s | | S r )r r r r r __enter__ s zPool.__enter__c C s | d S r )rr )r exc_typeZexc_valZexc_tbr r r __exit__ s z
Pool.__exit__)NNr NN)N)N)NNN)r )r )NNN)NNN)N),r r! r" rx staticmethodrY r warningswarnr r r3 rs r r rp r r ra r r r r r r r r r r r r classmethodru r| r r r+ r= rr r( r r r r r r r r r sv
P
-
;
5c @ s@ e Zd Zdd Zdd Zdd Zddd Zdd
d Zd d
ZdS )r c C s> || _ t | _tt| _|j| _|| _|| _ | | j| j< d S r )
r^ rt ZEvent_eventnext job_counterr rf _callback_error_callback)r r r r r r r r s
zApplyResult.__init__c C s
| j S r )r Zis_setr r r r ready s zApplyResult.readyc C s | std| | jS )Nz{0!r} not ready)r rl r9 _successr r r r
successful s zApplyResult.successfulNc C s | j | d S r )r r r r r r r r s zApplyResult.waitc C s, | | | st| jr"| jS | jd S r )r r r r _valuer r r r r; s
zApplyResult.getc C sZ |\| _ | _| jr$| j r$| | j | jr<| j s<| | j | j | j| j= d | _d S r ) r r r r r setrf r r^ r rL r r r r r s
zApplyResult._set)N)N) r r! r" r r r r r; r r r r r r s
r c @ s e Zd Zdd Zdd ZdS )r c C sh t j| |||d d| _d g| | _|| _|dkrNd| _| j | j| j = n|| t
|| | _d S )Nr Tr ) r r r r
_chunksize _number_leftr r rf r bool)r r r lengthr r r r r r s
zMapResult.__init__c C s | j d8 _ |\}}|rv| jrv|| j|| j |d | j < | j dkr| jrZ| | j | j| j= | j d | _ nL|s| jrd| _|| _| j dkr| j
r|
| j | j| j= | j d | _ d S )Nr r F) r r r r r rf r r r r^ r )r rL Zsuccess_resultsuccessrO r r r r $ s&
zMapResult._setN)r r! r" r r r r r r r s
r c @ s: e Zd Zdd Zdd Zd ddZeZdd Zd
d ZdS )
r c C sT || _ tt | _tt| _|j| _t
| _ d| _ d | _
i | _| | j| j< d S r )r^ rt Z ConditionZLock_condr r r rf collectionsdeque_items_index_length _unsorted)r r r r r r B s
zIMapIterator.__init__c C s | S r r r r r r __iter__M s zIMapIterator.__iter__Nc C s | j z| j }W nz tk
r | j| jkr>d | _td | j | z| j }W n2 tk
r | j| jkrd | _td t d Y nX Y nX W 5 Q R X |\}}|r|S |d S r )
r r popleft
IndexErrorr r r^
StopIterationr r )r r rW r r/ r r r r P s& zIMapIterator.nextc C s | j | j|krn| j| | jd7 _| j| jkrb| j| j}| j| | jd7 _q,| j n
|| j|< | j| jkr| j| j = d | _
W 5 Q R X d S Nr ) r r r r r popnotifyr rf r r^ r r r r r h s
zIMapIterator._setc C sB | j 2 || _| j| jkr4| j | j| j= d | _W 5 Q R X d S r )r r r r rf r r^ )r r r r r r y s
zIMapIterator._set_length)N) r r! r" r r r __next__r r r r r r r @ s
r c @ s e Zd Zdd ZdS )r c C sV | j F | j| | jd7 _| j | j| jkrH| j| j= d | _W 5 Q R X d S r ) r r r r r r rf r r^ r r r r r s
zIMapUnorderedIterator._setN)r r! r" r r r r r r s r c @ sV e Zd ZdZedd ZdddZdd Zd
d Zed d
Z edd Z
dd Z dS )r Fc O s ddl m} |||S )Nr rX )ZdummyrY )rZ r rN rY r r r rY s zThreadPool.ProcessNr c C s t | ||| d S r )r r )r r rG rH r r r r s zThreadPool.__init__c C s, t | _t | _| jj| _| jj| _d S r )rb rc rv rw r: r} r; r r r r r ra s
zThreadPool._setup_queuesc C s
| j jgS r )re r> r r r r rs s zThreadPool._get_sentinelsc C s g S r r r r r r r s z ThreadPool._get_worker_sentinelsc C sF z| j dd qW n tjk
r( Y nX t|D ]}| d q2d S )NF)block)r; rb r r r: )rE r r rL r r r r s z ThreadPool._help_stuff_finishc C s t | d S r )r r )r r r r r r r r s z ThreadPool._wait_for_updates)NNr ) r r! r" rx r rY r ra rs r r r r r r r r s
)Nr NF))__all__r r rj rb rt r r% r r r$ r r r Z
connectionr r
r r r
countr r r rB r r# r* r- rR rC dictrS objectr r Z AsyncResultr r r r r r r r <module>
sN
- =)+E