U Nv f @s dZddlZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z ddlm Z ddl mZddlmZddlmZddlZddlZdd lmZdd lmZmZdd lmZdd lmZmZdd lmZddlmZmZddl m!Z!m"Z"m#Z#m$Z$m%Z%edk rddl&m'Z'ddl(m)Z)z ddl*Z*Wne+k rLdZ*YnXzddl,m-Z-Wne+k rxdZ-YnXz ddl.Z.Wne+k rdZ.YnXddl/m0Z0ddl/m1Z1ddl/m2Z2ddl/m3Z3ddl/m4Z4ddl5m6Z6m7Z7ddl5m8Z8ddl5m9Z9ddl5m:Z:ddl5m;Z;mZ>e=?Z@e@AdddgeBe=CZDeDd d!e=D7ZDedkr~gZEndd"gZEeEd#gZFeGed$reDHeId%e=e>ZJd&d'ZKd(d)ZLd*d+ZMGd,d-d-eNZOdMd/d0ZPd1d2ZQdNd3d4ZRd5d6ZSd7d8ZTd9d:ZUd;d<ZVe!d=e8e9gejWj!d>d?d@e;d@dAfdBgdCdDdEgdFedGdHZXe!dIeDe!dJddKd@dLge!dMdKdNdOgdPdQZYe!dIeDdRdSZZdTdUZ[ee!dVdWdXdYdZd[d\gd]d^Z\ee!dId"dd#gd_d`Z]dadbZ^e!dce=e!dde=dedfZ_dgdhZ`edidjZadkdlZbe!dJddKdmgdndoZce!dIeFdpdqZdedrdsZee!dIeFdtduZfee!dIeFdvdwZgee!dIeEdxdyZhdzd{Zie!dIe=e!d|dd}d~ddddddddddg fdd}dddd~dddddddg fgddZjee!dIeFddZkddZlee!dIeEddZmddZnddZoddZpee!dIeFejWjqddddZrGddde0ZsddZte!dIeDddZuddZvddZwe$edk ddddZxddZyeFddZzez{dd!e|dmDee!dIeze!d=e9e8gddZ}Gddde0Z~e!d=e8e9gddZe!d=e8e9gddZddZddZGdddejj4Ze:deee!dId#d"ddge!d=e8e9gddZee!dJdKd@dge!dIeFe!d=e8e9gddZddZee!deFe!deFe!d=e8e9gddĄZee!d=e8e9ge!ddd.gddȄZe!ddd@dgdd̄Ze!ddddddddddddg ddلZeddۄZeee!dIeEdd݄ZdZee!dIeEddZdZdZdZdZee!dIeEedkrgnd%ge!deeege!ddddgddZdjejejejdZeddZdjejejejdZeddZddZddZddZeee!dIeEddZddZeeddZddZddZejW!dJddKdgddZe!dIeDe!dJddKdLd@gddZddZee!dIe@e!dJddKdLd@gd d Ze!dIe@e!dJddKdLd@gd d Ze!dIe@e!dJddKdLd@gd dZe!dIe@e!dJddKdLd@gddZe!ddddgddZeee!dIeEddZe!dIedk rdd#gndd"d#gddZee!dIeEddZee!d=e8e9gddZe!d=e8e9gdd Zd!d"ZdOd#d$Zee!dId"d#ge!d=e8e9gd%d&Zee$e.dkd'de!d=e8e9gd(d)ZdPd*d+ZejWje!d=e8e9ge!dIedk r.d#gnd"d#gd,d-Zd.d/Ze!dIedk rjdd"gndge$e-dkd0dd1d2Ze$e.dk d3de!d=e8e9gd4d5Ze!d=e8e9gd6d7Zd8d9Zd:d;Zd<d=Zd>d?Zeee!dJdKddLd@gd@dAZeee!dBddKddge!dJdKd@ge!d=e8e9gdCdDZee!dJdKd@ge!dEdFdGdHge!d=e8e9gdIdJZee!dJdKdd@gdKdLZdS(Qz Test the parallel module. N)format_exceptionsqrt)sleep) PicklingError) nullcontext TimeoutErrorparallel)dumpload)mp)np with_numpy)with_multiprocessing)IS_PYPY force_gc_pypy) parametrizeraisescheck_subprocess_callskipifwarns)get_reusable_executor)Queue) parallel_sum)SequentialBackend)ThreadingBackend)MultiprocessingBackend)ParallelBackendBase) LokyBackend)Paralleldelayed)parallel_config)parallel_backend)register_parallel_backend)effective_n_jobs cpu_count)rBACKENDSDEFAULT_BACKENDmultiprocessingcCsg|]}t|qS)r)).0Z backend_strr,r,=/tmp/pip-unpacked-wheel-dylwa62s/joblib/test/test_parallel.py Msr/loky threading get_contextspawncCst|dt|ddS)N_poolZ_workers)getattrbackendr,r,r. get_workers[sr8cCs||SNr,)xyr,r,r.division_sr<cCs|dSNr,r:r,r,r.squarecsr@c@seZdZdZddZdS)MyExceptionWithFinickyInitz1An exception class with non trivial __init__ cCsdSr9r,)selfabcdr,r,r.__init__jsz#MyExceptionWithFinickyInit.__init__N__name__ __module__ __qualname____doc__rGr,r,r,r.rAgsrAFcCs"|dkr|rtddddnt|S)NrCrDrErF)rA ValueError)r:custom_exceptionr,r,r.exception_raiserns rPcCstdtdS)Ng?)timerKeyboardInterruptr?r,r,r.interrupt_raiserus rScCs|d||S)zO A module-level function so that it can be spawn with multiprocessing. r>r,)r:r;zr,r,r.fzsrUcCsttdSNr)typer get_active_backendr,r,r,r._active_backend_typesrYcCst||dddtdDS)Nn_jobsr7css|]}tt|VqdSr9r#r@r-ir,r,r. sz parallel_func..r"range) inner_n_jobsr7r,r,r. parallel_funcsrdcCstdkstdSrV)r(AssertionErrorr,r,r,r.test_cpu_countsrfcCstdkstdSrVr'rer,r,r,r.test_effective_n_jobssrhcontextzbackend_n_jobs, expected_n_jobs)r`r`r[Nrz positive-intz negative-intNone)idsc Cs@|d|dtdd|ks tW5QRXtdddksverbose dcCs:ddtdDt|||dddtdDks6tdS)NcSsg|] }t|qSr,r@r-r:r,r,r.r/sz(test_simple_parallel..)r[r7rqcss|]}tt|VqdSr9r\rur,r,r.r_sz'test_simple_parallel..rbr"re)r7r[rqr,r,r.test_simple_parallelsrxc Cs~|jtdddtjdd6}td|ddd td D}|d d d gksRtW5QRXdd|D}t|d ksztdS)NnameZ!some_new_name_for_the_main_thread)targetryvalueTrecordr>rZcss|]}tt|VqdSr9r\rur,r,r.r_sz6test_main_thread_renamed_no_warning..r`rrcSsg|]}dt|jkr|qS)zworker timeout)strmessager-wr,r,r.r/sz7test_main_thread_renamed_no_warning..) setattrr1current_threadwarningscatch_warningsr"rbrelen)r7 monkeypatchwarninforesultsr,r,r.#test_main_thread_renamed_no_warnings rc Cstjdd}tdt||dW5QRXdd|D}|r|rtdd|D}ttjd d rpt|d kn t|d k}|o|Sd S|rt dSdS) NTr|always)r7rccSsg|] }|jqSr,)rrr,r,r.r/sz*_assert_warning_nested..css|]}d|jdkVqdS)zbacked parallel loops cannotrN)args)r-Zeachr,r,r.r_sz)_assert_warning_nested..ZnogilFr) rr simplefilterrdallr5sysflagsrre)r7rcexpectedrZwarnings_are_correctZwarnings_have_the_right_lengthr,r,r._assert_warning_nesteds   rz%parent_backend,child_backend,expected)r0r+T)r0r0F)r+r+T)r+r0T)r1r+T)r1r0Tcstd|dfddtdDtd|dfddtdD}|dkrttrft|sftjdd t|stn t|stdS) Nr>rZc3s |]}ttdddVqdS)rFr7rcrNr#rr-_ child_backendr,r.r_s z0test_nested_parallel_warnings..rvc3s |]}ttddVqdS)r>rNrrrrr,r.r_s r1zThis test often fails in PyPy.reason)r"rbranypytestxfailrer)parent_backendrrresr,rr.test_nested_parallel_warningss   rcCs>dg}dd}tj||fd}|||ds:tdS)NFc SsTtjdd"}tddddtdDW5QRXtt|t|dk|d<dS) NTr|r>rkcss|]}ttdVqdS)皙?Nr#rrr,r,r.r_szPtest_background_thread_parallelism..background_thread..r~r)rrr"rbprintr)is_run_parallelrr,r,r.background_thread s  z=test_background_thread_parallelism..background_thread)rzrr)r1Threadstartjoinre)r7rrtr,r,r."test_background_thread_parallelisms rcCs"td|dddtdDdS)Nr>rZcss|]}ttdVqdS){Gz?Nr\rr,r,r.r_sznested_loop..rar6r,r,r. nested_loopsrrrcs&td|dfddtdDdS)Nr>rZc3s|]}ttVqdSr9)r#rrrr,r.r_!sz#test_nested_loop..ra)rrr,rr.test_nested_loopsrcCstdSr9)rNr6r,r,r.raise_exception%src CsLtt:tddd"}|ttdttdgW5QRXW5QRXdS)Nr>r0rZ)rrNr"r#rrr r,r,r.)test_nested_loop_with_exception_with_loky)s    rcs<tddtdddfddtdDs8tdS) z1Input is mutable when using the threading backendrv)maxsizer>r1rZc3s|]}tjdVqdSrN)r#putrqr,r.r_4sz1test_mutate_input_with_threads..N)rr"rbfullrer,r,rr.test_mutate_input_with_threads1s  rr`cCs6td}dd|Dt|ddd|Dks2tdS)z.Check the keyword argument processing of pmap. cSsg|]}t|ddqSrr;rUrur,r,r.r/=sz(test_parallel_kwargs..rkcss|]}tt|ddVqdSrrNr#rUrur,r,r.r_>sz'test_parallel_kwargs..Nrw)r[lstr,r,r.test_parallel_kwargs9s rc Cstd}dd|D}td|d^}|j}||dd|DksDt||dd|Dks^ttdk r|t|t|jks|tW5QRXtdk rt|jdkst||d d|Dksttdk rt|jdkstdS) NrcSsg|]}t|ddqSrrrur,r,r.r/Dsz4test_parallel_as_context_manager..r~rZcss|]}tt|ddVqdSrrrur,r,r.r_Msz3test_parallel_as_context_manager..css|]}tt|ddVqdSrrrur,r,r.r_Nscss|]}tt|ddVqdSrrrur,r,r.r_Zs)rbr"_backendrerr8)r7rrpZmanaged_backendr,r,r. test_parallel_as_context_managerAs rc sNGdddtttdd(tdddfdd td DW5QRXd S) ze Check that pmap captures the errors when it is passed an object that cannot be pickled. c@seZdZddZdS)z1test_parallel_pickling..UnpicklableObjectcSs tddS)NZ123) RuntimeErrorrBr,r,r. __reduce__esz.UnpicklableObject.__reduce__N)rIrJrKrr,r,r,r.UnpicklableObjectdsrzthe task to sendmatchr>r0rZc3s|]}ttVqdSr9r#idrrr,r.r_isz)test_parallel_pickling..rN)objectrrr"rbr,r,rr.test_parallel_pickling_s rcCs0ttd|ddddtdDdks,tdS)Nr>r[r7timeoutcss|]}ttdVqdS)gMbP?Nrrur,r,r.r_psz0test_parallel_timeout_success..r)rr"rbrer6r,r,r.test_parallel_timeout_successms rc Cs8tt&td|ddddtdDW5QRXdS)Nr>rrcss|]}ttdVqdSrNrrur,r,r.r_ysz-test_parallel_timeout_fail..r)rr r"rbr6r,r,r.test_parallel_timeout_failts rc Csdtdk rtt&td|dddtddDW5QRXtt td|ddddDW5QRXtd|d:}t|jdk stt|j}tt|ddtddDW5QRXt|jdk stt|j|k std dt d D|d d t d Dkstt|j}tt|d ddDW5QRXt|jdk sZtt|j|k sntddt d D|dd t d Dkst|j |j |j |j fW5QRXt|jdkstn,tttdddddDW5QRXtt$tddddtddDW5QRXtt$tddddd t dDW5QRXdS)Nr>rZcSsg|]\}}tt||qSr,r#r<r-r:r;r,r,r.r/sz&test_error_capture..)rr)rrcSsg|]}tt|qSr,r#rSrur,r,r.r/scSsg|]\}}tt||qSr,rrr,r,r.r/scSsg|]}t|ddqSrrrur,r,r.r/srcss|]}tt|ddVqdSrrrur,r,r.r_sz%test_error_capture..cSsg|]}tt|qSr,rrur,r,r.r/scSsg|]}t|ddqSrrrur,r,r.r/scss|]}tt|ddVqdSrrrur,r,r.r_srkcSsg|]}tt|qSr,rrur,r,r.r/scSsg|]\}}tt||qSr,rrr,r,r.r/sr)r[rqcss|]}tt|ddVqdS)T)rONr#rPr]r,r,r.r_sr)rrZeroDivisionErrorr"ziprRr8rrerb _iteratingZn_completed_tasksZn_dispatched_tasks _abortingrA)r7r Zoriginal_workersr,r,r.test_error_capture}sh                rcCs|d|dS)Nz Consumed %s)append)queueitemr,r,r.consumersrzbatch_size, expected_queue Produced 0z Consumed 0z Produced 1z Consumed 1z Produced 2z Consumed 2 Produced 3z Consumed 3z Produced 4z Consumed 4z Produced 5z Consumed 5r~csTtfdd}td||dfdd|D|ks@ttdksPtdS) zC Test that with only one job, Parallel does act as a iterator. c3s&tdD]}d||VqdSNz Produced %irbrr^rr,r.producers z'test_dispatch_one_job..producerrr[ batch_sizer7c3s|]}tt|VqdSr9r#rrurr,r.r_sz(test_dispatch_one_job.. N)listr"rer)r7rZexpected_queuerr,rr.test_dispatch_one_jobs  rcst}|fdd}tddd|dfdd|Dt}|d d ksXt|d d d }|dksvt|d}||ksttdkstd S)zW Check that using pre_dispatch Parallel does indeed dispatch items lazily. c3s&tdD]}d||VqdSrrrrr,r.rs z/test_dispatch_multiprocessing..producerr>rr`)r[r pre_dispatchr7c3s|]}ttdVqdS)rNrrrr,r.r_sz0test_dispatch_multiprocessing..rrNr~z Consumed anyrjrr)rManagerrr"reindexr)r7managerrZqueue_contentsZfirst_consumption_indexZproduced_3_indexr,rr.test_dispatch_multiprocessings    rc CsFtdddd.}|ddtdD|jdks8tW5QRXdS) Nr>autor1rcss|]}tt|VqdSr9rr]r,r,r.r_ sz/test_batching_auto_threading..rr"rbrZcompute_batch_sizere)rr,r,r.test_batching_auto_threadingsrc CsFtdd|d.}|ddtdD|jdks8tW5QRXdS)Nr>rrcss|]}tt|VqdSr9rr]r,r,r.r_sz2test_batching_auto_subprocesses..rrr)r7rr,r,r.test_batching_auto_subprocesses src Cs8tt&tddddddtdDW5QRXdS) zCMake sure that exception raised during dispatch are indeed capturedr>r)r[rrqcss|]}tt|VqdSr9rr]r,r,r.r_sz*test_exception_dispatch..rN)rrNr"rbr,r,r,r.test_exception_dispatchs rcCs tddddtdDdS)Nr>rkcss|]}tt|VqdSr9rr-jr,r,r.r_!sz(nested_function_inner..rrarr,r,r.nested_function_inner srcCs tddddtdDdS)Nr>rkcss|]}tt|VqdSr9)r#rrr,r,r.r_&sz(nested_function_outer..rrarr,r,r.nested_function_outer%srz'https://github.com/joblib/loky/pull/255rc Cstt$}td|dddtdDW5QRXt|j|j|j}d|}d|ksZt d|ksft d |ksrt t|jtkst d S) zEnsure errors for nested joblib cases gets propagated We rely on the Python 3 built-in __cause__ system that already report this kind of information to the user. r>rZcss|]}tt|VqdSr9)r#rr]r,r,r.r_4sz1test_nested_exception_dispatch..rrrrPN) rrNr"rbrrWr{tbrre)r7excinfoZ report_linesreportr,r,r.test_nested_exception_dispatch*s     rc@s$eZdZdZdddZd ddZdS) FakeParallelBackendz8Pretends to run concurrently while running sequentially.rNcKs|||_||_|Sr9)r'r[r )rBr[r Z backend_argsr,r,r. configureEs zFakeParallelBackend.configurecCs"|dkrttd|d}|S)Nrr)maxrr(rBr[r,r,r.r'Jsz$FakeParallelBackend.effective_n_jobs)rN)r)rIrJrKrLrr'r,r,r,r.rBs rc CsttddtddW5QRXttddtddW5QRXW5QRXttddtddW5QRXW5QRXdS)NzInvalid backend:rz unit-testingr6)rrNr"r$r,r,r,r.test_invalid_backendPs  r c Cs:tt}td|dW5QRXdt|jks6tdS)NrrZz&n_jobs == 0 in Parallel has no meaning)rrNr"Z_initialize_backendrr{re)r7rr,r,r.test_invalid_njobs]s r cCs8z*tdtdtksttdtks(tW5td=XdS)NZ test_backend)r)r&rrer,r,r,r.test_register_parallel_backendds   r cCsNttkstz$tdtdddttks0tW5tt_XttksJtdS)Nr1T)Z make_default)rYDefaultBackendrer*r r&r)rr,r,r,r.test_overwrite_default_backendms r zOnly without multiprocessingc Csfttdd"tddddtdDW5QRXtddtddtdDW5QRXdS) Nz)joblib backend '.*' is not available on.*rr0r6css|]}tt|VqdSr9r\r]r,r,r.r_}sz2test_backend_no_multiprocessing..r`css|]}tt|VqdSr9r\r]r,r,r.r_s)r UserWarningr"rbr$r,r,r,r.test_backend_no_multiprocessingys & rc Cs||ddt\}}|dks&ttddks6tt}|jdksJt|dkrvt|tksbtt|jtkstn|dkrt|t kstt|jt kstnX|dkrt|t kstt|jt kstn,| drt|t kstt|jt kstW5QRXdS)Nr`rkr+r0r1Ztest_) r rXrer'r"r[rWrrr!r startswithr)ri backend_nameactive_backend active_n_jobsrr,r,r.check_backend_context_managers$   rcCsg|] }d|qS)ztest_backend_%dr,r]r,r,r.r/scsl|tkr|t|tttks$tt|ttksr1rZc3s |]}|stt|VqdSr9)r#r)r-rDrir,r.r_sz/test_backend_context_manager..) r)setitemrrYr rerr" all_backends_for_context_manager)rr7rir,rr.test_backend_context_managers rc@seZdZdZdddZdS)ParameterizedParallelBackendz9Pretends to run conncurrently while running sequentially.NcCs|dkrtd||_dS)Nzparam should not be None)rNparam)rBrr,r,r.rGsz%ParameterizedParallelBackend.__init__)NrHr,r,r,r.rsrc Cs|tdtttkst|ddddtt\}}t|tksHt|j dksVt|dksbtt }|j dksvt|j |kst|ddt dD}W5QRX|dd t dDkstttkstdS) NZ param_backend*r`)rr[css|]}tt|VqdSr9r#rr]r,r,r.r_sz=test_parameterized_backend_context_manager..rvcSsg|] }t|qSr,rr]r,r,r.r/sz>test_parameterized_backend_context_manager..)rr)rrYr rer rXrWrr"r[rrb)rrirrrrr,r,r.*test_parameterized_backend_context_managers   rc Csttkst|tddddtt\}}t|tks>t|jdksLt|dksXtt}|j dkslt|j |kszt|ddt dD}W5QRX|ddt dDkstttkstdS) N+)rrvrkcss|]}tt|VqdSr9rr]r,r,r.r_szFtest_directly_parameterized_backend_context_manager..cSsg|] }t|qSr,rr]r,r,r.r/szGtest_directly_parameterized_backend_context_manager..) rYr rerr rXrWrr"r[rrb)rirrrrr,r,r.3test_directly_parameterized_backend_context_managers   rcCstdtS)Nr)rosgetpidr,r,r,r.sleep_and_return_pidsr"cCs<ttksttdks ttddddtdDS)Nrr>rkcss|]}ttVqdSr9)r#r"rr,r,r.r_sz"get_nested_pids..)rYrrer"_effective_n_jobsrbr,r,r,r.get_nested_pidss r$cs eZdZdZfddZZS) MyBackendz:Backend to test backward compatibility with older backendscstt|dSrV)superr%get_nested_backendr __class__r,r.r'szMyBackend.get_nested_backend)rIrJrKrLr' __classcell__r,r,r(r.r%sr%Zback_compat_backendc CsR||@tddddtdD}|D]}tt|dks*tq*W5QRXdS)Nr>rkcss|]}ttVqdSr9)r#r$rr,r,r.r_ sz6test_nested_backend_context_manager..rr)r"rbrsetre)rir7Z pid_groupsZ pid_groupr,r,r.#test_nested_backend_context_managers  r,c shddtddfddtdD|d*tddfddtdDW5QRXdS) NcSs0tt|kstt|}t|ks,tdSr9)rYr)rer'r"r#)Zexpected_backend_typeZexpected_n_jobr,r,r.check_nested_backendsz?test_nested_backend_in_sequential..check_nested_backendrrkc3s|]}ttdVqdSr)r#r*r)r-r,r.r_&sz4test_nested_backend_in_sequential..rc3s|]}tVqdSr9r#rr7r-r[r,r.r_,sra)r7r[rir,r/r.!test_nested_backend_in_sequentials r0c CsF||4}|tkr|d}|tkr*|d}|j|ks8tW5QRXdS)Nr7r)r$r% nesting_levelre)ri inner_backendZexpected_levelctxr7r,r,r.check_nesting_level2s  r4 outer_backendr2c sjt|dtd|dfddtdDdd$tfddtdDW5QRXdS) Nrr>rZc3s|]}ttdVqdSrr#r4rrir2r,r.r_Csz-test_backend_nesting_level..rrkc3s|]}ttdVqdSrr6rr7r,r.r_Is)r4r"rb)rir5r2r,r7r.test_backend_nesting_level;s r8with_retrieve_callbackTc sddlGfdddt}td|dd|dZ}tddfd d td D|tkrt|d jd kstt|tkr|djd kstW5QRXdS)Nrcs"eZdZdZZjddZdS)z)test_retrieval_context..MyBackendrcss|jd7_dVdSrlrrr,r,r.retrieval_contextWsz;test_retrieval_context..MyBackend.retrieval_contextN)rIrJrKr^Zsupports_retrieve_callbackcontextmanagerr:r,) contextlibr9r,r.r%Ssr%Z retrievalcSstddddt|DS)Nr>rkcss|]}tt|VqdSr9rr]r,r,r.r__sz>test_retrieval_context..nested_call..ra)nr,r,r. nested_call^sz+test_retrieval_context..nested_callr>rkc3s|]}t|VqdSr9r.r])r>r,r.r_bsz)test_retrieval_context..rvr7r) r<rr&r"rbr$r^rer%)rir9r%r3r,)r<r>r9r.test_retrieval_contextMs  r?rgQ?c Cs"ttt|dW5QRXdS)Nr)rrNr"r@r,r,r.test_invalid_batch_sizeos rAz)n_tasks, n_jobs, pre_dispatch, batch_size)r>r>rr)r>r>r[r)rr>r[r)ir>r[r)rr~r[r)rr[r)r 2 * n_jobsr)rrr)rErrDrM)rBrrDrcCsJ|||d}ddt|D}tf|ddt|D}||ksFtdS)N)r[rrcSsg|] }t|qSr,rtr]r,r,r.r/sz0test_dispatch_race_condition..css|]}tt|VqdSr9r\r]r,r,r.r_sz/test_dispatch_race_condition..rw)Zn_tasksr[rrparamsrrr,r,r.test_dispatch_race_conditionus rGcCs8t}tddd}|jd}|}||ks4tdS)Nr>r+rZri)rget_start_methodr"Z _backend_argsgetre)Zmp_start_methodrriZ start_methodr,r,r.test_default_mp_contexts   rJcs^|dkrtd}tjd}|ddtjtd|dfddt dDdS) Nr+r3rir>rZc3s |]}ttjjVqdSr9)r#rdotTr]rCr,r.r_szAtest_no_blas_crash_or_freeze_with_subprocesses..) rr2rrandomZ RandomStaterandnrKrLr"rb)r7rngr,rMr..test_no_blas_crash_or_freeze_with_subprocessess   rQafrom joblib import Parallel, delayed def square(x): return x ** 2 backend = "{}" if backend == "spawn": from multiprocessing import get_context backend = get_context(backend) print(Parallel(n_jobs=2, backend=backend)( delayed(square)(i) for i in range(5))) cCsB|dkrtdkrtdt|}ttjd|gddddS)Nr+forkzVRequire fork start method to use interactively defined functions with multiprocessing.z-cr\[0, 1, 4, 9, 16\]r stdout_regex) rrHrskip,UNPICKLABLE_CALLABLE_SCRIPT_TEMPLATE_NO_MAINformatrr executable)r7coder,r,r.2test_parallel_with_interactively_defined_functionss   r[aimport sys # Make sure that joblib is importable in the subprocess launching this # script. This is needed in case we run the tests from the joblib root # folder without having installed joblib sys.path.insert(0, {joblib_root_folder!r}) from joblib import Parallel, delayed def run(f, x): return f(x) {define_func} if __name__ == "__main__": backend = "{backend}" if backend == "spawn": from multiprocessing import get_context backend = get_context(backend) callable_position = "{callable_position}" if callable_position == "delayed": print(Parallel(n_jobs=2, backend=backend)( delayed(square)(i) for i in range(5))) elif callable_position == "args": print(Parallel(n_jobs=2, backend=backend)( delayed(run)(square, i) for i in range(5))) else: print(Parallel(n_jobs=2, backend=backend)( delayed(run)(f=square, x=i) for i in range(5))) z!def square(x): return x ** 2 zcdef gen_square(): def square(x): return x ** 2 return square square = gen_square() zsquare = lambda x: x ** 2 define_funccallable_positionr#rkwargsc Csv|dkr$|tkstjdkr$tdtj|||tj tj t j d}| d}| |ttj|jgddddS) N)r+r3win32zNot picklable with pickle)r\r7r]joblib_root_folderzunpicklable_func_script.pyrrSrT) SQUARE_MAINrplatformrrV)UNPICKLABLE_CALLABLE_SCRIPT_TEMPLATE_MAINrXr pathdirnamejoblib__file__rwriterrYstrpath)r7r\r]tmpdirrZZ code_filer,r,r.0test_parallel_with_unpicklable_functions_in_argss&    rkaimport sys import faulthandler # Make sure that joblib is importable in the subprocess launching this # script. This is needed in case we run the tests from the joblib root # folder without having installed joblib sys.path.insert(0, {joblib_root_folder!r}) from joblib import Parallel, delayed from functools import partial class MyClass: '''Class defined in the __main__ namespace''' def __init__(self, value): self.value = value def square(x, ignored=None, ignored2=None): '''Function defined in the __main__ namespace''' return x.value ** 2 square2 = partial(square, ignored2='something') # Here, we do not need the `if __name__ == "__main__":` safeguard when # using the default `loky` backend (even on Windows). # To make debugging easier faulthandler.dump_traceback_later(30, exit=True) # The following baroque function call is meant to check that joblib # introspection rightfully uses cloudpickle instead of the (faster) pickle # module of the standard library when necessary. In particular cloudpickle is # necessary for functions and instances of classes interactively defined in the # __main__ module. print(Parallel(backend="loky", n_jobs=2)( delayed(square2)(MyClass(i), ignored=[dict(a=MyClass(1))]) for i in range(5) )) )r`cCs.|d}|tttj|jgddddS)Nz(joblib_interactively_defined_function.pyrS)rUr)rrh5INTERACTIVE_DEFINED_FUNCTION_AND_CLASS_SCRIPT_CONTENTrrrYrirjscriptr,r,r.7test_parallel_with_interactively_defined_functions_loky=s   roaimport sys # Make sure that joblib is importable in the subprocess launching this # script. This is needed in case we run the tests from the joblib root # folder without having installed joblib sys.path.insert(0, {joblib_root_folder!r}) from joblib import Parallel, delayed, hash import multiprocessing as mp mp.util.log_to_stderr(5) class MyList(list): '''MyList is interactively defined by MyList.append is a built-in''' def __hash__(self): # XXX: workaround limitation in cloudpickle return hash(self).__hash__() l = MyList() print(Parallel(backend="loky", n_jobs=2)( delayed(l.append)(i) for i in range(3) )) cCs0|d}|tttj|jgdddddS)Nz)joblib_interactive_bound_method_script.pyz\[None, None, None\]Z LokyProcess)rUZ stderr_regexr)rrh9INTERACTIVELY_DEFINED_SUBCLASS_WITH_METHOD_SCRIPT_CONTENTrrrYrirmr,r,r.:test_parallel_with_interactively_defined_bound_method_lokyes   rrcCs"tg}tdd|gkstdS)Nr>rk)iterr"re)Zexhausted_iteratorr,r,r.%test_parallel_with_exhausted_iteratorosrtcCsttddS)z+Helper function to force gc in each worker.rN)rrQrr,r,r,r._cleanup_workertsrucCs"t|tjstdt||S)Nz#Expected np.memmap instance, got %r) isinstancermemmap TypeErrorrWcopyrMr,r,r. check_memmapzs  rzcCsdd}tdd|ddd|dD}t||t|D]\}}tj||q:td d|dd d|dD}t||t|D]\}}tj||qdS) Ncss(t|D]}tjdtjd|VqdS)NrZdtype)rbronesZfloat32)r=r^r,r,r.generate_arrayss zBtest_auto_memmap_on_arrays_from_generator..generate_arraysr>rr[ max_nbytesr7css|]}tt|VqdSr9r#rzr-rCr,r,r.r_sz.rsr~css|]}tt|VqdSr9rrr,r,r.r_s)r"rrrtestingassert_array_equal)r7r}rresultrr,r,r.)test_auto_memmap_on_arrays_from_generatorsrcCs|Sr9r,)argr,r,r.identitysrcs|dj}tj}tj|ddtj|ddg}t||t|ddt ddfdd d D\}t d tj svt d j |kst tj||dS) Nz test.mmapuint8r{r)Z mmap_moder>rkc3s|]}ttVqdSr9)r#rrrwr,r.r_sz.test_memmap_with_big_offset..)rr)rrimmapZALLOCATIONGRANULARITYrzerosr|r r r"rvrwreoffsetrr)rjfnamesizeobjrr,rr.test_memmap_with_big_offsets   rc Csttjdd$}tdddddtdDW5QRXt|dksFt|d}t|jts^tt |jd ksptdS) NTr|r)r[rcss|]}tt|VqdSr9r\r]r,r,r.r_szFtest_warning_about_timeout_not_supported_by_backend..2rzThe backend class 'SequentialBackend' does not support timeout. You have set 'timeout=1' in Parallel but the 'timeout' parameter will not be used.) rrr"rbrrervrrr)rrr,r,r.3test_warning_about_timeout_not_supported_by_backends(rcCs |||<|Sr9r,) input_listrr{r,r,r.set_list_valuesrcsVdgdt|dddfddtdD}t|}tddt|DsRtdS) Nrrv generatorr1)r[ return_asr7c3s|]}tt||VqdSr9)r#rr]rr,r.r_szPtest_parallel_return_order_with_return_as_generator_parameter..css|]\}}||kVqdSr9r,)r-vrr,r,r.r_s)r"rbrrrre)r[rr,rr.=test_parallel_return_order_with_return_as_generator_parameters  rc Csbdgdgd}tt(t}t||ddd|DW5QRXt|}|dks^tdS)NrCrrsrZcss|]}ttj|VqdSr9r#rQrr]r,r,r.r_sz%test_abort_backend..)rrxrQr"re)r[r7Zdelayst_startdtr,r,r.test_abort_backends  rcCstjtdtd}d|d<|S)NgAr{Fr)rr|intbool)rrr,r,r.get_large_objectsrc CsLt||dd4}|ddtdD}t|t|~tW5QRXdS)Nr)r[r7rcss|]}tt|VqdSr9)r#rr]r,r,r.r_sz/test_deadlock_with_generator..r)r"rbnextr)r7r[r rr,r,r.test_deadlock_with_generatorsrc Cs~ttddHt||dd}|ddtdD}t}|ddtd D}W5QRXt|d ksrtd ~tdS) N)This Parallel instance is already runningrrr7rcss|]}ttdVqdSrrrr,r,r.r_sz/test_multiple_generator_call..rcss|]}tt|VqdSr9rr]r,r,r.r_srsr>ZThe error should be raised immediatly when submitting a new task but it took more than 2s.)rrr"rbrQrer)r7r[r grZgen2r,r,r.test_multiple_generator_calls rc Cst||ddj}|ddtdD}t}ttdd|ddtd D}W5QRXt|d ksttd W5QRX~tdS) Nrrcss|]}ttdVqdSrrrr,r,r.r_sz7test_multiple_generator_call_managed..rrrcss|]}tt|VqdSr9rr]r,r,r.r_srsr>r)r"rbrQrrrer)r7r[r rrg2r,r,r.$test_multiple_generator_call_managed s rcCst||ddddtdD}t||ddddtddD}tddt|tdDsbttd dt|tddDstdS) Nrrcss|]}tt|dVqdSr>Nrr]r,r,r.r_)sz9test_multiple_generator_call_separated..rcss|]}tt|dVqdSrrr]r,r,r.r_,srcss|]\}}||kVqdSr9r,r-rr^r,r,r.r_0scss|]\}}||kVqdSr9r,rr,r,r.r_1s)r"rbrrre)r7r[rrr,r,r.&test_multiple_generator_call_separated%s rzbackend, error)r0T)r1F) sequentialFc Cs8|dkrtdkrtdtd|dd}|ddtdD}t|d d }|r^ttd d nt }|Xt }td|ddd dtddD}t t ddt |tddDstW5QRXt |dkstd}|jr|dkr|d7}t dqt |dkst|dkr4|js4tdS)Nr0zRequires multiprocessingr>rrcss|]}ttdVqdSrrr]r,r,r.r_Asz.rcSstdS)NzGenerator collected)rr,r,r,r.Bz;test_multiple_generator_call_separated_gc..z The executor underlying Parallelrcss|]}tt|dVqdSrrr]r,r,r.r_Msrcss|]\}}||kVqdSr9r,rr,r,r.r_Tsrvrr`rg?)rrrVr"rbweakreffinalizerrrrQrrrrealiverr#r)r7errorr rZg_wrr3rretryr,r,r.)test_multiple_generator_call_separated_gc4s0 , rc Cs|j}tdd||dR}|ddtjdgdDtt|dksNt|ddtdDW5QRXtd D]}t|sqt d qvtd tdd|d }|d dtjdgdD|ddtdDtd D]}t|sq t d qtd dS)Nr>r)r[rr7Z temp_foldercss|]}tt|VqdSr9rrr,r,r.r_qsz(test_memmapping_leaks..rrcss|]}ttVqdSr9r#rurr,r,r.r_{srsrz/temporary directory of Parallel was not removedr~css|]}tt|VqdSr9rrr,r,r.r_scss|]}ttVqdSr9rrr,r,r.r_s) rir"rrNrr listdirrerbr)r7rjrrr,r,r.test_memmapping_leakses*         rcCs<td|dddtdD}|ddtdDks8tdS)Nr>rZcss|]}tdd|VqdS)cSs|dSr=r,r?r,r,r.rrz2test_lambda_expression...Nr.r]r,r,r.r_sz)test_lambda_expression..rcSsg|] }|dqS)r>r,r]r,r,r.r/sz*test_lambda_expression..)r"rbre)r7rr,r,r.test_lambda_expressionsrcsd}d}d|td||d}|fddt|D|jj|jjksLt|jj|jjks`t|fddt|D|jj|jjkst|jj|jjkstd S) zCTest that a parallel backend correctly resets its batch statistics.r>ig@r)rqr[r7c3s|]}ttjVqdSr9rr]Z task_timer,r.r_sz6test_backend_batch_statistics_reset..c3s|]}ttjVqdSr9rr]rr,r.r_sN)r"rbrZ_effective_batch_sizeZ_DEFAULT_EFFECTIVE_BATCH_SIZEreZ_smoothed_batch_durationZ _DEFAULT_SMOOTHED_BATCH_DURATION)r7r[Zn_inputsrr,rr.#test_backend_batch_statistics_resets$rc CsdD]v}tt|djtks tt|dd}t|jtks>tt|dd}t|jtks\tt|dd}t|jtkstqtdd dd }t|jtkst|d dd0tdd }t|jtkst|jdkstW5QRX|d dd6td dd}t|jtks t|jd kstW5QRX|d dd4tdd }t|jtksRt|jdksbtW5QRX|d dd6td dd}t|jtkst|jd kstW5QRXdS)N)rr>rjrkthreadsr[prefer processes sharedmemr[requirer>r0)r[r7rrr`rr)rWr"rr rerr!r[)rir[rr,r,r.$test_backend_hinting_and_constraintss4       rc Cs@Gdddt}||Btddd}t|j|ks:ttddd}t|j|ksXtW5QRXGdd d t}||tddd}t|j|kst|\}}|d kst|d ksttddd d }t|jtkst|\}}d }||kst|d kstW5QRXtt t|ddW5QRXdS)Nc@s$eZdZdZdZddZddZdS)z[test_backend_hinting_and_constraints_with_custom_backends..MyCustomThreadingBackendTcSsdSr9r,rr,r,r. apply_asyncszgtest_backend_hinting_and_constraints_with_custom_backends..MyCustomThreadingBackend.apply_asynccSs|Sr9r,rr,r,r.r'szltest_backend_hinting_and_constraints_with_custom_backends..MyCustomThreadingBackend.effective_n_jobsNrIrJrKZsupports_sharedmemZ use_threadsrr'r,r,r,r.MyCustomThreadingBackendsrr>rrrrc@s$eZdZdZdZddZddZdS)z\test_backend_hinting_and_constraints_with_custom_backends..MyCustomProcessingBackendFcSsdSr9r,rr,r,r.rszhtest_backend_hinting_and_constraints_with_custom_backends..MyCustomProcessingBackend.apply_asynccSs|Sr9r,rr,r,r.r'szmtest_backend_hinting_and_constraints_with_custom_backends..MyCustomProcessingBackend.effective_n_jobsNrr,r,r,r.MyCustomProcessingBackendsrrr)r[rrqzUsing ThreadingBackend as joblib backend instead of MyCustomProcessingBackend as the latter does not provide shared memory semantics.r7r) r r"rWrreZ readouterrrstriprrN)Zcapsysrirrrouterrrr,r,r.9test_backend_hinting_and_constraints_with_custom_backendss*          rc CstttddW5QRXtttddW5QRXtttdddW5QRXtdk rtttdddW5QRXtttd ddW5QRXdS) Ninvalidrrrr)rrr0rr+)rrNr"rr,r,r,r.,test_invalid_backend_hinting_and_constraintss     rc sxtddd}t|jj|jjfg}dkr:|W5QRS|fddtdD}||dW5QRSQRXdS)zCPerform nested parallel calls and introspect the backend on the wayr>rkrc3s(|] }ttfddiVqdS)limitrN)r#_recursive_backend_infor]r^rr,r.r_7sz*_recursive_backend_info..rN)r"rWrrIr1rb)rr^rZ this_levelrr,rr.r0s rc Csr||dd t}W5QRXtdkr2d}d}nd}d}|d}|df|dfd|fd|fg}||ksntdS)Nr>rkrrrZBackendr)rr(titlere)rir7backend_types_and_levelsZsecond_level_backend_typeZ max_levelZtop_level_backend_typeZexpected_types_and_levelsr,r,r.test_nested_parallelism_limit<s  rzThis test requires daskc Cstjddd}tjtdtjd}tdD]H}|dt|d}W5QRXt|dks\t t dd |Ds*t q*|d t}W5QRXt|dkst t d d |Dst dS) Nr>)Z n_workersZthreads_per_workergcAr{dask)datar~css|]\}}|dkVqdSZDaskDistributedBackendNr,r-ryrr,r,r.r_`sz4test_nested_parallelism_with_dask..css|]\}}|dkVqdSrr,rr,r,r.r_gs) distributedZClientrr|rrrbrrrer)riclientrr^rr,r,r.!test_nested_parallelism_with_daskTs   rcCstddtdDS)z6A horrible function that does recursive parallel callscss|]}ttVqdSr9)r#_recursive_parallelr]r,r,r.r_msz&_recursive_parallel..r>ra)Z nesting_limitr,r,r.rksrc Csv||dd tt }tW5QRXW5QRX|j}|dkrdddlm}t||tfrdt dt|t srt dS)Nr>rkr0r)TerminatedWorkerErrorz1Loky worker crash when serializing RecursionError) r BaseExceptionrr{Z&joblib.externals.loky.process_executorrrvrrrRecursionErrorre)rir7rexcrr,r,r.test_thread_bomb_mitigationps    rcCs*i}dD]}tj|||<q|tdfS)N)OMP_NUM_THREADSOPENBLAS_NUM_THREADSMKL_NUM_THREADSZVECLIB_MAXIMUM_THREADSZNUMEXPR_NUM_THREADSZNUMBA_NUM_THREADS ENABLE_IPCrs)r environrIr)Zenv_varsvarr,r,r._run_parallel_sumsrzNeed OpenMP helper compiledcCstd|dddtdD}ttdd}|D]Z\}}||ksFt|D]<\}}|drr|t|kstqN|dks~t|dksNtqNq2dS) Nr>rZcss|]}ttVqdSr9)r#rrr,r,r.r_sz-test_parallel_thread_limit..rZ_THREADSr1)r"rbrr(reitemsendswithr)r7rZexpected_num_threadsZworker_env_varsZomp_num_threadsryr{r,r,r.test_parallel_thread_limits    rz%This test requires dask NOT installedc Cs$ttdd|dW5QRXdS)NzPlease install daskrr)rrNrr,r,r.)test_dask_backend_when_dask_not_installedsrc CsdGdddt}d}||<tjt|d"tddddtdDW5QRXW5QRXdS) Nc@s&eZdZddZdddZddZdS) z3test_zero_worker_backend..ZeroWorkerBackendc_sdSrVr,)rBrr^r,r,r.rsz=test_zero_worker_backend..ZeroWorkerBackend.configureNcSs tddS)NzNo worker availabler)rBfunccallbackr,r,r.rsz?test_zero_worker_backend..ZeroWorkerBackend.apply_asynccSsdSrVr,rr,r,r.r'szDtest_zero_worker_backend..ZeroWorkerBackend.effective_n_jobs)N)rIrJrKrrr'r,r,r,r.ZeroWorkerBackends rz&ZeroWorkerBackend has no active workerrr>rkcss|]}tt|VqdSr9rr]r,r,r.r_sz+test_zero_worker_backend..)rrrrr"rb)rirZ expected_msgr,r,r.test_zero_worker_backends   rcsdadddksttddfddtdD}t|dhksLtdadks^ttddfd dtdD}t|dhkstdS) Nzoriginal valuecSstSr9)MY_GLOBAL_VARIABLEr,r,r,r. check_globalssz@test_globals_update_at_each_parallel_call..check_globalsr>rkc3s|]}tVqdSr9r.r]rr,r.r_sz.z changed valuec3s|]}tVqdSr9r.r]rr,r.r_s)rrer"rbr+)Zworkers_global_variabler,rr.)test_globals_update_at_each_parallel_callsrcCs4ddl}|jdd}|||ddlm}|S)Nrrs)threadpool_info)ZnumpyrNrOrKZ threadpoolctlr)rrCrr,r,r._check_numpy_threadpool_limitss   rcCs8|D] }|d|dkr|dSqtd|dS)Nfilepath num_threadsz,An unexpected module was loaded in child: {})rNrX) child_module parent_info parent_moduler,r,r._parent_max_num_threads_fors rcCs@|D]6}|D],}t||}t|||h}|d|ks tq qdS)Nr)rminre)Z workers_inforrZchild_threadpool_inforZparent_max_num_threadsrr,r,r.check_child_num_threadssrcCsdt}t|dkrtjddtd|dddtdD}t|}tt|d }t |||dS) Nr&Need a version of numpy linked to BLASmsgr0)r7r[css|]}ttVqdSr9r#rr]r,r,r.r_sz;test_threadpool_limitation_in_child_loky..r>r rrrrVr"rbr'rr(r)r[rworkers_threadpool_infosexpected_child_num_threadsr,r,r.(test_threadpool_limitation_in_child_lokys  r inner_max_num_threadsc Cst}t|dkrtjdd|d|d"t|dddtd D}W5QRXt|}|dkrttt|d }n|}t |||dS) Nrrrr0r rkcss|]}ttVqdSr9rr]r,r,r.r_*sz>test_threadpool_limitation_in_child_context..r>rr)rir[r rr r r,r,r.+test_threadpool_limitation_in_child_contexts   rvar_namerrrc stddddtj}zdtj<t|dfddtd D}|ddgks`t|d d d (t|dfd dtd D}W5QRX|ddgkstW5|dkrtj=n |tj<XdS)NTZreusecSs tj|Sr9)r rrI)rr,r,r._get_envEsz>test_threadpool_limitation_in_child_override.._get_env4rkc3s|]}tVqdSr9r.r]rrr,r.r_Lsz?test_threadpool_limitation_in_child_override..r>r0rr c3s|]}tVqdSr9r.r]rr,r.r_Qsr)rshutdownr rrIr"rbre)rir[rZoriginal_var_valuerr,rr.,test_threadpool_limitation_in_child_override7s"    rcCsJdd}||tdd}tdD]"}||tdd}||ks"tq"dS)Ncs,tdt|dfddtdDdS)Nrrkc3s|]}ttVqdSr9)r#sumr]r?r,r.r_dszAtest_loky_reuse_workers..parallel_call..)rbr"rkr,r?r. parallel_callbsz.test_loky_reuse_workers..parallel_callTrr)rrbre)r[rZfirst_executorrexecutorr,r,r.test_loky_reuse_workers\s   r)F)rr)r`)N)rLr rrQrrrr1 tracebackrmathrrpicklerr<rr+r rrfr r r Zjoblib._multiprocessing_helpersrZjoblib.test.commonrrrrrZjoblib.testingrrrrrZjoblib.externals.lokyrrrposix ImportErrorZ _openmp_test_helper.parallel_sumrrZjoblib._parallel_backendsrrrr r!Zjoblib.parallelr"r#r$r%r&r'r(r)r*ryZRETURN_GENERATOR_BACKENDSpopsortedkeysZALL_VALID_BACKENDSZPROCESS_BACKENDSZPARALLEL_BACKENDShasattrrr2r r8r<r@ ExceptionrArPrSrUrYrdrfrhmarkrorxrrrrrrrrrrrrrrrrrrrrrrrrrrr r r r rrrextendrbrrrrr"r$Z_parallel_backendsr%r,r0r4r8r?rArGrJrQrWr[rcraZ SQUARE_LOCALZ SQUARE_LAMBDArkrXrdrergrlrorqrrrtrurzrrrrrrrrrrrrrrrrrrrrrrrZno_coverrrrrrrrrrr rrrr,r,r,r.s                                           D                               ( ,      ,+ . 6          !