U Nv fB@s@dZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl m Z ddlmZmZddlmZmZddlmZmZmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0ej1dkr"dKd d Z2dLd d Z3ndMd d Z2dNdd Z3ej4ej5Z4Z5ddZ6ddZ7ddZ8ddZ9ddZ:ddZ;ddZd!d"Z?d#d$Z@d%d&ZAd'd(ZBd)d*ZCd+d,ZDd-d.ZEd/d0ZFd1d2ZGd3d4ZHd5d6ZId7d8ZJd9d:ZKd;d<ZLd=d>ZMd?d@ZNdAdBZOdCdDZPdEdFZQdGdHZRGdIdJdJeZSdS)Oa New, fast version of the CloudPickler. This new CloudPickler class can now extend the fast C Pickler instead of the previous Python implementation of the Pickler class. Because this functionality is only available for Python versions 3.8+, a lot of backward-compatibility code is also removed. Note that the C Pickler subclassing API is CPython-specific. Therefore, some guards present in cloudpickle.py that were written to handle PyPy specificities are not present in cloudpickle_fast.py N)Enum)ChainMap OrderedDict)picklePickler)_extract_code_globals_BUILTIN_TYPE_NAMESDEFAULT_PROTOCOL_find_imported_submodules_get_cell_contents_should_pickle_by_reference _builtin_type_get_or_create_tracker_id_make_skeleton_class_make_skeleton_enum_extract_class_dictdynamic_subimport subimport_typevar_reduce _get_bases _make_cell_make_empty_cellCellType_is_parametrized_type_hintPYPYcell_set"parametrized_type_hint_getinitargs_create_parametrized_type_hintbuiltin_code_type_make_dict_keys_make_dict_values_make_dict_items_make_functioncCst|||d|dS)Serialize obj as bytes streamed into file protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to pickle.HIGHEST_PROTOCOL. This setting favors maximum communication speed between processes running the same Python version. Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure compatibility with older versions of Python. protocolbuffer_callbackN CloudPicklerdump)objfiler'r(r.Q/tmp/pip-unpacked-wheel-dylwa62s/joblib/externals/cloudpickle/cloudpickle_fast.pyr+-s r+c Cs@t.}t|||d}|||W5QRSQRXdS)Serialize obj as a string of bytes allocated in memory protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to pickle.HIGHEST_PROTOCOL. This setting favors maximum communication speed between processes running the same Python version. Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure compatibility with older versions of Python. r&NioBytesIOr*r+getvalue)r,r'r(r-cpr.r.r/dumps;s  r6cCst||d|dS)r%r'Nr))r,r-r'r.r.r/r+Ns c Cs>t,}t||d}|||W5QRSQRXdS)r0r7Nr1)r,r'r-r5r.r.r/r6Zs   cCsVi}d|jkr|j|d<|jdd}t|tr8||d<t||jt||t|dfS)N __slots____dict__) r9r8get isinstancepropertytype__name__rr)r,Z type_kwargsr9r.r.r/_class_getnewargsps   r?cCs,dd|D}|j|j|j||jt|dfS)NcSsi|]}|j|jqSr.namevalue.0er.r.r/ ~sz$_enum_getnewargs..) __bases__r> __qualname__ __module__r)r,membersr.r.r/_enum_getnewargs}s rKcCs|SNr.)retvalr.r.r/_file_reconstructorsrNc sjjjjjjjjd}tj }fdd|D}jdk r\t t t jnd}t j t|||d<||d<j}||fS)N)r>rH__annotations____kwdefaults__ __defaults__rI__doc__ __closure__cs"i|]}|jkr|j|qSr.) __globals__)rDkfuncr.r/rFs z&_function_getstate..r._cloudpickle_submodulesrT)r>rHrOrPrQrIrRrSr__code__listmapr r itertoolschainvaluesr9)rW slotstateZ f_globals_ref f_globalsZclosure_valuesstater.rVr/_function_getstates,  rbcCst|}|ddtt|tjr|dd|dd|dd|dd}|dkr|ddt|\}}}}dd|D|d<nd d|D|d<d |krt|jt r||jn|jD]}||dq|d d|ifS) N __weakref__ _abc_cache_abc_negative_cache_abc_negative_cache_version _abc_registry _abc_implcSsg|] }|qSr.r.)rDZsubclass_weakrefr.r.r/ sz#_class_getstate..cSsg|]}|qSr.r.)rDtype_r.r.r/risr8r9) rpop issubclassr=abcABCMeta _get_dumpr;r8str)r,clsdictregistry_rUr.r.r/_class_getstates*          rtcCsLt|\}}dd|D}dD]}||dq|D]}||q4||fS)NcSsi|]}|j|jqSr.r@rCr.r.r/rFsz"_enum_getstate..)_generate_next_value__member_names_ _member_map_ _member_type__value2member_map_)rtrk)r,rqr_rJattrnamememberr.r.r/_enum_getstates  r|cCst|drZ|j|j|j|j|j|j|j|j|j |j |j |j |j |j|j|j|j|jf}nHt|dr|j|j|j|j|j|j|j|j|j |j |j |j |j|j|j|jf}nt|dr|j|j|j|j|j|j|j|j|j|j|j |j |j |j|j|j|j|j|j|j|jf}nt|drb|j|j|j|j|j|j|j|j|j |j |j |j |j|j|j|jf}n@|j|j|j|j|j|j|j|j |j |j |j |j|j|j|jf}tj|fS)zcodeobject reducerco_exceptiontable co_linetableco_nmetaco_posonlyargcount)hasattr co_argcountrco_kwonlyargcount co_nlocals co_stacksizeco_flagsco_code co_constsco_names co_varnames co_filenameco_nameZ co_qualnameco_firstlinenor~r} co_freevars co_cellvarsZ co_framesizeZco_ndefaultargsr co_lnotabZco_exc_handlersZ co_jump_tableZ co_free2regZ co_cell2regtypesCodeType)r,argsr.r.r/ _code_reduces     rcCs8z |jWntk r&tdfYSXt|jffSdS)z?Cell (containing values of a function's free variables) reducerr.N) cell_contents ValueErrorrrr,r.r.r/ _cell_reduce3s  rcCs|j}t||ffSrL)__func__r=)r,Z orig_funcr.r.r/_classmethod_reduce=src CsFddl}t|drt|ds&td|tjkrrr.r.r/_getset_descriptor_reduceqsrcCstjt|ffSrL)rMappingProxyTypedictrr.r.r/_mappingproxy_reduceusrcCst|ffSrL)bytestobytesrr.r.r/_memoryview_reduceysrcCs<t|rt|jffS|j}|ddt|j|ffSdS)N __builtins__)r rr>r9copyrkr)r,rar.r.r/_module_reduce}s    rcCstj|j|jffSrL)r MethodTyper__self__rr.r.r/_method_reducesrcCstj|jffSrL)logging getLoggerrArr.r.r/_logger_reducesrcCs tjdfS)Nr.)rrrr.r.r/_root_logger_reducesrcCst|j|j|j|jffSrL)r<fgetfsetfdelrRrr.r.r/_property_reducesrcCstjt|ffSrL)weakrefWeakSetrZrr.r.r/_weakset_reducesrcCsFtdk r*t|tr*tt|t|ddtfStt|t|ddtfSdS)z Save a class that can't be stored as module global. This method is used to serialize classes that are defined inside functions, or that otherwise can't be serialized as attribute lookups from global modules. N) rrlrrKr|_class_setstaterr?rtrr.r.r/_dynamic_class_reducesrcCsj|tdkrtdfS|ttkr*ttffS|ttkr@ttffS|tkrVtt|ffSt|sft|StS)zCSelect the reducer depending on the dynamic nature of the class objNrL)r=EllipsisNotImplementedr rr rrr.r.r/ _class_reduces     rcCstt|ffSrLr rZrr.r.r/_dict_keys_reducesrcCstt|ffSrLr!rZrr.r.r/_dict_values_reducesrcCstt|ffSrLr"rrr.r.r/_dict_items_reducesrcCstt|dffSNTrrr.r.r/_odict_keys_reducesrcCstt|dffSrrrr.r.r/_odict_values_reducesrcCstt|dffSrrrr.r.r/_odict_items_reducesrc Cs|\}}|j||d}|d}|d|j|t|jd<|dk rt|D]<\}}z |j}Wntk rYqXYnXt|j ||qX| D]\}} t ||| qdS)aUpdate the state of a dynamic function. As __closure__ and __globals__ are readonly attributes of a function, we cannot rely on the native setstate routine of pickle.load_build, that calls setattr on items of the slotstate. Instead, we have to modify them inplace. rTrSrXrN) r9updaterkrTr enumeraterrrrSitemssetattr) r,rar_Z obj_globalsZ obj_closureicellrBrUvr.r.r/_function_setstates         rcCsX|\}}d}|D]"\}}|dkr*|}qt|||q|dk rT|D]}||qD|S)Nrh)rrregister)r,rar_rrrzattrsubclassr.r.r/r s rc@seZdZiZeee<eeej<e ee j <e ee j <eee<eee<eee<eee<eeej<eeej<eeej<eeej<eeej<e ee!j"<e#ee$j%<e&ee'j(<e)ee'j*<e+ee'j,<e-ee.e/0<e1ee.e/2<e3ee.e/4<eee5j6<eee5j7<eee5j8<eee5j9<e:ee;j<ZddZ?ddZ@eAjBd krXdd d ZCn dd d ZCeAjBd kreDseReduce a function that is not pickleable via attribute lookup.N)_function_getnewargsrbr#r)selfrWZnewargsrar.r.r/_dynamic_function_reduce=s   z%CloudPickler._dynamic_function_reducecCst|r tS||SdS)aReducer for function objects. If obj is a top-level attribute of a file-backed module, this reducer returns NotImplemented, making the CloudPickler fallback to traditional _pickle.Pickler routines to save obj. Otherwise, it reduces obj using a custom cloudpickle reducer designed specifically to handle dynamic functions. As opposed to cloudpickle.py, There no special handling for builtin pypy functions because cloudpickle_fast is CPython-specific. N)r rr)rr,r.r.r/_function_reduceDs zCloudPickler._function_reducecCs~|j}|jt|ji}|ikrDdD]}||jkr&|j|||<q&|jdkrTd}ntddtt|j D}||dd|fS)N) __package__r>__path____file__css|] }tVqdSrL)r)rDrsr.r.r/ qsz4CloudPickler._function_getnewargs..) rY globals_ref setdefaultidrTrStuplerangelenr)rrWcode base_globalsrUclosurer.r.r/rUs    z!CloudPickler._function_getnewargsc CsXzt||WStk rR}z&d|jdkr@d}t||nW5d}~XYnXdS)N recursionrz?Could not pickle object as excessively deep recursion required.)rr+ RuntimeErrorrrr)rr,rEmsgr.r.r/r+vszCloudPickler.dumpr$NcCs2|dkr t}tj||||di|_t||_dS)Nr&)r r__init__rintproto)rr-r'r(r.r.r/rszCloudPickler.__init__cCs4|dkr t}tj|||di|_t|ds0tdS)Nr7r)r rrrrAssertionError)rr-r'r.r.r/rs cCstjdddkr&t|r&tt|fSt|}zt|t}Wntk rTd}YnX|rbt|St |t j rx| |St SdS)a*Type-agnostic reducing callback for function and classes. For performance reasons, subclasses of the C _pickle.Pickler class cannot register custom reducers for functions and classes in the dispatch_table. Reducer for such types must instead implemented in the special reducer_override method. Note that method will be called for any object except a few builtin-types (int, lists, dicts etc.), which differs from reducers in the Pickler's dispatch_table, each of them being invoked for objects of a specific type only. This property comes in handy for classes: although most classes are instances of the ``type`` metaclass, some of them can be instances of other custom metaclasses (such as enum.EnumMeta for example). In particular, the metaclass will likely not be known in advance, and thus cannot be special-cased using an entry in the dispatch_table. reducer_override, among other things, allows us to register a reducer that will be called for any class, independently of its type. Notes: * reducer_override has the priority over dispatch_table-registered reducers. * reducer_override can be used to fix other limitations of cloudpickle for other types that suffered from type-specific reducers, such as Exceptions. See https://github.com/cloudpipe/cloudpickle/issues/248 NF)r version_inforrrr=rl TypeErrorrr;r FunctionTyperr)rr,tZ is_anyclassr.r.r/reducer_overrides    zCloudPickler.reducer_overridec Cs\|j}|j} |j||d|||d||||||| tj| tj| tjdS)N)ra listitems dictitemsr,)saver save_reducerTUPLE2REDUCEPOP) rrWrrarr state_setterr,rrr.r.r/_save_reduce_pickle5s   z!CloudPickler._save_reduce_pickle5cCs|tdkr|jtd|dS|ttkr:|jttf|dS|ttkrX|jttf|dS|tkrv|jtt|f|dStjdddkrt|r|jt t ||dnL|dk rt j |||dn2t ||ds|jt|d|int j |||ddS)z Save a "global". The name of this method is somewhat misleading: all types get dispatched here. NrLrrrrAr,)r=rrrr rrrrrrr save_globalr r r)rr,rApackr.r.r/r s.    zCloudPickler.save_globalcCsPt||drtj|||dStr6t|jtr6||S|j| |d|iSdS)z Registered with the dispatch to handle all function types. Determines what kind of function obj is (e.g. lambda, defined at interactive prompt, etc) and handles the pickling appropriately. r r,N) r rr rr;rYrsave_pypy_builtin_funcr r)rr,rAr.r.r/ save_function's  zCloudPickler.save_functioncCs4tj|ji|j|j|jf|jf}|j|d|idS)a Save pypy equivalent of builtin functions. PyPy does not have the concept of builtin-functions. Instead, builtin-functions are simple function instances, but with a builtin-code attribute. Most of the time, builtin functions should be pickled by attribute. But PyPy has flaky support for __qualname__, so some builtin functions such as float.__new__ will be classified as dynamic. For this reason only, we created this special routine. Because builtin-functions are not expected to have closure or globals, there is no additional hack (compared the one already implemented in pickle) to protect ourselves from reference cycles. A simple (reconstructor, newargs, obj.__dict__) tuple is save_reduced. Note also that PyPy improved their support for __qualname__ in v3.6, so this routing should be removed when cloudpickle supports only PyPy 3.6 and later. r,N)rrrYr>rQrSr9r)rr,rvr.r.r/r6sz#CloudPickler.save_pypy_builtin_func)NN)N)NNNNN)N)Pr>rIrHZ_dispatch_tabler classmethodrr2 TextIOWrapperrrLoggerr RootLoggerr memoryviewrr< staticmethodrrrrrrGetSetDescriptorTyper ModuleTyperrrrrrrrtypingTypeVarr_collections_abc dict_keysr dict_valuesr dict_itemsrr=rkeysrr^rrrmabstractmethodabstractclassmethodabstractstaticmethodabstractpropertyrcopyregdispatch_tablerrrr+rHIGHEST_PROTOCOLrrdispatchrrrr structr r rrrr.r.r.r/r*s`                  !    9  " r*)NN)NN)N)N)TrRrrmr$r2r\rrr(rrrenumr collectionsrrcompatrrZ cloudpicklerr r r r r rrrrrrrrrrrrrrrrrrr r!r"r#r&r+r6loadloadsr?rKrNrbrtr|rrrrrrrrrrrrrrrrrrrrrrrr*r.r.r.r/sf  x     '(> /  "