U Cv f @sddlmZddlZddlmZddlmZmZmZm Z m Z m Z m Z m Z ddlZddlZddlmZmZmZddlmZmZddlmZmZddlmZmZdd lm Z dd l!m"Z"m#Z#d Z$d Z%er8dd l&m'Z'ddl(m)Z)e e edfee*e*fZ+e edfZ,e ee#fZ-e eefZ.e e/e edfZ0e e)e'fZ1Gdddej2Z3Gdddej2Z4dddddZ5dSddddddZ6dddddZ7ddddd Z8dTdd!d"d#d$Z9dUddd%dd&d'Z:d(d)d*d+Z;dVddddd,d-Zd8d%d9d:d;Z?d8d%d9dd/d/d/d/d?d4d4d@dA dBdCZAdZdDd/d/d/d/d?d4d4dEdF dGdHZBd[d8d>d/d/d/dIdJdKdLZCdMdNZDdOdPZEdQdRZFdS)\) annotationsN) defaultdict) TYPE_CHECKINGAnyHashableIterableMappingMutableMappingTupleUnion)stringstimes variables)SerializationWarningpop_to)duck_array_opsindexing)_contains_datetime_like_objectscontains_cftime_datetimes)is_duck_dask_array) IndexVariableVariable) boundsZ grid_mappingZ climatologyZgeometryZnode_coordinatesZ node_countZpart_node_countZ interior_ring cell_measures formula_terms)rrAbstractDataStoreDataset.c@s0eZdZdZdZddZeddZddZd S) NativeEndiannessArrayaDecode arrays on the fly from non-native to native endianness This is useful for decoding arrays from netCDF3 files (which are all big endian) into native endianness, so they can be used with Cython functions, such as those found in bottleneck and pandas. >>> x = np.arange(5, dtype=">i2") >>> x.dtype dtype('>i2') >>> NativeEndiannessArray(x).dtype dtype('int16') >>> indexer = indexing.BasicIndexer((slice(None),)) >>> NativeEndiannessArray(x)[indexer].dtype dtype('int16') arraycCst||_dSNrZ as_indexabler!selfr!r&6/tmp/pip-unpacked-wheel-h316xyqg/xarray/conventions.py__init__QszNativeEndiannessArray.__init__cCst|jjjt|jjjSr")npdtyper!kindstritemsizer%r&r&r'r*TszNativeEndiannessArray.dtypecCstj|j||jdSNr*r)Zasarrayr!r*r%keyr&r&r' __getitem__Xsz!NativeEndiannessArray.__getitem__N __name__ __module__ __qualname____doc__ __slots__r(propertyr*r4r&r&r&r'r;s  rc@s0eZdZdZdZddZeddZddZd S) BoolTypeArrayaDecode arrays on the fly from integer to boolean datatype This is useful for decoding boolean arrays from integer typed netCDF variables. >>> x = np.array([1, 0, 1, 1, 0], dtype="i1") >>> x.dtype dtype('int8') >>> BoolTypeArray(x).dtype dtype('bool') >>> indexer = indexing.BasicIndexer((slice(None),)) >>> BoolTypeArray(x)[indexer].dtype dtype('bool') r cCst||_dSr"r#r$r&r&r'r(qszBoolTypeArray.__init__cCs tdS)Nbool)r)r*r.r&r&r'r*tszBoolTypeArray.dtypecCstj|j||jdSr/r1r2r&r&r'r4xszBoolTypeArray.__getitem__Nr5r&r&r&r'r<\s  r<r T_VarTuple)varreturncCs|j|j|j|jfSr")dimsdataattrscopyencodingr?r&r&r' _var_as_tuple|srGT_Name)r?namer@cCsd|jkr|jddtfkrt|\}}}}t|d}||jkrt|tjrt|jtjrd|j krd|j krt j d|dt ddt |}|j|d }t||||d d }|S) Nr*S1 _FillValueZ missing_valuezsaving variable zT with floating point data as an integer dtype without any _FillValue to use for NaNs  stacklevelr0Tfastpath)rEr,rGr)r*pop issubdtypeintegerfloatingrCwarningswarnrZaroundastyper)r?rIrArBrCrEr*r&r&r'maybe_encode_nonstring_dtypes(    rXcCs<d|jkr8d|jkr8t|jtjr8|jtj|jd<|S)NrK)rCrEr)rRr*rTtypenanrFr&r&r'maybe_default_fill_valuesr[cCs\|jtkrXd|jkrXd|jkrXt|\}}}}d|d<tj|ddd}t||||dd}|S)Nr*r=i1T)r*rDrO)r*r=rErCrGrrWr)r?rArBrCrEr&r&r'maybe_encode_boolssr]znp.dtype)rIr@cCs|jjdkrtd|jdkr(ttS|d|j}t|trJt tSt|t r^t t St |j}|jdkrx|St d|dS)z]Given an object array with no missing values, infer its dtype from its first element Oz1infer_type must be called on a dtype=object arrayr)rzXunable to infer dtype on variable {!r}; xarray cannot serialize arbitrary Python objectsN)r*r+ TypeErrorsizer)floatndim isinstancebytesr Zcreate_vlen_dtyper,r! ValueErrorformat)r!rIelementr*r&r&r' _infer_dtypes"         rhNonecCs,t|tr(t|tjr(td|dS)Nzvariable {!r} is a MultiIndex, which cannot yet be serialized to netCDF files (https://github.com/pydata/xarray/issues/1077). Use reset_index() to convert MultiIndex levels into coordinate variables instead.)rcrZto_indexpdZ MultiIndexNotImplementedErrorrf)r?rIr&r&r'ensure_not_multiindexs rlznp.typing.DTypeLiker0cCst|j|}||d<|S)zCreate a copy of an array with the given dtype. We use this instead of np.array() to ensure that custom object dtypes end up on the resulting array. .)r)emptyshape)rBr*resultr&r&r'_copy_with_dtypesrpc Cs|jjdkrt|\}}}}t|r>td|t|}t |}| r||}t ||}t |rtd} n4t |rd} n$t|tjstt}|tj} t||d}| ||<nt|t ||d}|jjdks|jjstt||||dd}|S)Nr^avariable {} has data in the form of a dask array with dtype=object, which means it is being loaded into memory to determine a data type that can be safely stored on disk. To avoid this, coerce this variable to a fixed-size dtype with astype() before saving it.r0TrO)r*r+rGrrUrVrfrZcomputerjZisnullanyrhr Zis_bytes_dtypeZis_unicode_dtyper)rRrTrarYrZrpmetadataAssertionErrorr) r?rIrArBrCrEmissingZnon_missing_valuesZinferred_dtypeZ fill_valuer&r&r'ensure_dtype_not_objects6          rwTr=)r? needs_copyrIr@cCst||dtttttfD]}|j||d}q.t ||d}t |}t |}t ||d}t D]}t|j|j|qn|S)a Converts an Variable into an Variable which follows some of the CF conventions: - Nans are masked using _FillValue (or the deprecated missing_value) - Rescaling via: scale_factor and add_offset - datetimes are converted to the CF 'units since time' format - dtype encodings are enforced. Parameters ---------- var : Variable A variable holding un-encoded data. Returns ------- out : Variable A variable which has been encoded as described above. rI)rlr CFDatetimeCoderCFTimedeltaCoderrCFScaleOffsetCoder CFMaskCoderUnsignedIntegerCoderencoderXr[r]rwCF_RELATED_DATArrErC)r?rxrIcoder attr_namer&r&r'encode_cf_variables   rrz bool | None) rIr?concat_charactersmask_and_scale decode_timesdecode_endiannessstack_char_dim use_cftimedecode_timedeltar@c Cs$t|r |S|j} |dkr|}|rF|r8tj||d}t|}|rtttt fD]} | j||d}q`|rt j||d}|rt j |dj||d}t |\} } } }|r| jjst| } | j} |d| d| kr| ddkr| d=t| } t| st| } t| | | |ddS)a Decodes a variable which may hold CF encoded information. This includes variables that have been masked and scaled, which hold CF style time variables (this is almost always the case if the dataset has been serialized) and which have strings encoded as character arrays. Parameters ---------- name : str Name of the variable. Used for better error messages. var : Variable A variable holding potentially CF encoded information. concat_characters : bool Should character arrays be concatenated to strings, for example: ["h", "e", "l", "l", "o"] -> "hello" mask_and_scale : bool Lazily scale (using scale_factor and add_offset) and mask (using _FillValue). If the _Unsigned attribute is present treat integer arrays as unsigned. decode_times : bool Decode cf times ("hours since 2000-01-01") to np.datetime64. decode_endianness : bool Decode arrays from non-native to native endianness. stack_char_dim : bool Whether to stack characters into bytes along the last dimension of this array. Passed as an argument because we need to look at the full dataset to figure out if this is appropriate. use_cftime : bool, optional Only relevant if encoded dates come from a standard calendar (e.g. "gregorian", "proleptic_gregorian", "standard", or not specified). If None (default), attempt to decode times to ``np.datetime64[ns]`` objects; if this is not possible, decode times to ``cftime.datetime`` objects. If True, always decode times to ``cftime.datetime`` objects, regardless of whether or not they can be represented using ``np.datetime64[ns]`` objects. If False, always decode times to ``np.datetime64[ns]`` objects; if this is not possible raise an error. Returns ------- out : Variable A variable holding the decoded equivalent of var. Nry)rr*r=T)rErP)rr*r ZCharacterArrayCoderdecodeZEncodedStringCoderrr~r}r|r r{rzZunpack_for_decodingZisnativer setdefaultr<rrZLazilyIndexedArrayr)rIr?rrrrrrrZoriginal_dtyperZ dimensionsrB attributesrEr&r&r'decode_cf_variable<s<9    r T_Variables)rr@cCs|D]t}|j}|d}t|to,d|k}|rd|kr|d|kr||dj}|d|dd|kr|d|dqdS)aAdds time attributes to time bounds variables. Variables handling time bounds ("Cell boundaries" in the CF conventions) do not necessarily carry the necessary attributes to be decoded. This copies the attributes from the time variable to the associated boundaries. See Also: http://cfconventions.org/Data/cf-conventions/cf-conventions-1.7/ cf-conventions.html#cell-boundaries https://github.com/pydata/xarray/issues/2565 unitssincercalendarN)valuesrCgetrcr,r)rvrCrhas_date_unitsZ bounds_attrsr&r&r'_update_bounds_attributess    rcCs|D]\}}|j}|j}d|ko.d|dk}t|jtjpFt|}|r||s|d|kr||d|kr|t d ||dt |rd|kr|d|kr||dj}| d|dd|kr| d|dqdS)aAdds time encoding to time bounds variables. Variables handling time bounds ("Cell boundaries" in the CF conventions) do not necessarily carry the necessary attributes to be decoded. This copies the encoding from the time variable to the associated bounds variable so that we write CF-compliant files. See Also: http://cfconventions.org/Data/cf-conventions/cf-conventions-1.7/ cf-conventions.html#cell-boundaries https://github.com/pydata/xarray/issues/2565 rrra3Variable '{0}' has datetime type and a bounds variable but {0}.encoding does not have units specified. The units encodings for '{0}' and '{1}' will be determined independently and may not be equal, counter to CF-conventions. If this is a concern, specify a units encoding for '{0}' before writing to a file.rN) itemsrCrEr)rRr*Z datetime64rrUrVrf UserWarningr)rrIrrCrErZis_datetime_typeZbounds_encodingr&r&r'_update_bounds_encodings>   rT_AttrsT_DropVariablesz*tuple[T_Variables, T_Attrs, set[Hashable]]) rrrrr decode_coordsdrop_variablesrrr@c  sttD]} | jD]} | | qqdddfdd } t} t|tr^|g}n |dkrjg}t|}|r~ti}  D]\}} ||krq|o| j dko| j dko| | jd }z t || ||||||d | |<Wn<t k r }zt|d |d |W5d}~XYnX|d kr| |j}d|kr|d}|}tfdd|Dr|| |jd<|d=| ||dkrtD]}||kr||}|tkr|}nLdd|dD}t|ddkrtjd|dddd|ddd}tfdd|Dr:|| |j|<| |n.fdd|D}tjd|dd|dd||=qq|rt|ddtrt|}|d}| || || fS) zK Decode several CF encoded variables. See: decode_cf_variable rr=)dimr@cs>|kr dS|D]$}|jjdks2||jdkrdSqdS)NFST)r*r+rA)rrZdimensions_used_byrr&r' stackables  z&decode_cf_variables..stackableNrJrr)rrrrrrzFailed to decode variable z: )T coordinatesallrc3s|]}|kVqdSr"r&).0krr&r' 8sz&decode_cf_variables..rcSsg|]}|D]}|qqSr&)split)rpartZ role_or_namer&r&r' Ds z'decode_cf_variables..:z Attribute sz malformedrMc3s|]}|kVqdSr"r&)rvar_namerr&r'rNscsg|]}|kr|qSr&r&)rZ proj_namerr&r'rRszVariable(s) referenced in z not in variables: )rlistrrAappendsetrcr,rrr*rbr ExceptionrYrCrrrEupdaterCF_RELATED_DATA_NEEDS_PARSINGlenrUrVrdictrQ)rrrrrrrrrrdr coord_namesnew_varsrreZ var_attrsZ coord_strZvar_coord_namesrZattr_valZ var_namesZroles_and_namesZ referenced_vars_not_in_variablesZcrdsr&rr'decode_cf_variabless      *            rT_DatasetOrAbstractstorer) objrrrrrrrr@c Csddlm}ddlm} t|| rF|j} |j} t|j} |j } |j }n4t||rr| \} } t} |j } | }ntdt| | |||||||d \} } }| | | d}||| | }|| ||_ |S)aDecode the given Dataset or Datastore according to CF conventions into a new Dataset. Parameters ---------- obj : Dataset or DataStore Object to decode. concat_characters : bool, optional Should character arrays be concatenated to strings, for example: ["h", "e", "l", "l", "o"] -> "hello" mask_and_scale : bool, optional Lazily scale (using scale_factor and add_offset) and mask (using _FillValue). decode_times : bool, optional Decode cf times (e.g., integers since "hours since 2000-01-01") to np.datetime64. decode_coords : bool or {"coordinates", "all"}, optional Controls which variables are set as coordinate variables: - "coordinates" or True: Set variables referred to in the ``'coordinates'`` attribute of the datasets or individual variables as coordinate variables. - "all": Set variables referred to in ``'grid_mapping'``, ``'bounds'`` and other attributes as coordinate variables. drop_variables : str or iterable, optional A variable or list of variables to exclude from being parsed from the dataset. This may be useful to drop variables with problems or inconsistent values. use_cftime : bool, optional Only relevant if encoded dates come from a standard calendar (e.g. "gregorian", "proleptic_gregorian", "standard", or not specified). If None (default), attempt to decode times to ``np.datetime64[ns]`` objects; if this is not possible, decode times to ``cftime.datetime`` objects. If True, always decode times to ``cftime.datetime`` objects, regardless of whether or not they can be represented using ``np.datetime64[ns]`` objects. If False, always decode times to ``np.datetime64[ns]`` objects; if this is not possible raise an error. decode_timedelta : bool, optional If True, decode variables and coordinates with time units in {"days", "hours", "minutes", "seconds", "milliseconds", "microseconds"} into timedelta objects. If False, leave them encoded as numbers. If None (default), assume the same value of decode_time. Returns ------- decoded : Dataset rrrz,can only decode Dataset or DataStore objects)rrr)rC)xarray.backends.commonrxarray.core.datasetrrc _variablesrCrcoords_closerEloadcloseZ get_encodingr_rZ set_coordsunion intersectionZ set_close)rrrrrrrrrrvarsrCZ extra_coordsrrErZdsr&r&r' decode_cfes<:         rztuple[T_Variables, T_Attrs])rrrrrr@cCst|||||\}}}||fS)a{ Decode a set of CF encoded variables and attributes. Parameters ---------- variables : dict A dictionary mapping from variable name to xarray.Variable attributes : dict A dictionary mapping from attribute name to value concat_characters : bool Should character arrays be concatenated to strings, for example: ["h", "e", "l", "l", "o"] -> "hello" mask_and_scale : bool Lazily scale (using scale_factor and add_offset) and mask (using _FillValue). decode_times : bool Decode cf times ("hours since 2000-01-01") to np.datetime64. Returns ------- decoded_variables : dict A dictionary mapping from variable name to xarray.Variable objects. decoded_attributes : dict A dictionary mapping from attribute name to values. See Also -------- decode_cf_variable )r)rrrrr_r&r&r' cf_decoders$ rcsPt|}t|D]6}t|trd|krtjd|tdd||q| }t t}t|D]~|j }| D]f\}||kr|j krt|tj kr|| tfddtDrx |qxqbdd| D}t}| D]\}} | j} | j} d | kr:d | kr:td |d d | krT| d dksnd | kr| d dkr| d d| d dqt| | d p| d } | s||rdfd d||D} | r| | d <d | kr|| d q|||rHt|}d |kr4td |dtndtt||d <||fS)N zcoordinate {!r} has a space in its name, which means it cannot be marked as a coordinate on disk and will be saved as a data variable insteadrMc3s(|] }|jkoj|kVqdSr")rEr)rr) coord_namerr&r'r sz&_encode_coordinates..cSsi|]\}}||jddqS)F)deep)rDrrrr&r&r' sz'_encode_coordinates..rz<'coordinates' found in both attrs and encoding for variable .c3s|]}|krt|VqdSr")r,)rr)not_technically_coordinatesr&r'r/sz$cannot serialize global coordinates zu because the global attribute 'coordinates' already exists. This may prevent faithful roundtrippingof xarray datasets)rrrcr,rUrVrfrdiscardrDrrAraddrsrrErCrerrQrjoinrrdifference_updatermap)rrnon_dim_coord_namesrIZglobal_coordinatesZvariable_coordinatesZ target_dimsrZwritten_coordsr?rErCZ coords_strZcoordinates_textr&)rrrr'_encode_coordinatess             rcCs&t|jt|j}t|j|j|dS)a1Encode coordinates on the given dataset object into variable specific and global attributes. When possible, this is done according to CF conventions. Parameters ---------- dataset : Dataset Object to encode. Returns ------- variables : dict attrs : dict )r)rrrArrrC)Zdatasetrr&r&r'encode_dataset_coordinatesOs rcCst|dd|D}|D]r}d|jkr:|jdnd}|r"||kr"dD]D}|||jkrN||jkrN||j||j|krN||j|qNq"||fS)a Encode a set of CF encoded variables and attributes. Takes a dicts of variables and attributes and encodes them to conform to CF conventions as much as possible. This includes masking, scaling, character array handling, and CF-time encoding. Parameters ---------- variables : dict A dictionary mapping from variable name to xarray.Variable attributes : dict A dictionary mapping from attribute name to value Returns ------- encoded_variables : dict A dictionary mapping from variable name to xarray.Variable, encoded_attributes : dict A dictionary mapping from attribute name to value See Also -------- decode_cf_variable, encode_cf_variable cSsi|]\}}|t||dqS)ry)rrr&r&r'rszcf_encoder..rN) rZ standard_nameZaxisZpositiverZ long_nameZ leap_monthZ leap_yearZ month_lengths)rrrrCrQ)rrrr?rattrr&r&r' cf_encoderes   r)N)N)N)N)TN)TTTTTNN)TTTTNNN)TTTTNNN)TTT)G __future__rrU collectionsrtypingrrrrrr r r Znumpyr)ZpandasrjZ xarray.codingr r rZxarray.coding.variablesrrZ xarray.corerrZxarray.core.commonrrZxarray.core.pycompatrZxarray.core.variablerrrrrrrrrr>rHrrr,rrZExplicitlyIndexedNDArrayMixinrr<rGrXr[r]rhrlrprwrrrrrrrrrrr&r&r&r's  (         !     //"g5"u c*`