U Cv fJ@sUddlmZddlZddlmZddlmZddlmZddlm Z ddl m Z m Z m Z mZmZmZmZmZmZmZmZmZmZmZmZddlZddlmZmZdd lm Z dd l!m"Z"m#Z#m$Z$dd l%m&Z&dd l'm(Z(dd l)m*Z*m+Z+m,Z,ddl-m.Z.ddl/m0Z0m1Z1m2Z2ddl3m4Z4ddl5m6Z6e rzddl7m8Z8Wne9k rVdZ8YnXddlm:Z:ddl!m;Z;ddlZ>m?Z?m@Z@edZAeeAedee;eBdfZCeeDee e feddfZEedZFdZGdZHejIjJejKejLjJejMjJejNejOjJejPejQjRdZSdddd ZTd!dd"d#ZUd$dd%d&ZVdd(d)d*d+d,d-ZWd.d/d0d1d2ZXdd3d4ZYd5d6ZZd7d8Z[d9d:Z\d;d<Z]d=d>Z^d.dd?d@Z_dAdBZ`dCdDZadEdFZbdddddddddddd'ddG dHdIdJdKdKdKdKdKdKdKdLdMd)dNd.dOdPdQZcdddddddddddd'ddG dHdIdJdKdKdKdKdKdKdKdLdMd)dNdRdOdSdTZddd[dJd\d]d^dId_d`d)dadbdcd.dd dedfZeejIjJejKejMjJdZfdgegdh<edd'dkd.dbdldmdndodpdqd)drd)dsdt dudvZhedd.d/dldmdndodpdqd)dwd)dxdt dydvZhedd'd'dzd.d{dldmdndodpdqdwdwd)d|dt d}dvZhedd.d{dldmdndodpdqdrdwd)d/dt d~dvZhdd.dbdldmdndodpdqd)d)d)ddt ddvZhdddZidddZjddZkddZledd.ddddnddrdKddd)ddddddZmedddddjdddd.ddddnddwdKddd)ddd|dddZmdd.ddddndd)dKddd)ddddddZmdS)) annotationsN)partial)glob)BytesIO)Number) TYPE_CHECKINGAnyCallableDictFinalHashableIterableLiteralMappingMutableMappingSequenceTypeUnioncastoverload)backends conventions)plugins)AbstractDataStore ArrayWriter_normalize_path)_get_scheduler)indexing)"_infer_concat_order_from_positions_nested_combinecombine_by_coords) DataArray)Dataset _get_chunk _maybe_chunk)Index) is_remote_uri)Delayed)BufferedIOBase)BackendEntrypoint)CombineAttrsOptions CompatOptions JoinOptionsNestedSequence)netcdf4scipyh5netcdf)pydappynio pseudonetcdfcfgribzarrauto)ZNETCDF4ZNETCDF4_CLASSICZ NETCDF3_64BITZNETCDF3_CLASSICZ__xarray_dataarray_name__Z__xarray_dataarray_variable__)r.r/r1r0r2r3r4r5zLiteral[('netcdf4', 'pydap')])returnc CsXzddl}d}WnBtk rRzddl}d}Wntk rLtdYnXYnX|S)Nrr.r1zFnetCDF4 or pydap is required for accessing remote datasets via OPeNDAP)netCDF4 ImportErrorr1 ValueError)r8enginer1r<7/tmp/pip-unpacked-wheel-h316xyqg/xarray/backends/api.py_get_default_engine_remote_uriWsr>zLiteral['scipy']cCs2zddl}d}Wntk r,tdYnX|S)Nrr/z)scipy is required for accessing .gz files)r/r9r:)r/r;r<r<r=_get_default_engine_gzjs r?zLiteral[('netcdf4', 'scipy')]c CsXzddl}d}WnBtk rRzddl}d}Wntk rLtdYnXYnX|S)Nrr.r/zKcannot read or write netCDF files without netCDF4-python or scipy installed)r8r9Zscipy.io.netcdfr:)r8r;r/r<r<r=_get_default_engine_netcdftsr@FstrboolT_NetcdfEngine)path allow_remoter7cCs,|rt|rtS|dr"tStSdS)Nz.gz)r&r>endswithr?r@)rDrEr<r<r=_get_default_engines   rGr"None)datasetr7cCs&dddd}|jD] }||qdS)z8DataArray.name and Dataset keys must be a string or Noner namecSs<t|tr |s8td|dn|dk r8td|ddS)Nz Invalid name zc for DataArray or Dataset key: string must be length 1 or greater for serialization to netCDF filesz` for DataArray or Dataset key: must be either a string or None for serialization to netCDF files) isinstancerAr: TypeErrorrJr<r<r= check_names   z+_validate_dataset_names..check_nameN) variables)rIrNkr<r<r=_validate_dataset_namess rQcCstttjtjttf}|r$|tjf7}dd}|j D]\}}||||q6|j D]$}|j D]\}}||||qdqVdS)a `attrs` must have a string key and a value which is either: a number, a string, an ndarray, a list/tuple of numbers/strings, or a numpy.bool_. Notes ----- A numpy.bool_ is only allowed when using the h5netcdf engine with `invalid_netcdf=True`. c Ssht|tr |s0td|dntd|dt||sdtd|d|ddd d |DdS) NzInvalid name for attr zF: string must be length 1 or greater for serialization to netCDF fileszInvalid name for attr: z3 must be a string for serialization to netCDF fileszInvalid value for attr z: zV. For serialization to netCDF files, its value must be of one of the following types: z, cSsg|] }|jqSr<)__name__).0vtyper<r<r= sz7_validate_attrs..check_attr..)rLrAr:rMjoin)rKvalue valid_typesr<r<r= check_attrs    $z#_validate_attrs..check_attrN) rArnpZndarraynumberlisttuplebool_attrsitemsrOvalues)rIinvalid_netcdfrXrYrPvvariabler<r<r=_validate_attrss  recKs@t|D]2}|dkr$||kr$d||<||dkr||q|S)NF)r\pop) decode_cfopen_backend_dataset_parametersdecodersdr<r<r=_resolve_decoders_kwargss    rkcCsPd}zt|}Wntk r*d}YnX|rLt|sLtjtj|}|SN)osfspathrMr&rDgetmtime expanduser)filename_or_objmtimerDr<r<r= _get_mtimes  rscCsB|jD]2\}}||jkr t|j}|r6t|}||_q dSrl)rOr`Z_indexesrZCopyOnWriteArray_dataZMemoryCachedArraydata)rIcacherKrdrur<r<r="_protect_dataset_variables_inplaces    rwcCs~|dS)z5Finalize this store by explicitly syncing and closingN)close)writestorer<r<r=_finalize_storesr{cCs|D] }|qdSrlr<)closerscloserr<r<r=_multi_file_closersr~c Ks<d|krtdt|f|}|W5QRSQRXdS)aPOpen, load into memory, and close a Dataset from a file or file-like object. This is a thin wrapper around :py:meth:`~xarray.open_dataset`. It differs from `open_dataset` in that it loads the Dataset into memory, closes the file, and returns the Dataset. In contrast, `open_dataset` keeps the file handle open and lazy loads its contents. All parameters are passed directly to `open_dataset`. See that documentation for further details. Returns ------- dataset : Dataset The newly created Dataset. See Also -------- open_dataset rv#cache has no effect in this contextN)rM open_datasetload)rqkwargsdsr<r<r= load_datasetsrc Ks<d|krtdt|f|}|W5QRSQRXdS)aOpen, load into memory, and close a DataArray from a file or file-like object containing a single data variable. This is a thin wrapper around :py:meth:`~xarray.open_dataarray`. It differs from `open_dataarray` in that it loads the Dataset into memory, closes the file, and returns the Dataset. In contrast, `open_dataarray` keeps the file handle open and lazy loads its contents. All parameters are passed directly to `open_dataarray`. See that documentation for further details. Returns ------- datarray : DataArray The newly created DataArray. See Also -------- open_dataarray rvrN)rMopen_dataarrayr)rqrdar<r<r=load_dataarraysrc Ksvddlm}t|}|||||f|} d| } i} |jD],\} } t| |}t| | ||| | |d| | <q>|| S)Nr)tokenizez open_dataset-)overwrite_encoded_chunks name_prefixtoken inline_array)Z dask.baserrsrOr`r#r$_replace) backend_dsrqr;chunksrr extra_tokensrrrrrrOrKvarZ var_chunksr<r<r= _chunk_ds5s"    rc Kst|ttfs&|dkr&td|dt|||dkr>|}nt||||||f|}||jd|jkrt|t t j frt ||jd<|S)N>Nr6z>> from functools import partial >>> def _preprocess(x, lon_bnds, lat_bnds): ... return x.sel(lon=slice(*lon_bnds), lat=slice(*lat_bnds)) ... >>> lon_bnds, lat_bnds = (-110, -105), (40, 45) >>> partial_func = partial(_preprocess, lon_bnds=lon_bnds, lat_bnds=lat_bnds) >>> ds = xr.open_mfdataset( ... "file_*.nc", concat_dim="time", preprocess=partial_func ... ) # doctest: +SKIP References ---------- .. [1] https://docs.xarray.dev/en/stable/dask.html .. [2] https://docs.xarray.dev/en/stable/dask.html#chunking-and-performance r5r)get_fs_token_pathszCThe use of remote URLs for opening zarr requires the package fsspecNrbrstorage_optionsF)moderexpandcsg|]}|qSr< get_mapper)rSrD)fsr<r=rUsz"open_mfdataset..zjcannot do wild-card matching for paths that are remote URLs unless engine='zarr' is specified. Got paths: z7. Instead, supply paths as an explicit list of strings.cSs&g|]}t|tjrt|n|qSr<)rLrmrrnrSpr<r<r=rUszno files to opennestedrzWhen combine='by_coords', passing a value for `concat_dim` has no effect. To manually combine along a specific dimension you should instead specify combine='nested' along with a value for `concat_dim`.)r;rcsg|]}|fqSr<r<r)open_ open_kwargsr<r=rUscsg|]}|dqS)rr<rSr)getattr_r<r=rUscsg|] }|qSr<r<r)rr<r=rUs)Z concat_dimsrrcoordsidsrVr)rrrrVrz<{} is an invalid option for the keyword argument ``combine``)%rLrAr&Z fsspec.corerr9getrZ_strip_protocolr:sortedrrmrrnOSErrorr!rr\keysrardaskdelayedrgetattrcomputerr formatrxrrr~rindexr_)rrrrrr;rrrrrVrrrre_Z tmp_pathsZcombined_ids_pathsrrdatasetsr|Zcombinedrr<)rrrrrr=open_mfdatasets<                  rzdict[T_NetcdfEngine, Callable]WRITEABLE_STORESwTrbzLiteral[('w', 'a')]zT_NetcdfTypes | Nonez str | NonezT_NetcdfEngine | Nonez+Mapping[Hashable, Mapping[str, Any]] | NonezIterable[Hashable] | Nonez Literal[True]z%tuple[ArrayWriter, AbstractDataStore]) rI path_or_filerrgroupr;runlimited_dimsr multifilerbr7c  CsdSrlr< rIrrrrr;rrrrrbr<r<r= to_netcdf'srzLiteral[False]bytesc CsdSrlr<rr<r<r=r:s)rrbzstr | os.PathLiker'c CsdSrlr<rr<r<r=rLsc CsdSrlr<rr<r<r=r_sz>tuple[ArrayWriter, AbstractDataStore] | bytes | Delayed | Nonec Cst|tjrt|}|dkr"i}|dkr^|dkr8d}n|dkrPtd|d|stdn(t|tr|dkrxt|}t|}nd}t |t || o|dkdz t |} Wn"t k rtd|YnX|dk r| }t} td d |jD} | o | d k}|r4|dkr4td |d | d|dk rB|nt}|rXtddni}| r|dkrv| |d<ntd|| ||||f|}|dkr|jdd}|dk rt|tst|ts|g}nt|}t}zbt|||||d|r|| r||fW2S|j|d}t|trF||WSW5| s^|r^|X|s~ddl}|t||SdS)zThis function creates an appropriate datastore for writing a dataset to disk as a netCDF file See `Dataset.to_netcdf` for full API docs. The ``multifile`` argument is only for the private use of save_mfdataset. Nr/z2invalid engine for creating bytes with to_netcdf: z8. Only the default engine or engine='scipy' is supportedzJto_netcdf() with compute=False is not yet implemented when returning bytesr0rz#unrecognized engine for to_netcdf: css|]}|jdk VqdSrl)r)rSrcr<r<r= szto_netcdf..)Z distributedmultiprocessingzWriting netCDF files with the z0 backend is not currently supported with dask's z schedulerT) autocloserbz0unrecognized option 'invalid_netcdf' for engine r)rrrr) rLrmrrnr:NotImplementedErrorrArGrrQrerKeyErrorupperranyrOrarrrrr r\rrx dump_to_storesyncgetvaluerrr{)rIrrrrr;rrrrrbZ store_openZ schedulerZ have_chunksrtargetrrzwriterwritesrr<r<r=rps               c Cs~|dkrt}|dkri}t|\}}t}|D]\} } | || _|| q6|rf|||\}}|j|||||ddS)z7Store dataset contents to a backends.*DataStore object.N)r)rrZencode_dataset_coordinatessetr`raddrz) rIrzrencoderrrrOr_Zcheck_encodingrPencr<r<r=rs  rc sdkr$tt|t|kr$td|D] }t|ts(tdt|q(|dkr`dgt|}tt|t|t|hdkrtdtfddt|||D\} } zfd d| D} W5r| D] } | qXs d dl  fd dt| | DSdS) a Write multiple datasets to disk as netCDF files simultaneously. This function is intended for use with datasets consisting of dask.array objects, in which case it can write the multiple datasets to disk simultaneously using a shared thread pool. When not using dask, it is no different than calling ``to_netcdf`` repeatedly. Parameters ---------- datasets : list of Dataset List of datasets to save. paths : list of str or list of path-like objects List of paths to which to save each corresponding dataset. mode : {"w", "a"}, optional Write ("w") or append ("a") mode. If mode="w", any existing file at these locations will be overwritten. format : {"NETCDF4", "NETCDF4_CLASSIC", "NETCDF3_64BIT", "NETCDF3_CLASSIC"}, optional **kwargs : additional arguments are passed along to ``to_netcdf`` File format for the resulting netCDF file: * NETCDF4: Data is stored in an HDF5 file, using netCDF4 API features. * NETCDF4_CLASSIC: Data is stored in an HDF5 file, using only netCDF 3 compatible API features. * NETCDF3_64BIT: 64-bit offset version of the netCDF 3 file format, which fully supports 2+ GB files, but is only compatible with clients linked against netCDF version 3.6.0 or later. * NETCDF3_CLASSIC: The classic netCDF 3 file format. It does not handle 2+ GB files very well. All formats are supported by the netCDF4-python library. scipy.io.netcdf only supports the last two formats. The default format is NETCDF4 if you are saving a file to disk and have the netCDF4-python library available. Otherwise, xarray falls back to using scipy to write netCDF files and defaults to the NETCDF3_64BIT format (scipy does not support netCDF4). groups : list of str, optional Paths to the netCDF4 group in each corresponding file to which to save datasets (only works for format="NETCDF4"). The groups will be created if necessary. engine : {"netcdf4", "scipy", "h5netcdf"}, optional Engine to use when writing netCDF files. If not provided, the default engine is chosen based on available dependencies, with a preference for "netcdf4" if writing to a file on disk. See `Dataset.to_netcdf` for additional information. compute : bool If true compute immediately, otherwise return a ``dask.delayed.Delayed`` object that can be computed later. Examples -------- Save a dataset into one netCDF per year of data: >>> ds = xr.Dataset( ... {"a": ("time", np.linspace(0, 1, 48))}, ... coords={"time": pd.date_range("2010-01-01", freq="M", periods=48)}, ... ) >>> ds Dimensions: (time: 48) Coordinates: * time (time) datetime64[ns] 2010-01-31 2010-02-28 ... 2013-12-31 Data variables: a (time) float64 0.0 0.02128 0.04255 0.06383 ... 0.9574 0.9787 1.0 >>> years, datasets = zip(*ds.groupby("time.year")) >>> paths = [f"{y}.nc" for y in years] >>> xr.save_mfdataset(datasets, paths) rzCcannot use mode='w' when writing multiple datasets to the same pathzDsave_mfdataset only supports writing Dataset objects, received type Nrzcmust supply lists of the same length for the datasets, paths and groups arguments to save_mfdatasetc s2g|]*\}}}t|||fddqS)T)rr)r)rSrrDr)rr;rrrr<r=rUms  z"save_mfdataset..csg|]}|jdqS)r)r)rSrrr<r=rU~srcs g|]\}}t||qSr<)rr{)rSrs)rr<r=rUs) rrr:rLr"rMtypeziprxrr) rrrrgroupsr;rrobjZwritersZstoresrzrr<)rrr;rrrr=save_mfdatasets<T    rcsttstdtD]`\}}||jkrTtdtdt|jt|tsltd|j dkr$tdq$fdd|j D}|rtd t d |d dS) Nz``region`` must be a dict, got z:all keys in ``region`` are not in Dataset dimensions, got z and z;all values in ``region`` must be slice objects, got region=>rNz?step on all slices in ``region`` must be 1 or None, got region=cs$g|]\}}t|js|qSr<)r intersectiondims)rSrPrcregionr<r=rUsz$_validate_region..zwhen setting `region` explicitly in to_zarr(), all variables in the dataset to write must have at least one dimension in common with the region's dimensions z, but that is not the case for some variables here. To drop these variables from this dataset before exporting to zarr, write: .drop_vars()) rLrrMrr`rr:r\slicesteprOr)rrrPrcZnon_matching_varsr<rr=_validate_regions,     rcs6|fdd}|jD]\}}|||qdS)zpIf variable exists in the store, confirm dtype of the data to append is compatible with existing dtype. cs||ksxt|jtjsxt|jtjsxt|jtjsx|jtkrDn4|j|jksxtd|d|jd|jddS)NzMismatched dtypes for variable zC between Zarr store on disk and dataset to append. Store has dtype z! but dataset to append has dtype r)rZZ issubdtypeZdtyper[Z datetime64r^objectr:)vnamerZ existing_varsr<r= check_dtypesz8_validate_datatypes_for_zarr_append..check_dtypeN)Z get_variablesrr`)zstorerIrrrr<rr=#_validate_datatypes_for_zarr_appends r z.MutableMapping | str | os.PathLike[str] | Nonez)MutableMapping | str | os.PathLike | Nonez%Literal[('w', 'w-', 'a', 'r+', None)]zMapping | NonezHashable | NonezMapping[str, slice] | Nonezdict[str, str] | Nonez int | Nonezbackends.ZarrStore)rIrz chunk_storerrrr consolidated append_dimr safe_chunksr zarr_versionr7cCsdSrlr<rIrzr r synchronizerrrrr r rrrrr<r<r=to_zarrsr)r r rrrrcCsdSrlr<rr<r<r=rszbackends.ZarrStore | Delayedcs|jD]}|jdkr |q t|}t|}| dkrD|}|}nNddlm}t|tslt dt |||f| }|dk r||f| }n|}|dkri}|dkr| dk rd}n| dk rd}nd}|dkr| dk rt d|d kr| dk rt d |d krt d |t || dk rNt || | dk rN| | krNt d | d| dkrht t|dd} |dkr| dkrd}|dkr|}d}nd}|p|dk}tjj|||||||| | | d| d }|d krzt||| dk r|}| |krt d| d|t|jD]$}||kr t d|dq |dkrzfdd|jD}|rzt d|dt}t||||d|j|d}|rt||nddl}|t||S|S)zThis function creates an appropriate datastore for writing a dataset to a zarr ztore See `Dataset.to_zarr` for full API docs. rNrz3store must be a string to use storage_options. Got ar+w-z2cannot set append_dim unless mode='a' or mode=None)rrz9cannot set region unless mode='a', mode='r+' or mode=None)rrrrzJThe only supported options for mode are 'w', 'w-', 'a' and 'r+', but mode=zYcannot list the same dimension in both ``append_dim`` and ``region`` with to_zarr(), got z in bothZ_store_versionF) rzrrrr consolidate_on_closer r Z write_regionr stacklevelrz append_dim=z0 does not match any existing dataset dimensions z variable z* already exists, but encoding was providedcsg|]}|kr|qSr<r<)rSrPZexisting_var_namesr<r=rU|szto_zarr..z,dataset contains non-pre-existing variables zt, which is not allowed in ``xarray.Dataset.to_zarr()`` with mode='r+'. To allow writing new variables, set mode='a'.)rr)rOrasizerrZfsspecrrLrAr:rrQrrrr ZarrStore open_groupr Zget_dimensionsrZ zarr_groupZ array_keysrrrrr{rr)rIrzr rrrrrr r rrrrrcZmapperZ chunk_mapperrZalready_consolidatedrr Z existing_dimsvar_nameZ new_namesrrrr<rr=rs                     )F)F) NNrNNrrrFrNr)NrNNNNNT) NrNNNNNTFF)rNNNNN) rNNNNNTFF) NrNNNNNTFF)NNNN)rNNNT) NNNNNNTNNNTNN)NNNNNN) NNNNNNTNNNTNN)n __future__rrm functoolsrriorZnumbersrtypingrrr r r r r rrrrrrrrZnumpyrZZxarrayrrZxarray.backendsrZxarray.backends.commonrrrZxarray.backends.locksrZ xarray.corerZxarray.core.combinerrr Zxarray.core.dataarrayr!Zxarray.core.datasetr"r#r$Zxarray.core.indexesr%Zxarray.core.utilsr&Z dask.delayedr'r9r(r)Zxarray.core.typesr*r+r,r-rCrArrrZ T_NetcdfTypesrrZNetCDF4DataStoreopenZScipyDataStoreZPydapDataStoreZ H5NetCDFStoreZ NioDataStoreZPseudoNetCDFDataStoreZCfGribDataStorerrZENGINESr>r?r@rGrQrerkrsrwr{r~rrrrrrrr__annotations__rrrrr rr<r<r<r=s     D             -  ).=.8(; *( *(&z   $%, .