U
    Cvf#                     @  s  d Z ddlmZ ddlZddlZddlmZmZ ddl	Z	ddl
mZ erZddlZddlZne	dZe	dZddlmZmZ dd	lmZ dd
lmZmZmZmZmZ ddlZddlmZmZ ddlm Z m!Z!m"Z"m#Z# ddl$m%Z% ddl&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0 e	dZ1eZe	j2dd Z3g Z4e,rBe45d e*rRe45d e)rbe45d dddddgddgdgdZ6ddddd gZ7e	j89d!e7d"d# Z:e0d$d% Z;e/e0d&d' Z<e	j89d!e7d(d) Z=e-e	j89d*d+d,ge	j89d-d+d,gd.d.d/d0d1d2Z>e+e	j8?d3d/d4d5d6Z@e.e	j8?d3d/d4d7d8ZAe	j8jBeejCed9k d:d;ed+d<d/d4d=d>ZDd/d4d?d@ZEe	j8jBeejCed9k d:d;ed+d<d/d4dAdBZFdS )Cz isort:skip_file     )annotationsN)AnyTYPE_CHECKING)Versiondaskdistributed)ClientLock)
futures_of)clustergen_clusterloopcleanuploop_in_thread)	HDF5_LOCKCombinedLock)
ON_WINDOWScreate_tmp_filecreate_tmp_geotiffopen_example_dataset)create_test_data)
assert_allcloseassert_identicalhas_h5netcdfhas_netCDF4requires_rasterio	has_scipyrequires_zarrrequires_cfgribrequires_cftimerequires_netCDF4z
dask.arrayc                 C  s   t | dS )Nztestfile.nc)strjoin)Ztmpdir r#   A/tmp/pip-unpacked-wheel-h316xyqg/xarray/tests/test_distributed.pytmp_netcdf_filename<   s    r%   scipynetcdf4h5netcdfNETCDF3_CLASSICZNETCDF3_64BIT_OFFSETZNETCDF3_64BIT_DATANETCDF4_CLASSICNETCDF4NETCDF3_64BIT)r'   r&   r(   )r'   r)   )r'   r*   )r'   r+   )r(   r+   )r&   r,   zengine,nc_formatc                 C  s   |t krtd dddd}t \}\}}t|d | d t |}|dkrtt |j	|||d	 W 5 Q R X W 5 Q R  W 5 Q R  d S |j	|||d	 t
j|||d
,}	t|	jjtjst|	 }
t||
 W 5 Q R X W 5 Q R X W 5 Q R X d S )Nengine not available         Zdim1dim2Zdim3addressr   r&   engineformatchunksr6   )ENGINESpytestskipr   r   r   chunkZraisesNotImplementedError	to_netcdfxropen_dataset
isinstancevar1datadaArrayAssertionErrorcomputer   r   r%   r6   Z	nc_formatr9   saboriginalrestoredcomputedr#   r#   r$   &test_dask_distributed_netcdf_roundtrip^   s0    
    rP   c                 C  s~   t  n\}\}}t|d | dJ tdtdi}|| t|}|jj	dks\t
W 5 Q R X W 5 Q R X W 5 Q R X d S )Nr3   r4   xr#   )r   r   r@   ZDatasetrE   zerosr?   rA   rQ   shaperG   )r   r%   rJ   rK   rL   rM   actualr#   r#   r$   ?test_dask_distributed_write_netcdf_with_dimensionless_variables~   s    
rU   c                 C  s   t jdddd}td}tj|j|jf}t j|||ddd}| d	 }|| t X\}\}}t	|d
 8 dD ],}	t j
||	d}
t|
d | W 5 Q R X qxW 5 Q R X W 5 Q R X d S )NZ20010101Z20010501Z360_day)calendard   )timeLontest)Zcoordsnameztest.ncr3   )FT)parallel)r@   Zcftime_rangenpZarangerandomsizeZ	DataArrayr?   r   r   Zopen_mfdatasetr   )Ztmp_pathTrY   rD   rE   	file_pathrJ   rK   rL   r\   tfr#   r#   r$   4test_open_mfdataset_can_open_files_with_cftime_index   s    

rc   c                 C  s   |t krtd dddd}t \}\}}t|d | d^ t }|j|||d tj|||d	,}	t	|	j
jtjs|t|	 }
t||
 W 5 Q R X W 5 Q R X W 5 Q R X d S )
Nr-   r.   r/   r0   r1   r3   r4   r5   r8   )r:   r;   r<   r   r   r   r?   r@   rA   rB   rC   rD   rE   rF   rG   rH   r   rI   r#   r#   r$   2test_dask_distributed_read_netcdf_integration_test   s     
  rd   consolidatedTFrH   boolNone)re   rH   returnc                 C  s  |r(t jddd ddi}dddii}ni  }}ddd	d
}t \}\}}t|d | d t |}	ttddr}
|	j|
fd|i|}|s|	  t
j|
fddd|,}t|jjtjst|	 }t|	| W 5 Q R X W 5 Q R X W 5 Q R X W 5 Q R X d S )NZzarrz
2.2.1.dev2)Z
minversionre   TZbackend_kwargsr.   r/      r1   r3   r4   z.zarrc)Zallow_cleanup_failuresuffixrH   autor8   )r;   importorskipr   r   r   r=   r   r   Zto_zarrrH   r@   rA   rB   rC   rD   rE   rF   rG   r   )r   re   rH   Zwrite_kwargsZread_kwargsr9   rJ   rK   rL   rM   filenameZmaybe_futuresrN   rO   r#   r#   r$   +test_dask_distributed_zarr_integration_test   sF      rn   z&ignore:deallocating CachingFileManager)rh   c                 C  s   t  \}}t v\}\}}ttX t|d | d< tj|ddid}t|j	t
js^t| }t|| W 5 Q R X W 5 Q R X W 5 Q R X W 5 Q R X d S )Nr3   r4   Zband   )r9   )r   r   r;   ZwarnsDeprecationWarningr   r@   Zopen_rasteriorB   rD   rE   rF   rG   rH   r   )r   Ztmp_fileexpectedrJ   rK   rL   Zda_tiffrT   r#   r#   r$   /test_dask_distributed_rasterio_integration_test   s    rr   c                 C  s   t  \}\}}t|d | dd tddddidF}tddd.}t|d	 jtjsZt| }t	|| W 5 Q R X W 5 Q R X W 5 Q R X W 5 Q R X d S )
Nr3   r4   zexample.gribZcfgribrX   ro   )r6   r9   )r6   t)
r   r   r   rB   rD   rE   rF   rG   rH   r   )r   rJ   rK   rL   Zdsrq   rT   r#   r#   r$   -test_dask_distributed_cfgrib_integration_test   s      rt   z	2022.02.0z-https://github.com/dask/distributed/pull/5739)	conditionreason)clientc           	        s
  t  }t|rt|ddid }t|s4tt|jsDtt|jsTt| }t|shtt|svtt|jstt|jstt	|
 t	|
 kstt|rtt|st| |}|I d H }t|rtt|d | |jstd S )Nr2   r.   
   )r   r   Zis_dask_collectionrG   r=   rC   Zvar2Zpersistr!   lenZ__dask_graph__r
   rH   r   Ztasks)	crJ   rK   rL   rQ   yzfuturewr#   r#   r$   
test_async   s&    

r   c                   C  s   t ttjjstd S )N)rB   r   r   utilsZSerializableLockrG   r#   r#   r#   r$   test_hdf5_lock  s    r   c                   s   ddd}t t tdtt gtt tdgfD ]P}| j|ttd|d}| |I d H  tt	|}t
|t
|ks0tq0d S )Nc              
   S  s$   | | d W  5 Q R  S Q R X d S )Nro   r#   )rQ   lockr#   r#   r$   f  s    z"test_serializable_locks.<locals>.fzfilename.ncrx   )r   )N)r   r	   r   maplistrangeZgatherpickleloadsdumpstyperG   )rz   rJ   rK   rL   r   r   ZfuturesZlock2r#   r#   r$   test_serializable_locks  s    
r   )G__doc__
__future__r   r   Znumpyr]   typingr   r   r;   Zpackaging.versionr   r   r   rl   Zdask.distributedr   r	   Zdistributed.clientr
   Zdistributed.utils_testr   r   r   r   r   Zxarrayr@   Zxarray.backends.locksr   r   Zxarray.tests.test_backendsr   r   r   r   Zxarray.tests.test_datasetr   Zxarray.testsr   r   r   r   r   r   r   r   r   r    rE   Zfixturer%   r:   appendZ
NC_FORMATSZENGINES_AND_FORMATSmarkZparametrizerP   rU   rc   rd   rn   filterwarningsrr   rt   Zxfail__version__r   r   r   r#   r#   r#   r$   <module>   s   


0




	





