U
    Nvf>                     @   s   d Z ddlmZ ddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlZddlZddlmZmZ ddlmZ ddlmZmZmZ ddlmZ e
d	d
ZG dd deZdd ZG dd dedZG dd deZG dd deeZ dS )z.Storage providers backends for Memory caching.    )PicklingErrorN)ABCMetaabstractmethod   )concurrency_safe_rename)mkdirpmemstr_to_bytes
rm_subdirs)numpy_pickleCacheItemInfozpath size last_accessc                   @   s   e Zd ZdZdS )CacheWarningz:Warning to capture dump failures except for PicklingError.N)__name__
__module____qualname____doc__ r   r   :/tmp/pip-unpacked-wheel-dylwa62s/joblib/_store_backends.pyr      s   r   c                 C   s,   t t }d||t }|| | |S )z>Writes an object into a unique file in a concurrency-safe way.z{}.thread-{}-pid-{})id	threadingcurrent_threadformatosgetpid)Zobject_to_writefilename
write_funcZ	thread_idtemporary_filenamer   r   r   concurrency_safe_write   s      
r   c                   @   sp   e Zd ZdZdZedd Zedd Zedd Zed	d
 Z	edd Z
edd Zede fddZdS )StoreBackendBasezaHelper Abstract Base Class which defines all methods that
       a StorageBackend must implement.Nc                 C   s   dS )a  Opens an item on the store and return a file-like object.

        This method is private and only used by the StoreBackendMixin object.

        Parameters
        ----------
        f: a file-like object
            The file-like object where an item is stored and retrieved
        mode: string, optional
            the mode in which the file-like object is opened allowed valued are
            'rb', 'wb'

        Returns
        -------
        a file-like object
        Nr   )selffmoder   r   r   
_open_item-   s    zStoreBackendBase._open_itemc                 C   s   dS )a  Checks if an item location exists in the store.

        This method is private and only used by the StoreBackendMixin object.

        Parameters
        ----------
        location: string
            The location of an item. On a filesystem, this corresponds to the
            absolute path, including the filename, of a file.

        Returns
        -------
        True if the item exists, False otherwise
        Nr   r   locationr   r   r   _item_exists@   s    zStoreBackendBase._item_existsc                 C   s   dS )a-  Moves an item from src to dst in the store.

        This method is private and only used by the StoreBackendMixin object.

        Parameters
        ----------
        src: string
            The source location of an item
        dst: string
            The destination location of an item
        Nr   )r   srcdstr   r   r   
_move_itemQ   s    zStoreBackendBase._move_itemc                 C   s   dS )zCreates a location on the store.

        Parameters
        ----------
        location: string
            The location in the store. On a filesystem, this corresponds to a
            directory.
        Nr   r"   r   r   r   create_location_   s    z StoreBackendBase.create_locationc                 C   s   dS )zClears a location on the store.

        Parameters
        ----------
        location: string
            The location in the store. On a filesystem, this corresponds to a
            directory or a filename absolute path
        Nr   r"   r   r   r   clear_locationj   s    zStoreBackendBase.clear_locationc                 C   s   dS )zReturns the whole list of items available in the store.

        Returns
        -------
        The list of items identified by their ids (e.g filename in a
        filesystem).
        Nr   r   r   r   r   	get_itemsu   s    zStoreBackendBase.get_itemsr   c                 C   s   dS )a  Configures the store.

        Parameters
        ----------
        location: string
            The base location used by the store. On a filesystem, this
            corresponds to a directory.
        verbose: int
            The level of verbosity of the store
        backend_options: dict
            Contains a dictionary of named parameters used to configure the
            store backend.
        Nr   )r   r#   verbosebackend_optionsr   r   r   	configure   s    zStoreBackendBase.configure)r   r   r   r   r#   r   r!   r$   r'   r(   r)   r+   dictr.   r   r   r   r   r   '   s    







	r   )	metaclassc                   @   s   e Zd ZdZd&ddZd'ddZdd	 Zd
d Zdd Zdd Z	dd Z
dd Zdd Zd(ddZdd Zdd Zdd Zd)ddZd*d d!Zd"d# Zd$d% ZdS )+StoreBackendMixina  Class providing all logic for managing the store in a generic way.

    The StoreBackend subclass has to implement 3 methods: create_location,
    clear_location and configure. The StoreBackend also has to provide
    a private _open_item, _item_exists and _move_item methods. The _open_item
    method has to have the same signature as the builtin open and return a
    file-like object.
    r   Nc           	   	   C   s   t jj| jf| }|dkrD|dk r4td| ntd|| t| dsRdn| j}t j|d}| |s|t	d| |dkr| 
|d	}t|}W 5 Q R X ntj||d
}|S )zKLoad an item from the store given its path as a list of
           strings.r   
   z{0}...z{0} from {1}	mmap_modeN
output.pklzANon-existing item (may have been cleared).
File %s does not existrb)r3   )r   pathjoinr#   printr   hasattrr3   r$   KeyErrorr!   r
   load)	r   r6   r,   msg	full_pathr3   r   r   itemr   r   r   	load_item   s$    
zStoreBackendMixin.load_itemc              
      s   zht jj jf| } |s* | t j|d}|dkrLtd|   fdd} ||| W n6 tk
r } zt	
d| dt W 5 d}~X Y nX dS )	zLDump an item in the store at the path given as a list of
           strings.r4   r2   zPersisting in %sc                    sj     |dT}ztj| | jd W n6 tk
rZ } ztd| dt W 5 d }~X Y nX W 5 Q R X d S )Nwb)compresszjUnable to cache to disk: failed to pickle output. In version 1.5 this will raise an exception. Exception: .)r!   r
   dumprA   r   warningswarnFutureWarning)to_writedest_filenamer   er*   r   r   r      s    
z/StoreBackendMixin.dump_item.<locals>.write_funcz`Unable to cache to disk. Possibly a race condition in the creation of the directory. Exception: rB   N)r   r6   r7   r#   r$   r(   r8   _concurrency_safe_write	ExceptionrD   rE   r   )r   r6   r>   r,   	item_pathr   r   rI   r   r*   r   	dump_item   s    


zStoreBackendMixin.dump_itemc                 C   s,   t jj| jf| }| |r(| | dS )z7Clear the item at the path, given as a list of strings.Nr   r6   r7   r#   r$   r)   )r   r6   rL   r   r   r   
clear_item   s    
zStoreBackendMixin.clear_itemc                 C   s,   t jj| jf| }t j|d}| |S )zLCheck if there is an item at the path, given as a list of
           stringsr4   )r   r6   r7   r#   r$   )r   r6   rL   r   r   r   r   contains_item   s    zStoreBackendMixin.contains_itemc                 C   s   dt jj| jf| iS )zReturn information about item.r#   r   r6   r7   r#   r   r6   r   r   r   get_item_info   s    zStoreBackendMixin.get_item_infoc              
   C   sr   z\t jj| jf| }t j|d}| |d$}t| dW  5 Q R  W S Q R X W n   i  Y S X dS )z"Return actual metadata of an item.metadata.jsonr5   utf-8N)	r   r6   r7   r#   r!   jsonloadsreaddecode)r   r6   rL   r   r   r   r   r   get_metadata   s    ,zStoreBackendMixin.get_metadatac                    s\   zJt jj jf| } | t j|d} fdd} ||| W n   Y nX dS )z Store metadata of a computation.rT   c              	      s2     |d}|t| d W 5 Q R X d S )Nr@   rU   )r!   writerV   dumpsencode)rG   rH   r   r*   r   r   r      s    z4StoreBackendMixin.store_metadata.<locals>.write_funcN)r   r6   r7   r#   r(   rJ   )r   r6   metadatarL   r   r   r   r*   r   store_metadata   s    
z StoreBackendMixin.store_metadatac                 C   s   t jj| jf| }| |S )z,Check cached function is available in store.)r   r6   r7   r#   Zobject_existsr   r6   	func_pathr   r   r   contains_path  s    zStoreBackendMixin.contains_pathc                 C   s,   t jj| jf| }| |r(| | dS )z0Clear all items with a common path in the store.NrN   r`   r   r   r   
clear_path  s    
zStoreBackendMixin.clear_pathc              	   C   sj   t jj| jf| }| |s(| | |dk	rft j|d}| |d}||d W 5 Q R X dS )&Store the code of the cached function.Nfunc_code.pyr@   rU   )	r   r6   r7   r#   r$   r(   r!   r[   r]   )r   r6   	func_codera   r   r   r   r   r   store_cached_func_code  s    

z(StoreBackendMixin.store_cached_func_codec              
   C   sf   |dg7 }t jj| jf| }z4| |d}| dW  5 Q R  W S Q R X W n    Y nX dS )rd   re   r5   rU   N)r   r6   r7   r#   r!   rX   rY   )r   r6   r   r   r   r   r   get_cached_func_code  s    
&z&StoreBackendMixin.get_cached_func_codec                 C   s   dt jj| jf| iS )z?Return information related to the cached function if it exists.r#   rQ   rR   r   r   r   get_cached_func_info!  s    z&StoreBackendMixin.get_cached_func_infoc                 C   s   |  | j dS )zClear the whole store content.N)r)   r#   r*   r   r   r   clear%  s    zStoreBackendMixin.clearc              	   C   sZ   |  |||}|D ]B}| jdkr.td| z| |j W q tk
rR   Y qX qdS )zX
        Remove the store's oldest files to enforce item, byte, and age limits.
        r2   zDeleting item {0}N)_get_items_to_deleter,   r8   r   r)   r6   OSError)r   bytes_limititems_limit	age_limititems_to_deleter>   r   r   r   enforce_store_limits)  s      
z&StoreBackendMixin.enforce_store_limitsc                 C   s  t |trt|}|  }tdd |D }|dk	r>|| }nd}|dk	rXt|| }nd}|dk	rtdd |D }tj | }	nd}	|dkr|dkr|	dks||	krg S |j	t
dd g }
d}d}|D ]H}||kr||kr|	dks|	|jk r q|
| ||j7 }|d7 }q|
S )	zW
        Get items to delete to keep the store under size, file, & age limits.
        c                 s   s   | ]}|j V  qd S N)size.0r>   r   r   r   	<genexpr>I  s     z9StoreBackendMixin._get_items_to_delete.<locals>.<genexpr>Nr   c                 s   s   | ]}|j V  qd S rr   )last_accessrt   r   r   r   rv   V  s     rw   )keyr   )
isinstancestrr   r+   sumlenmindatetimenowsortoperator
attrgetterrw   appendrs   )r   rm   rn   ro   itemsrs   Zto_delete_sizeZto_delete_itemsZ
older_itemdeadlinerp   Zsize_so_farZitems_so_farr>   r   r   r   rk   ?  sR    




z&StoreBackendMixin._get_items_to_deletec                 C   s   t |||}| || dS )z7Writes an object into a file in a concurrency-safe way.N)r   r'   )r   rG   r   r   r   r   r   r   rJ   w  s
     z)StoreBackendMixin._concurrency_safe_writec                 C   s   dj | jj| jdS )z/Printable representation of the store location.z#{class_name}(location="{location}"))
class_namer#   )r   	__class__r   r#   r*   r   r   r   __repr__}  s     zStoreBackendMixin.__repr__)r   N)r   )N)NN)NN)r   r   r   r   r?   rM   rO   rP   rS   rZ   r_   rb   rc   rg   rh   ri   rj   rq   rk   rJ   r   r   r   r   r   r1      s,   	

 


   
   
8r1   c                   @   sN   e Zd ZdZeeZeejj	Z
eeZdd Zdd Zdd Zdd
dZd	S )FileSystemStoreBackendz7A StoreBackend used with local or network file systems.c                 C   s&   || j krt| ntj|dd dS )zDelete location on store.T)ignore_errorsN)r#   r	   shutilrmtreer"   r   r   r   r)     s    

z%FileSystemStoreBackend.clear_locationc                 C   s   t | dS )zCreate object location on storeN)r   r"   r   r   r   r(     s    z&FileSystemStoreBackend.create_locationc           	         s   g }t | jD ]\ }}tdt j }|rt j d}zt j|}W n@ t	k
r   zt j }W n t	k
r   Y Y qY nX Y nX t
j
|}z( fdd|D }tdd |D }W n t	k
r   Y qY nX |t || q|S )z7Returns the whole list of items available in the store.z[a-f0-9]{32}r4   c                    s   g | ]}t j |qS r   )r   r6   r7   ru   fndirpathr   r   
<listcomp>  s   z4FileSystemStoreBackend.get_items.<locals>.<listcomp>c                 s   s   | ]}t j|V  qd S rr   )r   r6   getsizer   r   r   r   rv     s   z3FileSystemStoreBackend.get_items.<locals>.<genexpr>)r   walkr#   rematchr6   basenamer7   getatimerl   r~   fromtimestampr{   r   r   )	r   r   _	filenamesZis_cache_hash_dirZoutput_filenamerw   Zfull_filenamesdirsizer   r   r   r+     s8    



z FileSystemStoreBackend.get_itemsr   Nc                 C   sn   |dkri }|| _ tj| j s*t| j  |dd| _|d}| jr^|dk	r^tjddd || _	|| _
dS )zsConfigure the store backend.

        For this backend, valid store options are 'compress' and 'mmap_mode'
        NrA   Fr3   zSCompressed items cannot be memmapped in a filesystem store. Option will be ignored.   )
stacklevel)r#   r   r6   existsr   getrA   rD   rE   r3   r,   )r   r#   r,   r-   r3   r   r   r   r.     s    

z FileSystemStoreBackend.configure)r   N)r   r   r   r   staticmethodopenr!   r   r6   r   r$   r   r'   r)   r(   r+   r.   r   r   r   r   r     s   $r   )!r   pickler   r   r   os.pathr~   rV   r   rD   collectionsr   r   abcr   r   Z	backportsr   Zdiskr   r   r	    r
   
namedtupler   Warningr   r   r   objectr1   r   r   r   r   r   <module>   s0   
i t