B
    ̵a                @  s  d Z ddlmZ ddlmZ ddlZddlmZmZ ddl	Z	ddl
Z
ddlZddlmZ ddlmZmZmZmZmZmZ ddlZddlZddlmZmZ dd	lmZmZ dd
lm Z  ddl!m"Z"m#Z#m$Z$m%Z%m&Z& ddl'm(Z( ddl)m*Z* ddl+m,Z, ddl-m.Z. ddl/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9 ddl:m;Z; ddl<m=Z=m>Z>m?Z?m@Z@mAZAmBZBmCZCmDZDmEZEmFZF ddlGmHZHmIZImJZJ ddlKmL  mMZN ddlOmPZPmQZQ ddlRmSZS ddlTmUZU ddlVmWZWmXZX ddlYmZZZ ddl[m\Z\m]Z] erddl^m_Z_m`Z`maZa ddlVmbZb dZcdZddd Zed d! Zfd"d# ZgePZhd$d%d&d'ZiG d(d) d)ejZkG d*d+ d+ejZlG d,d- d-emZnd.ZoG d/d0 d0emZpd1ZqG d2d3 d3emZrd4Zsd5Ztd6d6d7d7d8Zue=dgiZvd9Zwd:Zxeyd;8 ejzd<d=ewej{d> ejzd?dexe|d6d7dgd> W dQ R X da}d=a~d@dA ZddEdFdEdGdHdIdHdIdJdKdLdEdEdMdNdOZddEdEdGdGdGdQdRdSZdTdTdIdUdVdWZG dXdY dYZG dZd[ d[ZG d\d] d]ZG d^d_ d_eZG d`da daeZG dbdc dceZG ddde deeZG dfdg dgZG dhdi dieZG djdk dkeZG dldm dmeZG dndo doeZG dpdq dqeZG drds dseZG dtdu dueZG dvdw dweZG dxdy dyeZG dzd{ d{eZG d|d} d}eZG d~d deZddd$dddddZdddddZddddIddddZdEddEdEd]dddZdEdEdEddddZdEdddddZddEdEddddZddEdEddddZddEdEdEdddZdEdEdEdddZdEdIdddZdEddEdddZdEdEdddZddddZG dd dZdS )zY
High level interface to PyTables for reading and writing pandas data structures
to disk
    )annotations)suppressN)datetzinfo)dedent)TYPE_CHECKINGAnyCallableHashableSequencecast)config
get_option)libwriters)	timezones)	ArrayLikeDtypeArgFrameOrSeriesFrameOrSeriesUnionShape)import_optional_dependency)patch_pickle)PerformanceWarning)cache_readonly)
ensure_objectis_categorical_dtypeis_complex_dtypeis_datetime64_dtypeis_datetime64tz_dtypeis_extension_array_dtypeis_list_likeis_string_dtypeis_timedelta64_dtypeneeds_i8_conversion)array_equivalent)
	DataFrameDatetimeIndexIndex
Int64Index
MultiIndexPeriodIndexSeriesTimedeltaIndexconcatisna)CategoricalDatetimeArrayPeriodArray)PyTablesExprmaybe_expression)extract_array)ensure_index)ArrayManagerBlockManager)stringify_path)adjoinpprint_thing)ColFileNode)Blockz0.15.2UTF-8c             C  s   t | tjr| d} | S )z(if we have bytes, decode them to unicodezUTF-8)
isinstancenpZbytes_decode)s rE   1lib/python3.7/site-packages/pandas/io/pytables.py_ensure_decodedu   s    
rG   c             C  s   | d krt } | S )N)_default_encoding)encodingrE   rE   rF   _ensure_encoding|   s    rJ   c             C  s   t | trt| } | S )z
    Ensure that an index / column name is a str (python 3); otherwise they
    may be np.string dtype. Non-string dtypes are passed through unchanged.

    https://github.com/pandas-dev/pandas/issues/13492
    )rA   str)namerE   rE   rF   _ensure_str   s    
rM   int)scope_levelc               sV   |d  t | ttfr* fdd| D } nt| r>t|  d} | dksNt| rR| S dS )z
    Ensure that the where is a Term or a list of Term.

    This makes sure that we are capturing the scope of variables that are
    passed create the terms here with a frame_level=2 (we are 2 levels down)
       c               s0   g | ](}|d k	rt |r(t| d dn|qS )NrP   )rO   )r4   Term).0Zterm)levelrE   rF   
<listcomp>   s   z _ensure_term.<locals>.<listcomp>)rO   N)rA   listtupler4   rQ   len)whererO   rE   )rS   rF   _ensure_term   s    	

rY   c               @  s   e Zd ZdS )PossibleDataLossErrorN)__name__
__module____qualname__rE   rE   rE   rF   rZ      s   rZ   c               @  s   e Zd ZdS )ClosedFileErrorN)r[   r\   r]   rE   rE   rE   rF   r^      s   r^   c               @  s   e Zd ZdS )IncompatibilityWarningN)r[   r\   r]   rE   rE   rE   rF   r_      s   r_   z
where criteria is being ignored as this version [%s] is too old (or
not-defined), read the file in and write it out to a new file to upgrade (with
the copy_to method)
c               @  s   e Zd ZdS )AttributeConflictWarningN)r[   r\   r]   rE   rE   rE   rF   r`      s   r`   zu
the [%s] attribute of the existing index is [%s] which conflicts with the new
[%s], resetting the attribute to None
c               @  s   e Zd ZdS )DuplicateWarningN)r[   r\   r]   rE   rE   rE   rF   ra      s   ra   z;
duplicate entries in table, taking most recently appended
z
your performance may suffer as PyTables will pickle object types that it cannot
map directly to c-types [inferred_type->%s,key->%s] [items->%s]
fixedtable)frb   trc   z;
: boolean
    drop ALL nan rows when appending to a table
z~
: format
    default format writing format, if None, then
    put will default to 'fixed' and append will default to 'table'
zio.hdfZdropna_tableF)Z	validatorZdefault_formatc           	   C  s8   t d kr4dd l} | a tt | jjdkaW d Q R X t S )Nr   strict)
_table_modtablesr   AttributeErrorfileZ_FILE_OPEN_POLICY!_table_file_open_policy_is_strict)rh   rE   rE   rF   _tables   s    
rl   aTrf   rK   r   z
int | Nonez
str | Noneboolzint | dict[str, int] | Nonezbool | Nonezbool | list[str] | None)keyvaluemode	complevelcomplibappendformatindexmin_itemsizedropnadata_columnserrorsrI   c          
     s   |r$ 	f
dd}n 	f
dd}t | } t| trzt| |||d}|| W dQ R X n||  dS )z+store this object, close it if we opened itc               s   | j 	 d
S )N)ru   rv   rw   nan_reprx   ry   rz   rI   )rt   )store)
ry   rx   rI   rz   ru   rv   ro   rw   r{   rp   rE   rF   <lambda>  s   zto_hdf.<locals>.<lambda>c               s   | j 	 d
S )N)ru   rv   rw   r{   ry   rz   rI   rx   )put)r|   )
ry   rx   rI   rz   ru   rv   ro   rw   r{   rp   rE   rF   r}   (  s   )rq   rr   rs   N)r9   rA   rK   HDFStore)path_or_bufro   rp   rq   rr   rs   rt   ru   rv   rw   r{   rx   ry   rz   rI   rd   r|   rE   )
ry   rx   rI   rz   ru   rv   ro   rw   r{   rp   rF   to_hdf  s     
r   r)rq   rz   startstop	chunksizec
             K  s  |dkrt d| d|dk	r,t|dd}t| trN| jsDtd| }d}nvt| } t| tshtd	yt	j
| }W n tt fk
r   d}Y nX |std
|  dt| f||d|
}d}yz|dkr&| }t|dkrt d|d }x*|dd D ]}t||st dqW |j}|j|||||||	|dS  t ttfk
r   t| ts~tt |  W dQ R X  Y nX dS )a  
    Read from the store, close it if we opened it.

    Retrieve pandas object stored in file, optionally based on where
    criteria.

    .. warning::

       Pandas uses PyTables for reading and writing HDF5 files, which allows
       serializing object-dtype data with pickle when using the "fixed" format.
       Loading pickled data received from untrusted sources can be unsafe.

       See: https://docs.python.org/3/library/pickle.html for more.

    Parameters
    ----------
    path_or_buf : str, path object, pandas.HDFStore
        Any valid string path is acceptable. Only supports the local file system,
        remote URLs and file-like objects are not supported.

        If you want to pass in a path object, pandas accepts any
        ``os.PathLike``.

        Alternatively, pandas accepts an open :class:`pandas.HDFStore` object.

    key : object, optional
        The group identifier in the store. Can be omitted if the HDF file
        contains a single pandas object.
    mode : {'r', 'r+', 'a'}, default 'r'
        Mode to use when opening the file. Ignored if path_or_buf is a
        :class:`pandas.HDFStore`. Default is 'r'.
    errors : str, default 'strict'
        Specifies how encoding and decoding errors are to be handled.
        See the errors argument for :func:`open` for a full list
        of options.
    where : list, optional
        A list of Term (or convertible) objects.
    start : int, optional
        Row number to start selection.
    stop  : int, optional
        Row number to stop selection.
    columns : list, optional
        A list of columns names to return.
    iterator : bool, optional
        Return an iterator object.
    chunksize : int, optional
        Number of rows to include in an iteration when using an iterator.
    **kwargs
        Additional keyword arguments passed to HDFStore.

    Returns
    -------
    item : object
        The selected object. Return type depends on the object stored.

    See Also
    --------
    DataFrame.to_hdf : Write a HDF file from a DataFrame.
    HDFStore : Low-level access to HDF files.

    Examples
    --------
    >>> df = pd.DataFrame([[1, 1.0, 'a']], columns=['x', 'y', 'z'])
    >>> df.to_hdf('./store.h5', 'data')
    >>> reread = pd.read_hdf('./store.h5')
    )r   zr+rm   zmode zG is not allowed while performing a read. Allowed modes are r, r+ and a.NrP   )rO   z&The HDFStore must be open for reading.Fz5Support for generic buffers has not been implemented.zFile z does not exist)rq   rz   Tr   z]Dataset(s) incompatible with Pandas data types, not table, or no datasets found in HDF5 file.z?key must be provided when HDF5 file contains multiple datasets.)rX   r   r   columnsiteratorr   
auto_close)
ValueErrorrY   rA   r   is_openOSErrorr9   rK   NotImplementedErrorospathexists	TypeErrorFileNotFoundErrorgroupsrW   _is_metadata_of_v_pathnameselectKeyErrorr   ri   close)r   ro   rq   rz   rX   r   r   r   r   r   kwargsr|   r   r   r   Zcandidate_only_groupZgroup_to_checkrE   rE   rF   read_hdf?  s`    O




r   r>   )groupparent_groupreturnc             C  sJ   | j |j krdS | }x0|j dkrD|j}||kr<|jdkr<dS |j}qW dS )zDCheck if a given group is a metadata group for a given parent_group.FrP   metaT)Z_v_depthZ	_v_parent_v_name)r   r   currentparentrE   rE   rF   r     s    
r   c               @  s  e Zd ZU dZded< ded< ded< ded	< dddddddZdd Zedd Zedd Z	ddddZ
ddddZddddZdddd Zddd!d"d#Zdd$d%d&Zdd$d'd(Zd)d* Zd+d, Zddd.d/d0d1Zd2d3 Zd4d5 ZeZddd6d7d8Zd9d: Zedd$d;d<Zddd=d>d?Zddd@dAZddddBdCdDZdddddEdFdGZddddddHdIdJZdddKdLdMZdddPddQdRddddSdTdUZ ddddVdWZ!dddPddQdXdRddYdZd[Z"dd\d]d^d_Z#dddd`dadbdcZ$ddde Z%ddgdhZ&ddid!djdkZ'ddld!dmdnZ(dddddpdqdrZ)dd$dsdtZ*dudv Z+dddwdxdyZ,dd{dddld|d}d~Z-dddPddQdddddZ.ddddZ/ddddddZ0ddd!ddZ1dS )r   aa	  
    Dict-like IO interface for storing pandas objects in PyTables.

    Either Fixed or Table format.

    .. warning::

       Pandas uses PyTables for reading and writing HDF5 files, which allows
       serializing object-dtype data with pickle when using the "fixed" format.
       Loading pickled data received from untrusted sources can be unsafe.

       See: https://docs.python.org/3/library/pickle.html for more.

    Parameters
    ----------
    path : str
        File path to HDF5 file.
    mode : {'a', 'w', 'r', 'r+'}, default 'a'

        ``'r'``
            Read-only; no data can be modified.
        ``'w'``
            Write; a new file is created (an existing file with the same
            name would be deleted).
        ``'a'``
            Append; an existing file is opened for reading and writing,
            and if the file does not exist it is created.
        ``'r+'``
            It is similar to ``'a'``, but the file must already exist.
    complevel : int, 0-9, default None
        Specifies a compression level for data.
        A value of 0 or None disables compression.
    complib : {'zlib', 'lzo', 'bzip2', 'blosc'}, default 'zlib'
        Specifies the compression library to be used.
        As of v0.20.2 these additional compressors for Blosc are supported
        (default if no compressor specified: 'blosc:blosclz'):
        {'blosc:blosclz', 'blosc:lz4', 'blosc:lz4hc', 'blosc:snappy',
         'blosc:zlib', 'blosc:zstd'}.
        Specifying a compression library which is not available issues
        a ValueError.
    fletcher32 : bool, default False
        If applying compression use the fletcher32 checksum.
    **kwargs
        These parameters will be passed to the PyTables open_file method.

    Examples
    --------
    >>> bar = pd.DataFrame(np.random.randn(10, 4))
    >>> store = pd.HDFStore('test.h5')
    >>> store['foo'] = bar   # write to HDF5
    >>> bar = store['foo']   # retrieve
    >>> store.close()

    **Create or load HDF5 file in-memory**

    When passing the `driver` option to the PyTables open_file method through
    **kwargs, the HDF5 file is loaded or created in-memory and will only be
    written when closed:

    >>> bar = pd.DataFrame(np.random.randn(10, 4))
    >>> store = pd.HDFStore('test.h5', driver='H5FD_CORE')
    >>> store['foo'] = bar
    >>> store.close()   # only now, data is written to disk
    zFile | None_handlerK   _moderN   
_complevelrn   _fletcher32rm   NFz
int | None)rq   rr   
fletcher32c             K  s   d|krt dtd}|d k	r@||jjkr@t d|jj d|d krX|d k	rX|jj}t|| _|d krnd}|| _d | _|r|nd| _	|| _
|| _d | _| jf d|i| d S )	Nru   z-format is not a defined argument for HDFStorerh   zcomplib only supports z compression.rm   r   rq   )r   r   filtersZall_complibsZdefault_complibr9   _pathr   r   r   _complibr   _filtersopen)selfr   rq   rr   rs   r   r   rh   rE   rE   rF   __init__/  s$    

zHDFStore.__init__c             C  s   | j S )N)r   )r   rE   rE   rF   
__fspath__Q  s    zHDFStore.__fspath__c             C  s   |    | jdk	st| jjS )zreturn the root nodeN)_check_if_openr   AssertionErrorroot)r   rE   rE   rF   r   T  s    zHDFStore.rootc             C  s   | j S )N)r   )r   rE   rE   rF   filename[  s    zHDFStore.filename)ro   c             C  s
   |  |S )N)get)r   ro   rE   rE   rF   __getitem___  s    zHDFStore.__getitem__c             C  s   |  || d S )N)r~   )r   ro   rp   rE   rE   rF   __setitem__b  s    zHDFStore.__setitem__c             C  s
   |  |S )N)remove)r   ro   rE   rE   rF   __delitem__e  s    zHDFStore.__delitem__)rL   c          	   C  sD   y
|  |S  ttfk
r"   Y nX tdt| j d| ddS )z$allow attribute access to get stores'z' object has no attribute 'N)r   r   r^   ri   typer[   )r   rL   rE   rE   rF   __getattr__h  s    
zHDFStore.__getattr__)ro   r   c             C  s8   |  |}|dk	r4|j}||ks0|dd |kr4dS dS )zx
        check for existence of this key
        can match the exact pathname or the pathnm w/o the leading '/'
        NrP   TF)get_noder   )r   ro   noderL   rE   rE   rF   __contains__r  s    
zHDFStore.__contains__)r   c             C  s   t |  S )N)rW   r   )r   rE   rE   rF   __len__~  s    zHDFStore.__len__c             C  s   t | j}t|  d| dS )Nz
File path: 
)r;   r   r   )r   ZpstrrE   rE   rF   __repr__  s    
zHDFStore.__repr__c             C  s   | S )NrE   )r   rE   rE   rF   	__enter__  s    zHDFStore.__enter__c             C  s   |    d S )N)r   )r   exc_type	exc_value	tracebackrE   rE   rF   __exit__  s    zHDFStore.__exit__pandasz	list[str])includer   c             C  s^   |dkrdd |   D S |dkrJ| jdk	s0tdd | jjddd	D S td
| ddS )a#  
        Return a list of keys corresponding to objects stored in HDFStore.

        Parameters
        ----------

        include : str, default 'pandas'
                When kind equals 'pandas' return pandas objects.
                When kind equals 'native' return native HDF5 Table objects.

                .. versionadded:: 1.1.0

        Returns
        -------
        list
            List of ABSOLUTE path-names (e.g. have the leading '/').

        Raises
        ------
        raises ValueError if kind has an illegal value
        r   c             S  s   g | ]
}|j qS rE   )r   )rR   nrE   rE   rF   rT     s    z!HDFStore.keys.<locals>.<listcomp>ZnativeNc             S  s   g | ]
}|j qS rE   )r   )rR   r   rE   rE   rF   rT     s    /Table)Z	classnamez8`include` should be either 'pandas' or 'native' but is 'r   )r   r   r   Z
walk_nodesr   )r   r   rE   rE   rF   keys  s    zHDFStore.keysc             C  s   t |  S )N)iterr   )r   rE   rE   rF   __iter__  s    zHDFStore.__iter__c             c  s"   x|   D ]}|j|fV  q
W dS )z'
        iterate on key->group
        N)r   r   )r   grE   rE   rF   items  s    zHDFStore.items)rq   c             K  s   t  }| j|krR| jdkr$|dkr$n(|dkrL| jrLtd| j d| j d|| _| jr`|   | jr| jdkrt  j| j| j| j	d| _
tr| jrd	}t||j| j| jf|| _d
S )a9  
        Open the file in the specified mode

        Parameters
        ----------
        mode : {'a', 'w', 'r', 'r+'}, default 'a'
            See HDFStore docstring or tables.open_file for info about modes
        **kwargs
            These parameters will be passed to the PyTables open_file method.
        )rm   w)r   zr+)r   zRe-opening the file [z] with mode [z] will delete the current file!r   )r   zGCannot open HDF5 file, which is already opened, even in read-only mode.N)rl   r   r   rZ   r   r   r   Filtersr   r   r   rk   r   Z	open_filer   )r   rq   r   rh   msgrE   rE   rF   r     s$    

zHDFStore.openc             C  s   | j dk	r| j   d| _ dS )z0
        Close the PyTables file handle
        N)r   r   )r   rE   rE   rF   r     s    

zHDFStore.closec             C  s   | j dkrdS t| j jS )zF
        return a boolean indicating whether the file is open
        NF)r   rn   Zisopen)r   rE   rE   rF   r     s    
zHDFStore.is_open)fsyncc          	   C  s@   | j dk	r<| j   |r<tt t| j   W dQ R X dS )a  
        Force all buffered modifications to be written to disk.

        Parameters
        ----------
        fsync : bool (default False)
          call ``os.fsync()`` on the file handle to force writing to disk.

        Notes
        -----
        Without ``fsync=True``, flushing may not guarantee that the OS writes
        to disk. With fsync, the operation will block until the OS claims the
        file has been written; however, other caching layers may still
        interfere.
        N)r   flushr   r   r   r   fileno)r   r   rE   rE   rF   r     s
    


zHDFStore.flushc          	   C  s>   t  . | |}|dkr*td| d| |S Q R X dS )z
        Retrieve pandas object stored in file.

        Parameters
        ----------
        key : str

        Returns
        -------
        object
            Same type as object stored in file.
        NzNo object named z in the file)r   r   r   _read_group)r   ro   r   rE   rE   rF   r     s
    
zHDFStore.get)ro   r   c	               st   |  |}	|	dkr"td| dt|dd}| |	   fdd}
t| |
|j|||||d
}| S )	a  
        Retrieve pandas object stored in file, optionally based on where criteria.

        .. warning::

           Pandas uses PyTables for reading and writing HDF5 files, which allows
           serializing object-dtype data with pickle when using the "fixed" format.
           Loading pickled data received from untrusted sources can be unsafe.

           See: https://docs.python.org/3/library/pickle.html for more.

        Parameters
        ----------
        key : str
            Object being retrieved from file.
        where : list or None
            List of Term (or convertible) objects, optional.
        start : int or None
            Row number to start selection.
        stop : int, default None
            Row number to stop selection.
        columns : list or None
            A list of columns that if not None, will limit the return columns.
        iterator : bool or False
            Returns an iterator.
        chunksize : int or None
            Number or rows to include in iteration, return an iterator.
        auto_close : bool or False
            Should automatically close the store when finished.

        Returns
        -------
        object
            Retrieved object from file.
        NzNo object named z in the filerP   )rO   c               s   j | || dS )N)r   r   rX   r   )read)_start_stop_where)r   rD   rE   rF   funcZ  s    zHDFStore.select.<locals>.func)rX   nrowsr   r   r   r   r   )r   r   rY   _create_storer
infer_axesTableIteratorr   
get_result)r   ro   rX   r   r   r   r   r   r   r   r   itrE   )r   rD   rF   r   "  s&    .

zHDFStore.select)ro   r   r   c             C  s8   t |dd}| |}t|ts(td|j|||dS )a  
        return the selection as an Index

        .. warning::

           Pandas uses PyTables for reading and writing HDF5 files, which allows
           serializing object-dtype data with pickle when using the "fixed" format.
           Loading pickled data received from untrusted sources can be unsafe.

           See: https://docs.python.org/3/library/pickle.html for more.


        Parameters
        ----------
        key : str
        where : list of Term (or convertible) objects, optional
        start : integer (defaults to None), row number to start selection
        stop  : integer (defaults to None), row number to stop selection
        rP   )rO   z&can only read_coordinates with a table)rX   r   r   )rY   
get_storerrA   r   r   read_coordinates)r   ro   rX   r   r   tblrE   rE   rF   select_as_coordinatesm  s
    

zHDFStore.select_as_coordinates)ro   columnr   r   c             C  s,   |  |}t|tstd|j|||dS )a~  
        return a single column from the table. This is generally only useful to
        select an indexable

        .. warning::

           Pandas uses PyTables for reading and writing HDF5 files, which allows
           serializing object-dtype data with pickle when using the "fixed" format.
           Loading pickled data received from untrusted sources can be unsafe.

           See: https://docs.python.org/3/library/pickle.html for more.

        Parameters
        ----------
        key : str
        column : str
            The column of interest.
        start : int or None, default None
        stop : int or None, default None

        Raises
        ------
        raises KeyError if the column is not found (or key is not a valid
            store)
        raises ValueError if the column can not be extracted individually (it
            is part of a data block)

        z!can only read_column with a table)r   r   r   )r   rA   r   r   read_column)r   ro   r   r   r   r   rE   rE   rF   select_column  s    #

zHDFStore.select_column)r   c
               s~  t |dd}t|ttfr.t|dkr.|d }t|trRj|||||||	dS t|ttfshtdt|sxtd|dkr|d }fdd	|D 	|}
d}xxt
|
|fgt|D ]\\}}|dkrtd
| d|jstd|j d|dkr|j}q|j|krtdqW dd	 D }tdd |D d   fdd}t|
||||||||	d
}|jddS )a  
        Retrieve pandas objects from multiple tables.

        .. warning::

           Pandas uses PyTables for reading and writing HDF5 files, which allows
           serializing object-dtype data with pickle when using the "fixed" format.
           Loading pickled data received from untrusted sources can be unsafe.

           See: https://docs.python.org/3/library/pickle.html for more.

        Parameters
        ----------
        keys : a list of the tables
        selector : the table to apply the where criteria (defaults to keys[0]
            if not supplied)
        columns : the columns I want back
        start : integer (defaults to None), row number to start selection
        stop  : integer (defaults to None), row number to stop selection
        iterator : bool, return an iterator, default False
        chunksize : nrows to include in iteration, return an iterator
        auto_close : bool, default False
            Should automatically close the store when finished.

        Raises
        ------
        raises KeyError if keys or selector is not found or keys is empty
        raises TypeError if keys is not a list or tuple
        raises ValueError if the tables are not ALL THE SAME DIMENSIONS
        rP   )rO   r   )ro   rX   r   r   r   r   r   r   zkeys must be a list/tuplez keys must have a non-zero lengthNc               s   g | ]}  |qS rE   )r   )rR   k)r   rE   rF   rT     s    z/HDFStore.select_as_multiple.<locals>.<listcomp>zInvalid table []zobject [z>] is not a table, and cannot be used in all select as multiplez,all tables must have exactly the same nrows!c             S  s   g | ]}t |tr|qS rE   )rA   r   )rR   xrE   rE   rF   rT     s    c             S  s   h | ]}|j d  d  qS )r   )non_index_axes)rR   re   rE   rE   rF   	<setcomp>  s    z.HDFStore.select_as_multiple.<locals>.<setcomp>c               s*    fddD }t |dd S )Nc               s   g | ]}|j  d qS ))rX   r   r   r   )r   )rR   re   )r   r   r   r   rE   rF   rT     s   z=HDFStore.select_as_multiple.<locals>.func.<locals>.<listcomp>F)axisverify_integrity)r.   _consolidate)r   r   r   Zobjs)r   r   tbls)r   r   r   rF   r     s    z)HDFStore.select_as_multiple.<locals>.func)rX   r   r   r   r   r   r   T)coordinates)rY   rA   rU   rV   rW   rK   r   r   r   r   	itertoolschainzipr   is_tablepathnamer   r   r   )r   r   rX   selectorr   r   r   r   r   r   rD   r   re   r   Z_tblsr   r   rE   )r   r   r   r   rF   select_as_multiple  s^    +

"

zHDFStore.select_as_multipleTrf   r   zint | dict[str, int] | Nonezlist[str] | None)ro   rp   rr   rw   ry   rz   track_timesrx   c             C  sH   |dkrt dpd}| |}| j|||||||||	|
||||d dS )a<  
        Store object in HDFStore.

        Parameters
        ----------
        key : str
        value : {Series, DataFrame}
        format : 'fixed(f)|table(t)', default is 'fixed'
            Format to use when storing object in HDFStore. Value can be one of:

            ``'fixed'``
                Fixed format.  Fast writing/reading. Not-appendable, nor searchable.
            ``'table'``
                Table format.  Write as a PyTables Table structure which may perform
                worse but allow more flexible operations like searching / selecting
                subsets of the data.
        append : bool, default False
            This will force Table format, append the input data to the existing.
        data_columns : list, default None
            List of columns to create as data columns, or True to use all columns.
            See `here
            <https://pandas.pydata.org/pandas-docs/stable/user_guide/io.html#query-via-data-columns>`__.
        encoding : str, default None
            Provide an encoding for strings.
        track_times : bool, default True
            Parameter is propagated to 'create_table' method of 'PyTables'.
            If set to False it enables to have the same h5 files (same hashes)
            independent on creation time.

            .. versionadded:: 1.1.0
        Nzio.hdf.default_formatrb   )ru   rv   rt   rs   rr   rw   r{   ry   rI   rz   r   rx   )r   _validate_format_write_to_group)r   ro   rp   ru   rv   rt   rs   rr   rw   r{   ry   rI   rz   r   rx   rE   rE   rF   r~   /  s$    0
zHDFStore.putc          
   C  s   t |dd}y| |}W n~ tk
r0    Y nj tk
rD    Y nV tk
r } z8|dk	rftd|| |}|dk	r|jdd dS W dd}~X Y nX t	|||r|j
jdd n|jstd|j|||dS dS )	a:  
        Remove pandas object partially by specifying the where condition

        Parameters
        ----------
        key : str
            Node to remove or delete rows from
        where : list of Term (or convertible) objects, optional
        start : integer (defaults to None), row number to start selection
        stop  : integer (defaults to None), row number to stop selection

        Returns
        -------
        number of rows removed (or None if not a Table)

        Raises
        ------
        raises KeyError if key is not a valid store

        rP   )rO   Nz5trying to remove a node with a non-None where clause!T)	recursivez7can only remove with where on objects written as tables)rX   r   r   )rY   r   r   r   	Exceptionr   r   Z	_f_removecomall_noner   r   delete)r   ro   rX   r   r   rD   errr   rE   rE   rF   r   s  s,    
zHDFStore.removezbool | None)ro   rp   rr   rw   rx   ry   rz   c             C  sl   |	dk	rt d|dkr td}|dkr4tdp2d}| |}| j|||||||||
|||||||d dS )a6  
        Append to Table in file. Node must already exist and be Table
        format.

        Parameters
        ----------
        key : str
        value : {Series, DataFrame}
        format : 'table' is the default
            Format to use when storing object in HDFStore.  Value can be one of:

            ``'table'``
                Table format. Write as a PyTables Table structure which may perform
                worse but allow more flexible operations like searching / selecting
                subsets of the data.
        append       : bool, default True
            Append the input data to the existing.
        data_columns : list of columns, or True, default None
            List of columns to create as indexed data columns for on-disk
            queries, or True to use all columns. By default only the axes
            of the object are indexed. See `here
            <https://pandas.pydata.org/pandas-docs/stable/user_guide/io.html#query-via-data-columns>`__.
        min_itemsize : dict of columns that specify minimum str sizes
        nan_rep      : str to use as str nan representation
        chunksize    : size to chunk the writing
        expectedrows : expected TOTAL row size of this table
        encoding     : default None, provide an encoding for str
        dropna : bool, default False
            Do not write an ALL nan row to the store settable
            by the option 'io.hdf.dropna_table'.

        Notes
        -----
        Does *not* check if data being appended overlaps with existing
        data in the table, so be careful
        Nz>columns is not a supported keyword in append, try data_columnszio.hdf.dropna_tablezio.hdf.default_formatrc   )ru   axesrv   rt   rs   rr   rw   r{   r   expectedrowsrx   ry   rI   rz   )r   r   r   r   )r   ro   rp   ru   r  rv   rt   rs   rr   r   rw   r{   r   r  rx   ry   rI   rz   rE   rE   rF   rt     s2    8
zHDFStore.appenddict)dc               s  |dk	rt dt|ts"td||kr2tdtttjttt	  d }d}	g }
x<|
 D ]0\}  dkr|	dk	rtd|}	qj|
  qjW |	dk	rڈj| }|t|
}t||}||||	< |dkr|| }|r2fdd| D }t|}x|D ]}||}qW j| |d	d}xt|
 D ]h\} ||kr^|nd}j |d
}|dk	r fdd|
 D nd}| j||f||d| qHW dS )a  
        Append to multiple tables

        Parameters
        ----------
        d : a dict of table_name to table_columns, None is acceptable as the
            values of one node (this will get all the remaining columns)
        value : a pandas object
        selector : a string that designates the indexable table; all of its
            columns will be designed as data_columns, unless data_columns is
            passed, in which case these are used
        data_columns : list of columns to create as data columns, or True to
            use all columns
        dropna : if evaluates to True, drop rows from all tables if any single
                 row in each table has all NaN. Default False.

        Notes
        -----
        axes parameter is currently not accepted

        Nztaxes is currently not accepted as a parameter to append_to_multiple; you can create the tables independently insteadzQappend_to_multiple must have a dictionary specified as the way to split the valuez=append_to_multiple requires a selector that is in passed dictr   z<append_to_multiple can only have one value in d that is Nonec             3  s    | ]} | j d djV  qdS )all)ZhowN)rx   rv   )rR   cols)rp   rE   rF   	<genexpr>L  s    z.HDFStore.append_to_multiple.<locals>.<genexpr>rw   )r   c               s   i | ]\}}| kr||qS rE   rE   )rR   ro   rp   )vrE   rF   
<dictcomp>\  s    z/HDFStore.append_to_multiple.<locals>.<dictcomp>)ry   rw   )r   rA   r  r   rU   setrangendim	_AXES_MAPr   r   extendr  
differencer(   sortedget_indexertakevaluesnextintersectionlocpopreindexrt   )r   r  rp   r   ry   r  rx   r   r   Z
remain_keyZremain_valuesr   orderedZorddZidxsZvalid_indexrv   rw   dcvalZfilteredrE   )r	  rp   rF   append_to_multiple  sN    
&


 zHDFStore.append_to_multiplez
str | None)ro   optlevelkindc             C  sB   t   | |}|dkrdS t|ts.td|j|||d dS )a  
        Create a pytables index on the table.

        Parameters
        ----------
        key : str
        columns : None, bool, or listlike[str]
            Indicate which columns to create an index on.

            * False : Do not create any indexes.
            * True : Create indexes on all columns.
            * None : Create indexes on all columns.
            * listlike : Create indexes on the given columns.

        optlevel : int or None, default None
            Optimization level, if None, pytables defaults to 6.
        kind : str or None, default None
            Kind of index, if None, pytables defaults to "medium".

        Raises
        ------
        TypeError: raises if the node is not a table
        Nz1cannot create table index on a Fixed format store)r   r  r  )rl   r   rA   r   r   create_index)r   ro   r   r  r  rD   rE   rE   rF   create_table_indexb  s    

zHDFStore.create_table_indexc             C  s<   t   |   | jdk	sttdk	s(tdd | j D S )z
        Return a list of all the top-level nodes.

        Each node returned is not a pandas storage object.

        Returns
        -------
        list
            List of objects.
        Nc             S  sP   g | ]H}t |tjjst|jd dsHt|ddsHt |tjjr|jdkr|qS )pandas_typeNrc   )	rA   rg   linkZLinkgetattr_v_attrsrc   r   r   )rR   r   rE   rE   rF   rT     s
   z#HDFStore.groups.<locals>.<listcomp>)rl   r   r   r   rg   walk_groups)r   rE   rE   rF   r     s    zHDFStore.groupsr   c             c  s   t   |   | jdk	sttdk	s(tx| j|D ]}t|jdddk	rNq6g }g }xP|j	 D ]B}t|jdd}|dkrt
|tjjr||j qb||j qbW |jd||fV  q6W dS )aS  
        Walk the pytables group hierarchy for pandas objects.

        This generator will yield the group path, subgroups and pandas object
        names for each group.

        Any non-pandas PyTables objects that are not a group will be ignored.

        The `where` group itself is listed first (preorder), then each of its
        child groups (following an alphanumerical order) is also traversed,
        following the same procedure.

        Parameters
        ----------
        where : str, default "/"
            Group where to start walking.

        Yields
        ------
        path : str
            Full path to a group (without trailing '/').
        groups : list
            Names (strings) of the groups contained in `path`.
        leaves : list
            Names (strings) of the pandas objects contained in `path`.
        Nr"  r   )rl   r   r   r   rg   r&  r$  r%  Z_v_childrenr  rA   r   ZGrouprt   r   r   rstrip)r   rX   r   r   ZleavesZchildr"  rE   rE   rF   walk  s     zHDFStore.walkzNode | Nonec             C  s~   |    |dsd| }| jdk	s(ttdk	s4ty| j| j|}W n tjjk
r`   dS X t	|tj
sztt||S )z9return the node with the key or None if it does not existr   N)r   
startswithr   r   rg   r   r   
exceptionsZNoSuchNodeErrorrA   r>   r   )r   ro   r   rE   rE   rF   r     s    
zHDFStore.get_nodezGenericFixed | Tablec             C  s8   |  |}|dkr"td| d| |}|  |S )z<return the storer object for a key, raise if not in the fileNzNo object named z in the file)r   r   r   r   )r   ro   r   rD   rE   rE   rF   r     s    

zHDFStore.get_storerr   )propindexesrr   r   c	          	   C  s   t |||||d}	|dkr&t|  }t|ttfs:|g}x|D ]}
| |
}|dk	r@|
|	krl|rl|	|
 | |
}t|trd}|rdd |j	D }|	j
|
||t|dd|jd q@|	j|
||jd q@W |	S )	a;  
        Copy the existing store to a new file, updating in place.

        Parameters
        ----------
        propindexes : bool, default True
            Restore indexes in copied file.
        keys : list, optional
            List of keys to include in the copy (defaults to all).
        overwrite : bool, default True
            Whether to overwrite (remove and replace) existing nodes in the new store.
        mode, complib, complevel, fletcher32 same as in HDFStore.__init__

        Returns
        -------
        open file handle of the new store
        )rq   rs   rr   r   NFc             S  s   g | ]}|j r|jqS rE   )
is_indexedrL   )rR   rm   rE   rE   rF   rT      s    z!HDFStore.copy.<locals>.<listcomp>ry   )rv   ry   rI   )rI   )r   rU   r   rA   rV   r   r   r   r   r  rt   r$  rI   r~   )r   rj   rq   r+  r   rs   rr   r   Z	overwriteZ	new_storer   rD   datarv   rE   rE   rF   copy  s2    





zHDFStore.copyc       
      C  s  t | j}t|  d| d}| jrt|  }t|rg }g }x|D ]}y<| |}|dk	r|t |j	pn| |t |pd W qH t
k
r    Y qH tk
r } z(|| t |}	|d|	 d W dd}~X Y qHX qHW |td||7 }n|d7 }n|d	7 }|S )
zg
        Print detailed information on the store.

        Returns
        -------
        str
        z
File path: r   Nzinvalid_HDFStore nodez[invalid_HDFStore node: r      ZEmptyzFile is CLOSED)r;   r   r   r   r  r   rW   r   rt   r   r   r   r:   )
r   r   outputZlkeysr   r  r   rD   ZdetailZdstrrE   rE   rF   info-  s.    



(
zHDFStore.infoc             C  s   | j st| j dd S )Nz file is not open!)r   r^   r   )r   rE   rE   rF   r   W  s    zHDFStore._check_if_open)ru   r   c          
   C  sJ   yt |  }W n4 tk
rD } ztd| d|W dd}~X Y nX |S )zvalidate / deprecate formatsz#invalid HDFStore format specified [r   N)_FORMAT_MAPlowerr   r   )r   ru   r  rE   rE   rF   r   [  s
    $zHDFStore._validate_formatUTF-8zFrameOrSeries | None)rp   rI   rz   r   c          
     s  dk	rt ttfstd fdd}ttjdd}ttjdd}|dkrƈdkrt  tdk	stt	tddst tj
jrd}d	}qtd
n$t trd}nd} dkr|d7 }d|kr&ttd}	y|	| }
W n. tk
r } z|d|W dd}~X Y nX |
| ||dS |dkrdk	r|dkr~tdd}|dk	r|jdkrld}n|jdkrd}nB|dkrtdd}|dk	r|jdkrd}n|jdkrd}ttttttd}y|| }
W n. tk
r } z|d|W dd}~X Y nX |
| ||dS )z"return a suitable class to operateNz(value must be None, Series, or DataFramec          	     s$   t d|  d dt d  S )Nz(cannot properly create the storer for: [z
] [group->z,value->z	,format->)r   r   )re   )ru   r   rp   rE   rF   errors  s    z&HDFStore._create_storer.<locals>.errorr"  
table_typerc   frame_tablegeneric_tablezKcannot create a storer if the object is not existing nor a value are passedseriesframeZ_table)r9  r:  _STORER_MAP)rI   rz   series_tablerv   rP   appendable_seriesappendable_multiseriesappendable_frameappendable_multiframe)r8  r=  r>  r?  r@  worm
_TABLE_MAP)rA   r,   r&   r   rG   r$  r%  rl   rg   r   rc   r   SeriesFixed
FrameFixedr   nlevelsGenericTableAppendableSeriesTableAppendableMultiSeriesTableAppendableFrameTableAppendableMultiFrameTable	WORMTable)r   r   ru   rp   rI   rz   r5  ZptZttr;  clsr  rv   rB  rE   )ru   r   rp   rF   r   e  sl    








zHDFStore._create_storer)ro   rp   rr   rw   rz   r   c             C  s   t |dd r|dks|rd S | ||}| j|||||d}|rr|jrZ|jrb|dkrb|jrbtd|jsz|  n|  |js|rtd|j||||||	|
||||||d t|t	r|r|j
|d d S )	Nemptyrc   )rI   rz   rb   zCan only append to Tablesz0Compression not supported on Fixed format stores)objr  rt   rs   rr   r   rw   r   r  rx   r{   ry   r   )r   )r$  _identify_groupr   r   	is_existsr   set_object_infowriterA   r   r   )r   ro   rp   ru   r  rv   rt   rs   rr   r   rw   r   r  rx   r{   ry   rI   rz   r   r   rD   rE   rE   rF   r     s8    

zHDFStore._write_to_groupr>   )r   c             C  s   |  |}|  | S )N)r   r   r   )r   r   rD   rE   rE   rF   r     s    
zHDFStore._read_group)ro   rt   r   c             C  sN   |  |}| jdk	st|dk	r8|s8| jj|dd d}|dkrJ| |}|S )z@Identify HDF5 group based on key, delete/create group if needed.NT)r   )r   r   r   remove_node_create_nodes_and_group)r   ro   rt   r   rE   rE   rF   rO    s    

zHDFStore._identify_groupc             C  sz   | j dk	st|d}d}xX|D ]P}t|s0q"|}|dsF|d7 }||7 }| |}|dkrn| j ||}|}q"W |S )z,Create nodes from key and return group name.Nr   )r   r   splitrW   endswithr   Zcreate_group)r   ro   pathsr   pnew_pathr   rE   rE   rF   rT    s    



z HDFStore._create_nodes_and_group)rm   NNF)r   )rm   )F)NNNNFNF)NNN)NN)NNNNNFNF)NTFNNNNNNrf   TF)NNN)NNTTNNNNNNNNNNrf   )NNF)NNN)r   )r   TNNNFT)NNr4  rf   )NTFNNNNNNFNNNrf   T)2r[   r\   r]   __doc____annotations__r   r   propertyr   r   r   r   r   r   r   r   r   r   r   r   r   r   Z	iteritemsr   r   r   r   r   r   r   r   r   r~   r   rt   r  r!  r   r(  r   r   r.  r1  r   r   r   r   r   rO  rT  rE   rE   rE   rF   r     s   
@   
"-	      E   &       t           5=              H  ]  #
0      4*   [              +r   c               @  sb   e Zd ZU dZded< ded< ded< dddd
dd
dddZdd Zdd Zdd
dddZdS )r   aa  
    Define the iteration interface on a table

    Parameters
    ----------
    store : HDFStore
    s     : the referred storer
    func  : the function to execute the query
    where : the where of the query
    nrows : the rows to iterate on
    start : the passed start value (default is None)
    stop  : the passed stop value (default is None)
    iterator : bool, default False
        Whether to use the default iterator.
    chunksize : the passed chunking value (default is 100000)
    auto_close : bool, default False
        Whether to automatically close the store at the end of iteration.
    z
int | Noner   r   r|   zGenericFixed | TablerD   NFrn   )r|   rD   r   r   r   c             C  s   || _ || _|| _|| _| jjrN|d kr,d}|d kr8d}|d krD|}t||}|| _|| _|| _d | _	|sr|	d k	r|	d kr~d}	t
|	| _nd | _|
| _d S )Nr   i )r|   rD   r   rX   r   minr   r   r   r   rN   r   r   )r   r|   rD   r   rX   r   r   r   r   r   r   rE   rE   rF   r   D  s,    
zTableIterator.__init__c             c  sz   | j }| jd krtdxT|| jk rlt|| j | j}| d d | j|| }|}|d kst|sdq|V  qW |   d S )Nz*Cannot iterate until get_result is called.)	r   r   r   r   r]  r   r   rW   r   )r   r   r   rp   rE   rE   rF   r   n  s    

zTableIterator.__iter__c             C  s   | j r| j  d S )N)r   r|   r   )r   rE   rE   rF   r   ~  s    zTableIterator.close)r   c             C  s   | j d k	r4t| jtstd| jj| jd| _| S |rft| jtsLtd| jj| j| j| j	d}n| j}| 
| j| j	|}|   |S )Nz0can only use an iterator or chunksize on a table)rX   z$can only read_coordinates on a table)rX   r   r   )r   rA   rD   r   r   r   rX   r   r   r   r   r   )r   r   rX   resultsrE   rE   rF   r     s    
zTableIterator.get_result)NNFNF)F)	r[   r\   r]   rZ  r[  r   r   r   r   rE   rE   rE   rF   r   ,  s   
	    r   c               @  sv  e Zd ZU dZdZdZdddgZded< ded< dJdd
dddZe	ddddZ
e	ddddZddddZddddZdddddZddddZe	ddd d!Zd"ddd#d$d%Zd&d' Ze	d(d) Ze	d*d+ Ze	d,d- Ze	d.d/ Zd0d1 ZdKd2d3Zd4d5 Zd6dd7d8d9ZdLd:d;Zdd<d=d>Zd?d@ ZdAdB ZdCdD Zd6dEdFdGZ d6dEdHdIZ!d	S )MIndexCola  
    an index column description class

    Parameters
    ----------
    axis   : axis which I reference
    values : the ndarray like converted values
    kind   : a string description of this type
    typ    : the pytables type
    pos    : the position in the pytables

    Tfreqtz
index_namerK   rL   cnameNz
str | None)rL   rc  c             C  s   t |tstd|| _|| _|| _|| _|p0|| _|| _|| _	|| _
|	| _|
| _|| _|| _|| _|| _|d k	r|| | t | jtstt | jtstd S )Nz`name` must be a str.)rA   rK   r   r  r  typrL   rc  r   posr`  ra  rb  r  rc   r   metadataset_posr   )r   rL   r  r  rd  rc  r   re  r`  ra  rb  r  rc   r   rf  rE   rE   rF   r     s(    


zIndexCol.__init__rN   )r   c             C  s   | j jS )N)rd  itemsize)r   rE   rE   rF   rh    s    zIndexCol.itemsizec             C  s   | j  dS )N_kind)rL   )r   rE   rE   rF   	kind_attr  s    zIndexCol.kind_attr)re  c             C  s$   || _ |dk	r | jdk	r || j_dS )z,set the position of this column in the TableN)re  rd  Z_v_pos)r   re  rE   rE   rF   rg    s    zIndexCol.set_posc          	   C  sF   t tt| j| j| j| j| jf}ddd t	dddddg|D S )	N,c             s  s    | ]\}}| d | V  qdS )z->NrE   )rR   ro   rp   rE   rE   rF   r    s   z$IndexCol.__repr__.<locals>.<genexpr>rL   rc  r   re  r  )
rV   mapr;   rL   rc  r   re  r  joinr   )r   temprE   rE   rF   r     s
     zIndexCol.__repr__r   rn   )otherr   c               s   t  fdddD S )zcompare 2 col itemsc             3  s&   | ]}t |d t  |d kV  qd S )N)r$  )rR   rm   )ro  r   rE   rF   r    s   z"IndexCol.__eq__.<locals>.<genexpr>)rL   rc  r   re  )r  )r   ro  rE   )ro  r   rF   __eq__  s    zIndexCol.__eq__c             C  s   |  | S )N)rp  )r   ro  rE   rE   rF   __ne__  s    zIndexCol.__ne__c             C  s"   t | jdsdS t| jj| jjS )z%return whether I am an indexed columnr  F)hasattrrc   r$  r  rc  r,  )r   rE   rE   rF   r,    s    zIndexCol.is_indexedz
np.ndarray)r  rI   rz   c       	      C  s   t |tjstt||jjdk	r.|| j }t| j	}t
||||}i }t| j|d< | jdk	rpt| j|d< t}t|jst|jrt}y||f|}W n0 tk
r   d|krd|d< ||f|}Y nX t|| j}||fS )zV
        Convert the data from this selection to the appropriate pandas type.
        NrL   r`  )rA   rB   ndarrayr   r   dtypefieldsrc  rG   r  _maybe_convertrb  r`  r(   r   r   r'   r   _set_tzra  )	r   r  r{   rI   rz   val_kindr   factoryZnew_pd_indexrE   rE   rF   convert	  s(    


zIndexCol.convertc             C  s   | j S )zreturn the values)r  )r   rE   rE   rF   	take_data/  s    zIndexCol.take_datac             C  s   | j jS )N)rc   r%  )r   rE   rE   rF   attrs3  s    zIndexCol.attrsc             C  s   | j jS )N)rc   description)r   rE   rE   rF   r}  7  s    zIndexCol.descriptionc             C  s   t | j| jdS )z!return my current col descriptionN)r$  r}  rc  )r   rE   rE   rF   col;  s    zIndexCol.colc             C  s   | j S )zreturn my cython values)r  )r   rE   rE   rF   cvalues@  s    zIndexCol.cvaluesc             C  s
   t | jS )N)r   r  )r   rE   rE   rF   r   E  s    zIndexCol.__iter__c             C  sP   t | jdkrLt|tr$|| j}|dk	rL| jj|k rLt j	|| j
d| _dS )z
        maybe set a string col itemsize:
            min_itemsize can be an integer or a dict with this columns name
            with an integer size
        stringN)rh  re  )rG   r  rA   r  r   rL   rd  rh  rl   	StringColre  )r   rw   rE   rE   rF   maybe_set_sizeH  s
    
zIndexCol.maybe_set_sizec             C  s   d S )NrE   )r   rE   rE   rF   validate_namesU  s    zIndexCol.validate_namesAppendableTable)handlerrt   c             C  s:   |j | _ |   | | | | | | |   d S )N)rc   validate_colvalidate_attrvalidate_metadatawrite_metadataset_attr)r   r  rt   rE   rE   rF   validate_and_setX  s    


zIndexCol.validate_and_setc             C  s^   t | jdkrZ| j}|dk	rZ|dkr*| j}|j|k rTtd| d| j d|j d|jS dS )z:validate this column: return the compared against itemsizer  Nz#Trying to store a string with len [z] in [z)] column but
this column has a limit of [zC]!
Consider using min_itemsize to preset the sizes on these columns)rG   r  r~  rh  r   rc  )r   rh  crE   rE   rF   r  `  s    
zIndexCol.validate_col)rt   c             C  sB   |r>t | j| jd }|d k	r>|| jkr>td| d| j dd S )Nzincompatible kind in col [z - r   )r$  r|  rj  r  r   )r   rt   Zexisting_kindrE   rE   rF   r  s  s
    zIndexCol.validate_attrc             C  s   x| j D ]}t| |d}|| ji }||}||kr|dk	r||kr|dkrt|||f }tj|tdd d||< t	| |d qt
d| j d| d| d| d		q|dk	s|dk	r|||< qW dS )
z
        set/update the info for this indexable with the key/value
        if there is a conflict raise/warn as needed
        N)r`  rb     )
stacklevelzinvalid info for [z] for [z], existing_value [z] conflicts with new value [r   )_info_fieldsr$  
setdefaultrL   r   attribute_conflict_docwarningswarnr`   setattrr   )r   r1  ro   rp   idxZexisting_valuewsrE   rE   rF   update_info|  s    
$zIndexCol.update_infoc             C  s$   | | j}|dk	r | j| dS )z!set my state from the passed infoN)r   rL   __dict__update)r   r1  r  rE   rE   rF   set_info  s    zIndexCol.set_infoc             C  s   t | j| j| j dS )zset the kind for this columnN)r  r|  rj  r  )r   rE   rE   rF   r    s    zIndexCol.set_attr)r  c             C  sB   | j dkr>| j}|| j}|dk	r>|dk	r>t||s>tddS )z:validate that kind=category does not change the categoriescategoryNzEcannot append a categorical with different categories to the existing)r   rf  read_metadatarc  r%   r   )r   r  Znew_metadataZcur_metadatarE   rE   rF   r    s    

zIndexCol.validate_metadatac             C  s   | j dk	r|| j| j  dS )zset the meta dataN)rf  r  rc  )r   r  rE   rE   rF   r    s    
zIndexCol.write_metadata)NNNNNNNNNNNNN)N)N)"r[   r\   r]   rZ  is_an_indexableis_data_indexabler  r[  r   r\  rh  rj  rg  r   rp  rq  r,  rz  r{  r|  r}  r~  r  r   r  r  r  r  r  r  r  r  r  r  rE   rE   rE   rF   r_    sV   

            &

	r_  c               @  s<   e Zd ZdZeddddZddddd	d
Zdd ZdS )GenericIndexColz:an index which is not represented in the data of the tablern   )r   c             C  s   dS )NFrE   )r   rE   rE   rF   r,    s    zGenericIndexCol.is_indexedz
np.ndarrayrK   )r  rI   rz   c             C  s2   t |tjstt|ttt|}||fS )z
        Convert the data from this selection to the appropriate pandas type.

        Parameters
        ----------
        values : np.ndarray
        nan_rep : str
        encoding : str
        errors : str
        )rA   rB   rs  r   r   r)   arangerW   )r   r  r{   rI   rz   rE   rE   rF   rz    s    zGenericIndexCol.convertc             C  s   d S )NrE   )r   rE   rE   rF   r    s    zGenericIndexCol.set_attrN)r[   r\   r]   rZ  r\  r,  rz  r  rE   rE   rE   rF   r    s   r  c                  s,  e Zd ZdZdZdZddgZd9ddd fd	d
ZeddddZ	eddddZ
ddddZdddddZddddZdd Zeddddd Zed!d" Zedd#d$d%d&Zeddd$d'd(Zed)d* Zed+d, Zed-d. Zed/d0 Zd1d2 Zd3ddd4d5d6Zd7d8 Z  ZS ):DataCola3  
    a data holding column, by definition this is not indexable

    Parameters
    ----------
    data   : the actual data
    cname  : the column name in the table to hold the data (typically
                values)
    meta   : a string description of the metadata
    metadata : the actual metadata
    Fra  r  NrK   zDtypeArg | None)rL   rt  c               s2   t  j|||||||||	|
|d || _|| _d S )N)rL   r  r  rd  re  rc  ra  r  rc   r   rf  )superr   rt  r-  )r   rL   r  r  rd  rc  re  ra  r  rc   r   rf  rt  r-  )	__class__rE   rF   r     s    zDataCol.__init__)r   c             C  s   | j  dS )N_dtype)rL   )r   rE   rE   rF   
dtype_attr	  s    zDataCol.dtype_attrc             C  s   | j  dS )N_meta)rL   )r   rE   rE   rF   	meta_attr	  s    zDataCol.meta_attrc          	   C  sF   t tt| j| j| j| j| jf}ddd t	dddddg|D S )	Nrk  c             s  s    | ]\}}| d | V  qdS )z->NrE   )rR   ro   rp   rE   rE   rF   r  	  s   z#DataCol.__repr__.<locals>.<genexpr>rL   rc  rt  r  shape)
rV   rl  r;   rL   rc  rt  r  r  rm  r   )r   rn  rE   rE   rF   r   	  s    zDataCol.__repr__r   rn   )ro  r   c               s   t  fdddD S )zcompare 2 col itemsc             3  s&   | ]}t |d t  |d kV  qd S )N)r$  )rR   rm   )ro  r   rE   rF   r   	  s   z!DataCol.__eq__.<locals>.<genexpr>)rL   rc  rt  re  )r  )r   ro  rE   )ro  r   rF   rp  	  s    zDataCol.__eq__r   )r-  c             C  s@   |d k	st | jd kst t|\}}|| _|| _t|| _d S )N)r   rt  _get_data_and_dtype_namer-  _dtype_to_kindr  )r   r-  
dtype_namerE   rE   rF   set_data$	  s    zDataCol.set_datac             C  s   | j S )zreturn the data)r-  )r   rE   rE   rF   r{  .	  s    zDataCol.take_datar<   )r  r   c             C  s   |j }|j}|j}|jdkr&d|jf}t|trJ|j}| j||j j	d}ntt
|sZt|rf| |}nXt|rz| |}nDt|rt j||d d}n&t|r| ||}n| j||j	d}|S )zW
        Get an appropriately typed and shaped pytables.Col object for values.
        rP   )r  r   )rh  r  )rt  rh  r  r  sizerA   r0   codesget_atom_datarL   r   r   get_atom_datetime64r#   get_atom_timedelta64r   rl   Z
ComplexColr"   get_atom_string)rL  r  rt  rh  r  r  atomrE   rE   rF   	_get_atom2	  s$    


zDataCol._get_atomc             C  s   t  j||d dS )Nr   )rh  r  )rl   r  )rL  r  rh  rE   rE   rF   r  R	  s    zDataCol.get_atom_stringz	type[Col])r  r   c             C  sR   | dr$|dd }d| d}n"| dr4d}n| }| d}tt |S )z0return the PyTables column class for this columnuint   NZUIntr<   periodInt64Col)r)  
capitalizer$  rl   )rL  r  Zk4Zcol_nameZkcaprE   rE   rF   get_atom_coltypeV	  s    


zDataCol.get_atom_coltypec             C  s   | j |d|d dS )N)r  r   )r  )r  )rL  r  r  rE   rE   rF   r  e	  s    zDataCol.get_atom_datac             C  s   t  j|d dS )Nr   )r  )rl   r  )rL  r  rE   rE   rF   r  i	  s    zDataCol.get_atom_datetime64c             C  s   t  j|d dS )Nr   )r  )rl   r  )rL  r  rE   rE   rF   r  m	  s    zDataCol.get_atom_timedelta64c             C  s   t | jdd S )Nr  )r$  r-  )r   rE   rE   rF   r  q	  s    zDataCol.shapec             C  s   | j S )zreturn my cython values)r-  )r   rE   rE   rF   r  u	  s    zDataCol.cvaluesc             C  s`   |r\t | j| jd}|dk	r2|t| jkr2tdt | j| jd}|dk	r\|| jkr\tddS )zAvalidate that we have the same order as the existing & same dtypeNz4appended items do not match existing items in table!z@appended items dtype do not match existing items dtype in table!)r$  r|  rj  rU   r  r   r  rt  )r   rt   Zexisting_fieldsZexisting_dtyperE   rE   rF   r  z	  s    zDataCol.validate_attrz
np.ndarray)r  rI   rz   c             C  s  t |tjstt||jjdk	r.|| j }| jdk	s<t| jdkr\t	|\}}t
|}n|}| j}| j}t |tjs|tt| j}| j}	| j}
| j}|dk	stt|}|dkrt||dd}n(|dkrtj|dd}n|dkr8ytjd	d
 |D td}W n. tk
r4   tjdd
 |D td}Y nX n|dkr|	}| }|dkrhtg tjd}n<t|}| r||  }||dk  |t j8  < tj|||
d}n8y|j|dd}W n$ t k
r   |jddd}Y nX t|dkrt!||||d}| j"|fS )aR  
        Convert the data from this selection to the appropriate pandas type.

        Parameters
        ----------
        values : np.ndarray
        nan_rep :
        encoding : str
        errors : str

        Returns
        -------
        index : listlike to become an Index
        data : ndarraylike to become a column
        N
datetime64T)coercetimedelta64zm8[ns])rt  r   c             S  s   g | ]}t |qS rE   )r   fromordinal)rR   r	  rE   rE   rF   rT   	  s    z#DataCol.convert.<locals>.<listcomp>c             S  s   g | ]}t |qS rE   )r   fromtimestamp)rR   r	  rE   rE   rF   rT   	  s    r  )
categoriesr  F)r.  Or  )r{   rI   rz   )#rA   rB   rs  r   r   rt  ru  rc  rd  r  r  r  rG   r   rf  r  ra  rw  asarrayobjectr   ravelr(   Zfloat64r/   anyastyperN   Zcumsum_valuesr0   Z
from_codesr   _unconvert_string_arrayr  )r   r  r{   rI   rz   	convertedr  r  r   rf  r  ra  rt  r  r  maskrE   rE   rF   rz  	  s^    








 zDataCol.convertc             C  sH   t | j| j| j t | j| j| j | jdk	s2tt | j| j| j dS )zset the data for this columnN)	r  r|  rj  r  r  r   rt  r   r  )r   rE   rE   rF   r  	  s    zDataCol.set_attr)NNNNNNNNNNNN)r[   r\   r]   rZ  r  r  r  r   r\  r  r  r   rp  r  r{  classmethodr  r  r  r  r  r  r  r  r  rz  r  __classcell__rE   rE   )r  rF   r    sB              
 er  c               @  sT   e Zd ZdZdZdd Zedd Zeddd	d
dZedd Z	edd Z
dS )DataIndexableColz+represent a data column that can be indexedTc             C  s   t | j stdd S )Nz-cannot have non-object label DataIndexableCol)r(   r  Z	is_objectr   )r   rE   rE   rF   r  	  s    zDataIndexableCol.validate_namesc             C  s   t  j|dS )N)rh  )rl   r  )rL  r  rh  rE   rE   rF   r  	  s    z DataIndexableCol.get_atom_stringrK   r<   )r  r   c             C  s   | j |d S )N)r  )r  )rL  r  r  rE   rE   rF   r  
  s    zDataIndexableCol.get_atom_datac             C  s
   t   S )N)rl   r  )rL  r  rE   rE   rF   r  
  s    z$DataIndexableCol.get_atom_datetime64c             C  s
   t   S )N)rl   r  )rL  r  rE   rE   rF   r  

  s    z%DataIndexableCol.get_atom_timedelta64N)r[   r\   r]   rZ  r  r  r  r  r  r  r  rE   rE   rE   rF   r  	  s   r  c               @  s   e Zd ZdZdS )GenericDataIndexableColz(represent a generic pytables data columnN)r[   r\   r]   rZ  rE   rE   rE   rF   r  
  s   r  c               @  s  e Zd ZU dZded< dZded< ded< ded	< ded
< ded< ded< ded< dZdLdddddddZeddddZ	eddddZ
edd Zdddd Zd!d" Zd#d$ Zed%d& Zed'd( Zed)d* Zed+d, Zeddd-d.Zeddd/d0Zed1d2 Zd3d4 Zd5d6 Zed7d8 Zeddd9d:Zed;d< Zd=d> ZdMd@dAZdBdC ZdNdDdDdEdFdGZdHdI ZdOdDdDdEdJdKZ d?S )PFixedz
    represent an object in my store
    facilitate read/write of various types of objects
    this is an abstract base class

    Parameters
    ----------
    parent : HDFStore
    group : Node
        The group node where the table resides.
    rK   pandas_kindrb   format_typeztype[FrameOrSeriesUnion]obj_typerN   r  rI   r   r   r>   r   rz   FUTF-8rf   )r   r   rI   rz   c             C  sZ   t |tstt|td k	s"tt |tjs:tt||| _|| _t|| _	|| _
d S )N)rA   r   r   r   rg   r>   r   r   rJ   rI   rz   )r   r   r   rI   rz   rE   rE   rF   r   ,
  s    
zFixed.__init__rn   )r   c             C  s*   | j d dko(| j d dko(| j d dk S )Nr   rP   
      )version)r   rE   rE   rF   is_old_version;
  s    zFixed.is_old_versionztuple[int, int, int]c             C  sb   t t| jjdd}y0tdd |dD }t|dkrB|d }W n tk
r\   d}Y nX |S )	zcompute and set our versionpandas_versionNc             s  s   | ]}t |V  qd S )N)rN   )rR   r   rE   rE   rF   r  D
  s    z Fixed.version.<locals>.<genexpr>.r  )r   )r   r   r   )rG   r$  r   r%  rV   rU  rW   ri   )r   r  rE   rE   rF   r  ?
  s    
zFixed.versionc             C  s   t t| jjdd S )Nr"  )rG   r$  r   r%  )r   rE   rE   rF   r"  K
  s    zFixed.pandas_typec             C  s^   |    | j}|dk	rXt|ttfrDddd |D }d| d}| jdd| d	S | jS )
z(return a pretty representation of myselfNrk  c             s  s   | ]}t |V  qd S )N)r;   )rR   r   rE   rE   rF   r  U
  s    z!Fixed.__repr__.<locals>.<genexpr>[r   z12.12z	 (shape->))r   r  rA   rU   rV   rm  r"  )r   rD   ZjshaperE   rE   rF   r   O
  s    zFixed.__repr__c             C  s   t | j| j_t t| j_dS )zset my pandas type & versionN)rK   r  r|  r"  _versionr  )r   rE   rE   rF   rQ  Z
  s    zFixed.set_object_infoc             C  s   t  | }|S )N)r.  )r   Znew_selfrE   rE   rF   r.  _
  s    
z
Fixed.copyc             C  s   | j S )N)r   )r   rE   rE   rF   r  c
  s    zFixed.shapec             C  s   | j jS )N)r   r   )r   rE   rE   rF   r   g
  s    zFixed.pathnamec             C  s   | j jS )N)r   r   )r   rE   rE   rF   r   k
  s    zFixed._handlec             C  s   | j jS )N)r   r   )r   rE   rE   rF   r   o
  s    zFixed._filtersc             C  s   | j jS )N)r   r   )r   rE   rE   rF   r   s
  s    zFixed._complevelc             C  s   | j jS )N)r   r   )r   rE   rE   rF   r   w
  s    zFixed._fletcher32c             C  s   | j jS )N)r   r%  )r   rE   rE   rF   r|  {
  s    zFixed.attrsc             C  s   dS )zset our object attributesNrE   )r   rE   rE   rF   	set_attrs
  s    zFixed.set_attrsc             C  s   dS )zget our object attributesNrE   )r   rE   rE   rF   	get_attrs
  s    zFixed.get_attrsc             C  s   | j S )zreturn my storable)r   )r   rE   rE   rF   storable
  s    zFixed.storablec             C  s   dS )NFrE   )r   rE   rE   rF   rP  
  s    zFixed.is_existsc             C  s   t | jdd S )Nr   )r$  r  )r   rE   rE   rF   r   
  s    zFixed.nrowsc             C  s   |dkrdS dS )z%validate against an existing storableNTrE   )r   ro  rE   rE   rF   validate
  s    zFixed.validateNc             C  s   dS )z+are we trying to operate on an old version?TrE   )r   rX   rE   rE   rF   validate_version
  s    zFixed.validate_versionc             C  s   | j }|dkrdS |   dS )zr
        infer the axes of my storer
        return a boolean indicating if we have a valid storer or not
        NFT)r  r  )r   rD   rE   rE   rF   r   
  s
    zFixed.infer_axesz
int | None)r   r   c             C  s   t dd S )Nz>cannot read on an abstract storer: subclasses should implement)r   )r   rX   r   r   r   rE   rE   rF   r   
  s    z
Fixed.readc             K  s   t dd S )Nz?cannot write on an abstract storer: subclasses should implement)r   )r   r   rE   rE   rF   rR  
  s    zFixed.writec             C  s0   t |||r$| jj| jdd dS tddS )zs
        support fully deleting the node in its entirety (only) - where
        specification must be None
        T)r   Nz#cannot delete on an abstract storer)r   r   r   rS  r   r   )r   rX   r   r   rE   rE   rF   r   
  s    zFixed.delete)r  rf   )N)NNNN)NNN)!r[   r\   r]   rZ  r[  r  r   r   r\  r  r  r"  r   rQ  r.  r  r   r   r   r   r   r|  r  r  r  rP  r   r  r  r   r   rR  r   rE   rE   rE   rF   r  
  sN   
 

   r  c               @  s&  e Zd ZU dZedediZdd e D Zg Z	de
d< dd	d
dZdd Zdd Zdd Zedd	ddZdd Zdd Zdd Zd9ddddddZd:dddd d!d"d#Zdd d$d%d&Zdd'd$d(d)Zd;dddd'd!d*d+Zd<d,ddd d-d.d/Zdd0d1d2d3Zd=dd4d5d6d7d8ZdS )>GenericFixedza generified fixed versiondatetimer  c             C  s   i | ]\}}||qS rE   rE   )rR   r   r	  rE   rE   rF   r
  
  s    zGenericFixed.<dictcomp>z	list[str]
attributesrK   )r   c             C  s   | j |dS )N )_index_type_mapr   )r   rL  rE   rE   rF   _class_to_alias
  s    zGenericFixed._class_to_aliasc             C  s   t |tr|S | j|tS )N)rA   r   _reverse_index_mapr   r(   )r   aliasrE   rE   rF   _alias_to_class
  s    
zGenericFixed._alias_to_classc             C  s   |  tt|dd}|tkr.d	dd}|}n|tkrFd
dd}|}n|}i }d|krn|d |d< |tkrnt}d|krt|d tr|d 	d|d< n|d |d< |tkst
||fS )Nindex_classr  c             S  s:   t j| j|d}tj|d d}|d k	r6|d|}|S )N)r`  )rL   UTC)r1   _simple_newr  r'   tz_localize
tz_convert)r  r`  ra  ZdtaresultrE   rE   rF   rd   
  s
    z*GenericFixed._get_index_factory.<locals>.fc             S  s   t j| |d}tj|d dS )N)r`  )rL   )r2   r  r+   )r  r`  ra  ZparrrE   rE   rF   rd   
  s    r`  ra  zutf-8)NN)NN)r  rG   r$  r'   r+   r(   r-   rA   bytesrC   r   )r   r|  r  rd   ry  r   rE   rE   rF   _get_index_factory
  s(    

zGenericFixed._get_index_factoryc             C  s$   |dk	rt d|dk	r t ddS )zE
        raise if any keywords are passed which are not-None
        Nzqcannot pass a column specification when reading a Fixed format store. this store must be selected in its entiretyzucannot pass a where specification when reading from a Fixed format store. this store must be selected in its entirety)r   )r   r   rX   rE   rE   rF   validate_read  s    zGenericFixed.validate_readrn   c             C  s   dS )NTrE   )r   rE   rE   rF   rP    s    zGenericFixed.is_existsc             C  s   | j | j_ | j| j_dS )zset our object attributesN)rI   r|  rz   )r   rE   rE   rF   r    s    
zGenericFixed.set_attrsc          	   C  sV   t t| jdd| _tt| jdd| _x(| jD ]}t| |tt| j|d q0W dS )zretrieve our attributesrI   Nrz   rf   )rJ   r$  r|  rI   rG   rz   r  r  )r   r   rE   rE   rF   r    s    zGenericFixed.get_attrsc             K  s   |    d S )N)r  )r   rN  r   rE   rE   rF   rR  #  s    zGenericFixed.writeNz
int | None)ro   r   r   c             C  s   ddl }t| j|}|j}t|dd}t||jrD|d || }nztt|dd}	t|dd}
|
dk	rxtj|
|	d}n||| }|	dkrt|d	d}t	||d
d}n|	dkrtj
|dd}|r|jS |S dS )z2read an array for the specified node (off of groupr   N
transposedF
value_typer  )rt  r  ra  T)r  r  zm8[ns])rh   r$  r   r%  rA   ZVLArrayrG   rB   rM  rw  r  T)r   ro   r   r   rh   r   r|  r  Zretrt  r  ra  rE   rE   rF   
read_array&  s&    zGenericFixed.read_arrayr(   )ro   r   r   r   c             C  sh   t t| j| d}|dkr.| j|||dS |dkrVt| j|}| j|||d}|S td| d S )N_varietymulti)r   r   regularzunrecognized index variety: )rG   r$  r|  read_multi_indexr   read_index_noder   )r   ro   r   r   Zvarietyr   rv   rE   rE   rF   
read_indexH  s    zGenericFixed.read_index)ro   rv   c             C  s   t |tr,t| j| dd | || nt| j| dd td|| j| j}| ||j	 t
| j|}|j|j_|j|j_t |ttfr| t||j_t |tttfr|j|j_t |tr|jd k	rt|j|j_d S )Nr  r  r  rv   )rA   r*   r  r|  write_multi_index_convert_indexrI   rz   write_arrayr  r$  r   r  r%  rL   r'   r+   r  r   r  r-   r`  ra  _get_tz)r   ro   rv   r  r   rE   rE   rF   write_indexV  s    



zGenericFixed.write_indexr*   c             C  s   t | j| d|j xtt|j|j|jD ]\}\}}}t|rLt	d| d| }t
||| j| j}| ||j t| j|}	|j|	j_||	j_t |	j| d| | | d| }
| |
| q.W d S )N_nlevelsz=Saving a MultiIndex with an extension dtype is not supported._level_name_label)r  r|  rE  	enumerater   levelsr  namesr    r   r   rI   rz   r  r  r$  r   r  r%  rL   )r   ro   rv   ilevlevel_codesrL   	level_keyZ
conv_levelr   	label_keyrE   rE   rF   r  m  s    "
zGenericFixed.write_multi_indexc             C  s   t | j| d}g }g }g }xxt|D ]l}| d| }	t | j|	}
| j|
||d}|| ||j | d| }| j|||d}|| q(W t|||ddS )Nr  r  )r   r   r  T)r	  r  r
  r   )	r$  r|  r  r   r  rt   rL   r  r*   )r   ro   r   r   rE  r	  r  r
  r  r  r   r  r  r  rE   rE   rF   r    s    
zGenericFixed.read_multi_indexr>   )r   r   r   r   c             C  s   ||| }d|j kr>t|j jdkr>tj|j j|j jd}t|j j}d }d|j krlt|j j	}t|}|j }| 
|\}}	|dkr|t||| j| jdfdti|	}
n|t||| j| jdf|	}
||
_	|
S )Nr  r   )rt  rL   r   )rI   rz   rt  )r%  rB   prodr  rM  r  rG   r  rM   rL   r  _unconvert_indexrI   rz   r  )r   r   r   r   r-  r  rL   r|  ry  r   rv   rE   rE   rF   r    s,    

zGenericFixed.read_index_noder   )ro   rp   c             C  sJ   t d|j }| j| j|| t| j|}t|j|j	_
|j|j	_dS )zwrite a 0-len array)rP   N)rB   rM  r  r   create_arrayr   r$  rK   rt  r%  r  r  )r   ro   rp   Zarrr   rE   rE   rF   write_array_empty  s
    zGenericFixed.write_array_emptyr   zIndex | None)ro   rN  r   c          	   C  s2  t |dd}|| jkr&| j| j| |jdk}d}t|jrFtd|s^t|dr^|j	}d}d }| j
d k	rtt t j|j}W d Q R X |d k	r|s| jj| j|||j| j
d}||d d < n| || nH|jjtjkrHtj|dd}	|rn*|	d	krnt|	||f }
tj|
td
d | j| j|t  }|| nt|jr~| j | j||!d dt"| j|j#_$nt%|jr| j | j||j& t"| j|}t'|j(|j#_(d|j#_$n\t)|jr| j | j||!d dt"| j|j#_$n&|r| || n| j | j|| |t"| j|j#_*d S )NT)Zextract_numpyr   Fz]Cannot store a category dtype in a HDF5 dataset that uses format="fixed". Use format="table".r  )r   )skipnar     )r  i8r  r  )+r5   r   r   rS  r  r   rt  r   rr  r  r   r   r   rl   ZAtomZ
from_dtypeZcreate_carrayr  r  r   rB   Zobject_r   infer_dtypeperformance_docr  r  r   Zcreate_vlarray
ObjectAtomrt   r   r  viewr$  r%  r  r   asi8r  ra  r#   r  )r   ro   rN  r   rp   Zempty_arrayr  r  Zcainferred_typer  Zvlarrr   rE   rE   rF   r    s`    







zGenericFixed.write_array)NN)NN)NN)NN)N)r[   r\   r]   rZ  r'   r+   r  r   r  r  r[  r  r  r  r  r\  rP  r  r  rR  r  r  r  r  r  r  r  r  rE   rE   rE   rF   r  
  s(   
.#%	r  c                  sN   e Zd ZU dZdgZded< edd Zddddd	d
Z fddZ	  Z
S )rC  r9  rL   r
   c          	   C  s,   yt | jjfS  ttfk
r&   d S X d S )N)rW   r   r  r   ri   )r   rE   rE   rF   r  +  s    zSeriesFixed.shapeNz
int | None)r   r   c             C  s<   |  || | jd||d}| jd||d}t||| jdS )Nrv   )r   r   r  )rv   rL   )r  r  r  r,   rL   )r   rX   r   r   r   rv   r  rE   rE   rF   r   2  s    zSeriesFixed.readc               s8   t  j|f| | d|j | d| |j| j_d S )Nrv   r  )r  rR  r  rv   r  rL   r|  )r   rN  r   )r  rE   rF   rR  >  s    zSeriesFixed.write)NNNN)r[   r\   r]   r  r  r[  r\  r  r   rR  r  rE   rE   )r  rF   rC  %  s   
	   rC  c                  sR   e Zd ZU ddgZded< eddddZdd	d	d
ddZ fddZ  Z	S )BlockManagerFixedr  nblocksrN   zShape | None)r   c             C  s   y| j }d}xFt| jD ]8}t| jd| d}t|dd }|d k	r||d 7 }qW | jj}t|dd }|d k	rt|d|d  }ng }|| |S  tk
r   d S X d S )Nr   block_itemsr  rP   )	r  r  r  r$  r   Zblock0_valuesrU   rt   ri   )r   r  r   r  r   r  rE   rE   rF   r  J  s"    
zBlockManagerFixed.shapeNz
int | None)r   r   c             C  s   |  || |  d}g }xJt| jD ]<}||kr>||fnd\}}	| jd| ||	d}
||
 q*W |d }g }xht| jD ]Z}| d| d}| jd| d||	d}||	| }t
|j||d d	}|| qW t|dkrt|dd
}|j|dd}|S t
|d |d d	S )Nr   )NNr   )r   r   r  r   r  rP   )r   rv   )r   F)r   r.  )r  r  Z_get_block_manager_axisr  r  r  rt   r  r  r  r&   r  rW   r.   r  )r   rX   r   r   r   Zselect_axisr  r  r   r   axr   Zdfs	blk_itemsr  dfoutrE   rE   rF   r   e  s(    zBlockManagerFixed.readc               s   t  j|f| t|jtr&|d}|j}| s<| }|j| j	_x>t
|jD ]0\}}|dkrp|jsptd| d| | qRW t|j| j	_xRt
|jD ]D\}}|j|j}| jd| d|j|d | d| d| qW d S )Nr  r   z/Columns index has to be unique for fixed formatr   r  )r   r   )r  rR  rA   _mgrr7   _as_managerZis_consolidatedZconsolidater  r|  r  r  Z	is_uniquer   r  rW   blocksr  r   r  mgr_locsr  r  )r   rN  r   r-  r  r!  blkr"  )r  rE   rF   rR    s     

zBlockManagerFixed.write)NNNN)
r[   r\   r]   r  r[  r\  r  r   rR  r  rE   rE   )r  rF   r  E  s   
    r  c               @  s   e Zd ZdZeZdS )rD  r:  N)r[   r\   r]   r  r&   r  rE   rE   rE   rF   rD    s   rD  c                  s  e Zd ZU dZdZdZded< ded< dZded	< d
Zded< ded< ded< ded< ded< ded< ddddd fddZ	e
ddddZddd d!Zdd"d#d$Zd%d& Ze
d'dd(d)Zd*d+d,d-d.Ze
d/dd0d1Ze
d'dd2d3Ze
d4d5 Ze
d6d7 Ze
d8d9 Ze
d:d; Ze
d<d= Ze
d/dd>d?Ze
d'dd@dAZe
dBdC ZdDddEdFZdGdH ZdIddJdKZdddLdMdNZddOdPdQdRZddSdTdUZ dVdW Z!dXdY Z"ddZd[Z#d\d] Z$e%d^d_ Z&dd`dadbdcZ'ddddddedfdgdhZ(e)d'didjdkZ*dldm Z+ddnd'dodpdqZ,e-dnd'drdsdtZ.ddudvdwdxZ/ddd'dddDdydzd{Z0dddddd|d}d~Z1dddddddddZ2  Z3S )r   a  
    represent a table:
        facilitate read/write of various types of tables

    Attrs in Table Node
    -------------------
    These are attributes that are store in the main table node, they are
    necessary to recreate these tables when read back in.

    index_axes    : a list of tuples of the (original indexing axis and
        index column)
    non_index_axes: a list of tuples of the (original index axis and
        columns on a non-indexing axis)
    values_axes   : a list of the columns which comprise the data of this
        table
    data_columns  : a list of the columns that we are allowing indexing
        (these become single columns in values_axes), or True to force all
        columns
    nan_rep       : the string to use for nan representations for string
        objects
    levels        : the names of levels
    metadata      : the names of the metadata columns
    Z
wide_tablerc   rK   r  r6  rP   zint | list[Hashable]r	  Tzlist[IndexCol]
index_axeszlist[tuple[int, Any]]r   zlist[DataCol]values_axesrU   ry   rf  r  r1  Nrf   r   r>   )r   r   rz   c               sP   t  j||||d |pg | _|p$g | _|p.g | _|p8g | _|	pBi | _|
| _d S )N)rI   rz   )r  r   r*  r   r+  ry   r1  r{   )r   r   r   rI   rz   r*  r   r+  ry   r1  r{   )r  rE   rF   r     s    




zTable.__init__)r   c             C  s   | j dd S )N_r   )r6  rU  )r   rE   rE   rF   table_type_short  s    zTable.table_type_shortc             C  s   |    t| jrd| jnd}d| d}d}| jrZddd | jD }d| d}dd	d | jD }| jd
| d| j d| j	 d| j
 d| d| dS )z(return a pretty representation of myselfrk  r  z,dc->[r   r  c             s  s   | ]}t |V  qd S )N)rK   )rR   r   rE   rE   rF   r    s    z!Table.__repr__.<locals>.<genexpr>r  c             s  s   | ]}|j V  qd S )N)rL   )rR   rm   rE   rE   rF   r    s    z12.12z (typ->z,nrows->z,ncols->z,indexers->[r  )r   rW   ry   rm  r  r  r*  r"  r-  r   ncols)r   Zjdcr  ZverZjverZjindex_axesrE   rE   rF   r     s    zTable.__repr__)r  c             C  s"   x| j D ]}||jkr|S qW dS )zreturn the axis for cN)r  rL   )r   r  rm   rE   rE   rF   r     s    
zTable.__getitem__c          
   C  s   |dkrdS |j | j kr2td|j  d| j  dxdD ]}t| |d}t||d}||kr8x@t|D ]4\}}|| }||krftd| d| d| dqfW td| d| d| dq8W dS )	z"validate against an existing tableNz'incompatible table_type with existing [z - r   )r*  r   r+  zinvalid combination of [z] on appending data [z] vs current table [)r6  r   r$  r  r   r   )r   ro  r  ZsvZovr  ZsaxZoaxrE   rE   rF   r    s     
zTable.validatern   c             C  s   t | jtS )z@the levels attribute is 1 or a list in the case of a multi-index)rA   r	  rU   )r   rE   rE   rF   is_multi_index   s    zTable.is_multi_indexr   z tuple[DataFrame, list[Hashable]])rN  r   c          
   C  sf   dd t |jjD }y| }W n, tk
rN } ztd|W dd}~X Y nX t|ts^t||fS )ze
        validate that we can store the multi-index; reset and return the
        new object
        c             S  s&   g | ]\}}|d k	r|nd| qS )NZlevel_rE   )rR   r  lrE   rE   rF   rT   -  s    z-Table.validate_multiindex.<locals>.<listcomp>zBduplicate names/columns in the multi-index when storing as a tableN)r  rv   r
  Zreset_indexr   rA   r&   r   )r   rN  r	  Z	reset_objr  rE   rE   rF   validate_multiindex%  s    zTable.validate_multiindexrN   c             C  s   t dd | jD S )z-based on our axes, compute the expected nrowsc             S  s   g | ]}|j jd  qS )r   )r  r  )rR   r  rE   rE   rF   rT   ;  s    z(Table.nrows_expected.<locals>.<listcomp>)rB   r  r*  )r   rE   rE   rF   nrows_expected8  s    zTable.nrows_expectedc             C  s
   d| j kS )zhas this table been createdrc   )r   )r   rE   rE   rF   rP  =  s    zTable.is_existsc             C  s   t | jdd S )Nrc   )r$  r   )r   rE   rE   rF   r  B  s    zTable.storablec             C  s   | j S )z,return the table group (this is my storable))r  )r   rE   rE   rF   rc   F  s    zTable.tablec             C  s   | j jS )N)rc   rt  )r   rE   rE   rF   rt  K  s    zTable.dtypec             C  s   | j jS )N)rc   r}  )r   rE   rE   rF   r}  O  s    zTable.descriptionc             C  s   t | j| jS )N)r   r   r*  r+  )r   rE   rE   rF   r  S  s    z
Table.axesc             C  s   t dd | jD S )z.the number of total columns in the values axesc             s  s   | ]}t |jV  qd S )N)rW   r  )rR   rm   rE   rE   rF   r  Z  s    zTable.ncols.<locals>.<genexpr>)sumr+  )r   rE   rE   rF   r.  W  s    zTable.ncolsc             C  s   dS )NFrE   )r   rE   rE   rF   is_transposed\  s    zTable.is_transposedc             C  s(   t tdd | jD dd | jD S )z@return a tuple of my permutated axes, non_indexable at the frontc             S  s   g | ]}t |d  qS )r   )rN   )rR   rm   rE   rE   rF   rT   e  s    z*Table.data_orientation.<locals>.<listcomp>c             S  s   g | ]}t |jqS rE   )rN   r   )rR   rm   rE   rE   rF   rT   f  s    )rV   r   r   r   r*  )r   rE   rE   rF   data_orientation`  s    zTable.data_orientationzdict[str, Any]c               sR   ddd dd j D } fddjD }fddjD }t|| | S )z<return a dict of the kinds allowable columns for this objectrv   r   )r   rP   c             S  s   g | ]}|j |fqS rE   )rc  )rR   rm   rE   rE   rF   rT   p  s    z$Table.queryables.<locals>.<listcomp>c               s   g | ]\}} | d fqS )NrE   )rR   r   r  )
axis_namesrE   rF   rT   q  s    c               s&   g | ]}|j t jkr|j|fqS rE   )rL   r  ry   rc  )rR   r	  )r   rE   rF   rT   s  s    )r*  r   r+  r  )r   Zd1Zd2Zd3rE   )r6  r   rF   
queryablesj  s
    
zTable.queryablesc             C  s   dd | j D S )zreturn a list of my index colsc             S  s   g | ]}|j |jfqS rE   )r   rc  )rR   r  rE   rE   rF   rT   }  s    z$Table.index_cols.<locals>.<listcomp>)r*  )r   rE   rE   rF   
index_colsz  s    zTable.index_colsz	list[str]c             C  s   dd | j D S )zreturn a list of my values colsc             S  s   g | ]
}|j qS rE   )rc  )rR   r  rE   rE   rF   rT     s    z%Table.values_cols.<locals>.<listcomp>)r+  )r   rE   rE   rF   values_cols  s    zTable.values_cols)ro   r   c             C  s   | j j}| d| dS )z)return the metadata pathname for this keyz/meta/z/meta)r   r   )r   ro   r   rE   rE   rF   _get_metadata_path  s    zTable._get_metadata_pathz
np.ndarray)ro   r  c             C  s0   t |}| jj| ||d| j| j| jd dS )z
        Write out a metadata array to the key as a fixed-format Series.

        Parameters
        ----------
        key : str
        values : ndarray
        rc   )ru   rI   rz   r{   N)r,   r   r~   r:  rI   rz   r{   )r   ro   r  rE   rE   rF   r    s    zTable.write_metadata)ro   c             C  s0   t t | jdd|ddk	r,| j| |S dS )z'return the meta data array for this keyr   N)r$  r   r   r   r:  )r   ro   rE   rE   rF   r    s    zTable.read_metadatac             C  sp   t | j| j_|  | j_|  | j_| j| j_| j| j_| j| j_| j| j_| j	| j_	| j
| j_
| j| j_dS )zset our table type & indexablesN)rK   r6  r|  r8  r9  r   ry   r{   rI   rz   r	  r1  )r   rE   rE   rF   r    s    





zTable.set_attrsc             C  s   t | jddpg | _t | jddp$g | _t | jddp8i | _t | jdd| _tt | jdd| _tt | jdd| _	t | jd	dpg | _
d
d | jD | _dd | jD | _dS )zretrieve our attributesr   Nry   r1  r{   rI   rz   rf   r	  c             S  s   g | ]}|j r|qS rE   )r  )rR   rm   rE   rE   rF   rT     s    z#Table.get_attrs.<locals>.<listcomp>c             S  s   g | ]}|j s|qS rE   )r  )rR   rm   rE   rE   rF   rT     s    )r$  r|  r   ry   r1  r{   rJ   rI   rG   rz   r	  
indexablesr*  r+  )r   rE   rE   rF   r    s    zTable.get_attrsc             C  s\   |dk	rX| j d dkrX| j d dkrX| j d dk rXtddd | j D  }t|t dS )	z+are we trying to operate on an old version?Nr   rP   r  r  r  c             s  s   | ]}t |V  qd S )N)rK   )rR   r   rE   rE   rF   r    s    z)Table.validate_version.<locals>.<genexpr>)r  incompatibility_docrm  r  r  r_   )r   rX   r  rE   rE   rF   r    s    *zTable.validate_versionc             C  sV   |dkrdS t |tsdS |  }x.|D ]&}|dkr6q(||kr(td| dq(W dS )z
        validate the min_itemsize doesn't contain items that are not in the
        axes this needs data_columns to be defined
        Nr  zmin_itemsize has the key [z%] which is not an axis or data_column)rA   r  r7  r   )r   rw   qr   rE   rE   rF   validate_min_itemsize  s    

zTable.validate_min_itemsizec               s   g }j jjxztjjD ]j\}\}}t|}|}|dk	rLdnd}| d}t|d}	t||||	|j||d}
||
 q W t	j
t|  fdd|fddtjjD  |S )	z/create/cache the indexables if they don't existNr  ri  )rL   r   re  r  rd  rc   r   rf  c               s   t |tstt}|krt}t|}t|j}t| dd }t| dd }t|}	|}t| dd }	||||| |  |j
|	||d
}
|
S )Nri  r  r  )
rL   rc  r  r  re  rd  rc   r   rf  rt  )rA   rK   r   r  r  r$  _maybe_adjust_namer  r  r  rc   )r  r  klassr  adj_namer  rt  r  mdr   rN  )base_posr  descr   table_attrsrE   rF   rd     s.    

zTable.indexables.<locals>.fc               s   g | ]\}} ||qS rE   rE   )rR   r  r  )rd   rE   rF   rT   !  s    z$Table.indexables.<locals>.<listcomp>)r}  rc   r|  r  r8  r$  r  r_  rt   r  ry   rW   r  r9  )r   _indexablesr  r   rL   r  rB  r   rj  r  	index_colrE   )rC  r  rD  rd   r   rE  rF   r;    s0    



# zTable.indexablesz
str | None)r  c          	   C  sV  |   sdS |dkrdS |dks(|dkr8dd | jD }t|ttfsL|g}i }|dk	r`||d< |dk	rp||d< | j}x|D ]}t|j|d}|dk	r |jr|j	}|j
}	|j}
|dk	r|
|kr|  n|
|d< |dk	r|	|kr|  n|	|d< |jsN|jdrtd	|jf | q||| jd
 d kr|td| d| d| dq|W dS )aZ  
        Create a pytables index on the specified columns.

        Parameters
        ----------
        columns : None, bool, or listlike[str]
            Indicate which columns to create an index on.

            * False : Do not create any indexes.
            * True : Create indexes on all columns.
            * None : Create indexes on all columns.
            * listlike : Create indexes on the given columns.

        optlevel : int or None, default None
            Optimization level, if None, pytables defaults to 6.
        kind : str or None, default None
            Kind of index, if None, pytables defaults to "medium".

        Raises
        ------
        TypeError if trying to create an index on a complex-type column.

        Notes
        -----
        Cannot index Time64Col or ComplexCol.
        Pytables must be >= 3.0.
        NFTc             S  s   g | ]}|j r|jqS rE   )r  rc  )rR   rm   rE   rE   rF   rT   H  s    z&Table.create_index.<locals>.<listcomp>r  r  complexzColumns containing complex values can be stored but cannot be indexed when using table format. Either use fixed format, set index=False, or do not include the columns containing complex values to data_columns when initializing the table.r   rP   zcolumn z/ is not a data_column.
In order to read column z: you must reload the dataframe 
into HDFStore and include z  with the data_columns argument.)r   r  rA   rV   rU   rc   r$  r  r,  rv   r  r  Zremove_indexr   r)  r   r   r   ri   )r   r   r  r  kwrc   r  r	  rv   Zcur_optlevelZcur_kindrE   rE   rF   r   %  sF    



zTable.create_indexz
int | Nonez!list[tuple[ArrayLike, ArrayLike]])r   r   r   c       	      C  s^   t | |||d}| }g }x<| jD ]2}|| j |j|| j| j| jd}|	| q$W |S )a  
        Create the axes sniffed from the table.

        Parameters
        ----------
        where : ???
        start : int or None, default None
        stop : int or None, default None

        Returns
        -------
        List[Tuple[index_values, column_values]]
        )rX   r   r   )r{   rI   rz   )
	Selectionr   r  r  r1  rz  r{   rI   rz   rt   )	r   rX   r   r   	selectionr  r^  rm   ZresrE   rE   rF   
_read_axesy  s    
zTable._read_axes)r  c             C  s   |S )zreturn the data for this objrE   )rL  rN  r  rE   rE   rF   
get_object  s    zTable.get_objectc               s   t |sg S |d \} | j|i }|ddkrL|rLtd| d| |dkr^t }n|dkrjg }t|trt|t|}|fdd	|	 D   fd
d	|D S )zd
        take the input data_columns and min_itemize and create a data
        columns spec
        r   r   r*   z"cannot use a multi-index on axis [z] with data_columns TNc               s    g | ]}|d kr| kr|qS )r  rE   )rR   r   )existing_data_columnsrE   rF   rT     s   z/Table.validate_data_columns.<locals>.<listcomp>c               s   g | ]}| kr|qS rE   rE   )rR   r  )axis_labelsrE   rF   rT     s    )
rW   r1  r   r   rU   rA   r  r  r  r   )r   ry   rw   r   r   r1  rE   )rO  rN  rF   validate_data_columns  s$    


zTable.validate_data_columnsr&   )rN  r  c       /        s  t ts,| jj}td| dt d dkr:dg fdd D  |  rzd}d	d | jD  t| j	}| j
}nd
}| j}	| jdkstt | jd krtdg }
|dkrd} fdddD d }j| }t|}|r<t|
}| j| d }tt|t|s<ttt|tt|r<|}|	|i }t|j|d< t|j|d< |
||f  d }j| }|}t||| j| j}||_|d | |	 |!| |g}t|}|dkstt|
dkstx"|
D ]}t"|d |d qW |jdk}| #|||
}| $|% }| &|||
| j'|\}}g }xt(t)||D ]\}\}}t*}d}|rt|dkr|d |krt+}|d }|dkst |t,std|r.|r.y| j'| }W nB t-t.fk
r* }  ztd| d| j' d| W dd} ~ X Y nX nd}|p@d| }!t/|!||||| j| j|d}"t0|!| j1}#|2|"}$t3|"j4j5}%d}&t6|"dddk	rt7|"j8}&d }' }(})t9|"j4r|"j:})d}'tj|"j;d
d< }(t=|"\}*}+||#|!t||$||%|&|)|'|(|+|*d},|, |	 ||, |d7 }qlW dd |D }-t| | j>| j| j| j||
||-|	|d
}.t?| drr| j@|._@|.A| |r|r|.B|  |.S )a0  
        Create and return the axes.

        Parameters
        ----------
        axes: list or None
            The names or numbers of the axes to create.
        obj : DataFrame
            The object to create axes on.
        validate: bool, default True
            Whether to validate the obj against an existing object already written.
        nan_rep :
            A value to use for string column nan_rep.
        data_columns : List[str], True, or None, default None
            Specify the columns that we want to create to allow indexing on.

            * True : Use all available columns.
            * None : Use no columns.
            * List[str] : Use the specified columns.

        min_itemsize: Dict[str, int] or None, default None
            The min itemsize for a column in bytes.
        z/cannot properly create the storer for: [group->z,value->r   Nr   c               s   g | ]}  |qS rE   )_get_axis_number)rR   rm   )rN  rE   rF   rT     s    z&Table._create_axes.<locals>.<listcomp>Tc             S  s   g | ]
}|j qS rE   )r   )rR   rm   rE   rE   rF   rT     s    Fr  rP   z<currently only support ndim-1 indexers in an AppendableTablenanc               s   g | ]}| kr|qS rE   rE   )rR   r   )r  rE   rF   rT     s    )r   rP   r
  r   z-cannot have non-object label DataIndexableColzIncompatible appended table [z]with existing table [Zvalues_block_)existing_colrw   r{   rI   rz   r   ra  r  )r.  )rL   rc  r  rd  re  r  ra  r  r   rf  rt  r-  c             S  s   g | ]}|j r|jqS rE   )r  rL   )rR   r~  rE   rE   rF   rT     s    )
r   r   rI   rz   r*  r   r+  ry   r1  r{   r	  )CrA   r&   r   r   r   r   r   r*  rU   ry   r{   r1  r  r   rW   r   r  r   r%   rB   Zarrayr  r  r
  r[   rt   Z_get_axis_namer   rI   rz   r   rg  r  r  _reindex_axisrP  rM  r   _get_blocks_and_itemsr+  r  r   r  r  rK   
IndexErrorr   _maybe_convert_for_string_atomr?  r  r  r  rt  rL   r$  r  ra  r   r  r  r  r  r   rr  r	  r>  r  )/r   r  rN  r  r{   ry   rw   r   table_existsZnew_infonew_non_index_axesr  rm   Zappend_axisZindexerZ
exist_axisr1  	axis_nameZ	new_indexZnew_index_axesjr  r:  r'  r"  Zvaxesr  r)  b_itemsr@  rL   rS  r  new_namedata_convertedrA  rd  r  ra  r   rf  r  r-  r  r~  ZdcsZ	new_tablerE   )r  rN  rF   _create_axes  s     










 "





zTable._create_axes)r:  rX  c             C  s  t | jtr| d} dd }| j}tt|}t|j}||}t|r|d \}	}
t	|

t	|}| j||	dj}t|j}||}x8|D ]0}| j|g|	dj}||j ||| qW |rzdd t||D }g }g }x|D ]}t|j}y&||\}}|| || W q ttfk
rl } z*dd	d
 |D }td| d|W d d }~X Y qX qW |}|}||fS )Nr  c               s    fdd j D S )Nc               s   g | ]} j |jqS rE   )r   r  r(  )rR   r)  )mgrrE   rF   rT     s    zFTable._get_blocks_and_items.<locals>.get_blk_items.<locals>.<listcomp>)r'  )r`  rE   )r`  rF   get_blk_items  s    z2Table._get_blocks_and_items.<locals>.get_blk_itemsr   )r   c             S  s"   i | ]\}}||ft | qS rE   )rV   tolist)rR   br\  rE   rE   rF   r
    s   z/Table._get_blocks_and_items.<locals>.<dictcomp>rk  c             s  s   | ]}t |V  qd S )N)r;   )rR   itemrE   rE   rF   r    s    z.Table._get_blocks_and_items.<locals>.<genexpr>z+cannot match existing table structure for [z] on appending data)rA   r%  r7   r&  r   r8   rU   r'  rW   r(   r  r  r  r   rV   r  r  rt   rV  r   rm  r   )r:  rX  rY  r+  ry   ra  r`  r'  r"  r   rO  Z
new_labelsr  Zby_itemsZ
new_blocksZnew_blk_itemsZear   rc  r\  r  ZjitemsrE   rE   rF   rU    sH    







zTable._get_blocks_and_itemsrJ  )rK  c       
        s   |dk	rt |}|dk	rRjrRtjt s.tx"jD ]}||kr6|d| q6W x jD ]\}}t ||| qZW |jdk	rx2|j	 D ]$\}} fdd}	|	|| qW  S )zprocess axes filtersNr   c               s   x j D ]} |} |}|d k	s,t| |krdjrJ|tj}||} j|d| S | |krt	t
 | j}t	|}t trd| }||} j|d| S qW td|  dd S )N)r   rP   zcannot find the field [z] for filtering!)Z_AXIS_ORDERSrQ  	_get_axisr   r/  unionr(   r	  r  r6   r$  r  rA   r&   r   )fieldfiltrZ  Zaxis_numberZaxis_valuesZtakersr  )rN  opr   rE   rF   process_filter  s"    




z*Table.process_axes.<locals>.process_filter)
rU   r/  rA   r	  r   insertr   rT  filterru   )
r   rN  rK  r   r   r   labelsrg  rh  rj  rE   )rN  ri  r   rF   process_axes  s    
!zTable.process_axes)rr   r   r  r   c             C  s   |dkrt | jd}d|d}dd | jD |d< |rj|dkrH| jpFd}t j|||pZ| jd	}||d
< n| jdk	r~| j|d
< |S )z:create the description of the table from the axes & valuesNi'  rc   )rL   r  c             S  s   i | ]}|j |jqS rE   )rd  rc  )rR   rm   rE   rE   rF   r
  8  s    z,Table.create_description.<locals>.<dictcomp>r}  	   )rr   rs   r   r   )maxr2  r  r   rl   r   r   r   )r   rs   rr   r   r  r  r   rE   rE   rF   create_description)  s    	




zTable.create_description)r   r   c       
      C  s   |  | |  sdS t| |||d}| }|jdk	rxR|j D ]D\}}}| j|| | d d}	|||	j	||   |j
 }qDW t|S )zf
        select coordinates (row numbers) from a table; return the
        coordinates object
        F)rX   r   r   NrP   )r   r   )r  r   rJ  select_coordsrl  ru   r   r]  rp  ilocr  r(   )
r   rX   r   r   rK  Zcoordsrg  ri  rh  r-  rE   rE   rF   r   H  s    

"zTable.read_coordinates)r   r   r   c             C  s   |    |  sdS |dk	r$tdx| jD ]v}||jkr,|jsPtd| dt| jj	|}|
| j |j||| | j| j| jd}tt|d |j|dS q,W td| d	dS )
zj
        return a single column from the table, generally only indexables
        are interesting
        FNz4read_column does not currently accept a where clausezcolumn [z=] can not be extracted individually; it is not data indexable)r{   rI   rz   rP   )rL   z] not found in the table)r  r   r   r  rL   r  r   r$  rc   r  r  r1  rz  r{   rI   rz   r,   rw  ra  r   )r   r   rX   r   r   rm   r  Z
col_valuesrE   rE   rF   r   b  s&    


zTable.read_column)Nrf   NNNNNN)N)NNN)NN)TNNN)N)NNN)NNN)4r[   r\   r]   rZ  r  r  r[  r	  r   r   r\  r-  r   r   r  r/  r1  r2  rP  r  rc   rt  r}  r  r.  r4  r5  r7  r8  r9  r:  r  r  r  r  r  r>  r   r;  r   rL  r  rM  rP  r_  staticmethodrU  rn  rq  r   r   r  rE   rE   )r  rF   r     s~   
       


JU!*    d?:   r   c               @  s.   e Zd ZdZdZd
dddddZdd	 ZdS )rK  z
    a write-once read-many table: this format DOES NOT ALLOW appending to a
    table. writing is a one-time operation the data are stored in a format
    that allows for searching the data on disk
    rA  Nz
int | None)r   r   c             C  s   t ddS )z[
        read the indices and the indexing array, calculate offset rows and return
        z!WORMTable needs to implement readN)r   )r   rX   r   r   r   rE   rE   rF   r     s    
zWORMTable.readc             K  s   t ddS )z
        write in a format that we can search later on (but cannot append
        to): write out the indices and the values using _write_array
        (e.g. a CArray) create an indexing table so that we can search
        z"WORMTable needs to implement writeN)r   )r   r   rE   rE   rF   rR    s    zWORMTable.write)NNNN)r[   r\   r]   rZ  r6  r   rR  rE   rE   rE   rF   rK    s      rK  c               @  sV   e Zd ZdZdZdddZddd	d
ddZdddddddZddddddZdS )r  z(support the new appendable table formatsZ
appendableNFTc             C  s   |s| j r| j| jd | j||||||d}x|jD ]}|  q8W |j s|j||||	d}|  ||d< |jj	|jf| |j
|j_
x|jD ]}||| qW |j||
d d S )Nrc   )r  rN  r  rw   r{   ry   )rs   rr   r   r  r   )rx   )rP  r   rS  r   r_  r  r  rq  r  Zcreate_tabler1  r|  r  
write_data)r   rN  r  rt   rs   rr   r   rw   r   r  rx   r{   ry   r   rc   rm   ZoptionsrE   rE   rF   rR    s0    

zAppendableTable.writez
int | Nonern   )r   rx   c               s  | j j}| j}g }|rXx@| jD ]6}t|jjdd}t|tj	r|
|jddd qW t|r|d }x|dd D ]}||@ }qvW | }nd}dd	 | jD }	t|	}
|
dkst|
d
d	 | jD }dd	 |D }g }xBt|D ]6\}}|f| j ||
|   j }|
|| | qW |dkr0d}tjt||| j d}|| d }xt|D ]t}|| t|d | |  krP | j| fdd	|	D |dk	r|  nd fdd	|D d q\W dS )z`
        we form the data into a 2-d including indexes,values,mask write chunk-by-chunk
        r   )r   Zu1F)r.  rP   Nc             S  s   g | ]
}|j qS rE   )r  )rR   rm   rE   rE   rF   rT     s    z.AppendableTable.write_data.<locals>.<listcomp>c             S  s   g | ]}|  qS rE   )r{  )rR   rm   rE   rE   rF   rT     s    c          	   S  s,   g | ]$}| tt|j|jd  qS )rP   )Z	transposerB   Zrollr  r  )rR   r	  rE   rE   rF   rT     s    i )rt  c               s   g | ]}|  qS rE   rE   )rR   rm   )end_istart_irE   rF   rT   #  s    c               s   g | ]}|  qS rE   rE   )rR   r	  )rv  rw  rE   rF   rT   %  s    )indexesr  r  )rt  r
  r2  r+  r/   r-  r  rA   rB   rs  rt   r  rW   r  r*  r   r  r  reshaperM  r]  r  write_data_chunk)r   r   rx   r
  r   Zmasksrm   r  mrx  nindexesr  bvaluesr  r	  Z	new_shaperowsZchunksrE   )rv  rw  rF   ru    sJ    


zAppendableTable.write_dataz
np.ndarrayzlist[np.ndarray]znp.ndarray | None)r~  rx  r  r  c             C  s   x|D ]}t |jsdS qW |d jd }|t|krHt j|| jd}| jj}t|}x t|D ]\}	}
|
|||	 < qbW x$t|D ]\}	}||||	|  < qW |dk	r| j	t
dd }| s|| }t|r| j| | j  dS )z
        Parameters
        ----------
        rows : an empty memory space where we are putting the chunk
        indexes : an array of the indexes
        mask : an array of the masks
        values : an array of the values
        Nr   )rt  F)r.  )rB   r  r  rW   rM  rt  r
  r  r  r  rn   r  rc   rt   r   )r   r~  rx  r  r  r	  r   r
  r|  r  r  r{  rE   rE   rF   rz  (  s&    
z AppendableTable.write_data_chunk)r   r   c             C  sf  |d kst |sf|d kr:|d kr:| j}| jj| jdd n(|d krH| j}| jj||d}| j  |S |  srd S | j}t	| |||d}|
 }t| }t |}	|	rb| }
t|
|
dk j}t |sdg}|d |	kr||	 |d dkr|dd | }xLt|D ]@}|t||}|j||jd  ||jd  d d |}qW | j  |	S )NT)r   )r   r   rP   r   r  )rW   r   r   rS  r   rc   Zremove_rowsr   r   rJ  rr  r,   Zsort_valuesdiffrU   rv   rt   rk  r  reversedr  r  )r   rX   r   r   r   rc   rK  r  Zsorted_seriesZlnr  r   Zpgr   r~  rE   rE   rF   r   T  sB    

"

zAppendableTable.delete)NFNNNNNNFNNT)F)NNN)	r[   r\   r]   rZ  r6  rR  ru  rz  r   rE   rE   rE   rF   r    s"              
.;,r  c               @  s`   e Zd ZU dZdZdZdZeZde	d< e
ddd	d
ZeddddZddddddZdS )rI  z(support the new appendable table formatsr7  r?  r  ztype[FrameOrSeriesUnion]r  rn   )r   c             C  s   | j d jdkS )Nr   rP   )r*  r   )r   rE   rE   rF   r4    s    z"AppendableFrameTable.is_transposed)r  c             C  s   |r
|j }|S )zthese are written transposed)r  )rL  rN  r  rE   rE   rF   rM    s    zAppendableFrameTable.get_objectNz
int | None)r   r   c               s6    |   sd S  j|||d}t jrH j jd d i ni } fddt jD }t|dkstt	|d }|| d }	g }
x^t jD ]N\}}| j
krq|| \}}|ddkrt|}nt|}|d}|d k	r|j|d	d
  jr$|}|}t|	t|	dd d}n|j}t|	t|	dd d}|}|jdkrpt|tjrp|d|jd f}t|tjrt|j||d}n.t|trt|||d}ntj|g||d}|j|jk st	|j|jf|
| qW t|
dkr|
d }nt|
dd}t |||d} j |||d}|S )N)rX   r   r   r   c               s"   g | ]\}}| j d  kr|qS )r   )r*  )rR   r  r!  )r   rE   rF   rT     s    z-AppendableFrameTable.read.<locals>.<listcomp>rP   r   r*   r
  T)inplacerL   )rL   )r   rv   )r   )rK  r   )!r  r   rL  rW   r   r1  r   r  r  r   r+  r*   Zfrom_tuplesr(   	set_namesr4  r$  r  r  rA   rB   rs  ry  r  r&   Z_from_arraysZdtypesrt  r  rt   r.   rJ  rn  )r   rX   r   r   r   r  r1  ZindsZindrv   Zframesr  rm   Z
index_valsr  r  r
  r  Zindex_Zcols_r#  rK  rE   )r   rF   r     sV    	
"

"
zAppendableFrameTable.read)NNNN)r[   r\   r]   rZ  r  r6  r  r&   r  r[  r\  r4  r  rM  r   rE   rE   rE   rF   rI    s   
	   rI  c                  sn   e Zd ZdZdZdZdZeZe	ddddZ
edd	d
dZd fdd	Zddddd fddZ  ZS )rG  z(support the new appendable table formatsr<  r=  r  rn   )r   c             C  s   dS )NFrE   )r   rE   rE   rF   r4    s    z#AppendableSeriesTable.is_transposed)r  c             C  s   |S )NrE   )rL  rN  r  rE   rE   rF   rM    s    z AppendableSeriesTable.get_objectNc               s<   t |ts|jpd}||}t jf ||j d|S )z+we are going to write this as a frame tabler  )rN  ry   )rA   r&   rL   Zto_framer  rR  r   rb  )r   rN  ry   r   rL   )r  rE   rF   rR    s    


zAppendableSeriesTable.writez
int | Noner,   )r   r   r   c               s   | j }|d k	rF|rFt| jts"tx"| jD ]}||kr*|d| q*W t j||||d}|rn|j| jdd |j	d d df }|j
dkrd |_
|S )Nr   )rX   r   r   r   T)r  r  )r/  rA   r	  rU   r   rk  r  r   	set_indexrs  rL   )r   rX   r   r   r   r/  r   rD   )r  rE   rF   r     s    
zAppendableSeriesTable.read)N)NNNN)r[   r\   r]   rZ  r  r6  r  r,   r  r\  r4  r  rM  rR  r   r  rE   rE   )r  rF   rG    s   	   rG  c                  s(   e Zd ZdZdZdZ fddZ  ZS )rH  z(support the new appendable table formatsr<  r>  c               s^   |j pd}| |\}| _t| jts*tt| j}|| t||_t	 j
f d|i|S )z+we are going to write this as a frame tabler  rN  )rL   r1  r	  rA   rU   r   rt   r(   r   r  rR  )r   rN  r   rL   Znewobjr  )r  rE   rF   rR  ,  s    



z AppendableMultiSeriesTable.write)r[   r\   r]   rZ  r  r6  rR  r  rE   rE   )r  rF   rH  &  s   rH  c               @  sd   e Zd ZU dZdZdZdZeZde	d< e
ddd	d
Ze
dd Zdd Zedd Zdd ZdS )rF  z:a table that read/writes the generic pytables table formatr7  r8  r  zlist[Hashable]r	  rK   )r   c             C  s   | j S )N)r  )r   rE   rE   rF   r"  @  s    zGenericTable.pandas_typec             C  s   t | jdd p| jS )Nrc   )r$  r   )r   rE   rE   rF   r  D  s    zGenericTable.storablec             C  sL   g | _ d| _g | _dd | jD | _dd | jD | _dd | jD | _dS )zretrieve our attributesNc             S  s   g | ]}|j r|qS rE   )r  )rR   rm   rE   rE   rF   rT   N  s    z*GenericTable.get_attrs.<locals>.<listcomp>c             S  s   g | ]}|j s|qS rE   )r  )rR   rm   rE   rE   rF   rT   O  s    c             S  s   g | ]
}|j qS rE   )rL   )rR   rm   rE   rE   rF   rT   P  s    )r   r{   r	  r;  r*  r+  ry   )r   rE   rE   rF   r  H  s    zGenericTable.get_attrsc       
   
   C  s   | j }| d}|dk	rdnd}tdd| j||d}|g}xlt|jD ]^\}}t|ts\tt	||}| |}|dk	r|dnd}t
|||g|| j||d}	||	 qFW |S )z0create the indexables from the table descriptionrv   Nr  r   )rL   r   rc   r   rf  )rL   re  r  rd  rc   r   rf  )r}  r  r  rc   r  Z_v_namesrA   rK   r   r$  r  rt   )
r   r  rB  r   rG  rF  r  r   r  r  rE   rE   rF   r;  R  s*    


zGenericTable.indexablesc             K  s   t dd S )Nz cannot write on an generic table)r   )r   r   rE   rE   rF   rR  u  s    zGenericTable.writeN)r[   r\   r]   rZ  r  r6  r  r&   r  r[  r\  r"  r  r  r   r;  rR  rE   rE   rE   rF   rF  7  s   

#rF  c                  s`   e Zd ZdZdZeZdZe	dZ
eddddZd fd
d	Zdddd fddZ  ZS )rJ  za frame with a multi-indexr@  r  z^level_\d+$rK   )r   c             C  s   dS )NZappendable_multirE   )r   rE   rE   rF   r-    s    z*AppendableMultiFrameTable.table_type_shortNc               s|   |d krg }n|dkr |j  }| |\}| _t| jts@tx"| jD ]}||krH|d| qHW t j	f ||d|S )NTr   )rN  ry   )
r   rb  r1  r	  rA   rU   r   rk  r  rR  )r   rN  ry   r   r   )r  rE   rF   rR    s    
zAppendableMultiFrameTable.writez
int | None)r   r   c               sD   t  j||||d}| j}|j fdd|jjD |_|S )N)rX   r   r   r   c               s    g | ]} j |rd n|qS )N)
_re_levelssearch)rR   rL   )r   rE   rF   rT     s    z2AppendableMultiFrameTable.read.<locals>.<listcomp>)r  r   r  r	  rv   r  r
  )r   rX   r   r   r   r#  )r  )r   rF   r     s
    zAppendableMultiFrameTable.read)N)NNNN)r[   r\   r]   rZ  r6  r&   r  r  recompiler  r\  r-  rR  r   r  rE   rE   )r  rF   rJ  y  s   
   rJ  r&   r(   )rN  r   rm  r   c             C  s   |  |}t|}|d k	r"t|}|d ks4||rB||rB| S t| }|d k	rlt| j|dd}||std d g| j }|||< | jt| } | S )NF)sort)	re  r6   Zequalsuniquer  slicer  r  rV   )rN  r   rm  ro  r!  ZslicerrE   rE   rF   rT    s    

rT  r   zstr | tzinfo)ra  r   c             C  s   t | }|S )z+for a tz-aware type, return an encoded zone)r   Zget_timezone)ra  ZzonerE   rE   rF   r    s    
r  znp.ndarray | Indexzstr | tzinfo | Noneznp.ndarray | DatetimeIndex)r  ra  r  r   c             C  s   t | tr"| jdks"| j|ks"t|dk	rtt | trB| j}| j} nd}|  } t|}t| |d} | d	|} n|rt
j| dd} | S )a  
    coerce the values to a DatetimeIndex if tz is set
    preserve the input shape if possible

    Parameters
    ----------
    values : ndarray or Index
    tz : str or tzinfo
    coerce : if we do not have a passed timezone, coerce to M8[ns] ndarray
    N)rL   r  zM8[ns])rt  )rA   r'   ra  r   rL   r  r  rG   r  r  rB   r  )r  ra  r  rL   rE   rE   rF   rw    s    

rw  )rL   rv   rI   rz   r   c          
   C  st  t | tst|j}t|\}}t|}t|}t |tsFt	|j
rlt| |||t|dd t|dd |dS t |tr~tdtj|dd}	t|}
|	dkrtjdd	 |
D tjd
}t| |dt  |dS |	dkrt|
||}|j
j}t| |dt ||dS |	dkr$t| ||||dS t |tjr>|j
tksBt|dksTt|t  }t| ||||dS d S )Nr`  ra  )r  r  rd  r`  ra  rb  zMultiIndex not supported here!F)r  r   c             S  s   g | ]}|  qS rE   )Z	toordinal)rR   r	  rE   rE   rF   rT     s    z"_convert_index.<locals>.<listcomp>)rt  )rb  r  )integerZfloating)r  r  rd  rb  r  )rA   rK   r   rL   r  r  r  r  r)   r$   rt  r_  r$  r*   r   r   r  rB   r  Zint32rl   Z	Time32Col_convert_string_arrayrh  r  rs  r  r  )rL   rv   rI   rz   rb  r  r  r  r  r  r  rh  rE   rE   rF   r     sL    








r   )r  rI   rz   r   c             C  s   |dkrt | }n|dkr$t| }n|dkrxytjdd | D td}W q tk
rt   tjdd | D td}Y qX nT|dkrt| }n@|d	krt| d ||d
}n&|dkrt| d }ntd| |S )Nr  r  r   c             S  s   g | ]}t |qS rE   )r   r  )rR   r	  rE   rE   rF   rT   0  s    z$_unconvert_index.<locals>.<listcomp>)rt  c             S  s   g | ]}t |qS rE   )r   r  )rR   r	  rE   rE   rF   rT   2  s    )r  floatr  )r{   rI   rz   r  r   zunrecognized index type )r'   r-   rB   r  r  r   r  )r-  r  rI   rz   rv   rE   rE   rF   r  '  s$    

 r  r?   z	list[str])rL   r  r   c             C  s  |j }|jtkr|S |jj}	tj|dd}
|
dkr<tdn&|
dkrNtdn|
dksb|	dksb|S |j|dd	}t|d
kst	|d }|j }tj|dd}
|
dkrxjt
|jd D ]X}||}tj|dd}
|
dkrt||kr|| nd| }td| d|
 dqW t||||j}|j}t|trXt|| pT|dpTd}t|pbd|}|d k	r||}|d k	r||kr|}|jd| dd}|S )NF)r  r   z+[date] is not implemented as a table columnr  z>too many timezones in this block, create separate data columnsr  r  )ZdowncastrP   r   zNo.zCannot serialize the column [z2]
because its data contents are not [string] but [z] object dtyper  z|S)r.  )r  rt  r  rL   r   r  r   ZfillnarW   r   r  r  Zigetr  ry  rh  rA   r  rN   r   rp  r  r  )rL   r  rS  rw   r{   rI   rz   r   r}  r  r  r'  r-  r  r~  Zerror_column_labelr^  rh  ZecirE   rE   rF   rW  @  sL    




	
 

rW  z
np.ndarray)r-  rI   rz   r   c             C  s\   t | r(t|  j||j| j} t|  }t	dt
|}tj| d| d} | S )a  
    Take a string-like that is object dtype and coerce to a fixed size string type.

    Parameters
    ----------
    data : np.ndarray[object]
    encoding : str
    errors : str
        Handler for encoding errors.

    Returns
    -------
    np.ndarray[fixed-length-string]
    rP   S)rt  )rW   r,   r  rK   encoder  ry  r  r   rp  
libwritersmax_len_string_arrayrB   r  )r-  rI   rz   Zensuredrh  rE   rE   rF   r    s    
r  c             C  s   | j }tj|  td} t| rvtt| }d| }t	| d t
r^t| jj||dj} n| j|ddjtdd} |dkrd}t| | | |S )	a*  
    Inverse of _convert_string_array.

    Parameters
    ----------
    data : np.ndarray[fixed-length-string]
    nan_rep : the storage repr of NaN
    encoding : str
    errors : str
        Handler for encoding errors.

    Returns
    -------
    np.ndarray[object]
        Decoded data.
    )rt  Ur   )rz   F)r.  NrR  )r  rB   r  r  r  rW   r  r  r   rA   r  r,   rK   rC   r  r  Z!string_array_replace_from_nan_repry  )r-  r{   rI   rz   r  rh  rt  rE   rE   rF   r    s    
r  )r  rx  rI   rz   c             C  s6   t |tstt|t|r2t|||}|| } | S )N)rA   rK   r   r   _need_convert_get_converter)r  rx  rI   rz   ZconvrE   rE   rF   rv    s
    rv  )r  rI   rz   c               s8   | dkrdd S | dkr& fddS t d|  d S )Nr  c             S  s   t j| ddS )NzM8[ns])rt  )rB   r  )r   rE   rE   rF   r}         z _get_converter.<locals>.<lambda>r  c               s   t | d  dS )N)r{   rI   rz   )r  )r   )rI   rz   rE   rF   r}     s   zinvalid kind )r   )r  rI   rz   rE   )rI   rz   rF   r    s
    r  )r  r   c             C  s   | dkrdS dS )N)r  r  TFrE   )r  rE   rE   rF   r    s    r  zSequence[int])rL   r  r   c             C  sl   t |tst|dk rtd|d dkrh|d dkrh|d dkrhtd| }|rh| d }d| } | S )	z
    Prior to 0.10.1, we named values blocks like: values_block_0 an the
    name values_0, adjust the given name if necessary.

    Parameters
    ----------
    name : str
    version : Tuple[int, int, int]

    Returns
    -------
    str
       z6Version is incorrect, expected sequence of 3 integers.r   rP   r  r  zvalues_block_(\d+)Zvalues_)rA   rK   rW   r   r  r  r   )rL   r  r{  ZgrprE   rE   rF   r?    s    $
r?  )	dtype_strr   c             C  s   t | } | ds| dr"d}n| dr2d}n| drBd}n| dsV| dr\d}nn| drld}n^| d	r|d
}nN| drd}n>| drd}n.| drd}n| dkrd}ntd|  d|S )zA
    Find the "kind" string describing the given dtype name.
    r  r  r  rH  rN   r  r  r  Z	timedeltar  rn   r  r  r  zcannot interpret dtype of [r   )rG   r)  r   )r  r  rE   rE   rF   r  
  s.    






r  r   )r-  c             C  sb   t | tr| j} | jjdd }| jjdkr@t| 	d} nt | t
rP| j} t| } | |fS )zJ
    Convert the passed data into a storable form and a dtype string.
    r  r   )r{  Mr  )rA   r0   r  rt  rL   rU  r  rB   r  r  r+   r  )r-  r  rE   rE   rF   r  +  s    


r  c               @  s<   e Zd ZdZdddddddZdd	 Zd
d Zdd ZdS )rJ  z
    Carries out a selection operation on a tables.Table object.

    Parameters
    ----------
    table : a Table object
    where : list of Terms (or convertible to)
    start, stop: indices to start and/or stop selection

    Nr   z
int | None)rc   r   r   c          	   C  sR  || _ || _|| _|| _d | _d | _d | _d | _t|rt	t
 tj|dd}|dksd|dkrt|}|jtjkr| j| j }}|d krd}|d kr| j j}t||| | _nVt|jjtjr| jd k	r|| jk  s | jd k	r|| jk rt
d|| _W d Q R X | jd krN| || _| jd k	rN| j \| _| _d S )NF)r  r  Zbooleanr   z3where must have index locations >= start and < stop)rc   rX   r   r   	conditionrl  Ztermsr   r!   r   r   r   r  rB   r  rt  Zbool_r   r  
issubclassr   r  r  generateZevaluate)r   rc   rX   r   r   ZinferredrE   rE   rF   r   M  s<    


zSelection.__init__c          
   C  s~   |dkrdS | j  }yt||| j jdS  tk
rx } z2d| }td| d| d}t||W dd}~X Y nX dS )z'where can be a : dict,list,tuple,stringN)r7  rI   rk  z-                The passed where expression: a*  
                            contains an invalid variable reference
                            all of the variable references must be a reference to
                            an axis (e.g. 'index' or 'columns'), or a data_column
                            The currently defined references are: z
                )	rc   r7  r3   rI   	NameErrorrm  r   r   r   )r   rX   r=  r  Zqkeysr   rE   rE   rF   r  |  s    
zSelection.generatec             C  sX   | j dk	r(| jjj| j  | j| jdS | jdk	rB| jj| jS | jjj| j| jdS )z(
        generate the selection
        N)r   r   )	r  rc   Z
read_whereru   r   r   r   r   r   )r   rE   rE   rF   r     s    

zSelection.selectc             C  s   | j | j }}| jj}|dkr$d}n|dk r4||7 }|dkrB|}n|dk rR||7 }| jdk	rx| jjj| j ||ddS | jdk	r| jS t	||S )z(
        generate the selection
        Nr   T)r   r   r  )
r   r   rc   r   r  Zget_where_listru   r   rB   r  )r   r   r   r   rE   rE   rF   rr    s     

zSelection.select_coords)NNN)r[   r\   r]   rZ  r   r  r   rr  rE   rE   rE   rF   rJ  A  s   
  *rJ  )rm   NNFNTNNNNrf   r@   )	Nr   rf   NNNNFN)N)F)rZ  Z
__future__r   
contextlibr   r.  r  r   r   r   r   r  textwrapr   typingr   r   r	   r
   r   r   r  ZnumpyrB   Zpandas._configr   r   Zpandas._libsr   r   r  Zpandas._libs.tslibsr   Zpandas._typingr   r   r   r   r   Zpandas.compat._optionalr   Zpandas.compat.pickle_compatr   Zpandas.errorsr   Zpandas.util._decoratorsr   Zpandas.core.dtypes.commonr   r   r   r   r   r    r!   r"   r#   r$   Zpandas.core.dtypes.missingr%   r   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   Zpandas.core.arraysr0   r1   r2   Zpandas.core.commonZcorecommonr   Z pandas.core.computation.pytablesr3   r4   Zpandas.core.constructionr5   Zpandas.core.indexes.apir6   Zpandas.core.internalsr7   r8   Zpandas.io.commonr9   Zpandas.io.formats.printingr:   r;   rh   r<   r=   r>   r?   r  rH   rG   rJ   rM   rQ   rY   r   rZ   r^   Warningr_   r<  r`   r  ra   Zduplicate_docr  r2  r  Z
dropna_docZ
format_docZconfig_prefixZregister_optionZis_boolZis_one_of_factoryrg   rk   rl   r   r   r   r   r   r_  r  r  r  r  r  r  rC  r  rD  r   rK  r  rI  rG  rH  rF  rJ  rT  r  rw  r   r  rW  r  r  rv  r  r  r?  r  r  rJ  rE   rE   rE   rF   <module>   s$   00
           (+                   Np      1  b _       l fd1B+	%=R'!