3
qd                @   s  d Z ddlZddlmZmZ ddlZddlZddlZddlm	Z	m
Z
mZmZmZmZmZmZ ddlZddlZddlmZmZ ddlmZmZ ddlmZ ddlmZmZm Z  dd	l!m"Z" dd
l#m$Z$ ddl%m&Z& ddl'm(Z( ddl)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2 ddl3m4Z4 ddl5m6Z6 ddl7m8Z8m9Z9m:Z:m;Z;m<Z<m=Z=m>Z>m?Z?m@Z@mAZA ddlBmCZCmDZDmEZE ddlFjGjHZIddlJmKZKmLZL ddlMmNZN ddlOmPZP ddlQmRZRmSZS e	rddlTmUZUmVZVmWZW dZXdZYdd ZZdd Z[dd Z\eKZ]e^dd d!Z_G d"d# d#e`ZaG d$d% d%e`ZbG d&d' d'ecZdd(ZeG d)d* d*ecZfd+ZgG d,d- d-ecZhd.Zid/Zjd0d0d1d1d2Zke8dgiZld3Zmd4Znejod58 ejpd6d7emejqd8 ejpd9denejrd0d1dgd8 W dQ R X dasd7atd:d; Zudeveevee^ eev eweev eweee^eeve^f f  eew eeeweev f  evevd?d@dAZxdevevee^ ee^ ee^ dCdDdEZydFdFewdGdHdIZzG dJdK dKZ{G dLdM dMZ|G dNdO dOZ}G dPdQ dQe}Z~G dRdS dSe}ZG dTdU dUeZG dVdW dWeZG dXdY dYZG dZd[ d[eZG d\d] d]eZG d^d_ d_eZG d`da daeZG dbdc dceZG ddde deeZG dfdg dgeZG dhdi dieZG djdk dkeZG dldm dmeZG dndo doeZG dpdq dqeZde8e^e:e8drdsdtZeeevef dudvdwZdeeje:f eeevef  eweeje9f dxdydzZeve:eveve}d{d|d}Zeveveveeje:f d~ddZevdddZejevevejdddZejevevejdddZejevevevdddZevevevdddZevewdddZevevdddZevevdddZedddZG dd dZdS )zY
High level interface to PyTables for reading and writing pandas data structures
to disk
    N)datetzinfo)TYPE_CHECKINGAnyDictListOptionalTupleTypeUnion)config
get_option)libwriters)	timezones)	ArrayLikeFrameOrSeriesLabel)import_optional_dependency)patch_pickle)PerformanceWarning)cache_readonly)	ensure_objectis_categorical_dtypeis_complex_dtypeis_datetime64_dtypeis_datetime64tz_dtypeis_extension_array_dtypeis_list_likeis_string_dtypeis_timedelta64_dtype)ABCExtensionArray)array_equivalent)
	DataFrameDatetimeIndexIndex
Int64Index
MultiIndexPeriodIndexSeriesTimedeltaIndexconcatisna)CategoricalDatetimeArrayPeriodArray)PyTablesExprmaybe_expression)ensure_index)stringify_path)adjoinpprint_thing)ColFileNodez0.15.2UTF-8c             C   s   t | tjr| jd} | S )z* if we have bytes, decode them to unicode zUTF-8)
isinstancenpbytes_decode)s r?   4/tmp/pip-build-7vycvbft/pandas/pandas/io/pytables.py_ensure_decodedF   s    
rA   c             C   s   | d krt } | S )N)_default_encoding)encodingr?   r?   r@   _ensure_encodingM   s    rD   c             C   s   t | trt| } | S )z
    Ensure that an index / column name is a str (python 3); otherwise they
    may be np.string dtype. Non-string dtypes are passed through unchanged.

    https://github.com/pandas-dev/pandas/issues/13492
    )r:   str)namer?   r?   r@   _ensure_strU   s    
rG   )scope_levelc             C   s   |d }t | ttfr^g }x<tdd | D ]*}t|sB|j| q*|jt||d q*W |} nt| rrt| |d} | dkst| r| S dS )z
    ensure that the where is a Term or a list of Term
    this makes sure that we are capturing the scope of variables
    that are passed
    create the terms here with a frame_level=2 (we are 2 levels down)
       c             S   s   | d k	S )Nr?   )xr?   r?   r@   <lambda>p   s    z_ensure_term.<locals>.<lambda>)rH   N)r:   listtuplefilterr1   appendTermlen)whererH   levelZwlistwr?   r?   r@   _ensure_termd   s    	rU   c               @   s   e Zd ZdS )PossibleDataLossErrorN)__name__
__module____qualname__r?   r?   r?   r@   rV   {   s   rV   c               @   s   e Zd ZdS )ClosedFileErrorN)rW   rX   rY   r?   r?   r?   r@   rZ      s   rZ   c               @   s   e Zd ZdS )IncompatibilityWarningN)rW   rX   rY   r?   r?   r?   r@   r[      s   r[   z
where criteria is being ignored as this version [%s] is too old (or
not-defined), read the file in and write it out to a new file to upgrade (with
the copy_to method)
c               @   s   e Zd ZdS )AttributeConflictWarningN)rW   rX   rY   r?   r?   r?   r@   r\      s   r\   zu
the [%s] attribute of the existing index is [%s] which conflicts with the new
[%s], resetting the attribute to None
c               @   s   e Zd ZdS )DuplicateWarningN)rW   rX   rY   r?   r?   r?   r@   r]      s   r]   z;
duplicate entries in table, taking most recently appended
z
your performance may suffer as PyTables will pickle object types that it cannot
map directly to c-types [inferred_type->%s,key->%s] [items->%s]
fixedtable)fr^   tr_   z;
: boolean
    drop ALL nan rows when appending to a table
z~
: format
    default format writing format, if None, then
    put will default to 'fixed' and append will default to 'table'
zio.hdfZdropna_tableF)Z	validatordefault_formatc              C   s>   t d kr:dd l} | a y| jjdkaW n tk
r8   Y nX t S )Nr   strict)
_table_modtablesfileZ_FILE_OPEN_POLICY!_table_file_open_policy_is_strictAttributeError)re   r?   r?   r@   _tables   s    ri   aTrc   )keyvaluemode	complevelcomplibrO   formatindexmin_itemsizedropnadata_columnserrorsrC   c          
      s   |r$ 	f
dd}n 	f	dd}t | } t| trxt| |||d}|| W dQ R X n||  dS )z- store this object, close it if we opened it c                s   | j 	 d
S )N)rp   rq   rr   nan_reprs   rt   ru   rC   )rO   )store)
rt   rs   rC   ru   rp   rq   rk   rr   rv   rl   r?   r@   rK      s   zto_hdf.<locals>.<lambda>c                s   | j  d	S )N)rp   rq   rr   rv   rt   ru   rC   )put)rw   )	rt   rC   ru   rp   rq   rk   rr   rv   rl   r?   r@   rK      s   )rm   rn   ro   N)r3   r:   rE   HDFStore)path_or_bufrk   rl   rm   rn   ro   rO   rp   rq   rr   rv   rs   rt   ru   rC   r`   rw   r?   )
rt   rs   rC   ru   rp   rq   rk   rr   rv   rl   r@   to_hdf   s     
r{   r)rm   ru   startstop	chunksizec
             K   s  |dkrt d| d|dk	r,t|dd}t| trN| jsDtd	| }d
}nvt| } t| tshtdyt	j
j| }W n tt fk
r   d
}Y nX |std|  dt| f||d|
}d}yz|dkr&|j }t|dkrt d|d }x*|dd D ]}t||st dqW |j}|j|||||||	|dS  t ttfk
r   t| tsy|j  W n tk
r   Y nX  Y nX dS )a
  
    Read from the store, close it if we opened it.

    Retrieve pandas object stored in file, optionally based on where
    criteria.

    .. warning::

       Pandas uses PyTables for reading and writing HDF5 files, which allows
       serializing object-dtype data with pickle when using the "fixed" format.
       Loading pickled data received from untrusted sources can be unsafe.

       See: https://docs.python.org/3/library/pickle.html for more.

    Parameters
    ----------
    path_or_buf : str, path object, pandas.HDFStore or file-like object
        Any valid string path is acceptable. The string could be a URL. Valid
        URL schemes include http, ftp, s3, and file. For file URLs, a host is
        expected. A local file could be: ``file://localhost/path/to/table.h5``.

        If you want to pass in a path object, pandas accepts any
        ``os.PathLike``.

        Alternatively, pandas accepts an open :class:`pandas.HDFStore` object.

        By file-like object, we refer to objects with a ``read()`` method,
        such as a file handler (e.g. via builtin ``open`` function)
        or ``StringIO``.
    key : object, optional
        The group identifier in the store. Can be omitted if the HDF file
        contains a single pandas object.
    mode : {'r', 'r+', 'a'}, default 'r'
        Mode to use when opening the file. Ignored if path_or_buf is a
        :class:`pandas.HDFStore`. Default is 'r'.
    errors : str, default 'strict'
        Specifies how encoding and decoding errors are to be handled.
        See the errors argument for :func:`open` for a full list
        of options.
    where : list, optional
        A list of Term (or convertible) objects.
    start : int, optional
        Row number to start selection.
    stop  : int, optional
        Row number to stop selection.
    columns : list, optional
        A list of columns names to return.
    iterator : bool, optional
        Return an iterator object.
    chunksize : int, optional
        Number of rows to include in an iteration when using an iterator.
    **kwargs
        Additional keyword arguments passed to HDFStore.

    Returns
    -------
    item : object
        The selected object. Return type depends on the object stored.

    See Also
    --------
    DataFrame.to_hdf : Write a HDF file from a DataFrame.
    HDFStore : Low-level access to HDF files.

    Examples
    --------
    >>> df = pd.DataFrame([[1, 1.0, 'a']], columns=['x', 'y', 'z'])
    >>> df.to_hdf('./store.h5', 'data')
    >>> reread = pd.read_hdf('./store.h5')
    r|   r+rj   zmode zG is not allowed while performing a read. Allowed modes are r, r+ and a.NrI   )rH   z&The HDFStore must be open for reading.Fz5Support for generic buffers has not been implemented.zFile z does not exist)rm   ru   Tr   z]Dataset(s) incompatible with Pandas data types, not table, or no datasets found in HDF5 file.z?key must be provided when HDF5 file contains multiple datasets.)rR   r}   r~   columnsiteratorr   
auto_close)r|   r   rj   )
ValueErrorrU   r:   ry   is_openIOErrorr3   rE   NotImplementedErrorospathexists	TypeErrorFileNotFoundErrorgroupsrQ   _is_metadata_of_v_pathnameselectKeyErrorcloserh   )rz   rk   rm   ru   rR   r}   r~   r   r   r   kwargsrw   r   r   r   Zcandidate_only_groupZgroup_to_checkr?   r?   r@   read_hdf  sd    S



r   r8   )groupparent_groupreturnc             C   sJ   | j |j krdS | }x0|j dkrD|j}||kr<|jdkr<dS |j}qW dS )zDCheck if a given group is a metadata group for a given parent_group.FrI   metaT)Z_v_depthZ	_v_parent_v_name)r   r   currentparentr?   r?   r@   r     s    
r   c               @   s.  e Zd ZU dZed ee	e
dveee e
dddZd	d
 Zedd Zedd ZedddZedddZedddZedddZee
dddZedddZeddd Zd!d" Zd#d$ Zdweee d&d'd(Zd)d* Zd+d, ZeZdxed-d.d/Zd0d1 Z ee
dd2d3Z!dye
d4d5d6Z"edd7d8Z#dzee
d9d:d;Z$d{eee ee d<d=d>Z%d|eeee ee d?d@dAZ&d}e
dBdCdDZ'd~ee(ee ee)ee*eef f  eee  ee
dGdHdIZ+deddJdKZ,dee(ee ee)ee*eef f  ee
 eee  edLdMdNZ-de*dOdPdQZ.deee ee dRdSdTZ/dUdV Z0ddXdYZ1eedZ dd[d\Z2ee)d dd_d`Z3de
ee e
dbdcddZ4eddedfZ5dgdh Z6eedidjdkZ7dee( eee)d dmdndoZ8dee(ee ee)ee*eef f  ee
dpdqdrZ9dZdsdtduZ:dS )ry   aa	  
    Dict-like IO interface for storing pandas objects in PyTables.

    Either Fixed or Table format.

    .. warning::

       Pandas uses PyTables for reading and writing HDF5 files, which allows
       serializing object-dtype data with pickle when using the "fixed" format.
       Loading pickled data received from untrusted sources can be unsafe.

       See: https://docs.python.org/3/library/pickle.html for more.

    Parameters
    ----------
    path : str
        File path to HDF5 file.
    mode : {'a', 'w', 'r', 'r+'}, default 'a'

        ``'r'``
            Read-only; no data can be modified.
        ``'w'``
            Write; a new file is created (an existing file with the same
            name would be deleted).
        ``'a'``
            Append; an existing file is opened for reading and writing,
            and if the file does not exist it is created.
        ``'r+'``
            It is similar to ``'a'``, but the file must already exist.
    complevel : int, 0-9, default None
        Specifies a compression level for data.
        A value of 0 or None disables compression.
    complib : {'zlib', 'lzo', 'bzip2', 'blosc'}, default 'zlib'
        Specifies the compression library to be used.
        As of v0.20.2 these additional compressors for Blosc are supported
        (default if no compressor specified: 'blosc:blosclz'):
        {'blosc:blosclz', 'blosc:lz4', 'blosc:lz4hc', 'blosc:snappy',
         'blosc:zlib', 'blosc:zstd'}.
        Specifying a compression library which is not available issues
        a ValueError.
    fletcher32 : bool, default False
        If applying compression use the fletcher32 checksum.
    **kwargs
        These parameters will be passed to the PyTables open_file method.

    Examples
    --------
    >>> bar = pd.DataFrame(np.random.randn(10, 4))
    >>> store = pd.HDFStore('test.h5')
    >>> store['foo'] = bar   # write to HDF5
    >>> bar = store['foo']   # retrieve
    >>> store.close()

    **Create or load HDF5 file in-memory**

    When passing the `driver` option to the PyTables open_file method through
    **kwargs, the HDF5 file is loaded or created in-memory and will only be
    written when closed:

    >>> bar = pd.DataFrame(np.random.randn(10, 4))
    >>> store = pd.HDFStore('test.h5', driver='H5FD_CORE')
    >>> store['foo'] = bar
    >>> store.close()   # only now, data is written to disk
    r7   rj   NF)rm   rn   
fletcher32c             K   s   d|krt dtd}|d k	r@||jjkr@t d|jj d|d krX|d k	rX|jj}t|| _|d krnd}|| _d | _|r|nd| _	|| _
|| _d | _| jf d|i| d S )	Nrp   z-format is not a defined argument for HDFStorere   zcomplib only supports z compression.rj   r   rm   )r   r   filtersZall_complibsZdefault_complibr3   _path_mode_handle
_complevel_complib_fletcher32_filtersopen)selfr   rm   rn   ro   r   r   re   r?   r?   r@   __init__	  s$    

zHDFStore.__init__c             C   s   | j S )N)r   )r   r?   r?   r@   
__fspath__+  s    zHDFStore.__fspath__c             C   s   | j   | jjS )z return the root node )_check_if_openr   root)r   r?   r?   r@   r   .  s    zHDFStore.rootc             C   s   | j S )N)r   )r   r?   r?   r@   filename4  s    zHDFStore.filename)rk   c             C   s
   | j |S )N)get)r   rk   r?   r?   r@   __getitem__8  s    zHDFStore.__getitem__c             C   s   | j || d S )N)rx   )r   rk   rl   r?   r?   r@   __setitem__;  s    zHDFStore.__setitem__c             C   s
   | j |S )N)remove)r   rk   r?   r?   r@   __delitem__>  s    zHDFStore.__delitem__)rF   c             C   sD   y
| j |S  ttfk
r"   Y nX tdt| j d| ddS )z& allow attribute access to get stores 'z' object has no attribute 'N)r   r   rZ   rh   typerW   )r   rF   r?   r?   r@   __getattr__A  s    
zHDFStore.__getattr__)rk   r   c             C   s8   | j |}|dk	r4|j}||ks0|dd |kr4dS dS )zx
        check for existence of this key
        can match the exact pathname or the pathnm w/o the leading '/'
        NrI   TF)get_noder   )r   rk   noderF   r?   r?   r@   __contains__K  s    
zHDFStore.__contains__)r   c             C   s   t | j S )N)rQ   r   )r   r?   r?   r@   __len__W  s    zHDFStore.__len__c             C   s   t | j}t|  d| dS )Nz
File path: 
)r5   r   r   )r   pstrr?   r?   r@   __repr__Z  s    
zHDFStore.__repr__c             C   s   | S )Nr?   )r   r?   r?   r@   	__enter__^  s    zHDFStore.__enter__c             C   s   | j   d S )N)r   )r   exc_type	exc_value	tracebackr?   r?   r@   __exit__a  s    zHDFStore.__exit__pandas)includer   c             C   s^   |dkrdd | j  D S |dkrJ| jdk	s0tdd | jjddd	D S td
| ddS )a!  
        Return a list of keys corresponding to objects stored in HDFStore.

        Parameters
        ----------

        include : str, default 'pandas'
                When kind equals 'pandas' return pandas objects
                When kind equals 'native' return native HDF5 Table objects

                .. versionadded:: 1.1.0

        Returns
        -------
        list
            List of ABSOLUTE path-names (e.g. have the leading '/').

        Raises
        ------
        raises ValueError if kind has an illegal value
        r   c             S   s   g | ]
}|j qS r?   )r   ).0nr?   r?   r@   
<listcomp>{  s    z!HDFStore.keys.<locals>.<listcomp>ZnativeNc             S   s   g | ]
}|j qS r?   )r   )r   r   r?   r?   r@   r     s    /Table)	classnamez8`include` should be either 'pandas' or 'native' but is 'r   )r   r   AssertionErrorZ
walk_nodesr   )r   r   r?   r?   r@   keysd  s    zHDFStore.keysc             C   s   t | j S )N)iterr   )r   r?   r?   r@   __iter__  s    zHDFStore.__iter__c             c   s"   x| j  D ]}|j|fV  q
W dS )z'
        iterate on key->group
        N)r   r   )r   gr?   r?   r@   items  s    zHDFStore.items)rm   c             K   s  t  }| j|krR| jdkr$|dkr$n(|dkrL| jrLtd| j d| j d|| _| jr`| j  | jr| jdkrt  j| j| j| j	d	| _
y|j| j| jf|| _W n tk
 r } z<d
t|krtd| j d |j| jdf|| _n W Y dd}~X n tk
rR } z8dt|kr>|j }td|j d| d}|W Y dd}~X nN tk
r } z0| jdkrdt|krtt|| W Y dd}~X nX dS )a9  
        Open the file in the specified mode

        Parameters
        ----------
        mode : {'a', 'w', 'r', 'r+'}, default 'a'
            See HDFStore docstring or tables.open_file for info about modes
        **kwargs
            These parameters will be passed to the PyTables open_file method.
        rj   rT   r|   r+zRe-opening the file [z] with mode [z] will delete the current file!r   )r   zcan not be writtenzOpening z in read-only modeNZFILE_OPEN_POLICYz
PyTables [zY] no longer supports opening multiple files
even in read-only mode on this HDF5 version [z]. You can accept this
and not open the same file multiple times at once,
upgrade the HDF5 version, or downgrade to PyTables 3.0.0 which allows
files to be opened multiple times at once
zUnable to open/create file)rj   rT   )r|   r   )rT   )ri   r   r   rV   r   r   r   Filtersr   r   r   	open_filer   r   rE   printr   Zget_hdf5_version__version__	Exception)r   rm   r   re   errZhdf_versionr?   r?   r@   r     s>    

zHDFStore.openc             C   s   | j dk	r| j j  d| _ dS )z0
        Close the PyTables file handle
        N)r   r   )r   r?   r?   r@   r     s    

zHDFStore.closec             C   s   | j dkrdS t| j jS )zF
        return a boolean indicating whether the file is open
        NF)r   boolZisopen)r   r?   r?   r@   r     s    
zHDFStore.is_open)fsyncc             C   sF   | j dk	rB| j j  |rBytj| j j  W n tk
r@   Y nX dS )a  
        Force all buffered modifications to be written to disk.

        Parameters
        ----------
        fsync : bool (default False)
          call ``os.fsync()`` on the file handle to force writing to disk.

        Notes
        -----
        Without ``fsync=True``, flushing may not guarantee that the OS writes
        to disk. With fsync, the operation will block until the OS claims the
        file has been written; however, other caching layers may still
        interfere.
        N)r   flushr   r   filenoOSError)r   r   r?   r?   r@   r     s    

zHDFStore.flushc             C   s>   t  . | j|}|dkr*td| d| j|S Q R X dS )z
        Retrieve pandas object stored in file.

        Parameters
        ----------
        key : str

        Returns
        -------
        object
            Same type as object stored in file.
        NzNo object named z in the file)r   r   r   _read_group)r   rk   r   r?   r?   r@   r     s
    
zHDFStore.get)rk   r   c	                st   | j |}	|	dkr"td| dt|dd}| j|	j   fdd}
t| |
|j|||||d
}|j S )	a  
        Retrieve pandas object stored in file, optionally based on where criteria.

        .. warning::

           Pandas uses PyTables for reading and writing HDF5 files, which allows
           serializing object-dtype data with pickle when using the "fixed" format.
           Loading pickled data received from untrusted sources can be unsafe.

           See: https://docs.python.org/3/library/pickle.html for more.

        Parameters
        ----------
        key : str
                Object being retrieved from file.
        where : list, default None
                List of Term (or convertible) objects, optional.
        start : int, default None
                Row number to start selection.
        stop : int, default None
                Row number to stop selection.
        columns : list, default None
                A list of columns that if not None, will limit the return columns.
        iterator : bool, default False
                Returns an iterator.
        chunksize : int, default None
                Number or rows to include in iteration, return an iterator.
        auto_close : bool, default False
            Should automatically close the store when finished.

        Returns
        -------
        object
            Retrieved object from file.
        NzNo object named z in the filerI   )rH   c                s   j | || dS )N)r}   r~   rR   r   )read)_start_stop_where)r   r>   r?   r@   funcR  s    zHDFStore.select.<locals>.func)rR   nrowsr}   r~   r   r   r   )r   r   rU   _create_storer
infer_axesTableIteratorr   
get_result)r   rk   rR   r}   r~   r   r   r   r   r   r   itr?   )r   r>   r@   r     s&    .

zHDFStore.select)rk   r}   r~   c             C   s8   t |dd}| j|}t|ts(td|j|||dS )a  
        return the selection as an Index

        .. warning::

           Pandas uses PyTables for reading and writing HDF5 files, which allows
           serializing object-dtype data with pickle when using the "fixed" format.
           Loading pickled data received from untrusted sources can be unsafe.

           See: https://docs.python.org/3/library/pickle.html for more.


        Parameters
        ----------
        key : str
        where : list of Term (or convertible) objects, optional
        start : integer (defaults to None), row number to start selection
        stop  : integer (defaults to None), row number to stop selection
        rI   )rH   z&can only read_coordinates with a table)rR   r}   r~   )rU   
get_storerr:   r   r   read_coordinates)r   rk   rR   r}   r~   tblr?   r?   r@   select_as_coordinatese  s
    

zHDFStore.select_as_coordinates)rk   columnr}   r~   c             C   s,   | j |}t|tstd|j|||dS )a~  
        return a single column from the table. This is generally only useful to
        select an indexable

        .. warning::

           Pandas uses PyTables for reading and writing HDF5 files, which allows
           serializing object-dtype data with pickle when using the "fixed" format.
           Loading pickled data received from untrusted sources can be unsafe.

           See: https://docs.python.org/3/library/pickle.html for more.

        Parameters
        ----------
        key : str
        column : str
            The column of interest.
        start : int or None, default None
        stop : int or None, default None

        Raises
        ------
        raises KeyError if the column is not found (or key is not a valid
            store)
        raises ValueError if the column can not be extracted individually (it
            is part of a data block)

        z!can only read_column with a table)r   r}   r~   )r   r:   r   r   read_column)r   rk   r   r}   r~   r   r?   r?   r@   select_column  s    #

zHDFStore.select_column)r   c
                s  t |dd}t|ttfr.t|dkr.|d }t|trRj|||||||	dS t|ttfshtdt|sxtd|dkr|d }fdd	|D j	|}
d}xzt
j|
|fgt|D ]^\}}|dkrtd
| d|j std|j d|dkr|j}q|j|krtdqW dd	 D }tdd |D d   fdd}t|
||||||||	d
}|jddS )a  
        Retrieve pandas objects from multiple tables.

        .. warning::

           Pandas uses PyTables for reading and writing HDF5 files, which allows
           serializing object-dtype data with pickle when using the "fixed" format.
           Loading pickled data received from untrusted sources can be unsafe.

           See: https://docs.python.org/3/library/pickle.html for more.

        Parameters
        ----------
        keys : a list of the tables
        selector : the table to apply the where criteria (defaults to keys[0]
            if not supplied)
        columns : the columns I want back
        start : integer (defaults to None), row number to start selection
        stop  : integer (defaults to None), row number to stop selection
        iterator : boolean, return an iterator, default False
        chunksize : nrows to include in iteration, return an iterator
        auto_close : bool, default False
            Should automatically close the store when finished.

        Raises
        ------
        raises KeyError if keys or selector is not found or keys is empty
        raises TypeError if keys is not a list or tuple
        raises ValueError if the tables are not ALL THE SAME DIMENSIONS
        rI   )rH   r   )rk   rR   r   r}   r~   r   r   r   zkeys must be a list/tuplez keys must have a non-zero lengthNc                s   g | ]} j |qS r?   )r   )r   k)r   r?   r@   r     s    z/HDFStore.select_as_multiple.<locals>.<listcomp>zInvalid table []zobject [z>] is not a table, and cannot be used in all select as multiplez,all tables must have exactly the same nrows!c             S   s   g | ]}t |tr|qS r?   )r:   r   )r   rJ   r?   r?   r@   r     s    c             S   s   h | ]}|j d  d  qS )r   )non_index_axes)r   ra   r?   r?   r@   	<setcomp>	  s    z.HDFStore.select_as_multiple.<locals>.<setcomp>c                s*    fddD }t |ddj S )Nc                s   g | ]}|j  d qS ))rR   r   r}   r~   )r   )r   ra   )r   r   r   r   r?   r@   r     s   z=HDFStore.select_as_multiple.<locals>.func.<locals>.<listcomp>F)axisverify_integrity)r+   _consolidate)r   r   r   objs)r   r   tbls)r   r   r   r@   r     s    z)HDFStore.select_as_multiple.<locals>.func)rR   r   r}   r~   r   r   r   T)coordinates)rU   r:   rL   rM   rQ   rE   r   r   r   r   	itertoolschainzipr   is_tablepathnamer   r   r   )r   r   rR   selectorr   r}   r~   r   r   r   r>   r   ra   r   Z_tblsr   r   r?   )r   r   r   r   r@   select_as_multiple  s^    +

"

zHDFStore.select_as_multipleTrc   )rk   rl   rn   rr   rt   ru   track_timesc             C   sF   |dkrt dpd}| j|}| j|||||||||	|
|||d dS )a  
        Store object in HDFStore.

        Parameters
        ----------
        key : str
        value : {Series, DataFrame}
        format : 'fixed(f)|table(t)', default is 'fixed'
            Format to use when storing object in HDFStore. Value can be one of:

            ``'fixed'``
                Fixed format.  Fast writing/reading. Not-appendable, nor searchable.
            ``'table'``
                Table format.  Write as a PyTables Table structure which may perform
                worse but allow more flexible operations like searching / selecting
                subsets of the data.
        append   : bool, default False
            This will force Table format, append the input data to the
            existing.
        data_columns : list, default None
            List of columns to create as data columns, or True to
            use all columns. See `here
            <https://pandas.pydata.org/pandas-docs/stable/user_guide/io.html#query-via-data-columns>`__.
        encoding : str, default None
            Provide an encoding for strings.
        dropna   : bool, default False, do not write an ALL nan row to
            The store settable by the option 'io.hdf.dropna_table'.
        track_times : bool, default True
            Parameter is propagated to 'create_table' method of 'PyTables'.
            If set to False it enables to have the same h5 files (same hashes)
            independent on creation time.

            .. versionadded:: 1.1.0
        Nzio.hdf.default_formatr^   )rp   rq   rO   ro   rn   rr   rv   rt   rC   ru   r   )r   _validate_format_write_to_group)r   rk   rl   rp   rq   rO   ro   rn   rr   rv   rt   rC   ru   r   r?   r?   r@   rx   '  s"    2
zHDFStore.putc             C   s   t |dd}y| j|}W n~ tk
r0    Y nj tk
rD    Y nV tk
r } z:|dk	rftd|| j|}|dk	r|jdd dS W Y dd}~X nX tj	|||r|j
jdd n|jstd|j|||dS dS )	a=  
        Remove pandas object partially by specifying the where condition

        Parameters
        ----------
        key : string
            Node to remove or delete rows from
        where : list of Term (or convertible) objects, optional
        start : integer (defaults to None), row number to start selection
        stop  : integer (defaults to None), row number to stop selection

        Returns
        -------
        number of rows removed (or None if not a Table)

        Raises
        ------
        raises KeyError if key is not a valid store

        rI   )rH   Nz5trying to remove a node with a non-None where clause!T)	recursivez7can only remove with where on objects written as tables)rR   r}   r~   )rU   r   r   r   r   r   r   Z	_f_removecomall_noner   r   delete)r   rk   rR   r}   r~   r>   r   r   r?   r?   r@   r   l  s,    
zHDFStore.remove)rk   rl   rn   rr   rs   rt   ru   c             C   sl   |	dk	rt d|dkr td}|dkr4tdp2d}| j|}| j|||||||||
|||||||d dS )a6  
        Append to Table in file. Node must already exist and be Table
        format.

        Parameters
        ----------
        key : str
        value : {Series, DataFrame}
        format : 'table' is the default
            Format to use when storing object in HDFStore.  Value can be one of:

            ``'table'``
                Table format. Write as a PyTables Table structure which may perform
                worse but allow more flexible operations like searching / selecting
                subsets of the data.
        append       : bool, default True
            Append the input data to the existing.
        data_columns : list of columns, or True, default None
            List of columns to create as indexed data columns for on-disk
            queries, or True to use all columns. By default only the axes
            of the object are indexed. See `here
            <https://pandas.pydata.org/pandas-docs/stable/user_guide/io.html#query-via-data-columns>`__.
        min_itemsize : dict of columns that specify minimum str sizes
        nan_rep      : str to use as str nan representation
        chunksize    : size to chunk the writing
        expectedrows : expected TOTAL row size of this table
        encoding     : default None, provide an encoding for str
        dropna : bool, default False
            Do not write an ALL nan row to the store settable
            by the option 'io.hdf.dropna_table'.

        Notes
        -----
        Does *not* check if data being appended overlaps with existing
        data in the table, so be careful
        Nz>columns is not a supported keyword in append, try data_columnszio.hdf.dropna_tablezio.hdf.default_formatr_   )rp   axesrq   rO   ro   rn   rr   rv   r   expectedrowsrs   rt   rC   ru   )r   r   r   r   )r   rk   rl   rp   r  rq   rO   ro   rn   r   rr   rv   r   r  rs   rt   rC   ru   r?   r?   r@   rO     s2    8
zHDFStore.append)dc                s  |dk	rt dt|ts"td||kr2tdtttjttt	  d }d}	g }
x<|j
 D ]0\}  dkr|	dk	rtd|}	qj|
j  qjW |	dk	rڈj| }|jt|
}t|j|}|j|||	< |dkr|| }|r2fdd|j D }t|}x|D ]}|j|}qW j| |jd	d}xt|j
 D ]h\} ||kr^|nd}j |d
}|dk	r fdd|j
 D nd}| j||f||d| qHW dS )a  
        Append to multiple tables

        Parameters
        ----------
        d : a dict of table_name to table_columns, None is acceptable as the
            values of one node (this will get all the remaining columns)
        value : a pandas object
        selector : a string that designates the indexable table; all of its
            columns will be designed as data_columns, unless data_columns is
            passed, in which case these are used
        data_columns : list of columns to create as data columns, or True to
            use all columns
        dropna : if evaluates to True, drop rows from all tables if any single
                 row in each table has all NaN. Default False.

        Notes
        -----
        axes parameter is currently not accepted

        Nztaxes is currently not accepted as a parameter to append_to_multiple; you can create the tables independently insteadzQappend_to_multiple must have a dictionary specified as the way to split the valuez=append_to_multiple requires a selector that is in passed dictr   z<append_to_multiple can only have one value in d that is Nonec             3   s    | ]} | j d djV  qdS )all)howN)rs   rq   )r   cols)rl   r?   r@   	<genexpr>E  s    z.HDFStore.append_to_multiple.<locals>.<genexpr>rr   )r   c                s   i | ]\}}| kr||qS r?   r?   )r   rk   rl   )vr?   r@   
<dictcomp>U  s    z/HDFStore.append_to_multiple.<locals>.<dictcomp>)rt   rr   )r   r:   dictr   rL   setrangendim	_AXES_MAPr   r   extendr  
differencer%   sortedget_indexertakevaluesnextintersectionlocpopreindexrO   )r   r  rl   r   rt   r  rs   r   r   Z
remain_keyZremain_valuesr   orderedZorddZidxsZvalid_indexrq   rr   dcvalfilteredr?   )r  rl   r@   append_to_multiple  sN    
&


 zHDFStore.append_to_multiple)rk   optlevelkindc             C   sB   t   | j|}|dkrdS t|ts.td|j|||d dS )a  
        Create a pytables index on the table.

        Parameters
        ----------
        key : str
        columns : None, bool, or listlike[str]
            Indicate which columns to create an index on.

            * False : Do not create any indexes.
            * True : Create indexes on all columns.
            * None : Create indexes on all columns.
            * listlike : Create indexes on the given columns.

        optlevel : int or None, default None
            Optimization level, if None, pytables defaults to 6.
        kind : str or None, default None
            Kind of index, if None, pytables defaults to "medium".

        Raises
        ------
        TypeError: raises if the node is not a table
        Nz1cannot create table index on a Fixed format store)r   r#  r$  )ri   r   r:   r   r   create_index)r   rk   r   r#  r$  r>   r?   r?   r@   create_table_index[  s    

zHDFStore.create_table_indexc             C   s"   t   | j  dd | jj D S )z
        Return a list of all the top-level nodes.

        Each node returned is not a pandas storage object.

        Returns
        -------
        list
            List of objects.
        c             S   sR   g | ]J}t |tjj rt|jd dsJt|ddsJt |tjjr|jdkr|qS )pandas_typeNr_   )	r:   rd   linkLinkgetattr_v_attrsr_   r   r   )r   r   r?   r?   r@   r     s
   z#HDFStore.groups.<locals>.<listcomp>)ri   r   r   walk_groups)r   r?   r?   r@   r     s    zHDFStore.groupsr   c             c   s   t   | j  x| jj|D ]}t|jdddk	r4qg }g }xP|jj D ]B}t|jdd}|dkr~t|t	j
jr|j|j qH|j|j qHW |jjd||fV  qW dS )au  
        Walk the pytables group hierarchy for pandas objects.

        This generator will yield the group path, subgroups and pandas object
        names for each group.

        Any non-pandas PyTables objects that are not a group will be ignored.

        The `where` group itself is listed first (preorder), then each of its
        child groups (following an alphanumerical order) is also traversed,
        following the same procedure.

        .. versionadded:: 0.24.0

        Parameters
        ----------
        where : str, default "/"
            Group where to start walking.

        Yields
        ------
        path : str
            Full path to a group (without trailing '/').
        groups : list
            Names (strings) of the groups contained in `path`.
        leaves : list
            Names (strings) of the pandas objects contained in `path`.
        r'  Nr   )ri   r   r   r,  r*  r+  Z_v_childrenr  r:   rd   r   GrouprO   r   r   rstrip)r   rR   r   r   Zleaveschildr'  r?   r?   r@   walk  s    zHDFStore.walkr8   c             C   s~   | j   |jdsd| }| jdk	s(ttdk	s4ty| jj| j|}W n tjjk
r`   dS X t	|tj
sztt||S )z; return the node with the key or None if it does not exist r   N)r   
startswithr   r   rd   r   r   
exceptionsZNoSuchNodeErrorr:   r8   r   )r   rk   r   r?   r?   r@   r     s    
zHDFStore.get_nodeGenericFixedr   c             C   s8   | j |}|dkr"td| d| j|}|j  |S )z> return the storer object for a key, raise if not in the file NzNo object named z in the file)r   r   r   r   )r   rk   r   r>   r?   r?   r@   r     s    

zHDFStore.get_storerrT   )propindexesrn   r   c	          	   C   s   t |||||d}	|dkr&t| j }t|ttfs:|g}x|D ]}
| j|
}|dk	r@|
|	krl|rl|	j|
 | j|
}t|trd}|rdd |j	D }|	j
|
||t|dd|jd q@|	j|
||jd q@W |	S )	a  
        Copy the existing store to a new file, updating in place.

        Parameters
        ----------
        propindexes: bool, default True
            Restore indexes in copied file.
        keys       : list of keys to include in the copy (defaults to all)
        overwrite  : overwrite (remove and replace) existing nodes in the
            new store (default is True)
        mode, complib, complevel, fletcher32 same as in HDFStore.__init__

        Returns
        -------
        open file handle of the new store
        )rm   ro   rn   r   NFc             S   s   g | ]}|j r|jqS r?   )
is_indexedrF   )r   rj   r?   r?   r@   r     s    z!HDFStore.copy.<locals>.<listcomp>rt   )rq   rt   rC   )rC   )ry   rL   r   r:   rM   r   r   r   r   r  rO   r*  rC   rx   )r   rf   rm   r4  r   ro   rn   r   	overwriteZ	new_storer   r>   datarq   r?   r?   r@   copy  s2    





zHDFStore.copyc       
      C   s  t | j}t|  d| d}| jrt| j }t|rg }g }x|D ]}y<| j|}|dk	r|jt |j	pn| |jt |pd W qH t
k
r    Y qH tk
r } z*|j| t |}	|jd|	 d W Y dd}~X qHX qHW |td||7 }n|d7 }n|d	7 }|S )
zg
        Print detailed information on the store.

        Returns
        -------
        str
        z
File path: r   Nzinvalid_HDFStore nodez[invalid_HDFStore node: r      EmptyzFile is CLOSED)r5   r   r   r   r  r   rQ   r   rO   r   r   r   r4   )
r   r   outputZlkeysr   r  r   r>   ZdetailZdstrr?   r?   r@   info"  s.    



(
zHDFStore.infoc             C   s   | j st| j dd S )Nz file is not open!)r   rZ   r   )r   r?   r?   r@   r   L  s    zHDFStore._check_if_open)rp   r   c             C   sJ   yt |j  }W n4 tk
rD } ztd| d|W Y dd}~X nX |S )z validate / deprecate formats z#invalid HDFStore format specified [r   N)_FORMAT_MAPlowerr   r   )r   rp   r   r?   r?   r@   r   P  s
    $zHDFStore._validate_formatUTF-8)rl   rC   ru   r   c          "      s$  dk	r t ttf r td fdd}ttjdd}ttjdd}|dkr̈dkrt  tdk	svt	tddst tj
jrd}d	}qtd
n(tdtdi}	|	t } dkr|d7 }d|kr,ttd}
y|
| }W n. tk
r } z|d|W Y dd}~X nX || ||dS |dkrƈdk	r|dkrtdd}|dk	r|jdkrrd}n|jdkrd}nB|dkrtdd}|dk	r|jdkrd}n|jdkrd}ttttttd}y|| }W n. tk
r } z|d|W Y dd}~X nX || ||dS )z$ return a suitable class to operate Nz(value must be None, Series, or DataFramec          	      s$   t d|  d dt d  S )Nz(cannot properly create the storer for: [z
] [group->z,value->z	,format->)r   r   )ra   )rp   r   rl   r?   r@   errorh  s    z&HDFStore._create_storer.<locals>.errorr'  
table_typer_   frame_tablegeneric_tablezKcannot create a storer if the object is not existing nor a value are passedseriesframeZ_table)rD  rE  _STORER_MAP)rC   ru   series_tablerq   rI   appendable_seriesappendable_multiseriesappendable_frameappendable_multiframe)rC  rH  rI  rJ  rK  worm
_TABLE_MAP)r:   r)   r#   r   rA   r*  r+  ri   rd   r   r_   r   r   SeriesFixed
FrameFixedr   nlevelsGenericTableAppendableSeriesTableAppendableMultiSeriesTableAppendableFrameTableAppendableMultiFrameTable	WORMTable)r   r   rp   rl   rC   ru   r@  ptttZ	_TYPE_MAPrF  clsr   rq   rM  r?   )rp   r   rl   r@   r   Z  sj    







zHDFStore._create_storer)rk   rl   rn   rr   ru   r   c             C   s~  | j |}| jd k	st|d k	r:| r:| jj|dd d }t|dd rV|dksR|rVd S |d kr|jd}d}xX|D ]P}t|sqr|}|jds|d7 }||7 }| j |}|d kr| jj||}|}qrW | j	|||||d}|r|j
 p|j
o|dko|jrtd|js |j  n|j  |j
 r8|r8td	|j||||||	|
||||||d
 t|trz|rz|j|d d S )NT)r  emptyr_   r   )rC   ru   r^   zCan only append to Tablesz0Compression not supported on Fixed format stores)objr  rO   ro   rn   r   rr   r   r  rs   rv   rt   r   )r   )r   r   r   remove_noder*  splitrQ   endswithZcreate_groupr   r   	is_existsr   set_object_infowriter:   r   r%  )r   rk   rl   rp   r  rq   rO   ro   rn   r   rr   r   r  rs   rv   rt   rC   ru   r   r   pathsr   pnew_pathr>   r?   r?   r@   r     s\    





zHDFStore._write_to_group)r   c             C   s   | j |}|j  |j S )N)r   r   r   )r   r   r>   r?   r?   r@   r     s    
zHDFStore._read_group)rj   NNF)r   )rj   )F)NNNNFNF)NNN)NN)NNNNNFNF)NTFNNNNNNrc   T)NNN)NNTTNNNNNNNNNNrc   )NNF)NNN)r   )r3  r   )rT   TNNNFT)NNr?  rc   )r3  r   )NTFNNNNNNFNNNrc   T);rW   rX   rY   __doc__r   r   rE   r   intr   r   r   r   r   propertyr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   	iteritemsr   r   r   r   r   r   r   r   r   r   r   r   rx   r   rO   r"  r&  r   r0  r   r   r8  r<  r   r   r   r   r   r?   r?   r?   r@   ry     s   
@   
"J      E   &       t          <7=              @H  ]  #
/      3*   [              2Ery   c               @   sf   e Zd ZU dZee ee	d 
dee	d eee edddZd	d
 Zdd ZdedddZdS )r   aa  
    Define the iteration interface on a table

    Parameters
    ----------
    store : HDFStore
    s     : the referred storer
    func  : the function to execute the query
    where : the where of the query
    nrows : the rows to iterate on
    start : the passed start value (default is None)
    stop  : the passed stop value (default is None)
    iterator : bool, default False
        Whether to use the default iterator.
    chunksize : the passed chunking value (default is 100000)
    auto_close : bool, default False
        Whether to automatically close the store at the end of iteration.
    r3  r   NF)rw   r>   r   r   r   c             C   s   || _ || _|| _|| _| jjrN|d kr,d}|d kr8d}|d krD|}t||}|| _|| _|| _d | _	|sr|	d k	r|	d kr~d}	t
|	| _nd | _|
| _d S )Nr   i )rw   r>   r   rR   r   minr   r}   r~   r   rf  r   r   )r   rw   r>   r   rR   r   r}   r~   r   r   r   r?   r?   r@   r   -  s,    
zTableIterator.__init__c             c   sj   | j }xV|| jk r\t|| j | j}| jd d | j|| }|}|d kst| rTq|V  qW | j  d S )N)r}   r~   ri  r   r   r   rQ   r   )r   r   r~   rl   r?   r?   r@   r   W  s    
zTableIterator.__iter__c             C   s   | j r| jj  d S )N)r   rw   r   )r   r?   r?   r@   r   g  s    zTableIterator.close)r   c             C   s   | j d k	r4t| jtstd| jj| jd| _| S |rft| jtsLtd| jj| j| j| j	d}n| j}| j
| j| j	|}| j  |S )Nz0can only use an iterator or chunksize on a table)rR   z$can only read_coordinates on a table)rR   r}   r~   )r   r:   r>   r   r   r   rR   r   r}   r~   r   r   )r   r   rR   resultsr?   r?   r@   r   k  s    
zTableIterator.get_result)r3  r   )NNFNF)r3  r   )F)rW   rX   rY   re  r   rf  r   ry   rw   r   r>   r   r   r   r   r   r?   r?   r?   r@   r     s   
	     r   c               @   st  e Zd ZU dZdZdZdddgZee	dBee
e ddd	Zeed
ddZeed
ddZedddZed
ddZeedddZed
ddZeed
ddZejeedddZdd Zed d! Zed"d# Zed$d% Zed&d' Zd(d) ZdCd*d+Z d,d- Z!d.ed/d0d1Z"dDd2d3Z#ed4d5d6Z$d7d8 Z%d9d: Z&d;d< Z'd.d=d>d?Z(d.d=d@dAZ)dS )EIndexCola  
    an index column description class

    Parameters
    ----------
    axis   : axis which I reference
    values : the ndarray like converted values
    kind   : a string description of this type
    typ    : the pytables type
    pos    : the position in the pytables

    Tfreqtz
index_nameN)rF   cnamec             C   s   t |tstd|| _|| _|| _|| _|p0|| _|| _|| _	|| _
|	| _|
| _|| _|| _|| _|| _|d k	r|| j| t | jtstt | jtstd S )Nz`name` must be a str.)r:   rE   r   r  r$  typrF   ro  r   posrl  rm  rn  r  r_   r   metadataset_posr   )r   rF   r  r$  rp  ro  r   rq  rl  rm  rn  r  r_   r   rr  r?   r?   r@   r     s(    


zIndexCol.__init__)r   c             C   s   | j jS )N)rp  itemsize)r   r?   r?   r@   rt    s    zIndexCol.itemsizec             C   s   | j  dS )N_kind)rF   )r   r?   r?   r@   	kind_attr  s    zIndexCol.kind_attr)rq  c             C   s$   || _ |dk	r | jdk	r || j_dS )z. set the position of this column in the Table N)rq  rp  Z_v_pos)r   rq  r?   r?   r@   rs    s    zIndexCol.set_posc             C   sF   t tt| j| j| j| j| jf}djdd t	dddddg|D S )	N,c             s   s    | ]\}}| d | V  qdS )z->Nr?   )r   rk   rl   r?   r?   r@   r    s   z$IndexCol.__repr__.<locals>.<genexpr>rF   ro  r   rq  r$  )
rM   mapr5   rF   ro  r   rq  r$  joinr   )r   tempr?   r?   r@   r     s
     zIndexCol.__repr__)otherr   c                s   t  fdddD S )z compare 2 col items c             3   s&   | ]}t |d t  |d kV  qd S )N)r*  )r   rj   )r{  r   r?   r@   r    s   z"IndexCol.__eq__.<locals>.<genexpr>rF   ro  r   rq  )rF   ro  r   rq  )r  )r   r{  r?   )r{  r   r@   __eq__  s    zIndexCol.__eq__c             C   s   | j | S )N)r|  )r   r{  r?   r?   r@   __ne__  s    zIndexCol.__ne__c             C   s"   t | jdsdS t| jj| jjS )z' return whether I am an indexed column r
  F)hasattrr_   r*  r
  ro  r5  )r   r?   r?   r@   r5    s    zIndexCol.is_indexed)r  rC   ru   c             C   s   t |tjstt||jjdk	r.|| j }t| j	}t
||||}t }t| j|d< | jdk	rrt| j|d< yt|f|}W n0 tk
r   d|krd|d< t|f|}Y nX t|| j}||fS )zV
        Convert the data from this selection to the appropriate pandas type.
        NrF   rl  )r:   r;   ndarrayr   r   dtypefieldsro  rA   r$  _maybe_convertr  rn  rl  r%   r   _set_tzrm  )r   r  rv   rC   ru   val_kindr   Znew_pd_indexr?   r?   r@   convert  s"    


zIndexCol.convertc             C   s   | j S )z return the values)r  )r   r?   r?   r@   	take_data  s    zIndexCol.take_datac             C   s   | j jS )N)r_   r+  )r   r?   r?   r@   attrs  s    zIndexCol.attrsc             C   s   | j jS )N)r_   description)r   r?   r?   r@   r    s    zIndexCol.descriptionc             C   s   t | j| jdS )z# return my current col description N)r*  r  ro  )r   r?   r?   r@   col  s    zIndexCol.colc             C   s   | j S )z return my cython values )r  )r   r?   r?   r@   cvalues$  s    zIndexCol.cvaluesc             C   s
   t | jS )N)r   r  )r   r?   r?   r@   r   )  s    zIndexCol.__iter__c             C   sP   t | jdkrLt|tr$|j| j}|dk	rL| jj|k rLt j	|| j
d| _dS )z
        maybe set a string col itemsize:
            min_itemsize can be an integer or a dict with this columns name
            with an integer size
        stringN)rt  rq  )rA   r$  r:   r  r   rF   rp  rt  ri   	StringColrq  )r   rr   r?   r?   r@   maybe_set_size,  s
    
zIndexCol.maybe_set_sizec             C   s   d S )Nr?   )r   r?   r?   r@   validate_names:  s    zIndexCol.validate_namesAppendableTable)handlerrO   c             C   s:   |j | _ | j  | j| | j| | j| | j  d S )N)r_   validate_colvalidate_attrvalidate_metadatawrite_metadataset_attr)r   r  rO   r?   r?   r@   validate_and_set=  s    


zIndexCol.validate_and_setc             C   s^   t | jdkrZ| j}|dk	rZ|dkr*| j}|j|k rTtd| d| j d|j d|jS dS )z< validate this column: return the compared against itemsize r  Nz#Trying to store a string with len [z] in [z)] column but
this column has a limit of [zC]!
Consider using min_itemsize to preset the sizes on these columns)rA   r$  r  rt  r   ro  )r   rt  cr?   r?   r@   r  E  s    
zIndexCol.validate_col)rO   c             C   sB   |r>t | j| jd }|d k	r>|| jkr>td| d| j dd S )Nzincompatible kind in col [z - r   )r*  r  rv  r$  r   )r   rO   Zexisting_kindr?   r?   r@   r  X  s
    zIndexCol.validate_attrc             C   s   x| j D ]}t| |d}|j| ji }|j|}||kr|dk	r||kr|dkrt|||f }tj|tdd d||< t	| |d qt
d| j d| d| d	| d
	q|dk	s|dk	r|||< qW dS )z
        set/update the info for this indexable with the key/value
        if there is a conflict raise/warn as needed
        Nrl  rn     )
stacklevelzinvalid info for [z] for [z], existing_value [z] conflicts with new value [r   )rl  rn  )_info_fieldsr*  
setdefaultrF   r   attribute_conflict_docwarningswarnr\   setattrr   )r   r<  rk   rl   idxZexisting_valuewsr?   r?   r@   update_infoa  s    
$zIndexCol.update_infoc             C   s$   |j | j}|dk	r | jj| dS )z# set my state from the passed info N)r   rF   __dict__update)r   r<  r  r?   r?   r@   set_info  s    zIndexCol.set_infoc             C   s   t | j| j| j dS )z set the kind for this column N)r  r  rv  r$  )r   r?   r?   r@   r    s    zIndexCol.set_attr)r  c             C   sD   | j dkr@| j}|j| j}|dk	r@|dk	r@t|| r@tddS )z< validate that kind=category does not change the categories categoryNzEcannot append a categorical with different categories to the existing)r   rr  read_metadataro  r"   r   )r   r  Znew_metadataZcur_metadatar?   r?   r@   r    s    
zIndexCol.validate_metadatac             C   s   | j dk	r|j| j| j  dS )z set the meta data N)rr  r  ro  )r   r  r?   r?   r@   r    s    
zIndexCol.write_metadata)NNNNNNNNNNNNN)N)N)*rW   rX   rY   re  is_an_indexableis_data_indexabler  rE   rF   ro  r   r   rg  rf  rt  rv  rs  r   r   r   r|  r}  r5  r;   r  r  r  r  r  r  r  r   r  r  r  r  r  r  r  r  r  r  r?   r?   r?   r@   rk    s\   

             

	 rk  c               @   s>   e Zd ZdZeedddZeje	e	dddZ
dd	 Zd
S )GenericIndexColz< an index which is not represented in the data of the table )r   c             C   s   dS )NFr?   )r   r?   r?   r@   r5    s    zGenericIndexCol.is_indexed)r  rC   ru   c             C   s2   t |tjstt|ttjt|}||fS )z
        Convert the data from this selection to the appropriate pandas type.

        Parameters
        ----------
        values : np.ndarray
        nan_rep : str
        encoding : str
        errors : str
        )r:   r;   r  r   r   r&   arangerQ   )r   r  rv   rC   ru   r?   r?   r@   r    s    zGenericIndexCol.convertc             C   s   d S )Nr?   )r   r?   r?   r@   r    s    zGenericIndexCol.set_attrN)rW   rX   rY   re  rg  r   r5  r;   r  rE   r  r  r?   r?   r?   r@   r    s
   r  c                   s0  e Zd ZdZdZdZddgZd2ed fddZe	ed	d
dZ
e	ed	ddZed	ddZeedddZedddZdd ZeeddddZedd Zeeed ddd Zeeddd!d"Zed#d$ Zed%d& Ze	d'd( Ze	d)d* Zd+d, Zejeed-d.d/Z d0d1 Z!  Z"S )3DataCola3  
    a data holding column, by definition this is not indexable

    Parameters
    ----------
    data   : the actual data
    cname  : the column name in the table to hold the data (typically
                values)
    meta   : a string description of the metadata
    metadata : the actual metadata
    Frm  r  N)rF   c                s2   t  j|||||||||	|
|d || _|| _d S )N)rF   r  r$  rp  rq  ro  rm  r  r_   r   rr  )superr   r  r7  )r   rF   r  r$  rp  ro  rq  rm  r  r_   r   rr  r  r7  )	__class__r?   r@   r     s    zDataCol.__init__)r   c             C   s   | j  dS )N_dtype)rF   )r   r?   r?   r@   
dtype_attr  s    zDataCol.dtype_attrc             C   s   | j  dS )N_meta)rF   )r   r?   r?   r@   	meta_attr  s    zDataCol.meta_attrc             C   sF   t tt| j| j| j| j| jf}djdd t	dddddg|D S )	Nrw  c             s   s    | ]\}}| d | V  qdS )z->Nr?   )r   rk   rl   r?   r?   r@   r    s   z#DataCol.__repr__.<locals>.<genexpr>rF   ro  r  r$  shape)
rM   rx  r5   rF   ro  r  r$  r  ry  r   )r   rz  r?   r?   r@   r     s    zDataCol.__repr__)r{  r   c                s   t  fdddD S )z compare 2 col items c             3   s&   | ]}t |d t  |d kV  qd S )N)r*  )r   rj   )r{  r   r?   r@   r  	  s   z!DataCol.__eq__.<locals>.<genexpr>rF   ro  r  rq  )rF   ro  r  rq  )r  )r   r{  r?   )r{  r   r@   r|  	  s    zDataCol.__eq__)r7  c             C   s@   |d k	st | jd kst t|\}}|| _|| _t|| _d S )N)r   r  _get_data_and_dtype_namer7  _dtype_to_kindr$  )r   r7  
dtype_namer?   r?   r@   set_data	  s    zDataCol.set_datac             C   s   | j S )z return the data )r7  )r   r?   r?   r@   r  	  s    zDataCol.take_datar6   )r  r   c             C   s   |j }|j}|j}|jdkr&d|jf}t|trJ|j}| j||j j	d}ntt
|sZt|rf| j|}nXt|rz| j|}nDt|rt j||d d}n&t|r| j||}n| j||j	d}|S )zW
        Get an appropriately typed and shaped pytables.Col object for values.
        rI   )r$  r   )rt  r  )r  rt  r  r  sizer:   r-   codesget_atom_datarF   r   r   get_atom_datetime64r    get_atom_timedelta64r   ri   Z
ComplexColr   get_atom_string)rY  r  r  rt  r  r  atomr?   r?   r@   	_get_atom	  s$    


zDataCol._get_atomc             C   s   t  j||d dS )Nr   )rt  r  )ri   r  )rY  r  rt  r?   r?   r@   r  6	  s    zDataCol.get_atom_string)r$  r   c             C   sR   |j dr$|dd }d| d}n"|j dr4d}n|j }| d}tt |S )z2 return the PyTables column class for this column uint   NZUIntr6   periodInt64Col)r1  
capitalizer*  ri   )rY  r$  Zk4Zcol_nameZkcapr?   r?   r@   get_atom_coltype:	  s    


zDataCol.get_atom_coltypec             C   s   | j |d|d dS )N)r$  r   )r  )r  )rY  r  r$  r?   r?   r@   r  I	  s    zDataCol.get_atom_datac             C   s   t  j|d dS )Nr   )r  )ri   r  )rY  r  r?   r?   r@   r  M	  s    zDataCol.get_atom_datetime64c             C   s   t  j|d dS )Nr   )r  )ri   r  )rY  r  r?   r?   r@   r  Q	  s    zDataCol.get_atom_timedelta64c             C   s   t | jdd S )Nr  )r*  r7  )r   r?   r?   r@   r  U	  s    zDataCol.shapec             C   s   | j S )z return my cython values )r7  )r   r?   r?   r@   r  Y	  s    zDataCol.cvaluesc             C   s`   |r\t | j| jd}|dk	r2|t| jkr2tdt | j| jd}|dk	r\|| jkr\tddS )zAvalidate that we have the same order as the existing & same dtypeNz4appended items do not match existing items in table!z@appended items dtype do not match existing items dtype in table!)r*  r  rv  rL   r  r   r  r  )r   rO   Zexisting_fieldsZexisting_dtyper?   r?   r@   r  ^	  s    zDataCol.validate_attr)r  rC   ru   c             C   s  t |tjstt||jjdk	r.|| j }| jdk	s<t| jdkr\t	|\}}t
|}n|}| j}| j}t |tjs|tt| j}| j}	| j}
| j}|dk	stt|}|dkrt||dd}n(|dkrtj|dd}n|dkr8ytjd	d
 |D td}W n. tk
r4   tjdd
 |D td}Y nX n|dkr|	}|j }|dkrhtg tjd}n<t|}|j r||  }||dk  |jtj j8  < tj|||
d}n8y|j|dd}W n$ t k
r   |jddd}Y nX t|dkrt!||||d}| j"|fS )aR  
        Convert the data from this selection to the appropriate pandas type.

        Parameters
        ----------
        values : np.ndarray
        nan_rep :
        encoding : str
        errors : str

        Returns
        -------
        index : listlike to become an Index
        data : ndarraylike to become a column
        N
datetime64T)coercetimedelta64zm8[ns])r  r   c             S   s   g | ]}t j|qS r?   )r   fromordinal)r   r  r?   r?   r@   r   	  s    z#DataCol.convert.<locals>.<listcomp>c             S   s   g | ]}t j|qS r?   )r   fromtimestamp)r   r  r?   r?   r@   r   	  s    r  rI   )
categoriesr  F)r8  Or  )rv   rC   ru   )#r:   r;   r  r   r   r  r  ro  rp  r  r  r$  rA   r   rr  r  rm  r  asarrayobjectr   ravelr%   Zfloat64r,   anyastyperf  Zcumsum_valuesr-   Z
from_codesr   _unconvert_string_arrayr  )r   r  rv   rC   ru   	convertedr  r$  r   rr  r  rm  r  r  r  maskr?   r?   r@   r  k	  s^    








 zDataCol.convertc             C   sH   t | j| j| j t | j| j| j | jdk	s2tt | j| j| j dS )z set the data for this column N)	r  r  rv  r  r  r   r  r   r  )r   r?   r?   r@   r  	  s    zDataCol.set_attr)NNNNNNNNNNNN)#rW   rX   rY   re  r  r  r  rE   r   rg  r  r  r   r   r   r|  r   r  r  classmethodr  r  r
   r  r  r  r  r  r  r  r;   r  r  r  __classcell__r?   r?   )r  r@   r    sL              
gr  c               @   sT   e Zd ZdZdZdd Zedd Zeeddd	d
Z	edd Z
edd ZdS )DataIndexableColz- represent a data column that can be indexed Tc             C   s   t | jj stdd S )Nz-cannot have non-object label DataIndexableCol)r%   r  	is_objectr   )r   r?   r?   r@   r  	  s    zDataIndexableCol.validate_namesc             C   s   t  j|dS )N)rt  )ri   r  )rY  r  rt  r?   r?   r@   r  	  s    z DataIndexableCol.get_atom_stringr6   )r$  r   c             C   s   | j |d S )N)r$  )r  )rY  r  r$  r?   r?   r@   r  	  s    zDataIndexableCol.get_atom_datac             C   s
   t  j S )N)ri   r  )rY  r  r?   r?   r@   r  	  s    z$DataIndexableCol.get_atom_datetime64c             C   s
   t  j S )N)ri   r  )rY  r  r?   r?   r@   r  	  s    z%DataIndexableCol.get_atom_timedelta64N)rW   rX   rY   re  r  r  r  r  rE   r  r  r  r?   r?   r?   r@   r  	  s   r  c               @   s   e Zd ZdZdS )GenericDataIndexableColz* represent a generic pytables data column N)rW   rX   rY   re  r?   r?   r?   r@   r  	  s   r  c               @   s  e Zd ZU dZedZeeee	e
f  eeededZd=edeeddd	Zeed
ddZeeeeef d
ddZedd Zed
ddZdd Zdd Zedd Zedd Zedd Z edd Z!eed
dd Z"eed
d!d"Z#ed#d$ Z$d%d& Z%d'd( Z&ed)d* Z'eed
d+d,Z(ed-d. Z)d/d0 Z*d>d2d3Z+d4d5 Z,d?e-e e-e d6d7d8Z.d9d: Z/d@e-e e-e d6d;d<Z0d1S )AFixedz
    represent an object in my store
    facilitate read/write of various types of objects
    this is an abstract base class

    Parameters
    ----------
    parent : HDFStore
    group : Node
        The group node where the table resides.
    r^   r8   FUTF-8rc   )r   r   rC   ru   c             C   sZ   t |tstt|td k	s"tt |tjs:tt||| _|| _t|| _	|| _
d S )N)r:   ry   r   r   rd   r8   r   r   rD   rC   ru   )r   r   r   rC   ru   r?   r?   r@   r   
  s    
zFixed.__init__)r   c             C   s*   | j d dko(| j d dko(| j d dk S )Nr   rI   
      )version)r   r?   r?   r@   is_old_version!
  s    zFixed.is_old_versionc             C   sb   t t| jjdd}y0tdd |jdD }t|dkrB|d }W n tk
r\   d	}Y nX |S )
z compute and set our version pandas_versionNc             s   s   | ]}t |V  qd S )N)rf  )r   rJ   r?   r?   r@   r  *
  s    z Fixed.version.<locals>.<genexpr>.r  r   )r   )r   r   r   )rA   r*  r   r+  rM   r]  rQ   rh   )r   r  r?   r?   r@   r  %
  s    
zFixed.versionc             C   s   t t| jjdd S )Nr'  )rA   r*  r   r+  )r   r?   r?   r@   r'  1
  s    zFixed.pandas_typec             C   s^   | j   | j}|dk	rXt|ttfrDdjdd |D }d| d}| jdd| d	S | jS )
z* return a pretty representation of myself Nrw  c             s   s   | ]}t |V  qd S )N)r5   )r   rJ   r?   r?   r@   r  ;
  s    z!Fixed.__repr__.<locals>.<genexpr>[r   z12.12z	 (shape->))r   r  r:   rL   rM   ry  r'  )r   r>   Zjshaper?   r?   r@   r   5
  s    zFixed.__repr__c             C   s   t | j| j_t t| j_dS )z set my pandas type & version N)rE   pandas_kindr  r'  _versionr  )r   r?   r?   r@   r`  @
  s    zFixed.set_object_infoc             C   s   t j | }|S )N)r8  )r   Znew_selfr?   r?   r@   r8  E
  s    
z
Fixed.copyc             C   s   | j S )N)r   )r   r?   r?   r@   r  I
  s    zFixed.shapec             C   s   | j jS )N)r   r   )r   r?   r?   r@   r   M
  s    zFixed.pathnamec             C   s   | j jS )N)r   r   )r   r?   r?   r@   r   Q
  s    zFixed._handlec             C   s   | j jS )N)r   r   )r   r?   r?   r@   r   U
  s    zFixed._filtersc             C   s   | j jS )N)r   r   )r   r?   r?   r@   r   Y
  s    zFixed._complevelc             C   s   | j jS )N)r   r   )r   r?   r?   r@   r   ]
  s    zFixed._fletcher32c             C   s   | j jS )N)r   r+  )r   r?   r?   r@   r  a
  s    zFixed.attrsc             C   s   dS )z set our object attributes Nr?   )r   r?   r?   r@   	set_attrse
  s    zFixed.set_attrsc             C   s   dS )z get our object attributes Nr?   )r   r?   r?   r@   	get_attrsi
  s    zFixed.get_attrsc             C   s   | j S )z return my storable )r   )r   r?   r?   r@   storablem
  s    zFixed.storablec             C   s   dS )NFr?   )r   r?   r?   r@   r_  r
  s    zFixed.is_existsc             C   s   t | jdd S )Nr   )r*  r  )r   r?   r?   r@   r   v
  s    zFixed.nrowsc             C   s   |dkrdS dS )z' validate against an existing storable NTr?   )r   r{  r?   r?   r@   validatez
  s    zFixed.validateNc             C   s   dS )z- are we trying to operate on an old version? Tr?   )r   rR   r?   r?   r@   validate_version
  s    zFixed.validate_versionc             C   s   | j }|dkrdS | j  dS )zr
        infer the axes of my storer
        return a boolean indicating if we have a valid storer or not
        NFT)r  r  )r   r>   r?   r?   r@   r   
  s
    zFixed.infer_axes)r}   r~   c             C   s   t dd S )Nz>cannot read on an abstract storer: subclasses should implement)r   )r   rR   r   r}   r~   r?   r?   r@   r   
  s    z
Fixed.readc             K   s   t dd S )Nz?cannot write on an abstract storer: subclasses should implement)r   )r   r   r?   r?   r@   ra  
  s    zFixed.writec             C   s0   t j|||r$| jj| jdd dS tddS )zs
        support fully deleting the node in its entirety (only) - where
        specification must be None
        T)r  Nz#cannot delete on an abstract storer)r  r  r   r\  r   r   )r   rR   r}   r~   r?   r?   r@   r  
  s    zFixed.delete)r  rc   )N)NNNN)NNN)1rW   rX   rY   re  rE   r  format_typer
   r   r#   r)   obj_typerf  r  rC   ry   r   r   ru   r   r   rg  r   r  r	   r  r'  r   r`  r8  r  r   r   r   r   r   r  r  r  r  r_  r   r  r  r   r   r   ra  r  r?   r?   r?   r@   r  	  sX   
 

   r  c               @   sJ  e Zd ZU dZedediZdd ej D Zg Z	e
e 	edddZd	d
 Zdd Zdd ZeedddZdd Zdd Zdd Zd/eee ee dddZd0eee ee edddZeeddd Zeedd!d"Zd1eee ee edd#d$Zd2d%ee ee ed&d'd(Zee d)d*d+Z!d3ee ee d,d-d.Z"dS )4r3  z a generified fixed version datetimer  c             C   s   i | ]\}}||qS r?   r?   )r   r   r  r?   r?   r@   r  
  s    zGenericFixed.<dictcomp>)r   c             C   s   | j j|dS )N )_index_type_mapr   )r   rY  r?   r?   r@   _class_to_alias
  s    zGenericFixed._class_to_aliasc             C   s   t |tr|S | jj|tS )N)r:   r   _reverse_index_mapr   r%   )r   aliasr?   r?   r@   _alias_to_class
  s    
zGenericFixed._alias_to_classc             C   s0   |t krddd}|S |tkr,ddd}|S |S )Nc             S   s:   t j| j|d}tj|d d}|d k	r6|jdj|}|S )N)rl  )rF   UTC)r.   _simple_newr  r$   tz_localize
tz_convert)r  rl  rm  Zdtaresultr?   r?   r@   r`   
  s
    z*GenericFixed._get_index_factory.<locals>.fc             S   s   t j| |d}tj|d dS )N)rl  )rF   )r/   r  r(   )r  rl  rm  Zparrr?   r?   r@   r`   
  s    )NN)NN)r$   r(   )r   klassr`   r?   r?   r@   _get_index_factory
  s    

zGenericFixed._get_index_factoryc             C   s$   |dk	rt d|dk	r t ddS )zE
        raise if any keywords are passed which are not-None
        Nzqcannot pass a column specification when reading a Fixed format store. this store must be selected in its entiretyzucannot pass a where specification when reading from a Fixed format store. this store must be selected in its entirety)r   )r   r   rR   r?   r?   r@   validate_read
  s    zGenericFixed.validate_readc             C   s   dS )NTr?   )r   r?   r?   r@   r_  
  s    zGenericFixed.is_existsc             C   s   | j | j_ | j| j_dS )z set our object attributes N)rC   r  ru   )r   r?   r?   r@   r  
  s    
zGenericFixed.set_attrsc          	   C   sV   t t| jdd| _tt| jdd| _x(| jD ]}t| |tt| j|d q0W dS )z retrieve our attributes rC   Nru   rc   )rD   r*  r  rC   rA   ru   
attributesr  )r   r   r?   r?   r@   r  
  s    zGenericFixed.get_attrsc             K   s   | j   d S )N)r  )r   r[  r   r?   r?   r@   ra  
  s    zGenericFixed.writeN)rk   r}   r~   c             C   s   ddl }t| j|}|j}t|dd}t||jrD|d || }nztt|dd}	t|dd}
|
dk	rxtj|
|	d}n||| }|	dkrt|d	d}t	||d
d}n|	dkrtj
|dd}|r|jS |S dS )z4 read an array for the specified node (off of group r   N
transposedF
value_typer  )r  r  rm  T)r  r  zm8[ns])re   r*  r   r+  r:   ZVLArrayrA   r;   rZ  r  r  T)r   rk   r}   r~   re   r   r  r  retr  r  rm  r?   r?   r@   
read_array
  s&    zGenericFixed.read_array)rk   r}   r~   r   c             C   sh   t t| j| d}|dkr.| j|||dS |dkrVt| j|}| j|||d}|S td| d S )N_varietymulti)r}   r~   regularzunrecognized index variety: )rA   r*  r  read_multi_indexr   read_index_noder   )r   rk   r}   r~   Zvarietyr   rq   r?   r?   r@   
read_index  s    zGenericFixed.read_index)rk   rq   c             C   s   t |tr,t| j| dd | j|| nt| j| dd td|| j| j}| j||j	 t
| j|}|j|j_|j|j_t |ttfr| jt||j_t |tttfr|j|j_t |tr|jd k	rt|j|j_d S )Nr  r  r  rq   )r:   r'   r  r  write_multi_index_convert_indexrC   ru   write_arrayr  r*  r   r$  r+  rF   r$   r(   r  r   index_classr*   rl  rm  _get_tz)r   rk   rq   r  r   r?   r?   r@   write_index)  s    



zGenericFixed.write_indexc             C   s   t | j| d|j xtt|j|j|jD ]\}\}}}t|rLt	d| d| }t
||| j| j}| j||j t| j|}	|j|	j_||	j_t |	j| d| | | d| }
| j|
| q.W d S )N_nlevelsz=Saving a MultiIndex with an extension dtype is not supported._level_name_label)r  r  rP  	enumerater   levelsr  namesr   r   r  rC   ru   r  r  r*  r   r$  r+  rF   )r   rk   rq   ilevlevel_codesrF   	level_keyZ
conv_levelr   	label_keyr?   r?   r@   r
  @  s    "
zGenericFixed.write_multi_indexc             C   s   t | j| d}g }g }g }xxt|D ]l}| d| }	t | j|	}
| j|
||d}|j| |j|j | d| }| j|||d}|j| q(W t|||ddS )Nr  r  )r}   r~   r  T)r  r  r  r   )	r*  r  r  r   r  rO   rF   r  r'   )r   rk   r}   r~   rP  r  r  r  r  r  r   r  r  r  r?   r?   r@   r  Y  s    
zGenericFixed.read_multi_indexr8   )r   r}   r~   r   c             C   s>  ||| }d|j kr>tj|j jdkr>tj|j j|j jd}t|j j}d }d|j krlt|j j	}t|}| j
tt|j dd}| j|}i }	d|j kr|j d |	d< d|j krt|j d tr|j d jd	|	d< n|j d |	d< |d
kr|t||| j| jdfdti|	}
n|t||| j| jdf|	}
||
_	|
S )Nr  r   )r  rF   r  r  rl  rm  zutf-8r   )rC   ru   r  )r+  r;   prodr  rZ  r   rA   r$  rG   rF   r  r*  r  r:   bytesr=   _unconvert_indexrC   ru   r  )r   r   r}   r~   r7  r$  rF   r  factoryr   rq   r?   r?   r@   r  p  s<    





zGenericFixed.read_index_node)rk   rl   c             C   sJ   t jd|j }| jj| j|| t| j|}t|j|j	_
|j|j	_dS )z write a 0-len array rI   N)rI   )r;   rZ  r  r   create_arrayr   r*  rE   r  r+  r   r  )r   rk   rl   Zarrr   r?   r?   r@   write_array_empty  s
    zGenericFixed.write_array_empty)rk   rl   r   c             C   sJ  t |tjtfstt||| jkr6| jj| j| |j	dk}d}t
|jrVtd|snt|drn|j}d}d }| jd k	ryt jj|j}W n tk
r   Y nX |d k	r|s| jj| j|||j| jd}||d d < n| j|| nJ|jjtjkr`tj|dd}|rn*|dkrnt|||f }	tj|	td	d
 | jj| j|t j }
|
j | n|rt| j|| nt!|jr| jj"| j||j#d dt$| j|j%_&nt'|jr| jj"| j||j( t$| j|}t)|j*|j%_*d|j%_&nHt+|jr$| jj"| j||j#d dt$| j|j%_&n| jj"| j|| |t$| j|j%_,d S )Nr   Fz]Cannot store a category dtype in a HDF5 dataset that uses format="fixed". Use format="table".r  T)r   )skipnar     )r  i8r  r  )-r:   r;   r  r!   r   r   r   r   r\  r  r   r  r   r~  r  r   ri   ZAtomZ
from_dtyper   Zcreate_carrayr  r!  Zobject_r   infer_dtypeperformance_docr  r  r   Zcreate_vlarray
ObjectAtomrO   r   r   viewr*  r+  r   r   asi8r  rm  r    r  )r   rk   rl   r   Zempty_arrayr  r  cainferred_typer  Zvlarrr   r?   r?   r@   r    sb    






zGenericFixed.write_array)NN)NN)NN)NN)N)#rW   rX   rY   re  r$   r(   r  r   r  r  r   rE   r  r  r  r  rg  r   r_  r  r  ra  r   rf  r  r%   r	  r  r'   r
  r  r  r   r!  r  r?   r?   r?   r@   r3  
  s0   
%2	r3  c                   sR   e Zd ZU dZdgZeedd Zde	e
 e	e
 dddZ fd	d
Z  ZS )rN  rD  rF   c             C   s,   yt | jjfS  ttfk
r&   d S X d S )N)rQ   r   r  r   rh   )r   r?   r?   r@   r    s    zSeriesFixed.shapeN)r}   r~   c             C   s<   | j || | jd||d}| jd||d}t||| jdS )Nrq   )r}   r~   r  )rq   rF   )r  r	  r  r)   rF   )r   rR   r   r}   r~   rq   r  r?   r?   r@   r     s    zSeriesFixed.readc                s:   t  j|f| | jd|j | jd|j |j| j_d S )Nrq   r  )r  ra  r  rq   r  r  rF   r  )r   r[  r   )r  r?   r@   ra    s    zSeriesFixed.write)NNNN)rW   rX   rY   r  r  r   rF   rg  r  r   rf  r   ra  r  r?   r?   )r  r@   rN    s   
	   rN  c                   sP   e Zd ZU ddgZeedd Zdee ee dddZ	 fd	d
Z
  ZS )BlockManagerFixedr  nblocksc             C   s   y| j }d}xFt| jD ]8}t| jd| d}t|dd }|d k	r||d 7 }qW | jj}t|dd }|d k	rt|d|d  }ng }|j| |S  tk
r   d S X d S )Nr   block_itemsr  rI   )	r  r  r-  r*  r   Zblock0_valuesrL   rO   rh   )r   r  r   r  r   r  r?   r?   r@   r  &  s"    
zBlockManagerFixed.shapeN)r}   r~   c             C   s   | j || | j jd}g }xJt| jD ]<}||kr>||fnd\}}	| jd| ||	d}
|j|
 q*W |d }g }xht| jD ]Z}| jd| d}| jd| d||	d}||j	| }t
|j||d d}|j| qW t|dkrt|dd	}|j|d
d}|S t
|d |d dS )Nr   r   )r}   r~   r.  r/  r  rI   )r   rq   )r   F)r   r8  )NN)r  r  Z_get_block_manager_axisr  r  r	  rO   r-  r  r  r#   r  rQ   r+   r  )r   rR   r   r}   r~   Zselect_axisr  r  r   r   axr   dfs	blk_itemsr  dfoutr?   r?   r@   r   A  s(    zBlockManagerFixed.readc                s   t  j|f| |j}|j s&|j }|j| j_x@t|jD ]2\}}|dkr\|j	 r\t
d| jd| | q<W t|j| j_xRt|jD ]D\}}|jj|j}| jd| d|j|d | jd| d| qW d S )Nr   z/Columns index has to be unique for fixed formatr   r.  r  )r   r/  )r  ra  _mgrZis_consolidatedZconsolidater  r  r  r  Z	is_uniquer   r  rQ   blocksr-  r   r  mgr_locsr  r  )r   r[  r   r7  r  r0  blkr2  )r  r?   r@   ra  f  s    
zBlockManagerFixed.write)NNNN)rW   rX   rY   r  rf  r-  rg  r  r   r   ra  r  r?   r?   )r  r@   r,  !  s   
    r,  c               @   s   e Zd ZdZeZdS )rO  rE  N)rW   rX   rY   r  r#   r  r?   r?   r?   r@   rO  {  s   rO  c                   s  e Zd ZU dZdZdZeedZdZ	e
e e
eeef  e
e e
e
eddeded	 fd
dZeedddZedddZedddZdd ZeedddZdd Z eedddZ!eedddZ"edd Z#ed d! Z$ed"d# Z%ed$d% Z&ed&d' Z'eedd(d)Z(eedd*d+Z)ed,d- Z*eeef dd.d/Z+d0d1 Z,e
e dd2d3Z-eed4d5d6Z.ee/j0d7d8d9Z1ed:d;d<Z2d=d> Z3d?d@ Z4dedAdBZ5dCdD Z6e7dEdF Z8dfe9e dGdHdIZ:dge9e e9e e
ee;e;f  dJdKdLZ<e=edMdNdOZ>dPdQ Z?dhe@edRdSdTZAeBdUdV ZCdidWdXdYdZZDe9e ee9e eeef d[d\d]ZEdje9e e9e d^d_d`ZFdkee9e e9e dadbdcZG  ZHS )lr   a  
    represent a table:
        facilitate read/write of various types of tables

    Attrs in Table Node
    -------------------
    These are attributes that are store in the main table node, they are
    necessary to recreate these tables when read back in.

    index_axes    : a list of tuples of the (original indexing axis and
        index column)
    non_index_axes: a list of tuples of the (original index axis and
        columns on a non-indexing axis)
    values_axes   : a list of the columns which comprise the data of this
        table
    data_columns  : a list of the columns that we are allowing indexing
        (these become single columns in values_axes), or True to force all
        columns
    nan_rep       : the string to use for nan representations for string
        objects
    levels        : the names of levels
    metadata      : the names of the metadata columns
    Z
wide_tabler_   rI   TNrc   r8   )r   r   ru   c                sR   t  j||||d |pg | _|p$g | _|p.g | _|p8g | _|	pDt | _|
| _d S )N)rC   ru   )	r  r   
index_axesr   values_axesrt   r  r<  rv   )r   r   r   rC   ru   r9  r   r:  rt   r<  rv   )r  r?   r@   r     s    



zTable.__init__)r   c             C   s   | j jdd S )N_r   )rA  r]  )r   r?   r?   r@   table_type_short  s    zTable.table_type_shortc             C   s   | j   t| jrdj| jnd}d| d}d}| jrZdjdd | jD }d| d}djd	d | jD }| jd
| d| j d| j	 d| j
 d| d| dS )z* return a pretty representation of myself rw  r  z,dc->[r   r  c             s   s   | ]}t |V  qd S )N)rE   )r   rJ   r?   r?   r@   r    s    z!Table.__repr__.<locals>.<genexpr>r  c             s   s   | ]}|j V  qd S )N)rF   )r   rj   r?   r?   r@   r    s    z12.12z (typ->z,nrows->z,ncols->z,indexers->[r  )r   rQ   rt   ry  r  r  r9  r'  r<  r   ncols)r   Zjdcr  verZjverZjindex_axesr?   r?   r@   r     s    zTable.__repr__)r  c             C   s"   x| j D ]}||jkr|S qW dS )z return the axis for c N)r  rF   )r   r  rj   r?   r?   r@   r     s    
zTable.__getitem__c          
   C   s   |dkrdS |j | j kr2td|j  d| j  dxdD ]}t| |d}t||d}||kr8x@t|D ]4\}}|| }||krftd| d	| d
| dqfW td| d	| d
| dq8W dS )z$ validate against an existing table Nz'incompatible table_type with existing [z - r   r9  r   r:  zinvalid combination of [z] on appending data [z] vs current table [)r9  r   r:  )rA  r   r*  r  r   r   )r   r{  r  svovr  saxZoaxr?   r?   r@   r    s     
zTable.validatec             C   s   t | jtS )z@the levels attribute is 1 or a list in the case of a multi-index)r:   r  rL   )r   r?   r?   r@   is_multi_index  s    zTable.is_multi_indexc             C   sT   dd t |jjD }y|j |fS  tk
rN } ztd|W Y dd}~X nX dS )ze
        validate that we can store the multi-index; reset and return the
        new object
        c             S   s&   g | ]\}}|d k	r|nd| qS )NZlevel_r?   )r   r  lr?   r?   r@   r     s    z-Table.validate_multiindex.<locals>.<listcomp>zBduplicate names/columns in the multi-index when storing as a tableN)r  rq   r  Zreset_indexr   )r   r[  r  r   r?   r?   r@   validate_multiindex  s    zTable.validate_multiindexc             C   s   t jdd | jD S )z/ based on our axes, compute the expected nrows c             S   s   g | ]}|j jd  qS )r   )r  r  )r   r  r?   r?   r@   r     s    z(Table.nrows_expected.<locals>.<listcomp>)r;   r  r9  )r   r?   r?   r@   nrows_expected  s    zTable.nrows_expectedc             C   s
   d| j kS )z has this table been created r_   )r   )r   r?   r?   r@   r_    s    zTable.is_existsc             C   s   t | jdd S )Nr_   )r*  r   )r   r?   r?   r@   r    s    zTable.storablec             C   s   | j S )z. return the table group (this is my storable) )r  )r   r?   r?   r@   r_     s    zTable.tablec             C   s   | j jS )N)r_   r  )r   r?   r?   r@   r    s    zTable.dtypec             C   s   | j jS )N)r_   r  )r   r?   r?   r@   r  "  s    zTable.descriptionc             C   s   t j| j| jS )N)r   r   r9  r:  )r   r?   r?   r@   r  &  s    z
Table.axesc             C   s   t dd | jD S )z0 the number of total columns in the values axes c             s   s   | ]}t |jV  qd S )N)rQ   r  )r   rj   r?   r?   r@   r  -  s    zTable.ncols.<locals>.<genexpr>)sumr:  )r   r?   r?   r@   r=  *  s    zTable.ncolsc             C   s   dS )NFr?   )r   r?   r?   r@   is_transposed/  s    zTable.is_transposedc             C   s(   t tjdd | jD dd | jD S )z@return a tuple of my permutated axes, non_indexable at the frontc             S   s   g | ]}t |d  qS )r   )rf  )r   rj   r?   r?   r@   r   8  s    z*Table.data_orientation.<locals>.<listcomp>c             S   s   g | ]}t |jqS r?   )rf  r   )r   rj   r?   r?   r@   r   9  s    )rM   r   r   r   r9  )r   r?   r?   r@   data_orientation3  s    zTable.data_orientationc                sR   ddd dd j D } fddjD }fddjD }t|| | S )z> return a dict of the kinds allowable columns for this object rq   r   )r   rI   c             S   s   g | ]}|j |fqS r?   )ro  )r   rj   r?   r?   r@   r   C  s    z$Table.queryables.<locals>.<listcomp>c                s   g | ]\}} | d fqS )Nr?   )r   r   r  )
axis_namesr?   r@   r   D  s    c                s&   g | ]}|j t jkr|j|fqS r?   )rF   r  rt   ro  )r   r  )r   r?   r@   r   F  s    )r9  r   r:  r  )r   Zd1Zd2Zd3r?   )rI  r   r@   
queryables=  s
    
zTable.queryablesc             C   s   dd | j D S )z  return a list of my index cols c             S   s   g | ]}|j |jfqS r?   )r   ro  )r   r  r?   r?   r@   r   P  s    z$Table.index_cols.<locals>.<listcomp>)r9  )r   r?   r?   r@   
index_colsM  s    zTable.index_colsc             C   s   dd | j D S )z! return a list of my values cols c             S   s   g | ]
}|j qS r?   )ro  )r   r  r?   r?   r@   r   T  s    z%Table.values_cols.<locals>.<listcomp>)r:  )r   r?   r?   r@   values_colsR  s    zTable.values_cols)rk   r   c             C   s   | j j}| d| dS )z+ return the metadata pathname for this key z/meta/z/meta)r   r   )r   rk   r   r?   r?   r@   _get_metadata_pathV  s    zTable._get_metadata_path)rk   r  c             C   s0   t |}| jj| j||d| j| j| jd dS )z
        Write out a metadata array to the key as a fixed-format Series.

        Parameters
        ----------
        key : str
        values : ndarray
        r_   )rp   rC   ru   rv   N)r)   r   rx   rM  rC   ru   rv   )r   rk   r  r?   r?   r@   r  [  s    	zTable.write_metadata)rk   c             C   s0   t t | jdd|ddk	r,| jj| j|S dS )z) return the meta data array for this key r   N)r*  r   r   r   rM  )r   rk   r?   r?   r@   r  n  s    zTable.read_metadatac             C   sp   t | j| j_| j | j_| j | j_| j| j_| j| j_| j| j_| j| j_| j	| j_	| j
| j_
| j| j_dS )z! set our table type & indexables N)rE   rA  r  rK  rL  r   rt   rv   rC   ru   r  r<  )r   r?   r?   r@   r  t  s    





zTable.set_attrsc             C   s   t | jddpg | _t | jddp$g | _t | jddp:t | _t | jdd| _tt | jdd| _t	t | jdd| _
t | jd	dpg | _d
d | jD | _dd | jD | _dS )z retrieve our attributes r   Nrt   r<  rv   rC   ru   rc   r  c             S   s   g | ]}|j r|qS r?   )r  )r   rj   r?   r?   r@   r     s    z#Table.get_attrs.<locals>.<listcomp>c             S   s   g | ]}|j s|qS r?   )r  )r   rj   r?   r?   r@   r     s    )r*  r  r   rt   r  r<  rv   rD   rC   rA   ru   r  
indexablesr9  r:  )r   r?   r?   r@   r    s    zTable.get_attrsc             C   s\   |dk	rX| j d dkrX| j d dkrX| j d dk rXtdjdd | j D  }tj|t dS )	z- are we trying to operate on an old version? Nr   rI   r  r  r  c             S   s   g | ]}t |qS r?   )rE   )r   rJ   r?   r?   r@   r     s    z*Table.validate_version.<locals>.<listcomp>)r  incompatibility_docry  r  r  r[   )r   rR   r  r?   r?   r@   r    s    *zTable.validate_versionc             C   s^   |dkrdS t |tsdS | j }x6|j D ]*\}}|dkr>q,||kr,td| dq,W dS )z
        validate the min_itemsize doesn't contain items that are not in the
        axes this needs data_columns to be defined
        Nr  zmin_itemsize has the key [z%] which is not an axis or data_column)r:   r  rJ  r   r   )r   rr   qr   r  r?   r?   r@   validate_min_itemsize  s    
zTable.validate_min_itemsizec                s   g }j jjxztjjD ]j\}\}}t|}j|}|dk	rLdnd}| d}t|d}	t||||	|j||d}
|j|
 q W t	j
t|  fdd|jfddtjjD  |S )	z1 create/cache the indexables if they don't exist Nr  ru  )rF   r   rq  r$  rp  r_   r   rr  c                s   t |tstt}|krt}t|}t|j}t| dd }t| dd }t|}j	|}t| dd }	||||| |  |j
|	||d
}
|
S )Nru  r  r  )
rF   ro  r  r$  rq  rp  r_   r   rr  r  )r:   rE   r   r  r  r*  _maybe_adjust_namer  r  r  r_   )r  r  r  r  adj_namer  r  r$  mdr   r[  )base_posr  descr   table_attrsr?   r@   r`     s.    

zTable.indexables.<locals>.fc                s   g | ]\}} ||qS r?   r?   )r   r  r  )r`   r?   r@   r     s    z$Table.indexables.<locals>.<listcomp>)r  r_   r  r  rK  r*  r  rk  rO   r  rt   rQ   r  rL  )r   _indexablesr  r   rF   r  rT  r   rv  r$  	index_colr?   )rU  r  rV  r`   r   rW  r@   rN    s0    



# zTable.indexables)r$  c          	   C   sX  | j  sdS |dkrdS |dks(|dkr8dd | jD }t|ttfsL|g}t }|dk	rb||d< |dk	rr||d< | j}x|D ]}t|j|d}|dk	r"|j	r|j
}|j}	|j}
|dk	r|
|kr|j  n|
|d< |dk	r|	|kr|j  n|	|d< |j	sP|jjdrtd	|jf | q~|| jd
 d kr~td| d| d| dq~W dS )aZ  
        Create a pytables index on the specified columns.

        Parameters
        ----------
        columns : None, bool, or listlike[str]
            Indicate which columns to create an index on.

            * False : Do not create any indexes.
            * True : Create indexes on all columns.
            * None : Create indexes on all columns.
            * listlike : Create indexes on the given columns.

        optlevel : int or None, default None
            Optimization level, if None, pytables defaults to 6.
        kind : str or None, default None
            Kind of index, if None, pytables defaults to "medium".

        Raises
        ------
        TypeError if trying to create an index on a complex-type column.

        Notes
        -----
        Cannot index Time64Col or ComplexCol.
        Pytables must be >= 3.0.
        NFTc             S   s   g | ]}|j r|jqS r?   )r  ro  )r   rj   r?   r?   r@   r     s    z&Table.create_index.<locals>.<listcomp>r#  r$  complexzColumns containing complex values can be stored but cannot be indexed when using table format. Either use fixed format, set index=False, or do not include the columns containing complex values to data_columns when initializing the table.r   rI   zcolumn z/ is not a data_column.
In order to read column z: you must reload the dataframe 
into HDFStore and include z  with the data_columns argument.)r   r  r:   rM   rL   r  r_   r*  r
  r5  rq   r#  r$  Zremove_indexr   r1  r   r%  r   rh   )r   r   r#  r$  kwr_   r  r  rq   Zcur_optlevelZcur_kindr?   r?   r@   r%    sF    



zTable.create_index)r}   r~   r   c       	      C   s^   t | |||d}|j }g }x<| jD ]2}|j| j |j|| j| j| jd}|j	| q$W |S )a  
        Create the axes sniffed from the table.

        Parameters
        ----------
        where : ???
        start : int or None, default None
        stop : int or None, default None

        Returns
        -------
        List[Tuple[index_values, column_values]]
        )rR   r}   r~   )rv   rC   ru   )
	Selectionr   r  r  r<  r  rv   rC   ru   rO   )	r   rR   r}   r~   	selectionr  rj  rj   resr?   r?   r@   
_read_axesH  s    
zTable._read_axes)r  c             C   s   |S )z return the data for this obj r?   )rY  r[  r  r?   r?   r@   
get_objectj  s    zTable.get_objectc                s   t |sg S |d \} | jj|t }|jddkrN|rNtd| d| |dkr`t }n|dkrlg }t|trt|t|}|jfdd	|j	 D   fd
d	|D S )zd
        take the input data_columns and min_itemize and create a data
        columns spec
        r   r   r'   z"cannot use a multi-index on axis [z] with data_columns TNc                s    g | ]}|d kr| kr|qS )r  r?   )r   r   )existing_data_columnsr?   r@   r     s   z/Table.validate_data_columns.<locals>.<listcomp>c                s   g | ]}| kr|qS r?   r?   )r   r  )axis_labelsr?   r@   r     s    )
rQ   r<  r   r  r   rL   r:   r  r  r   )r   rt   rr   r   r   r<  r?   )rb  ra  r@   validate_data_columnso  s$    


zTable.validate_data_columns)r[  r  c       0         s  t ts,| jj}td| dt d dkr:dg fdd D  | j rzd}d	d | jD  t| j	}| j
}nd
}| j}	| jdkstt | jd krtdg }
|dkrd} fddd D d }j| }t|}|r<t|
}| j| d }ttj|tj|s<ttjt|tjt|r<|}|	j|i }t|j|d< t|j|d< |
j||f  d }j| }j|}t||| j| j}||_|jd |j |	 |j!| |g}t|}|dkstt|
dkstx"|
D ]}t"|d |d qW dd }|jdk}| j#|||
}| j$|j% }| j&|||
| j'|\}}g }xt(t)||D ]\}\}}t*}d}|rt|dkr|d |krt+}|d }|dkpt |t,std|r6|r6y| j'| } W nB t-t.fk
r2 }! z td| d| j' d|!W Y dd}!~!X nX nd} |pHd| }"t/|"|| ||| j| jd}#t0|"| j1}$|j2|#}%t3|#j4j5}&t6|#drt7|#j8nd}'d }( })}*t9|#j4r|#j:}*d}(tj|#j;d
dj< })t=|#\}+},||$|"t||%||&|'|*|(|)|,|+d}-|-j |	 |j|- |d7 }qtW dd |D }.t| | j>| j| j| j||
||.|	|d
}/t6| drr| j?|/_?|/j@| |r|r|/jA|  |/S )!a0  
        Create and return the axes.

        Parameters
        ----------
        axes: list or None
            The names or numbers of the axes to create.
        obj : DataFrame
            The object to create axes on.
        validate: bool, default True
            Whether to validate the obj against an existing object already written.
        nan_rep :
            A value to use for string column nan_rep.
        data_columns : List[str], True, or None, default None
            Specify the columns that we want to create to allow indexing on.

            * True : Use all available columns.
            * None : Use no columns.
            * List[str] : Use the specified columns.

        min_itemsize: Dict[str, int] or None, default None
            The min itemsize for a column in bytes.
        z/cannot properly create the storer for: [group->z,value->r   Nr   c                s   g | ]} j |qS r?   )_get_axis_number)r   rj   )r[  r?   r@   r     s    z&Table._create_axes.<locals>.<listcomp>Tc             S   s   g | ]
}|j qS r?   )r   )r   rj   r?   r?   r@   r     s    Fr  rI   z<currently only support ndim-1 indexers in an AppendableTablenanc                s   g | ]}| kr|qS r?   r?   )r   rJ   )r  r?   r@   r     s    r  r   c                s    fdd|D S )Nc                s   g | ]} j j|jqS r?   )r   r  r7  )r   r8  )mgrr?   r@   r     s    z=Table._create_axes.<locals>.get_blk_items.<locals>.<listcomp>r?   )rf  r6  r?   )rf  r@   get_blk_items  s    z)Table._create_axes.<locals>.get_blk_itemsz-cannot have non-object label DataIndexableColzIncompatible appended table [z]with existing table [Zvalues_block_)existing_colrr   rv   rC   ru   rm  r  )r8  )rF   ro  r  rp  rq  r$  rm  r  r   rr  r  r7  c             S   s   g | ]}|j r|jqS r?   )r  rF   )r   r  r?   r?   r@   r   g  s    )
r   r   rC   ru   r9  r   r:  rt   r<  rv   r  )r   rI   )Br:   r#   r   r   r   r   r   r9  rL   rt   rv   r<  r  r   rQ   r   r  r   r"   r;   arrayr  r  r  rW   rO   Z_get_axis_namer  rC   ru   r   rs  r  r  _reindex_axisrc  r`  r   _get_blocks_and_itemsr:  r  r   r  r  rE   
IndexErrorr   _maybe_convert_for_string_atomrR  r  r  r  r  rF   r~  r  rm  r   r  r  r  r  r   r  rQ  r  )0r   r  r[  r  rv   rt   rr   r   table_existsZnew_infonew_non_index_axesr  rj   Zappend_axisindexerZ
exist_axisr<  	axis_nameZ	new_indexZnew_index_axesjrg  r  	block_objr6  r2  Zvaxesr  bb_itemsr  rF   rh  r   new_namedata_convertedrS  rp  r$  rm  r   rr  r  r7  r  r  ZdcsZ	new_tabler?   )r  r[  r@   _create_axes  s     










 "





zTable._create_axesc             C   sd  dd }| j j}|| j |}t|r|d \}}	t|	jt|}
| j|
|dj }t|j}|||}x<|D ]4}| j|g|dj }|j|j |j|||j qlW |r\dd t||D }g }g }x|D ]}t	|j
}y&|j|\}}|j| |j| W q ttfk
rN } z,djdd	 |D }td
| d|W Y d d }~X qX qW |}|}||fS )Nc                s    fdd|D S )Nc                s   g | ]} j j|jqS r?   )r   r  r7  )r   r8  )rf  r?   r@   r     s    zFTable._get_blocks_and_items.<locals>.get_blk_items.<locals>.<listcomp>r?   )rf  r6  r?   )rf  r@   rg    s    z2Table._get_blocks_and_items.<locals>.get_blk_itemsr   )r   c             S   s"   i | ]\}}||ft |j qS r?   )rM   tolist)r   rt  ru  r?   r?   r@   r    s   z/Table._get_blocks_and_items.<locals>.<dictcomp>rw  c             s   s   | ]}t |V  qd S )N)r5   )r   itemr?   r?   r@   r    s    z.Table._get_blocks_and_items.<locals>.<genexpr>z+cannot match existing table structure for [z] on appending data)r5  r6  rQ   r%   r  r  rL   r  r   rM   r  r  rO   rl  r   ry  r   )rs  rn  ro  r:  rt   rg  r6  r2  r   rb  Z
new_labelsrf  r  Zby_itemsZ
new_blocksZnew_blk_itemsZear   rt  ru  r   Zjitemsr?   r?   r@   rk    s@    





zTable._get_blocks_and_itemsr\  )r]  c       
         s   |dk	rt |}|dk	rRjrRtjt s.tx"jD ]}||kr6|jd| q6W x jD ]\}}t ||| qZW |jdk	rx2|jj	 D ]$\}} fdd}	|	|| qW  S )z process axes filters Nr   c                s   x j D ]} j|} j|}|d k	s,t| |krdjrJ|jtj}||} j|d| S | |krt	t
 | j}t	|}t trd| }||} j|d| S qW td|  dd S )N)r   rI   zcannot find the field [z] for filtering!)Z_AXIS_ORDERSrd  	_get_axisr   rB  unionr%   r  r  r2   r*  r  r:   r#   r   )fieldfiltrq  Zaxis_numberZaxis_valuesZtakersr  )r[  opr   r?   r@   process_filter  s"    




z*Table.process_axes.<locals>.process_filter)
rL   rB  r:   r  r   insertr   rj  rN   rp   )
r   r[  r]  r   r   r   labelsr}  r~  r  r?   )r[  r  r   r@   process_axes  s    
!zTable.process_axes)rn   r   r  r   c             C   s   |dkrt | jd}td|d}dd | jD |d< |rl|dkrJ| jpHd}t j|||p\| jd	}||d
< n| jdk	r| j|d
< |S )z< create the description of the table from the axes & values Ni'  r_   )rF   r  c             S   s   i | ]}|j |jqS r?   )rp  ro  )r   rj   r?   r?   r@   r    s    z,Table.create_description.<locals>.<dictcomp>r  	   )rn   ro   r   r   )	maxrE  r  r  r   ri   r   r   r   )r   ro   rn   r   r  r  r   r?   r?   r@   create_description  s    	



zTable.create_description)r}   r~   c       
      C   s   | j | | j sdS t| |||d}|j }|jdk	rxR|jj D ]D\}}}| j||j |j d d}	|||	j	||j   |j
 }qDW t|S )zf
        select coordinates (row numbers) from a table; return the
        coordinates object
        F)rR   r}   r~   NrI   )r}   r~   )r  r   r\  select_coordsrN   rp   r   ri  r  ilocr  r%   )
r   rR   r}   r~   r]  Zcoordsr}  r  r~  r7  r?   r?   r@   r   
  s    

"zTable.read_coordinates)r   r}   r~   c             C   s   | j   | j sdS |dk	r$tdx| jD ]v}||jkr,|jsPtd| dt| jj	|}|j
| j |j||| | j| j| jd}tt|d |j|dS q,W td| d	dS )
zj
        return a single column from the table, generally only indexables
        are interesting
        FNz4read_column does not currently accept a where clausezcolumn [z=] can not be extracted individually; it is not data indexable)rv   rC   ru   rI   )rF   z] not found in the table)r  r   r   r  rF   r  r   r*  r_   r
  r  r<  r  rv   rC   ru   r)   r  rm  r   )r   r   rR   r}   r~   rj   r  Z
col_valuesr?   r?   r@   r   $  s&    


zTable.read_column)Nrc   NNNNNN)N)NNN)NN)TNNN)N)NNN)NNN)IrW   rX   rY   re  r  r  rE   rA  r  r   r   rk  r9  r	   rf  r   r   r  r:  rt   rr  r   r<  ry   r   rg  r<  r   r   r  r   rB  rD  rE  r_  r  r_   r  r  r  r=  rG  rH  rJ  rK  rL  rM  r;   r  r  r  r  r  r  rQ  r   rN  r   r%  r   r_  r  r`  rc  r#   rx  staticmethodrk  r  r  r   r   r  r?   r?   )r  r@   r     s   
       


JU +    d1=  r   c               @   s6   e Zd ZdZdZd	ee ee dddZdd ZdS )
rV  z
    a write-once read-many table: this format DOES NOT ALLOW appending to a
    table. writing is a one-time operation the data are stored in a format
    that allows for searching the data on disk
    rL  N)r}   r~   c             C   s   t ddS )z[
        read the indices and the indexing array, calculate offset rows and return
        z!WORMTable needs to implement readN)r   )r   rR   r   r}   r~   r?   r?   r@   r   Z  s    
zWORMTable.readc             K   s   t ddS )z
        write in a format that we can search later on (but cannot append
        to): write out the indices and the values using _write_array
        (e.g. a CArray) create an indexing table so that we can search
        z"WORMTable needs to implement writeN)r   )r   r   r?   r?   r@   ra  f  s    zWORMTable.write)NNNN)	rW   rX   rY   re  rA  r   rf  r   ra  r?   r?   r?   r@   rV  Q  s      rV  c               @   sv   e Zd ZdZdZdddZdee edd	d
Z	e
jee
j ee
j ee
j dddZdee ee dddZdS )r  z* support the new appendable table formats Z
appendableNFTc             C   s   | r| j r| jj| jd | j||||||d}x|jD ]}|j  q:W |j s|j||||	d}|j  ||d< |jj	|jf| |j
|j_
x|jD ]}|j|| qW |j||
d d S )Nr_   )r  r[  r  rr   rv   rt   )ro   rn   r   r  r   )rs   )r_  r   r\  r   rx  r  r  r  r  Zcreate_tabler<  r  r  
write_data)r   r[  r  rO   ro   rn   r   rr   r   r  rs   rv   rt   r   r_   rj   optionsr?   r?   r@   ra  t  s0    
zAppendableTable.write)r   rs   c                s  | j j}| j}g }|rXx@| jD ]6}t|jjdd}t|tj	r|j
|jddd qW t|r|d }x|dd D ]}||@ }qvW |j }nd}dd	 | jD }	t|	}
|
dkst|
d
d	 | jD }dd	 |D }g }xBt|D ]6\}}|f| j ||
|   j }|j
|| j| qW |dkr0d}tjt||| j d}|| d }xt|D ]t}|| t|d | |  krP | j| fdd	|	D |dk	r|  nd fdd	|D d q\W dS )z`
        we form the data into a 2-d including indexes,values,mask write chunk-by-chunk
        r   )r   u1F)r8  rI   Nc             S   s   g | ]
}|j qS r?   )r  )r   rj   r?   r?   r@   r     s    z.AppendableTable.write_data.<locals>.<listcomp>c             S   s   g | ]}|j  qS r?   )r  )r   rj   r?   r?   r@   r     s    c             S   s,   g | ]$}|j tjtj|j|jd  qS )rI   )Z	transposer;   Zrollr  r  )r   r  r?   r?   r@   r     s    i )r  c                s   g | ]}|  qS r?   r?   )r   rj   )end_istart_ir?   r@   r     s    c                s   g | ]}|  qS r?   r?   )r   r  )r  r  r?   r@   r     s    )indexesr  r  )r  r  rE  r:  r,   r7  r  r:   r;   r  rO   r  rQ   r  r9  r   r  r  reshaperZ  ri  r  write_data_chunk)r   r   rs   r  r   masksrj   r  mr  nindexesr  Zbvaluesr  r  Z	new_shaperowschunksr?   )r  r  r@   r    sJ    


zAppendableTable.write_data)r  r  r  r  c             C   s   x|D ]}t j|jsdS qW |d jd }|t|krHt j|| jd}| jj}t|}x t|D ]\}	}
|
|||	 < qbW x$t|D ]\}	}||||	|  < qW |dk	r|j j	t
dd }|j s|| }t|r| jj| | jj  dS )z
        Parameters
        ----------
        rows : an empty memory space where we are putting the chunk
        indexes : an array of the indexes
        mask : an array of the masks
        values : an array of the values
        Nr   )r  F)r8  )r;   r  r  rQ   rZ  r  r  r  r  r  r   r  r_   rO   r   )r   r  r  r  r  r  r   r  r  r  r  r  r?   r?   r@   r    s&    
z AppendableTable.write_data_chunk)r}   r~   c             C   sh  |d kst | rh|d kr<|d kr<| j}| jj| jdd n(|d krJ| j}| jj||d}| jj  |S | j std S | j}t	| |||d}|j
 }t|j }t |}	|	rd|j }
t|
|
dk j}t |sdg}|d |	kr|j|	 |d dkr|jdd |j }xLt|D ]@}|jt||}|j||jd  ||jd  d d |}qW | jj  |	S )NT)r  )r}   r~   rI   r   r  r  )rQ   r   r   r\  r   r_   Zremove_rowsr   r   r\  r  r)   Zsort_valuesdiffrL   rq   rO   r  r  reversedr  r  )r   rR   r}   r~   r   r_   r]  r  Zsorted_serieslnr  r   Zpgr   r  r?   r?   r@   r    sB    

"

zAppendableTable.delete)NFNNNNNNFNNT)F)NNN)rW   rX   rY   re  rA  ra  r   rf  r   r  r;   r  r   r  r  r?   r?   r?   r@   r  o  s(              
.?(r  c               @   sp   e Zd ZU dZdZdZdZeZe	e
eef  eedddZeedd	d
Zdee ee dddZdS )rT  z* support the new appendable table formats rB  rJ  r  )r   c             C   s   | j d jdkS )Nr   rI   )r9  r   )r   r?   r?   r@   rG  `  s    z"AppendableFrameTable.is_transposed)r  c             C   s   |r
|j }|S )z these are written transposed )r  )rY  r[  r  r?   r?   r@   r`  d  s    zAppendableFrameTable.get_objectN)r}   r~   c                s:   j |  j sd S  j|||d}t jrJ jj jd d t nt } fddt j	D }t|dksxt
|d }|| d }	g }
x^t j	D ]N\}}| jkrq|| \}}|jddkrtj|}nt|}|jd}|d k	r|j|d	d
  jr*|}|}t|	t|	dd d}n|j}t|	t|	dd d}|}|jdkrvt|tjrv|jd|jd f}t|tjrt|j||d}n,t|trt|||d}nt|g||d}|j|jkj st
|j|jf|
j| qW t|
dkr
|
d }nt|
dd}t |||d} j |||d}|S )N)rR   r}   r~   r   c                s"   g | ]\}}| j d  kr|qS )r   )r9  )r   r  r0  )r   r?   r@   r     s    z-AppendableFrameTable.read.<locals>.<listcomp>rI   r   r'   r  T)inplacerF   )rF   )r   rq   )r   )r]  r   )!r  r   r_  rQ   r   r<  r   r  r  r  r   r:  r'   from_tuplesr%   	set_namesrG  r*  r  r  r:   r;   r  r  r  r#   Zdtypesr  r  rO   r+   r\  r  )r   rR   r   r}   r~   r  r<  Zindsindrq   framesr  rj   Z
index_valsr  r
  r  r  Zindex_Zcols_r3  r]  r?   )r   r@   r   k  sV    	
$


"
zAppendableFrameTable.read)NNNN)rW   rX   rY   re  r  rA  r  r#   r  r
   r   r)   rg  r   rG  r  r`  r   rf  r   r?   r?   r?   r@   rT  X  s   
   rT  c                   sv   e Zd ZdZdZdZdZeZe	e
dddZee
dd	d
Zd fdd	Zdee ee ed fddZ  ZS )rR  z* support the new appendable table formats rG  rH  r  )r   c             C   s   dS )NFr?   )r   r?   r?   r@   rG    s    z#AppendableSeriesTable.is_transposed)r  c             C   s   |S )Nr?   )rY  r[  r  r?   r?   r@   r`    s    z AppendableSeriesTable.get_objectNc                s<   t |ts|jpd}|j|}t jf ||jj d|S )z- we are going to write this as a frame table r  )r[  rt   )r:   r#   rF   Zto_framer  ra  r   ry  )r   r[  rt   r   rF   )r  r?   r@   ra    s    


zAppendableSeriesTable.write)r}   r~   r   c                s   | j }|d k	rF|rFt| jts"tx"| jD ]}||kr*|jd| q*W t j||||d}|rn|j| jdd |j	d d df }|j
dkrd |_
|S )Nr   )rR   r   r}   r~   T)r  r  )rB  r:   r  rL   r   r  r  r   	set_indexr  rF   )r   rR   r   r}   r~   rB  r   r>   )r  r?   r@   r     s    
zAppendableSeriesTable.read)N)NNNN)rW   rX   rY   re  r  rA  r  r)   r  rg  r   rG  r  r`  ra  r   rf  r   r  r?   r?   )r  r@   rR    s   	   rR  c                   s(   e Zd ZdZdZdZ fddZ  ZS )rS  z* support the new appendable table formats rG  rI  c                sJ   |j pd}| j|\}| _t| j}|j| ||_t jf d|i|S )z- we are going to write this as a frame table r  r[  )rF   rD  r  rL   rO   r   r  ra  )r   r[  r   rF   r
  )r  r?   r@   ra    s    


z AppendableMultiSeriesTable.write)rW   rX   rY   re  r  rA  ra  r  r?   r?   )r  r@   rS    s   rS  c               @   sZ   e Zd ZdZdZdZdZeZe	e
dddZe	dd	 Zd
d Zedd Zdd ZdS )rQ  z< a table that read/writes the generic pytables table format rB  rC  r  )r   c             C   s   | j S )N)r  )r   r?   r?   r@   r'    s    zGenericTable.pandas_typec             C   s   t | jdd p| jS )Nr_   )r*  r   )r   r?   r?   r@   r  	  s    zGenericTable.storablec             C   sL   g | _ d| _g | _dd | jD | _dd | jD | _dd | jD | _dS )z retrieve our attributes Nc             S   s   g | ]}|j r|qS r?   )r  )r   rj   r?   r?   r@   r     s    z*GenericTable.get_attrs.<locals>.<listcomp>c             S   s   g | ]}|j s|qS r?   )r  )r   rj   r?   r?   r@   r     s    c             S   s   g | ]
}|j qS r?   )rF   )r   rj   r?   r?   r@   r     s    )r   rv   r  rN  r9  r:  rt   )r   r?   r?   r@   r    s    zGenericTable.get_attrsc       
   
   C   s   | j }| jd}|dk	rdnd}tdd| j||d}|g}xlt|jD ]^\}}t|ts\tt	||}| j|}|dk	r|dnd}t
|||g|| j||d}	|j|	 qFW |S )z2 create the indexables from the table description rq   Nr  r   )rF   r   r_   r   rr  )rF   rq  r  rp  r_   r   rr  )r  r  r  r_   r  Z_v_namesr:   rE   r   r*  r  rO   )
r   r  rT  r   rY  rX  r  r   r  r  r?   r?   r@   rN    s*    


zGenericTable.indexablesc             K   s   t dd S )Nz cannot write on an generic table)r   )r   r   r?   r?   r@   ra  :  s    zGenericTable.writeN)rW   rX   rY   re  r  rA  r  r#   r  rg  rE   r'  r  r  r   rN  ra  r?   r?   r?   r@   rQ    s   
#rQ  c                   sh   e Zd ZdZdZeZdZej	dZ
eedddZd fd	d
	Zdee ee d fddZ  ZS )rU  z a frame with a multi-index rK  r  z^level_\d+$)r   c             C   s   dS )NZappendable_multir?   )r   r?   r?   r@   r<  F  s    z*AppendableMultiFrameTable.table_type_shortNc                sl   |d krg }n|dkr |j j }| j|\}| _x"| jD ]}||kr8|jd| q8W t jf ||d|S )NTr   )r[  rt   )r   ry  rD  r  r  r  ra  )r   r[  rt   r   r   )r  r?   r@   ra  J  s    
zAppendableMultiFrameTable.write)r}   r~   c                sD   t  j||||d}|j j}|jj fdd|jjD |_|S )N)rR   r   r}   r~   c                s    g | ]} j j|rd n|qS )N)
_re_levelssearch)r   rC  )r   r?   r@   r   b  s    z2AppendableMultiFrameTable.read.<locals>.<listcomp>)r  r   r  r  rq   r  r  )r   rR   r   r}   r~   r3  )r  )r   r@   r   U  s
    zAppendableMultiFrameTable.read)N)NNNN)rW   rX   rY   re  rA  r#   r  r  recompiler  rg  rE   r<  ra  r   rf  r   r  r?   r?   )r  r@   rU  >  s   
   rU  )r[  r   r  r   c             C   s   | j |}t|}|d k	r"t|}|d ks4|j|rB|j|rB| S t|j }|d k	rlt|j j|dd}|j|std d g| j }|||< | jt| } | S )NF)sort)	r{  r2   equalsuniquer  slicer  r  rM   )r[  r   r  r{  r0  Zslicerr?   r?   r@   rj  h  s    

rj  )rm  r   c             C   s   t j| }|S )z- for a tz-aware type, return an encoded zone )r   Zget_timezone)rm  zoner?   r?   r@   r    s    
r  )r  rm  r  r   c             C   sz   t | tr"| jdks"| j|ks"t|dk	rdt| dd}| j } t|}t| |d} | jdj|} n|rvt	j
| dd} | S )a  
    coerce the values to a DatetimeIndex if tz is set
    preserve the input shape if possible

    Parameters
    ----------
    values : ndarray or Index
    tz : str or tzinfo
    coerce : if we do not have a passed timezone, coerce to M8[ns] ndarray
    NrF   )rF   r  zM8[ns])r  )r:   r$   rm  r   r*  r  rA   r  r  r;   r  )r  rm  r  rF   r?   r?   r@   r    s    
r  )rF   rq   rC   ru   r   c          
   C   sj  t | tst|j}t|\}}t|}tj|}t |trbt	| |||t
|dd t
|dd |dS t |trttdtj|dd}	tj|}
|	dkrtjdd	 |
D tjd
}t	| |dt j |dS |	dk rt|
||}|jj}t	| |dt j||dS |	dkrt	| ||||dS t |tjr4|jtks8t|dksJt|t j }t	| ||||dS d S )Nrl  rm  )r  r$  rp  rl  rm  rn  zMultiIndex not supported here!F)r"  r   c             S   s   g | ]}|j  qS r?   )	toordinal)r   r  r?   r?   r@   r     s    z"_convert_index.<locals>.<listcomp>)r  )rn  r  integerfloating)r  r$  rp  rn  r  )r  r  )r:   rE   r   rF   r  r  r  r  r&   rk  r*  r'   r   r   r%  r;   r  Zint32ri   Z	Time32Col_convert_string_arrayr  rt  r  r  r  r'  )rF   rq   rC   ru   rn  r  r  r$  r  r+  r  rt  r?   r?   r@   r    sL    









r  )r$  rC   ru   r   c             C   s   |dkrt | }n|dkr$t| }n|dkrxytjdd | D td}W q tk
rt   tjdd | D td}Y qX nT|dkrtj| }n@|d
krt| d ||d}n&|dkrtj| d }ntd| |S )Nr  r  r   c             S   s   g | ]}t j|qS r?   )r   r  )r   r  r?   r?   r@   r     s    z$_unconvert_index.<locals>.<listcomp>)r  c             S   s   g | ]}t j|qS r?   )r   r  )r   r  r?   r?   r@   r     s    r  floatr  )rv   rC   ru   r  r   zunrecognized index type )r  r  )r$   r*   r;   r  r  r   r  )r7  r$  rC   ru   rq   r?   r?   r@   r    s$    

 r  )rF   c             C   s  |j s|jS |jj}tj|jdd}|dkr6tdn(|dkrHtdn|dkpV|dks^|jS |j|dd	}t|t	r~|d
 }|j}	tj|	dd}|dkrx\t
t|jd
 D ]F}
|j|
}tj|dd}|dkr|jj|
 }td| d| dqW t|	||j|	j}|j|jks*t|j|jf|j}t|tr\t|j| pX|jdpXd
}t|pfd
|}|d k	r|j|}||kr|}|jd| dd}|S )NF)r"  r   z+[date] is not implemented as a table columnr  z>too many timezones in this block, create separate data columnsr  r  )Zdowncastr   zCannot serialize the column [z!] because
its data contents are [z] object dtyper  z|S)r8  )r  r  r  rF   r   r%  r   Zfillnar:   rL   r  rQ   r  igetr7  rp  r  r  r   rt  r  rf  r   r  r  r  )rF   r.  rh  rr   rv   rC   ru   r  r+  r7  r  r  r  rw  rt  Zecir?   r?   r@   rm    sH    


 


rm  )r7  rC   ru   r   c             C   s\   t | r(t| j jj||jj| j} t| j }t	dt
j|}tj| d| d} | S )a  
    Take a string-like that is object dtype and coerce to a fixed size string type.

    Parameters
    ----------
    data : np.ndarray[object]
    encoding : str
    errors : str
        Handler for encoding errors.

    Returns
    -------
    np.ndarray[fixed-length-string]
    rI   S)r  )rQ   r)   r  rE   encoder  r  r  r   r  
libwritersmax_len_string_arrayr;   r  )r7  rC   ru   ensuredrt  r?   r?   r@   r  ?  s    
r  c             C   s   | j }tj| j td} t| rvtjt| }d| }t	| d t
r^t| jj||dj} n| j|ddjtdd} |dkrd}tj| |} | j|S )	a*  
    Inverse of _convert_string_array.

    Parameters
    ----------
    data : np.ndarray[fixed-length-string]
    nan_rep : the storage repr of NaN
    encoding : str
    errors : str
        Handler for encoding errors.

    Returns
    -------
    np.ndarray[object]
        Decoded data.
    )r  Ur   )ru   F)r8  Nre  )r  r;   r  r  r  rQ   r  r  r   r:   r  r)   rE   r=   r  r  Z!string_array_replace_from_nan_repr  )r7  rv   rC   ru   r  rt  r  r?   r?   r@   r  ^  s    
r  )r  r  rC   ru   c             C   s6   t |tstt|t|r2t|||}|| } | S )N)r:   rE   r   r   _need_convert_get_converter)r  r  rC   ru   convr?   r?   r@   r    s
    r  )r$  rC   ru   c                s8   | dkrdd S | dkr& fddS t d|  d S )Nr  c             S   s   t j| ddS )NzM8[ns])r  )r;   r  )rJ   r?   r?   r@   rK     s    z _get_converter.<locals>.<lambda>r  c                s   t | d  dS )N)rv   rC   ru   )r  )rJ   )rC   ru   r?   r@   rK     s   zinvalid kind )r   )r$  rC   ru   r?   )rC   ru   r@   r    s
    r  )r$  r   c             C   s   | dkrdS dS )Nr  r  TF)r  r  r?   )r$  r?   r?   r@   r    s    r  )rF   r   c             C   sh   yN|d dkrL|d dkrL|d dkrLt jd| }|rL|j d }d| } W n tk
rb   Y nX | S )z
    Prior to 0.10.1, we named values blocks like: values_block_0 an the
    name values_0, adjust the given name if necessary.

    Parameters
    ----------
    name : str
    version : Tuple[int, int, int]

    Returns
    -------
    str
    r   rI   r  r  zvalues_block_(\d+)Zvalues_)r  r  r   rl  )rF   r  r  grpr?   r?   r@   rR    s    $rR  )	dtype_strr   c             C   s   t | } | jds| jdr"d}n| jdr2d}n| jdrBd}n| jdsV| jdr\d}nn| jdrld}n^| jd	r|d
}nN| jdrd}n>| jdrd}n.| jdrd}n| dkrd}ntd|  d|S )zA
    Find the "kind" string describing the given dtype name.
    r  r  r  rZ  rf  r  r  r  	timedeltar  r   r  r  r  zcannot interpret dtype of [r   )rA   r1  r   )r  r$  r?   r?   r@   r    s.    






r  )r7  c             C   sb   t | tr| j} | jjjdd }| jjdkr@tj| j	d} nt | t
rP| j} tj| } | |fS )zJ
    Convert the passed data into a storable form and a dtype string.
    r  r   r  Mr$  )r  r  )r:   r-   r  r  rF   r]  r$  r;   r  r(  r(   r)  )r7  r  r?   r?   r@   r    s    


r  c               @   sD   e Zd ZdZdeee ee dddZdd Zdd	 Z	d
d Z
dS )r\  z
    Carries out a selection operation on a tables.Table object.

    Parameters
    ----------
    table : a Table object
    where : list of Terms (or convertible to)
    start, stop: indices to start and/or stop selection

    N)r_   r}   r~   c             C   sP  || _ || _|| _|| _d | _d | _d | _d | _t|ryt	j
|dd}|dksZ|dkrtj|}|jtjkr| j| j }}|d krd}|d kr| j j}tj||| | _nNt|jjtjr| jd k	r|| jk j s| jd k	r|| jkj rtd|| _W n tk
r   Y nX | jd krL| j|| _| jd k	rL| jj \| _| _d S )NF)r"  r  booleanr   z3where must have index locations >= start and < stop)r_   rR   r}   r~   	conditionrN   Ztermsr   r   r   r%  r;   r  r  Zbool_r   r  
issubclassr   r  r  r   generateevaluate)r   r_   rR   r}   r~   inferredr?   r?   r@   r     s@    


zSelection.__init__c             C   sv   |dkrdS | j j }yt||| j jdS  tk
rp } z,dj|j }td| d| d|W Y dd}~X nX dS )z) where can be a : dict,list,tuple,string N)rJ  rC   rw  zThe passed where expression: z
            contains an invalid variable reference
            all of the variable references must be a reference to
            an axis (e.g. 'index' or 'columns'), or a data_column
            The currently defined references are: r   )r_   rJ  r0   rC   	NameErrorry  r   r   )r   rR   rP  r   Zqkeysr?   r?   r@   r  ,  s    
zSelection.generatec             C   sX   | j dk	r(| jjj| j j | j| jdS | jdk	rB| jjj| jS | jjj| j| jdS )z(
        generate the selection
        N)r}   r~   )	r  r_   Z
read_whererp   r}   r~   r   r   r   )r   r?   r?   r@   r   B  s    

zSelection.selectc             C   s   | j | j }}| jj}|dkr$d}n|dk r4||7 }| jdkrD|}n|dk rT||7 }| jdk	rz| jjj| jj ||ddS | jdk	r| jS tj	||S )z(
        generate the selection
        Nr   T)r}   r~   r  )
r}   r~   r_   r   r  Zget_where_listrp   r   r;   r  )r   r}   r~   r   r?   r?   r@   r  N  s     


zSelection.select_coords)NNN)rW   rX   rY   re  r   r   rf  r   r  r   r  r?   r?   r?   r@   r\    s   
  -r\  )rj   NNFNTNNNNrc   r9   )	Nr|   rc   NNNNFN)N)F)re  r8  r  r   r   r   r   r  typingr   r   r   r   r   r	   r
   r   r  numpyr;   Zpandas._configr   r   Zpandas._libsr   r   r  Zpandas._libs.tslibsr   Zpandas._typingr   r   r   Zpandas.compat._optionalr   Zpandas.compat.pickle_compatr   Zpandas.errorsr   Zpandas.util._decoratorsr   Zpandas.core.dtypes.commonr   r   r   r   r   r   r   r   r    Zpandas.core.dtypes.genericr!   Zpandas.core.dtypes.missingr"   r   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   Zpandas.core.arraysr-   r.   r/   Zpandas.core.commoncorecommonr  Z pandas.core.computation.pytablesr0   r1   Zpandas.core.indexes.apir2   Zpandas.io.commonr3   Zpandas.io.formats.printingr4   r5   re   r6   r7   r8   r  rB   rA   rD   rG   rP   rf  rU   r   rV   rZ   Warningr[   rO  r\   r  r]   Zduplicate_docr&  r=  r  Z
dropna_docZ
format_docZconfig_prefixZregister_optionZis_boolZis_one_of_factoryrd   rg   ri   rE   r   r{   r   r   ry   r   rk  r  r  r  r  r  r3  rN  r,  rO  r   rV  r  rT  rR  rS  rQ  rU  rj  r  r  r  r  r  rm  r  r  r  r  r  rR  r  r  r\  r?   r?   r?   r@   <module>   s   (,0
           \*        $           ]q    ! 3  V Z       X jd1A*	<A %!