HOME


sh-3ll 1.0
DIR:/usr/local/lib64/python3.6/site-packages/pandas/io/__pycache__/
Upload File :
Current File : //usr/local/lib64/python3.6/site-packages/pandas/io/__pycache__/pytables.cpython-36.pyc
3

���h���@s�dZddlZddlmZmZddlZddlZddlZddlm	Z	m
Z
mZmZm
Z
mZmZmZddlZddlZddlmZmZddlmZmZddlmZddlmZmZm Z dd	l!m"Z"dd
l#m$Z$ddl%m&Z&ddl'm(Z(dd
l)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2ddl3m4Z4ddl5m6Z6ddl7m8Z8m9Z9m:Z:m;Z;m<Z<m=Z=m>Z>m?Z?m@Z@mAZAddlBmCZCmDZDmEZEddlFjGjHZIddlJmKZKmLZLddlMmNZNddlOmPZPddlQmRZRmSZSe	�r�ddlTmUZUmVZVmWZWdZXdZYdd�ZZdd�Z[dd�Z\eKZ]e^d�d d!�Z_Gd"d#�d#e`�ZaGd$d%�d%e`�ZbGd&d'�d'ec�Zdd(ZeGd)d*�d*ec�Zfd+ZgGd,d-�d-ec�Zhd.Zid/Zjd0d0d1d1d2�Zke8dgiZld3Zmd4Znejod5��8ejpd6d7emejqd8�ejpd9denejrd0d1dg�d8�WdQRXdasd7atd:d;�Zud�eveeve
e^e
evewe
evewe
ee^eeve^ffe
ewe
eeweevfevevd?�
d@dA�Zxd�eveve
e^e
e^e
e^dC�dDdE�ZydFdFewdG�dHdI�ZzGdJdK�dK�Z{GdLdM�dM�Z|GdNdO�dO�Z}GdPdQ�dQe}�Z~GdRdS�dSe}�ZGdTdU�dUe�Z�GdVdW�dWe��Z�GdXdY�dY�Z�GdZd[�d[e��Z�Gd\d]�d]e��Z�Gd^d_�d_e��Z�Gd`da�dae��Z�Gdbdc�dce��Z�Gddde�dee��Z�Gdfdg�dge��Z�Gdhdi�die��Z�Gdjdk�dke��Z�Gdldm�dme��Z�Gdndo�doe��Z�Gdpdq�dqe��Z�d�e8e^e:e8dr�dsdt�Z�eeevefdu�dvdw�Z�d�eej�e:fe
eevefeweej�e9fdx�dydz�Z�eve:eveve}d{�d|d}�Z�eveveveej�e:fd~�dd��Z�evd��d�d��Z�ej�evevej�d��d�d��Z�ej�evevej�d��d�d��Z�ej�evevevd��d�d��Z�evevevd��d�d��Z�evewd��d�d��Z�evevd��d�d��Z�evevd��d�d��Z�ed��d�d��Z�Gd�d��d��Z�dS)�zY
High level interface to PyTables for reading and writing pandas data structures
to disk
�N)�date�tzinfo)�
TYPE_CHECKING�Any�Dict�List�Optional�Tuple�Type�Union)�config�
get_option)�lib�writers)�	timezones)�	ArrayLike�
FrameOrSeries�Label)�import_optional_dependency)�patch_pickle)�PerformanceWarning)�cache_readonly)	�
ensure_object�is_categorical_dtype�is_complex_dtype�is_datetime64_dtype�is_datetime64tz_dtype�is_extension_array_dtype�is_list_like�is_string_dtype�is_timedelta64_dtype)�ABCExtensionArray)�array_equivalent)
�	DataFrame�
DatetimeIndex�Index�
Int64Index�
MultiIndex�PeriodIndex�Series�TimedeltaIndex�concat�isna)�Categorical�
DatetimeArray�PeriodArray)�PyTablesExpr�maybe_expression)�ensure_index)�stringify_path)�adjoin�pprint_thing)�Col�File�Nodez0.15.2�UTF-8cCst|tj�r|jd�}|S)z* if we have bytes, decode them to unicode zUTF-8)�
isinstance�np�bytes_�decode)�s�r?�4/tmp/pip-build-5_djhm0z/pandas/pandas/io/pytables.py�_ensure_decodedFs
rAcCs|dkrt}|S)N)�_default_encoding)�encodingr?r?r@�_ensure_encodingMsrDcCst|t�rt|�}|S)z�
    Ensure that an index / column name is a str (python 3); otherwise they
    may be np.string dtype. Non-string dtypes are passed through unchanged.

    https://github.com/pandas-dev/pandas/issues/13492
    )r:�str)�namer?r?r@�_ensure_strUs
rG)�scope_levelcCs�|d}t|ttf�r^g}x<tdd�|�D]*}t|�sB|j|�q*|jt||d��q*W|}nt|�rrt||d�}|dks�t|�r�|SdS)z�
    ensure that the where is a Term or a list of Term
    this makes sure that we are capturing the scope of variables
    that are passed
    create the terms here with a frame_level=2 (we are 2 levels down)
    �cSs|dk	S)Nr?)�xr?r?r@�<lambda>psz_ensure_term.<locals>.<lambda>)rHN)r:�list�tuple�filterr1�append�Term�len)�whererH�levelZwlist�wr?r?r@�_ensure_termds	rUc@seZdZdS)�PossibleDataLossErrorN)�__name__�
__module__�__qualname__r?r?r?r@rV{srVc@seZdZdS)�ClosedFileErrorN)rWrXrYr?r?r?r@rZsrZc@seZdZdS)�IncompatibilityWarningN)rWrXrYr?r?r?r@r[�sr[z�
where criteria is being ignored as this version [%s] is too old (or
not-defined), read the file in and write it out to a new file to upgrade (with
the copy_to method)
c@seZdZdS)�AttributeConflictWarningN)rWrXrYr?r?r?r@r\�sr\zu
the [%s] attribute of the existing index is [%s] which conflicts with the new
[%s], resetting the attribute to None
c@seZdZdS)�DuplicateWarningN)rWrXrYr?r?r?r@r]�sr]z;
duplicate entries in table, taking most recently appended
z�
your performance may suffer as PyTables will pickle object types that it cannot
map directly to c-types [inferred_type->%s,key->%s] [items->%s]
�fixed�table)�fr^�tr_z;
: boolean
    drop ALL nan rows when appending to a table
z~
: format
    default format writing format, if None, then
    put will default to 'fixed' and append will default to 'table'
zio.hdfZdropna_tableF)Z	validator�default_formatcCs>tdkr:ddl}|ay|jjdkaWntk
r8YnXtS)Nr�strict)�
_table_mod�tables�fileZ_FILE_OPEN_POLICY�!_table_file_open_policy_is_strict�AttributeError)rer?r?r@�_tables�sri�aTrc)
�key�value�mode�	complevel�complibrO�format�index�min_itemsize�dropna�data_columns�errorsrCc
s�|r$����������	f
dd�}n���������	f	dd�}t|�}t|t�rxt||||d��}||�WdQRXn||�dS)z- store this object, close it if we opened it cs|j��	��������d�
S)N)rprqrr�nan_reprsrtrurC)rO)�store)
rtrsrCrurprqrkrrrvrlr?r@rK�szto_hdf.<locals>.<lambda>cs|j���������d�	S)N)rprqrrrvrtrurC)�put)rw)	rtrCrurprqrkrrrvrlr?r@rK�s)rmrnroN)r3r:rE�HDFStore)�path_or_bufrkrlrmrnrorOrprqrrrvrsrtrurCr`rwr?)
rtrsrCrurprqrkrrrvrlr@�to_hdf�s 
r{�r)rmru�start�stop�	chunksizec
Ks�|dkrtd|�d���|dk	r,t|dd�}t|t�rN|jsDtd	��|}d
}nvt|�}t|t�shtd��yt	j
j|�}
Wnttfk
r�d
}
YnX|
s�t
d|�d
���t|f||d�|
��}d}yz|dk�r&|j�}t|�dkr�td��|d}x*|dd�D]}t||��std���qW|j}|j|||||||	|d�Stttfk
�r�t|t��s�y|j�Wntk
�r�YnX�YnXdS)a

    Read from the store, close it if we opened it.

    Retrieve pandas object stored in file, optionally based on where
    criteria.

    .. warning::

       Pandas uses PyTables for reading and writing HDF5 files, which allows
       serializing object-dtype data with pickle when using the "fixed" format.
       Loading pickled data received from untrusted sources can be unsafe.

       See: https://docs.python.org/3/library/pickle.html for more.

    Parameters
    ----------
    path_or_buf : str, path object, pandas.HDFStore or file-like object
        Any valid string path is acceptable. The string could be a URL. Valid
        URL schemes include http, ftp, s3, and file. For file URLs, a host is
        expected. A local file could be: ``file://localhost/path/to/table.h5``.

        If you want to pass in a path object, pandas accepts any
        ``os.PathLike``.

        Alternatively, pandas accepts an open :class:`pandas.HDFStore` object.

        By file-like object, we refer to objects with a ``read()`` method,
        such as a file handler (e.g. via builtin ``open`` function)
        or ``StringIO``.
    key : object, optional
        The group identifier in the store. Can be omitted if the HDF file
        contains a single pandas object.
    mode : {'r', 'r+', 'a'}, default 'r'
        Mode to use when opening the file. Ignored if path_or_buf is a
        :class:`pandas.HDFStore`. Default is 'r'.
    errors : str, default 'strict'
        Specifies how encoding and decoding errors are to be handled.
        See the errors argument for :func:`open` for a full list
        of options.
    where : list, optional
        A list of Term (or convertible) objects.
    start : int, optional
        Row number to start selection.
    stop  : int, optional
        Row number to stop selection.
    columns : list, optional
        A list of columns names to return.
    iterator : bool, optional
        Return an iterator object.
    chunksize : int, optional
        Number of rows to include in an iteration when using an iterator.
    **kwargs
        Additional keyword arguments passed to HDFStore.

    Returns
    -------
    item : object
        The selected object. Return type depends on the object stored.

    See Also
    --------
    DataFrame.to_hdf : Write a HDF file from a DataFrame.
    HDFStore : Low-level access to HDF files.

    Examples
    --------
    >>> df = pd.DataFrame([[1, 1.0, 'a']], columns=['x', 'y', 'z'])
    >>> df.to_hdf('./store.h5', 'data')
    >>> reread = pd.read_hdf('./store.h5')
    r|�r+rjzmode zG is not allowed while performing a read. Allowed modes are r, r+ and a.NrI)rHz&The HDFStore must be open for reading.Fz5Support for generic buffers has not been implemented.zFile z does not exist)rmruTrz]Dataset(s) incompatible with Pandas data types, not table, or no datasets found in HDF5 file.z?key must be provided when HDF5 file contains multiple datasets.)rRr}r~�columns�iteratorr�
auto_close)r|r�rj)�
ValueErrorrUr:ry�is_open�IOErrorr3rE�NotImplementedError�os�path�exists�	TypeError�FileNotFoundError�groupsrQ�_is_metadata_of�_v_pathname�select�KeyError�closerh)rzrkrmrurRr}r~r�r�r�kwargsrwr�r�r�Zcandidate_only_groupZgroup_to_checkr?r?r@�read_hdfsdS



r�r8)�group�parent_group�returncCsJ|j|jkrdS|}x0|jdkrD|j}||kr<|jdkr<dS|j}qWdS)zDCheck if a given group is a metadata group for a given parent_group.FrI�metaT)Z_v_depthZ	_v_parent�_v_name)r�r��current�parentr?r?r@r��s
r�c@s.eZdZUdZedee	e
dveeee
d�dd�Zd	d
�Z
edd��Zed
d��Zed�dd�Zed�dd�Zed�dd�Zed�dd�Zee
d�dd�Zed�dd�Zed�dd �Zd!d"�Zd#d$�Zdweeed&�d'd(�Zd)d*�Zd+d,�ZeZdxed-�d.d/�Zd0d1�Z ee
d�d2d3��Z!dye
d4�d5d6�Z"ed�d7d8�Z#dzee
d9�d:d;�Z$d{eeeeed<�d=d>�Z%d|eeeeeed?�d@dA�Z&d}e
dB�dCdD�Z'd~ee(eeee)ee*eeffeeeee
dG�dHdI�Z+ded�dJdK�Z,d�ee(eeee)ee*eeffee
eeeedL�dMdN�Z-d�e*dO�dPdQ�Z.d�eeeeedR�dSdT�Z/dUdV�Z0d�dXdY�Z1eedZd�d[d\�Z2ee)d�d�d_d`�Z3d�e
eee
db�dcdd�Z4ed�dedf�Z5dgdh�Z6eedi�djdk�Z7d�ee(eee)d�dm�dndo�Z8d�ee(eeee)ee*eeffee
dp�dqdr�Z9dZds�dtdu�Z:dS)�ryaa	
    Dict-like IO interface for storing pandas objects in PyTables.

    Either Fixed or Table format.

    .. warning::

       Pandas uses PyTables for reading and writing HDF5 files, which allows
       serializing object-dtype data with pickle when using the "fixed" format.
       Loading pickled data received from untrusted sources can be unsafe.

       See: https://docs.python.org/3/library/pickle.html for more.

    Parameters
    ----------
    path : str
        File path to HDF5 file.
    mode : {'a', 'w', 'r', 'r+'}, default 'a'

        ``'r'``
            Read-only; no data can be modified.
        ``'w'``
            Write; a new file is created (an existing file with the same
            name would be deleted).
        ``'a'``
            Append; an existing file is opened for reading and writing,
            and if the file does not exist it is created.
        ``'r+'``
            It is similar to ``'a'``, but the file must already exist.
    complevel : int, 0-9, default None
        Specifies a compression level for data.
        A value of 0 or None disables compression.
    complib : {'zlib', 'lzo', 'bzip2', 'blosc'}, default 'zlib'
        Specifies the compression library to be used.
        As of v0.20.2 these additional compressors for Blosc are supported
        (default if no compressor specified: 'blosc:blosclz'):
        {'blosc:blosclz', 'blosc:lz4', 'blosc:lz4hc', 'blosc:snappy',
         'blosc:zlib', 'blosc:zstd'}.
        Specifying a compression library which is not available issues
        a ValueError.
    fletcher32 : bool, default False
        If applying compression use the fletcher32 checksum.
    **kwargs
        These parameters will be passed to the PyTables open_file method.

    Examples
    --------
    >>> bar = pd.DataFrame(np.random.randn(10, 4))
    >>> store = pd.HDFStore('test.h5')
    >>> store['foo'] = bar   # write to HDF5
    >>> bar = store['foo']   # retrieve
    >>> store.close()

    **Create or load HDF5 file in-memory**

    When passing the `driver` option to the PyTables open_file method through
    **kwargs, the HDF5 file is loaded or created in-memory and will only be
    written when closed:

    >>> bar = pd.DataFrame(np.random.randn(10, 4))
    >>> store = pd.HDFStore('test.h5', driver='H5FD_CORE')
    >>> store['foo'] = bar
    >>> store.close()   # only now, data is written to disk
    r7rjNF)rmrn�
fletcher32cKs�d|krtd��td�}|dk	r@||jjkr@td|jj�d���|dkrX|dk	rX|jj}t|�|_|dkrnd}||_d|_|r�|nd|_	||_
||_d|_|j
fd|i|��dS)	Nrpz-format is not a defined argument for HDFStorerezcomplib only supports z
 compression.rjrrm)r�r�filtersZall_complibsZdefault_complibr3�_path�_mode�_handle�
_complevel�_complib�_fletcher32�_filters�open)�selfr�rmrnror�r�rer?r?r@�__init__	s$

zHDFStore.__init__cCs|jS)N)r�)r�r?r?r@�
__fspath__+szHDFStore.__fspath__cCs|j�|jjS)z return the root node )�_check_if_openr��root)r�r?r?r@r�.sz
HDFStore.rootcCs|jS)N)r�)r�r?r?r@�filename4szHDFStore.filename)rkcCs
|j|�S)N)�get)r�rkr?r?r@�__getitem__8szHDFStore.__getitem__cCs|j||�dS)N)rx)r�rkrlr?r?r@�__setitem__;szHDFStore.__setitem__cCs
|j|�S)N)�remove)r�rkr?r?r@�__delitem__>szHDFStore.__delitem__)rFcCsDy
|j|�Sttfk
r"YnXtdt|�j�d|�d���dS)z& allow attribute access to get stores �'z' object has no attribute 'N)r�r�rZrh�typerW)r�rFr?r?r@�__getattr__As
zHDFStore.__getattr__)rkr�cCs8|j|�}|dk	r4|j}||ks0|dd�|kr4dSdS)zx
        check for existence of this key
        can match the exact pathname or the pathnm w/o the leading '/'
        NrITF)�get_noder�)r�rk�noderFr?r?r@�__contains__Ks
zHDFStore.__contains__)r�cCst|j��S)N)rQr�)r�r?r?r@�__len__WszHDFStore.__len__cCst|j�}t|��d|�d�S)Nz
File path: �
)r5r�r�)r��pstrr?r?r@�__repr__Zs
zHDFStore.__repr__cCs|S)Nr?)r�r?r?r@�	__enter__^szHDFStore.__enter__cCs|j�dS)N)r�)r��exc_type�	exc_value�	tracebackr?r?r@�__exit__aszHDFStore.__exit__�pandas)�includer�cCs^|dkrdd�|j�D�S|dkrJ|jdk	s0t�dd�|jjddd	�D�Std
|�d���dS)a!
        Return a list of keys corresponding to objects stored in HDFStore.

        Parameters
        ----------

        include : str, default 'pandas'
                When kind equals 'pandas' return pandas objects
                When kind equals 'native' return native HDF5 Table objects

                .. versionadded:: 1.1.0

        Returns
        -------
        list
            List of ABSOLUTE path-names (e.g. have the leading '/').

        Raises
        ------
        raises ValueError if kind has an illegal value
        r�cSsg|]
}|j�qSr?)r�)�.0�nr?r?r@�
<listcomp>{sz!HDFStore.keys.<locals>.<listcomp>ZnativeNcSsg|]
}|j�qSr?)r�)r�r�r?r?r@r��s�/�Table)�	classnamez8`include` should be either 'pandas' or 'native' but is 'r�)r�r��AssertionErrorZ
walk_nodesr�)r�r�r?r?r@�keysdsz
HDFStore.keyscCst|j��S)N)�iterr�)r�r?r?r@�__iter__�szHDFStore.__iter__ccs"x|j�D]}|j|fVq
WdS)z'
        iterate on key->group
        N)r�r�)r��gr?r?r@�items�szHDFStore.items)rmcKs�t�}|j|krR|jdkr$|dkr$n(|dkrL|jrLtd|j�d|j�d���||_|jr`|j�|jr�|jdkr�t�j|j|j|j	d	�|_
y|j|j|jf|�|_Wn�t
k
�r�}z<d
t|�kr�td|j�d��|j|jdf|�|_n�WYd
d
}~Xn�tk
�rR}z8dt|�k�r>|j�}td|j�d|�d��}|�WYd
d
}~XnNtk
�r�}z0|jdk�r�dt|�k�r�t
t|��|��WYd
d
}~XnXd
S)a9
        Open the file in the specified mode

        Parameters
        ----------
        mode : {'a', 'w', 'r', 'r+'}, default 'a'
            See HDFStore docstring or tables.open_file for info about modes
        **kwargs
            These parameters will be passed to the PyTables open_file method.
        rjrTr|�r+zRe-opening the file [z
] with mode [z] will delete the current file!r)r�zcan not be writtenzOpening z in read-only modeNZFILE_OPEN_POLICYz
PyTables [zY] no longer supports opening multiple files
even in read-only mode on this HDF5 version [z�]. You can accept this
and not open the same file multiple times at once,
upgrade the HDF5 version, or downgrade to PyTables 3.0.0 which allows
files to be opened multiple times at once
zUnable to open/create file)rjrT)r|r�)rT)rir�r�rVr�r�r��Filtersr�r�r��	open_filer�r�rE�printr�Zget_hdf5_version�__version__�	Exception)r�rmr�re�errZhdf_versionr?r?r@r��s>

z
HDFStore.opencCs|jdk	r|jj�d|_dS)z0
        Close the PyTables file handle
        N)r�r�)r�r?r?r@r��s

zHDFStore.closecCs|jdkrdSt|jj�S)zF
        return a boolean indicating whether the file is open
        NF)r��boolZisopen)r�r?r?r@r��s
zHDFStore.is_open)�fsynccCsF|jdk	rB|jj�|rBytj|jj��Wntk
r@YnXdS)a�
        Force all buffered modifications to be written to disk.

        Parameters
        ----------
        fsync : bool (default False)
          call ``os.fsync()`` on the file handle to force writing to disk.

        Notes
        -----
        Without ``fsync=True``, flushing may not guarantee that the OS writes
        to disk. With fsync, the operation will block until the OS claims the
        file has been written; however, other caching layers may still
        interfere.
        N)r��flushr�r��fileno�OSError)r�r�r?r?r@r��s

zHDFStore.flushcCs>t��.|j|�}|dkr*td|�d���|j|�SQRXdS)z�
        Retrieve pandas object stored in file.

        Parameters
        ----------
        key : str

        Returns
        -------
        object
            Same type as object stored in file.
        NzNo object named z in the file)rr�r��_read_group)r�rkr�r?r?r@r�s


zHDFStore.get)rkr�c	st|j|�}	|	dkr"td|�d���t|dd�}|j|	���j���fdd�}
t|�|
|�j|||||d�
}|j�S)	a
        Retrieve pandas object stored in file, optionally based on where criteria.

        .. warning::

           Pandas uses PyTables for reading and writing HDF5 files, which allows
           serializing object-dtype data with pickle when using the "fixed" format.
           Loading pickled data received from untrusted sources can be unsafe.

           See: https://docs.python.org/3/library/pickle.html for more.

        Parameters
        ----------
        key : str
                Object being retrieved from file.
        where : list, default None
                List of Term (or convertible) objects, optional.
        start : int, default None
                Row number to start selection.
        stop : int, default None
                Row number to stop selection.
        columns : list, default None
                A list of columns that if not None, will limit the return columns.
        iterator : bool, default False
                Returns an iterator.
        chunksize : int, default None
                Number or rows to include in iteration, return an iterator.
        auto_close : bool, default False
            Should automatically close the store when finished.

        Returns
        -------
        object
            Retrieved object from file.
        NzNo object named z in the filerI)rHcs�j|||�d�S)N)r}r~rRr�)�read)�_start�_stop�_where)r�r>r?r@�funcRszHDFStore.select.<locals>.func)rR�nrowsr}r~r�rr�)r�r�rU�_create_storer�
infer_axes�
TableIteratorr��
get_result)r�rkrRr}r~r�r�rr�r�r��itr?)r�r>r@r�s&.

zHDFStore.select)rkr}r~cCs8t|dd�}|j|�}t|t�s(td��|j|||d�S)a�
        return the selection as an Index

        .. warning::

           Pandas uses PyTables for reading and writing HDF5 files, which allows
           serializing object-dtype data with pickle when using the "fixed" format.
           Loading pickled data received from untrusted sources can be unsafe.

           See: https://docs.python.org/3/library/pickle.html for more.


        Parameters
        ----------
        key : str
        where : list of Term (or convertible) objects, optional
        start : integer (defaults to None), row number to start selection
        stop  : integer (defaults to None), row number to stop selection
        rI)rHz&can only read_coordinates with a table)rRr}r~)rU�
get_storerr:r�r��read_coordinates)r�rkrRr}r~�tblr?r?r@�select_as_coordinateses


zHDFStore.select_as_coordinates)rk�columnr}r~cCs,|j|�}t|t�std��|j|||d�S)a~
        return a single column from the table. This is generally only useful to
        select an indexable

        .. warning::

           Pandas uses PyTables for reading and writing HDF5 files, which allows
           serializing object-dtype data with pickle when using the "fixed" format.
           Loading pickled data received from untrusted sources can be unsafe.

           See: https://docs.python.org/3/library/pickle.html for more.

        Parameters
        ----------
        key : str
        column : str
            The column of interest.
        start : int or None, default None
        stop : int or None, default None

        Raises
        ------
        raises KeyError if the column is not found (or key is not a valid
            store)
        raises ValueError if the column can not be extracted individually (it
            is part of a data block)

        z!can only read_column with a table)r�r}r~)r�r:r�r��read_column)r�rkr�r}r~r�r?r?r@�
select_column�s#

zHDFStore.select_column)r�c
s�t|dd�}t|ttf�r.t|�dkr.|d}t|t�rR�j||�|||||	d�St|ttf�shtd��t|�sxtd��|dkr�|d}�fdd	�|D���j	|�}
d}xzt
j|
|fgt�|��D]^\}}
|dkr�t
d
|
�d���|j�s�td|j�d
���|dk�r|j}q�|j|kr�td��q�Wdd	��D�}tdd�|D��d����fdd�}t�|
||||||||	d�
}|jdd�S)a�
        Retrieve pandas objects from multiple tables.

        .. warning::

           Pandas uses PyTables for reading and writing HDF5 files, which allows
           serializing object-dtype data with pickle when using the "fixed" format.
           Loading pickled data received from untrusted sources can be unsafe.

           See: https://docs.python.org/3/library/pickle.html for more.

        Parameters
        ----------
        keys : a list of the tables
        selector : the table to apply the where criteria (defaults to keys[0]
            if not supplied)
        columns : the columns I want back
        start : integer (defaults to None), row number to start selection
        stop  : integer (defaults to None), row number to stop selection
        iterator : boolean, return an iterator, default False
        chunksize : nrows to include in iteration, return an iterator
        auto_close : bool, default False
            Should automatically close the store when finished.

        Raises
        ------
        raises KeyError if keys or selector is not found or keys is empty
        raises TypeError if keys is not a list or tuple
        raises ValueError if the tables are not ALL THE SAME DIMENSIONS
        rI)rHr)rkrRr�r}r~r�rr�zkeys must be a list/tuplez keys must have a non-zero lengthNcsg|]}�j|��qSr?)r�)r��k)r�r?r@r��sz/HDFStore.select_as_multiple.<locals>.<listcomp>zInvalid table [�]zobject [z>] is not a table, and cannot be used in all select as multiplez,all tables must have exactly the same nrows!cSsg|]}t|t�r|�qSr?)r:r�)r�rJr?r?r@r�scSsh|]}|jdd�qS)r)�non_index_axes)r�rar?r?r@�	<setcomp>	sz.HDFStore.select_as_multiple.<locals>.<setcomp>cs*����fdd��D�}t|�dd�j�S)Ncsg|]}|j����d��qS))rRr�r}r~)r�)r�ra)r�r�r�r�r?r@r�sz=HDFStore.select_as_multiple.<locals>.func.<locals>.<listcomp>F)�axis�verify_integrity)r+�_consolidate)r�r�r��objs)r�r��tbls)r�r�r�r@r�sz)HDFStore.select_as_multiple.<locals>.func)rRr�r}r~r�rr�T)�coordinates)rUr:rLrMrQrEr�r�r�r��	itertools�chain�zipr��is_table�pathnamer�r�r�)r�r�rR�selectorr�r}r~r�rr�r>r�rar�Z_tblsr�r�r?)r�r�r�r�r@�select_as_multiple�s^+

"


zHDFStore.select_as_multipleTrc)rkrlrnrrrtru�track_timescCsF|dkrtd�pd}|j|�}|j|||||||||	|
|||
d�
dS)a�
        Store object in HDFStore.

        Parameters
        ----------
        key : str
        value : {Series, DataFrame}
        format : 'fixed(f)|table(t)', default is 'fixed'
            Format to use when storing object in HDFStore. Value can be one of:

            ``'fixed'``
                Fixed format.  Fast writing/reading. Not-appendable, nor searchable.
            ``'table'``
                Table format.  Write as a PyTables Table structure which may perform
                worse but allow more flexible operations like searching / selecting
                subsets of the data.
        append   : bool, default False
            This will force Table format, append the input data to the
            existing.
        data_columns : list, default None
            List of columns to create as data columns, or True to
            use all columns. See `here
            <https://pandas.pydata.org/pandas-docs/stable/user_guide/io.html#query-via-data-columns>`__.
        encoding : str, default None
            Provide an encoding for strings.
        dropna   : bool, default False, do not write an ALL nan row to
            The store settable by the option 'io.hdf.dropna_table'.
        track_times : bool, default True
            Parameter is propagated to 'create_table' method of 'PyTables'.
            If set to False it enables to have the same h5 files (same hashes)
            independent on creation time.

            .. versionadded:: 1.1.0
        Nzio.hdf.default_formatr^)rprqrOrornrrrvrtrCrur�)r
�_validate_format�_write_to_group)r�rkrlrprqrOrornrrrvrtrCrur�r?r?r@rx's"2
zHDFStore.putcCs�t|dd�}y|j|�}Wn~tk
r0�Ynjtk
rD�YnVtk
r�}z:|dk	rftd�|�|j|�}|dk	r�|jdd�dSWYdd}~XnXtj	|||�r�|j
jdd�n|js�td��|j|||d�SdS)	a=
        Remove pandas object partially by specifying the where condition

        Parameters
        ----------
        key : string
            Node to remove or delete rows from
        where : list of Term (or convertible) objects, optional
        start : integer (defaults to None), row number to start selection
        stop  : integer (defaults to None), row number to stop selection

        Returns
        -------
        number of rows removed (or None if not a Table)

        Raises
        ------
        raises KeyError if key is not a valid store

        rI)rHNz5trying to remove a node with a non-None where clause!T)�	recursivez7can only remove with where on objects written as tables)rRr}r~)
rUr�r�r�r�r�r�Z	_f_remove�com�all_noner�r��delete)r�rkrRr}r~r>r�r�r?r?r@r�ls,
zHDFStore.remove)rkrlrnrrrsrtrucCsl|	dk	rtd��|dkr td�}|dkr4td�p2d}|j|�}|j|||||||||
|||
||||d�dS)a6
        Append to Table in file. Node must already exist and be Table
        format.

        Parameters
        ----------
        key : str
        value : {Series, DataFrame}
        format : 'table' is the default
            Format to use when storing object in HDFStore.  Value can be one of:

            ``'table'``
                Table format. Write as a PyTables Table structure which may perform
                worse but allow more flexible operations like searching / selecting
                subsets of the data.
        append       : bool, default True
            Append the input data to the existing.
        data_columns : list of columns, or True, default None
            List of columns to create as indexed data columns for on-disk
            queries, or True to use all columns. By default only the axes
            of the object are indexed. See `here
            <https://pandas.pydata.org/pandas-docs/stable/user_guide/io.html#query-via-data-columns>`__.
        min_itemsize : dict of columns that specify minimum str sizes
        nan_rep      : str to use as str nan representation
        chunksize    : size to chunk the writing
        expectedrows : expected TOTAL row size of this table
        encoding     : default None, provide an encoding for str
        dropna : bool, default False
            Do not write an ALL nan row to the store settable
            by the option 'io.hdf.dropna_table'.

        Notes
        -----
        Does *not* check if data being appended overlaps with existing
        data in the table, so be careful
        Nz>columns is not a supported keyword in append, try data_columnszio.hdf.dropna_tablezio.hdf.default_formatr_)rp�axesrqrOrornrrrvr�expectedrowsrsrtrCru)r�r
r�r)r�rkrlrprrqrOrornr�rrrvrrrsrtrCrur?r?r@rO�s28
zHDFStore.append)�dcs�|dk	rtd��t|t�s"td��||kr2td��ttt�j��ttt	����d}d}	g}
x<|j
�D]0\}��dkr�|	dk	r�td��|}	qj|
j��qjW|	dk	rڈj|}|j
t|
��}
t|j|
��}
|j|
�||	<|dkr�||}|�r2�fdd�|j�D�}t|�}x|D]}|j|�}�qW�j|�|jd	d�}xt|j
�D]h\}�||k�r^|nd}�j�|d
�}|dk	�r��fdd�|j
�D�nd}|j||f||d
�|���qHWdS)a
        Append to multiple tables

        Parameters
        ----------
        d : a dict of table_name to table_columns, None is acceptable as the
            values of one node (this will get all the remaining columns)
        value : a pandas object
        selector : a string that designates the indexable table; all of its
            columns will be designed as data_columns, unless data_columns is
            passed, in which case these are used
        data_columns : list of columns to create as data columns, or True to
            use all columns
        dropna : if evaluates to True, drop rows from all tables if any single
                 row in each table has all NaN. Default False.

        Notes
        -----
        axes parameter is currently not accepted

        Nztaxes is currently not accepted as a parameter to append_to_multiple; you can create the tables independently insteadzQappend_to_multiple must have a dictionary specified as the way to split the valuez=append_to_multiple requires a selector that is in passed dictrz<append_to_multiple can only have one value in d that is Nonec3s |]}�|jdd�jVqdS)�all)�howN)rsrq)r��cols)rlr?r@�	<genexpr>Esz.HDFStore.append_to_multiple.<locals>.<genexpr>rr)r�csi|]\}}|�kr||�qSr?r?)r�rkrl)�vr?r@�
<dictcomp>Usz/HDFStore.append_to_multiple.<locals>.<dictcomp>)rtrr)r�r:�dictr�rL�set�range�ndim�	_AXES_MAPr�r��extendr�
differencer%�sorted�get_indexer�take�values�next�intersection�loc�pop�reindexrO)r�rrlr�rtrrsr�r�Z
remain_keyZ
remain_valuesr��orderedZorddZidxsZvalid_indexrqrr�dc�val�filteredr?)rrlr@�append_to_multiple�sN
&


 zHDFStore.append_to_multiple)rk�optlevel�kindcCsBt�|j|�}|dkrdSt|t�s.td��|j|||d�dS)a�
        Create a pytables index on the table.

        Parameters
        ----------
        key : str
        columns : None, bool, or listlike[str]
            Indicate which columns to create an index on.

            * False : Do not create any indexes.
            * True : Create indexes on all columns.
            * None : Create indexes on all columns.
            * listlike : Create indexes on the given columns.

        optlevel : int or None, default None
            Optimization level, if None, pytables defaults to 6.
        kind : str or None, default None
            Kind of index, if None, pytables defaults to "medium".

        Raises
        ------
        TypeError: raises if the node is not a table
        Nz1cannot create table index on a Fixed format store)r�r#r$)rir�r:r�r��create_index)r�rkr�r#r$r>r?r?r@�create_table_index[s

zHDFStore.create_table_indexcCs"t�|j�dd�|jj�D�S)z�
        Return a list of all the top-level nodes.

        Each node returned is not a pandas storage object.

        Returns
        -------
        list
            List of objects.
        cSsRg|]J}t|tjj�rt|jdd�sJt|dd�sJt|tjj�r|jdkr|�qS)�pandas_typeNr_)	r:rd�link�Link�getattr�_v_attrsr_r�r�)r�r�r?r?r@r��s
z#HDFStore.groups.<locals>.<listcomp>)rir�r��walk_groups)r�r?r?r@r��szHDFStore.groupsr�ccs�t�|j�x�|jj|�D]�}t|jdd�dk	r4qg}g}xP|jj�D]B}t|jdd�}|dkr~t|t	j
j�r�|j|j
�qH|j|j
�qHW|jjd�||fVqWdS)au
        Walk the pytables group hierarchy for pandas objects.

        This generator will yield the group path, subgroups and pandas object
        names for each group.

        Any non-pandas PyTables objects that are not a group will be ignored.

        The `where` group itself is listed first (preorder), then each of its
        child groups (following an alphanumerical order) is also traversed,
        following the same procedure.

        .. versionadded:: 0.24.0

        Parameters
        ----------
        where : str, default "/"
            Group where to start walking.

        Yields
        ------
        path : str
            Full path to a group (without trailing '/').
        groups : list
            Names (strings) of the groups contained in `path`.
        leaves : list
            Names (strings) of the pandas objects contained in `path`.
        r'Nr�)rir�r�r,r*r+Z_v_childrenrr:rdr��GrouprOr�r��rstrip)r�rRr�r�Zleaves�childr'r?r?r@�walk�sz
HDFStore.walkr8cCs~|j�|jd�sd|}|jdk	s(t�tdk	s4t�y|jj|j|�}Wntjjk
r`dSXt	|tj
�sztt|���|S)z; return the node with the key or None if it does not exist r�N)r��
startswithr�r�rdr�r��
exceptionsZNoSuchNodeErrorr:r8r�)r�rkr�r?r?r@r��s
zHDFStore.get_node�GenericFixedr�cCs8|j|�}|dkr"td|�d���|j|�}|j�|S)z> return the storer object for a key, raise if not in the file NzNo object named z in the file)r�r�r�r�)r�rkr�r>r?r?r@r��s

zHDFStore.get_storerrT)�propindexesrnr�c		Cs�t|||||d�}	|dkr&t|j��}t|ttf�s:|g}x�|D]�}
|j|
�}|dk	r@|
|	krl|rl|	j|
�|j|
�}t|t�r�d}
|r�dd�|j	D�}
|	j
|
||
t|dd�|jd�q@|	j
|
||jd�q@W|	S)	a
        Copy the existing store to a new file, updating in place.

        Parameters
        ----------
        propindexes: bool, default True
            Restore indexes in copied file.
        keys       : list of keys to include in the copy (defaults to all)
        overwrite  : overwrite (remove and replace) existing nodes in the
            new store (default is True)
        mode, complib, complevel, fletcher32 same as in HDFStore.__init__

        Returns
        -------
        open file handle of the new store
        )rmrornr�NFcSsg|]}|jr|j�qSr?)�
is_indexedrF)r�rjr?r?r@r�sz!HDFStore.copy.<locals>.<listcomp>rt)rqrtrC)rC)ryrLr�r:rMr�r�r�r�rrOr*rCrx)r�rfrmr4r�rornr��	overwriteZ	new_storer�r>�datarqr?r?r@�copy�s2





z
HDFStore.copyc
Cst|j�}t|��d|�d�}|j�rt|j��}t|�r�g}g}x�|D]�}y<|j|�}|dk	r�|jt|j	pn|��|jt|p�d��WqHt
k
r��YqHtk
r�}z*|j|�t|�}	|jd|	�d��WYdd}~XqHXqHW|td||�7}n|d7}n|d	7}|S)
zg
        Print detailed information on the store.

        Returns
        -------
        str
        z
File path: r�Nzinvalid_HDFStore nodez[invalid_HDFStore node: r���EmptyzFile is CLOSED)
r5r�r�r�rr�rQr�rOr�r�r�r4)
r�r��outputZlkeysr�rr�r>ZdetailZdstrr?r?r@�info"s.



(
z
HDFStore.infocCs|jst|j�d���dS)Nz file is not open!)r�rZr�)r�r?r?r@r�LszHDFStore._check_if_open)rpr�cCsJyt|j�}Wn4tk
rD}ztd|�d��|�WYdd}~XnX|S)z validate / deprecate formats z#invalid HDFStore format specified [r�N)�_FORMAT_MAP�lowerr�r�)r�rpr�r?r?r@r�Ps
$zHDFStore._validate_format�UTF-8)rlrCrur�c"s$�dk	r t�ttf�r td�����fdd�}tt�jdd��}tt�jdd��}|dkr̈dkr�t�tdk	svt	�t�dd�s�t�tj
j�r�d}d	}q�td
��n(tdtdi}	|	t��}�dkr�|d
7}d|k�r,t
td�}
y|
|}Wn.tk
�r}z|d�|�WYdd}~XnX||�||d�S|dk�rƈdk	�r�|dk�r�t�dd�}
|
dk	�r�|
jdk�rrd}n|
jdk�r�d}nB|dk�r�t�dd�}
|
dk	�r�|
jdk�r�d}n|
jdk�r�d}ttttttd�}y||}Wn.tk
�r}z|d�|�WYdd}~XnX||�||d�S)z$ return a suitable class to operate Nz(value must be None, Series, or DataFramec	s$td|�d��dt���d����S)Nz(cannot properly create the storer for: [z
] [group->z,value->z	,format->)r�r�)ra)rpr�rlr?r@�errorhsz&HDFStore._create_storer.<locals>.errorr'�
table_typer_�frame_table�
generic_tablezKcannot create a storer if the object is not existing nor a value are passed�series�frameZ_table)rDrE�_STORER_MAP)rCru�series_tablerqrI�appendable_series�appendable_multiseries�appendable_frame�appendable_multiframe)rCrHrIrJrK�worm�
_TABLE_MAP)r:r)r#r�rAr*r+rirdr�r_r�r��SeriesFixed�
FrameFixedr��nlevels�GenericTable�AppendableSeriesTable�AppendableMultiSeriesTable�AppendableFrameTable�AppendableMultiFrameTable�	WORMTable)r�r�rprlrCrur@�pt�ttZ	_TYPE_MAPrF�clsr�rqrMr?)rpr�rlr@r�Zsj







zHDFStore._create_storer)rkrlrnrrrur�cCs~|j|�}|jdk	st�|dk	r:|r:|jj|dd�d}t|dd�rV|dksR|rVdS|dkr�|jd�}d}xX|D]P}t|�s�qr|}|jd�s�|d7}||7}|j|�}|dkr�|jj||�}|}qrW|j	|||||d�}|�r|j
p�|j
o�|dko�|j�rtd��|j�s |j
�n|j
�|j
�r8|�r8td	��|j||||||	|
|||
|||d
�
t|t��rz|�rz|j|d�dS)NT)r�emptyr_r�)rCrur^zCan only append to Tablesz0Compression not supported on Fixed format stores)
�objrrOrornr�rrrrrsrvrtr�)r�)r�r�r��remove_noder*�splitrQ�endswithZcreate_groupr�r��	is_existsr��set_object_info�writer:r�r%)r�rkrlrprrqrOrornr�rrrrrsrvrtrCrur�r��pathsr��p�new_pathr>r?r?r@r�s\





zHDFStore._write_to_group)r�cCs|j|�}|j�|j�S)N)r�r�r�)r�r�r>r?r?r@r�s
zHDFStore._read_group)rjNNF)r�)rj)F)NNNNFNF)NNN)NN)NNNNNFNF)NTFNNNNNNrcT)NNN)NNTTNNNNNNNNNNrc)NNF)NNN)r�)r3r�)rTTNNNFT)NNr?rc)r3r�)NTFNNNNNNFNNNrcT);rWrXrY�__doc__rr�rEr��intr�r�r�r�r��propertyr�r�r�r�r�r�r�r�r�r�r�rr�r�r��	iteritemsr�r�r�r�r�r�r�r�r�rrrrxr�rOr"r&r�r0r�r�r8r<r�r�r�rr�r?r?r?r@ry�s�
@
"JE&t<7=@H]#
/
3*
[2Eryc@sfeZdZUdZeeee	d
dee	deeeed�dd�Zd	d
�Z
dd�Zded
�dd�ZdS)r�aa
    Define the iteration interface on a table

    Parameters
    ----------
    store : HDFStore
    s     : the referred storer
    func  : the function to execute the query
    where : the where of the query
    nrows : the rows to iterate on
    start : the passed start value (default is None)
    stop  : the passed stop value (default is None)
    iterator : bool, default False
        Whether to use the default iterator.
    chunksize : the passed chunking value (default is 100000)
    auto_close : bool, default False
        Whether to automatically close the store at the end of iteration.
    r3r�NF)rwr>r�rr�cCs�||_||_||_||_|jjrN|dkr,d}|dkr8d}|dkrD|}t||�}||_||_||_d|_	|sr|	dk	r�|	dkr~d}	t
|	�|_nd|_|
|_dS)Nri��)
rwr>r�rRr��minr�r}r~r�rfrr�)r�rwr>r�rRr�r}r~r�rr�r?r?r@r�-s,

zTableIterator.__init__ccsj|j}xV||jkr\t||j|j�}|jdd|j||��}|}|dkst|�rTq|VqW|j�dS)N)r}r~rirr�r�rQr�)r�r�r~rlr?r?r@r�Ws
zTableIterator.__iter__cCs|jr|jj�dS)N)r�rwr�)r�r?r?r@r�gszTableIterator.close)r�cCs�|jdk	r4t|jt�std��|jj|jd�|_|S|rft|jt�sLtd��|jj|j|j|j	d�}n|j}|j
|j|j	|�}|j�|S)Nz0can only use an iterator or chunksize on a table)rRz$can only read_coordinates on a table)rRr}r~)rr:r>r�r�r�rRr�r}r~r�r�)r�r�rR�resultsr?r?r@r�ks
zTableIterator.get_result)r3r�)NNFNF)r3r�)F)rWrXrYrerrfrryrwrr>r�r�r�r�r�r?r?r?r@r�s
	 r�c
@steZdZUdZdZdZdddgZee	dBee
ed�dd	�Zee
d
�dd��Zeed
�d
d��Ze
d�dd�Zed
�dd�Zeed�dd�Zed
�dd�Zeed
�dd��Zejeed�dd�Zdd�Zed d!��Zed"d#��Zed$d%��Zed&d'��Zd(d)�ZdCd*d+�Z d,d-�Z!d.ed/�d0d1�Z"dDd2d3�Z#ed4�d5d6�Z$d7d8�Z%d9d:�Z&d;d<�Z'd.d=�d>d?�Z(d.d=�d@dA�Z)dS)E�IndexCola
    an index column description class

    Parameters
    ----------
    axis   : axis which I reference
    values : the ndarray like converted values
    kind   : a string description of this type
    typ    : the pytables type
    pos    : the position in the pytables

    T�freq�tz�
index_nameN)rF�cnamecCs�t|t�std��||_||_||_||_|p0||_||_||_	||_
|	|_|
|_||_
||_|
|_||_|dk	r||j|�t|jt�s�t�t|jt�s�t�dS)Nz`name` must be a str.)r:rEr�rr$�typrFror��posrlrmrnrr_r��metadata�set_posr�)r�rFrr$rpror�rqrlrmrnrr_r�rrr?r?r@r��s(


zIndexCol.__init__)r�cCs|jjS)N)rp�itemsize)r�r?r?r@rt�szIndexCol.itemsizecCs|j�d�S)N�_kind)rF)r�r?r?r@�	kind_attr�szIndexCol.kind_attr)rqcCs$||_|dk	r |jdk	r ||j_dS)z. set the position of this column in the Table N)rqrpZ_v_pos)r�rqr?r?r@rs�szIndexCol.set_poscCsFttt|j|j|j|j|jf��}djdd�t	dddddg|�D��S)	N�,css |]\}}|�d|��VqdS)z->Nr?)r�rkrlr?r?r@r�sz$IndexCol.__repr__.<locals>.<genexpr>rFror�rqr$)
rM�mapr5rFror�rqr$�joinr�)r��tempr?r?r@r��s
 zIndexCol.__repr__)�otherr�cst��fdd�dD��S)z compare 2 col items c3s&|]}t�|d�t�|d�kVqdS)N)r*)r�rj)r{r�r?r@r�sz"IndexCol.__eq__.<locals>.<genexpr>rFror�rq)rFror�rq)r)r�r{r?)r{r�r@�__eq__�szIndexCol.__eq__cCs|j|�S)N)r|)r�r{r?r?r@�__ne__�szIndexCol.__ne__cCs"t|jd�sdSt|jj|j�jS)z' return whether I am an indexed column r
F)�hasattrr_r*r
ror5)r�r?r?r@r5�szIndexCol.is_indexed)rrCrucCs�t|tj�stt|���|jjdk	r.||j}t|j	�}t
||||�}t�}t|j�|d<|j
dk	rrt|j
�|d<yt|f|�}Wn0tk
r�d|kr�d|d<t|f|�}YnXt||j�}||fS)zV
        Convert the data from this selection to the appropriate pandas type.
        NrFrl)r:r;�ndarrayr�r��dtype�fieldsrorAr$�_maybe_convertrrnrlr%r��_set_tzrm)r�rrvrCru�val_kindr�Znew_pd_indexr?r?r@�convert�s"


zIndexCol.convertcCs|jS)z return the values)r)r�r?r?r@�	take_dataszIndexCol.take_datacCs|jjS)N)r_r+)r�r?r?r@�attrsszIndexCol.attrscCs|jjS)N)r_�description)r�r?r?r@r�szIndexCol.descriptioncCst|j|jd�S)z# return my current col description N)r*r�ro)r�r?r?r@�colszIndexCol.colcCs|jS)z return my cython values )r)r�r?r?r@�cvalues$szIndexCol.cvaluescCs
t|j�S)N)r�r)r�r?r?r@r�)szIndexCol.__iter__cCsPt|j�dkrLt|t�r$|j|j�}|dk	rL|jj|krLt�j	||j
d�|_dS)z�
        maybe set a string col itemsize:
            min_itemsize can be an integer or a dict with this columns name
            with an integer size
        �stringN)rtrq)rAr$r:rr�rFrprtri�	StringColrq)r�rrr?r?r@�maybe_set_size,s

zIndexCol.maybe_set_sizecCsdS)Nr?)r�r?r?r@�validate_names:szIndexCol.validate_names�AppendableTable)�handlerrOcCs:|j|_|j�|j|�|j|�|j|�|j�dS)N)r_�validate_col�
validate_attr�validate_metadata�write_metadata�set_attr)r�r�rOr?r?r@�validate_and_set=s


zIndexCol.validate_and_setcCs^t|j�dkrZ|j}|dk	rZ|dkr*|j}|j|krTtd|�d|j�d|j�d���|jSdS)z< validate this column: return the compared against itemsize r�Nz#Trying to store a string with len [z] in [z)] column but
this column has a limit of [zC]!
Consider using min_itemsize to preset the sizes on these columns)rAr$r�rtr�ro)r�rt�cr?r?r@r�Es
zIndexCol.validate_col)rOcCsB|r>t|j|jd�}|dk	r>||jkr>td|�d|j�d���dS)Nzincompatible kind in col [z - r�)r*r�rvr$r�)r�rOZ
existing_kindr?r?r@r�Xs
zIndexCol.validate_attrcCs�x�|jD]�}t||d�}|j|ji�}|j|�}||kr�|dk	r�||kr�|dkr�t|||f}tj|tdd�d||<t	||d�q�t
d|j�d|�d|�d	|�d
�	��q|dk	s�|dk	r|||<qWdS)z
        set/update the info for this indexable with the key/value
        if there is a conflict raise/warn as needed
        Nrlrn�)�
stacklevelzinvalid info for [z] for [z], existing_value [z] conflicts with new value [r�)rlrn)�_info_fieldsr*�
setdefaultrFr��attribute_conflict_doc�warnings�warnr\�setattrr�)r�r<rkrl�idxZexisting_value�wsr?r?r@�update_infoas
$zIndexCol.update_infocCs$|j|j�}|dk	r |jj|�dS)z# set my state from the passed info N)r�rF�__dict__�update)r�r<r�r?r?r@�set_info�szIndexCol.set_infocCst|j|j|j�dS)z set the kind for this column N)r�r�rvr$)r�r?r?r@r��szIndexCol.set_attr)r�cCsD|jdkr@|j}|j|j�}|dk	r@|dk	r@t||�r@td��dS)z< validate that kind=category does not change the categories �categoryNzEcannot append a categorical with different categories to the existing)r�rr�
read_metadataror"r�)r�r�Znew_metadataZcur_metadatar?r?r@r��s
zIndexCol.validate_metadatacCs|jdk	r|j|j|j�dS)z set the meta data N)rrr�ro)r�r�r?r?r@r��s
zIndexCol.write_metadata)
NNNNNNNNNNNNN)N)N)*rWrXrYre�is_an_indexable�is_data_indexabler�rErFrorr�rgrfrtrvrsr�rr�r|r}r5r;rr�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r?r?r?r@rk�s\

 

	 rkc@s>eZdZdZeed�dd��Zeje	e	d�dd�Z
dd	�Zd
S)�GenericIndexColz< an index which is not represented in the data of the table )r�cCsdS)NFr?)r�r?r?r@r5�szGenericIndexCol.is_indexed)rrCrucCs2t|tj�stt|���ttjt|���}||fS)z�
        Convert the data from this selection to the appropriate pandas type.

        Parameters
        ----------
        values : np.ndarray
        nan_rep : str
        encoding : str
        errors : str
        )r:r;rr�r�r&�arangerQ)r�rrvrCrur?r?r@r��szGenericIndexCol.convertcCsdS)Nr?)r�r?r?r@r��szGenericIndexCol.set_attrN)rWrXrYrergr�r5r;rrEr�r�r?r?r?r@r��s
r�cs0eZdZdZdZdZddgZd2ed��fdd�
Ze	ed	�d
d��Z
e	ed	�dd
��Zed	�dd�Ze
ed�dd�Zed�dd�Zdd�Zeedd�dd��Zedd��Zeeedd�dd ��Zeedd�d!d"��Zed#d$��Zed%d&��Ze	d'd(��Ze	d)d*��Zd+d,�Zejeed-�d.d/�Z d0d1�Z!�Z"S)3�DataCola3
    a data holding column, by definition this is not indexable

    Parameters
    ----------
    data   : the actual data
    cname  : the column name in the table to hold the data (typically
                values)
    meta   : a string description of the metadata
    metadata : the actual metadata
    FrmrN)rFc
s2t�j|||||||||	|
|d�||_|
|_dS)N)rFrr$rprqrormrr_r�rr)�superr�r�r7)r�rFrr$rprorqrmrr_r�rrr�r7)�	__class__r?r@r��szDataCol.__init__)r�cCs|j�d�S)N�_dtype)rF)r�r?r?r@�
dtype_attr�szDataCol.dtype_attrcCs|j�d�S)N�_meta)rF)r�r?r?r@�	meta_attr�szDataCol.meta_attrcCsFttt|j|j|j|j|jf��}djdd�t	dddddg|�D��S)	Nrwcss |]\}}|�d|��VqdS)z->Nr?)r�rkrlr?r?r@r�sz#DataCol.__repr__.<locals>.<genexpr>rFror�r$�shape)
rMrxr5rFror�r$r�ryr�)r�rzr?r?r@r��szDataCol.__repr__)r{r�cst��fdd�dD��S)z compare 2 col items c3s&|]}t�|d�t�|d�kVqdS)N)r*)r�rj)r{r�r?r@r	sz!DataCol.__eq__.<locals>.<genexpr>rFror�rq)rFror�rq)r)r�r{r?)r{r�r@r|	szDataCol.__eq__)r7cCs@|dk	st�|jdkst�t|�\}}||_||_t|�|_dS)N)r�r��_get_data_and_dtype_namer7�_dtype_to_kindr$)r�r7�
dtype_namer?r?r@�set_data	szDataCol.set_datacCs|jS)z return the data )r7)r�r?r?r@r�	szDataCol.take_datar6)rr�cCs�|j}|j}|j}|jdkr&d|jf}t|t�rJ|j}|j||jj	d�}ntt
|�sZt|�rf|j|�}nXt
|�rz|j|�}nDt|�r�t�j||dd�}n&t|�r�|j||�}n|j||j	d�}|S)zW
        Get an appropriately typed and shaped pytables.Col object for values.
        rI)r$r)rtr�)r�rtr�r�sizer:r-�codes�
get_atom_datarFrr�get_atom_datetime64r �get_atom_timedelta64rriZ
ComplexColr�get_atom_string)rYrr�rtr�r��atomr?r?r@�	_get_atom	s$


zDataCol._get_atomcCst�j||dd�S)Nr)rtr�)rir�)rYr�rtr?r?r@r�6	szDataCol.get_atom_string)r$r�cCsR|jd�r$|dd�}d|�d�}n"|jd�r4d}n|j�}|�d�}tt�|�S)z2 return the PyTables column class for this column �uint�NZUIntr6�period�Int64Col)r1�
capitalizer*ri)rYr$Zk4Zcol_nameZkcapr?r?r@�get_atom_coltype:	s


zDataCol.get_atom_coltypecCs|j|d�|dd�S)N)r$r)r�)r�)rYr�r$r?r?r@r�I	szDataCol.get_atom_datacCst�j|dd�S)Nr)r�)rir�)rYr�r?r?r@r�M	szDataCol.get_atom_datetime64cCst�j|dd�S)Nr)r�)rir�)rYr�r?r?r@r�Q	szDataCol.get_atom_timedelta64cCst|jdd�S)Nr�)r*r7)r�r?r?r@r�U	sz
DataCol.shapecCs|jS)z return my cython values )r7)r�r?r?r@r�Y	szDataCol.cvaluescCs`|r\t|j|jd�}|dk	r2|t|j�kr2td��t|j|jd�}|dk	r\||jkr\td��dS)zAvalidate that we have the same order as the existing & same dtypeNz4appended items do not match existing items in table!z@appended items dtype do not match existing items dtype in table!)r*r�rvrLrr�r�r�)r�rOZexisting_fieldsZexisting_dtyper?r?r@r�^	szDataCol.validate_attr)rrCrucCst|tj�stt|���|jjdk	r.||j}|jdk	s<t�|jdkr\t	|�\}}t
|�}n|}|j}|j}t|tj�s|t�t|j
�}|j}	|j}
|j}|dk	s�t�t|�}|dkr�t||dd�}�n(|dkr�tj|dd�}�n|dk�r8ytjd	d
�|D�td�}Wn.tk
�r4tjdd
�|D�td�}YnXn�|dk�r�|	}
|j�}|
dk�rhtgtjd�}
n<t|
�}|j��r�|
|}
||dk|jt�j�j8<tj||
|
d�}n8y|j|dd�}Wn$t k
�r�|jddd�}YnXt|�dk�rt!||||d�}|j"|fS)aR
        Convert the data from this selection to the appropriate pandas type.

        Parameters
        ----------
        values : np.ndarray
        nan_rep :
        encoding : str
        errors : str

        Returns
        -------
        index : listlike to become an Index
        data : ndarraylike to become a column
        N�
datetime64T)�coerce�timedelta64zm8[ns])r�rcSsg|]}tj|��qSr?)r�fromordinal)r�rr?r?r@r��	sz#DataCol.convert.<locals>.<listcomp>cSsg|]}tj|��qSr?)r�
fromtimestamp)r�rr?r?r@r��	sr�rI)�
categoriesrF)r8�Or�)rvrCru���)#r:r;rr�r�r�r�rorpr�r�r$rAr�rrrrmr��asarray�objectr��ravelr%Zfloat64r,�any�astyperfZcumsum�_valuesr-Z
from_codesr��_unconvert_string_arrayr)r�rrvrCru�	convertedr�r$r�rrrrmr�r�r��maskr?r?r@r�k	s^








 zDataCol.convertcCsHt|j|j|j�t|j|j|j�|jdk	s2t�t|j|j|j�dS)z set the data for this column N)	r�r�rvrr�r�r�r�r�)r�r?r?r@r��	szDataCol.set_attr)NNNNNNNNNNNN)#rWrXrYrer�r�r�rEr�rgr�r�r�rr�r|rr�r��classmethodr�r�r
r�r�r�r�r�r�r�r;rr�r��
__classcell__r?r?)r�r@r��sL


gr�c@sTeZdZdZdZdd�Zedd��Zeedd�d	d
��Z	edd��Z
ed
d��ZdS)�DataIndexableColz- represent a data column that can be indexed TcCst|j�j�std��dS)Nz-cannot have non-object label DataIndexableCol)r%r�	is_objectr�)r�r?r?r@r��	szDataIndexableCol.validate_namescCst�j|d�S)N)rt)rir�)rYr�rtr?r?r@r��	sz DataIndexableCol.get_atom_stringr6)r$r�cCs|j|d��S)N)r$)r�)rYr�r$r?r?r@r��	szDataIndexableCol.get_atom_datacCs
t�j�S)N)rir�)rYr�r?r?r@r��	sz$DataIndexableCol.get_atom_datetime64cCs
t�j�S)N)rir�)rYr�r?r?r@r��	sz%DataIndexableCol.get_atom_timedelta64N)rWrXrYrer�r�r�r�rEr�r�r�r?r?r?r@r��	sr�c@seZdZdZdS)�GenericDataIndexableColz* represent a generic pytables data column N)rWrXrYrer?r?r?r@r��	sr�c@s�eZdZUdZedZeeee	e
fe
eededZd=edeed�dd	�Zeed
�dd��Zeeeeefd
�d
d��Zedd��Zed
�dd�Zdd�Zdd�Zedd��Zedd��Zedd��Z edd��Z!eed
�dd ��Z"eed
�d!d"��Z#ed#d$��Z$d%d&�Z%d'd(�Z&ed)d*��Z'eed
�d+d,��Z(ed-d.��Z)d/d0�Z*d>d2d3�Z+d4d5�Z,d?e-ee-ed6�d7d8�Z.d9d:�Z/d@e-ee-ed6�d;d<�Z0d1S)A�Fixedz�
    represent an object in my store
    facilitate read/write of various types of objects
    this is an abstract base class

    Parameters
    ----------
    parent : HDFStore
    group : Node
        The group node where the table resides.
    r^r8F�UTF-8rc)r�r�rCrucCsZt|t�stt|���tdk	s"t�t|tj�s:tt|���||_||_t|�|_	||_
dS)N)r:ryr�r�rdr8r�r�rDrCru)r�r�r�rCrur?r?r@r�
s
zFixed.__init__)r�cCs*|jddko(|jddko(|jddkS)NrrI�
�)�version)r�r?r?r@�is_old_version!
szFixed.is_old_versioncCsbtt|jjdd��}y0tdd�|jd�D��}t|�dkrB|d}Wntk
r\d	}YnX|S)
z compute and set our version �pandas_versionNcss|]}t|�VqdS)N)rf)r�rJr?r?r@r*
sz Fixed.version.<locals>.<genexpr>�.r�r)r)rrr)rAr*r�r+rMr]rQrh)r�r�r?r?r@r�%
s
z
Fixed.versioncCstt|jjdd��S)Nr')rAr*r�r+)r�r?r?r@r'1
szFixed.pandas_typecCs^|j�|j}|dk	rXt|ttf�rDdjdd�|D��}d|�d�}|jd�d|�d	�S|jS)
z* return a pretty representation of myself Nrwcss|]}t|�VqdS)N)r5)r�rJr?r?r@r;
sz!Fixed.__repr__.<locals>.<genexpr>�[r�z12.12z	 (shape->�))r�r�r:rLrMryr')r�r>Zjshaper?r?r@r�5
szFixed.__repr__cCst|j�|j_tt�|j_dS)z set my pandas type & version N)rE�pandas_kindr�r'�_versionr�)r�r?r?r@r`@
szFixed.set_object_infocCstj|�}|S)N)r8)r�Znew_selfr?r?r@r8E
s
z
Fixed.copycCs|jS)N)r�)r�r?r?r@r�I
szFixed.shapecCs|jjS)N)r�r�)r�r?r?r@r�M
szFixed.pathnamecCs|jjS)N)r�r�)r�r?r?r@r�Q
sz
Fixed._handlecCs|jjS)N)r�r�)r�r?r?r@r�U
szFixed._filterscCs|jjS)N)r�r�)r�r?r?r@r�Y
szFixed._complevelcCs|jjS)N)r�r�)r�r?r?r@r�]
szFixed._fletcher32cCs|jjS)N)r�r+)r�r?r?r@r�a
szFixed.attrscCsdS)z set our object attributes Nr?)r�r?r?r@�	set_attrse
szFixed.set_attrscCsdS)z get our object attributes Nr?)r�r?r?r@�	get_attrsi
szFixed.get_attrscCs|jS)z return my storable )r�)r�r?r?r@�storablem
szFixed.storablecCsdS)NFr?)r�r?r?r@r_r
szFixed.is_existscCst|jdd�S)Nr�)r*r�)r�r?r?r@r�v
szFixed.nrowscCs|dkrdSdS)z' validate against an existing storable NTr?)r�r{r?r?r@�validatez
szFixed.validateNcCsdS)z- are we trying to operate on an old version? Tr?)r�rRr?r?r@�validate_version�
szFixed.validate_versioncCs|j}|dkrdS|j�dS)zr
        infer the axes of my storer
        return a boolean indicating if we have a valid storer or not
        NFT)r�r�)r�r>r?r?r@r��
s
zFixed.infer_axes)r}r~cCstd��dS)Nz>cannot read on an abstract storer: subclasses should implement)r�)r�rRr�r}r~r?r?r@r��
sz
Fixed.readcKstd��dS)Nz?cannot write on an abstract storer: subclasses should implement)r�)r�r�r?r?r@ra�
szFixed.writecCs0tj|||�r$|jj|jdd�dStd��dS)zs
        support fully deleting the node in its entirety (only) - where
        specification must be None
        T)rNz#cannot delete on an abstract storer)rrr�r\r�r�)r�rRr}r~r?r?r@r�
szFixed.delete)r�rc)N)NNNN)NNN)1rWrXrYrerEr��format_typer
rr#r)�obj_typerfrrCryr�r�rur�r�rgr�r�r	r�r'r�r`r8r�r�r�r�r�r�r�r�r�r�r_r�r�r�r�rr�rarr?r?r?r@r��	sX



r�c@sJeZdZUdZedediZdd�ej�D�ZgZ	e
e	ed�dd�Zd	d
�Z
dd�Zd
d�Zeed�dd��Zdd�Zdd�Zdd�Zd/eeeeed�dd�Zd0eeeeeed�dd�Zeed�dd �Zeed�d!d"�Zd1eeeeeed�d#d$�Zd2d%eeeeed&�d'd(�Zee d)�d*d+�Z!d3ee eed,�d-d.�Z"dS)4r3z a generified fixed version �datetimer�cCsi|]\}}||�qSr?r?)r�r�rr?r?r@r
�
szGenericFixed.<dictcomp>)r�cCs|jj|d�S)N�)�_index_type_mapr�)r�rYr?r?r@�_class_to_alias�
szGenericFixed._class_to_aliascCst|t�r|S|jj|t�S)N)r:r��_reverse_index_mapr�r%)r��aliasr?r?r@�_alias_to_class�
s
zGenericFixed._alias_to_classcCs0|tkrddd�}|S|tkr,ddd�}|S|S)NcSs:tj|j|d�}tj|dd�}|dk	r6|jd�j|�}|S)N)rl)rF�UTC)r.�_simple_newrr$�tz_localize�
tz_convert)rrlrmZdta�resultr?r?r@r`�
s
z*GenericFixed._get_index_factory.<locals>.fcSstj||d�}tj|dd�S)N)rl)rF)r/r�r()rrlrmZparrr?r?r@r`�
s)NN)NN)r$r()r��klassr`r?r?r@�_get_index_factory�
s

zGenericFixed._get_index_factorycCs$|dk	rtd��|dk	r td��dS)zE
        raise if any keywords are passed which are not-None
        Nzqcannot pass a column specification when reading a Fixed format store. this store must be selected in its entiretyzucannot pass a where specification when reading from a Fixed format store. this store must be selected in its entirety)r�)r�r�rRr?r?r@�
validate_read�
szGenericFixed.validate_readcCsdS)NTr?)r�r?r?r@r_�
szGenericFixed.is_existscCs|j|j_|j|j_dS)z set our object attributes N)rCr�ru)r�r?r?r@r��
s
zGenericFixed.set_attrsc	CsVtt|jdd��|_tt|jdd��|_x(|jD]}t||tt|j|d���q0WdS)z retrieve our attributes rCNrurc)rDr*r�rCrAru�
attributesr�)r�r�r?r?r@r��
szGenericFixed.get_attrscKs|j�dS)N)r�)r�r[r�r?r?r@ra�
szGenericFixed.writeN)rkr}r~cCs�ddl}t|j|�}|j}t|dd�}t||j�rD|d||�}nztt|dd��}	t|dd�}
|
dk	rxtj|
|	d�}n|||�}|	dkr�t|d	d�}t	||d
d�}n|	dkr�tj
|d
d�}|r�|jS|SdS)z4 read an array for the specified node (off of group rN�
transposedF�
value_typer�)r�r�rmT)r�r�zm8[ns])rer*r�r+r:ZVLArrayrAr;rZr�r��T)r�rkr}r~rer�r�r��retr�r�rmr?r?r@�
read_array�
s&zGenericFixed.read_array)rkr}r~r�cCshtt|j|�d���}|dkr.|j|||d�S|dkrVt|j|�}|j|||d�}|Std|����dS)N�_variety�multi)r}r~�regularzunrecognized index variety: )rAr*r��read_multi_indexr��read_index_noder�)r�rkr}r~Zvarietyr�rqr?r?r@�
read_indexszGenericFixed.read_index)rkrqcCs�t|t�r,t|j|�d�d�|j||�n�t|j|�d�d�td||j|j�}|j||j	�t
|j|�}|j|j
_|j|j
_t|ttf�r�|jt|��|j
_t|tttf�r�|j|j
_t|t�r�|jdk	r�t|j�|j
_dS)Nrrrrq)r:r'r�r��write_multi_index�_convert_indexrCru�write_arrayrr*r�r$r+rFr$r(r�r��index_classr*rlrm�_get_tz)r�rkrqr�r�r?r?r@�write_index)s



zGenericFixed.write_indexcCs�t|j|�d�|j�x�tt|j|j|j��D]�\}\}}}t|�rLt	d��|�d|��}t
|||j|j�}|j
||j�t|j|�}	|j|	j_||	j_t|	j|�d|��|�|�d|��}
|j
|
|�q.WdS)N�_nlevelsz=Saving a MultiIndex with an extension dtype is not supported.�_level�_name�_label)r�r�rP�	enumerater��levelsr��namesrr�rrCrurrr*r�r$r+rF)r�rkrq�i�lev�level_codesrF�	level_keyZ
conv_levelr��	label_keyr?r?r@r
@s"
zGenericFixed.write_multi_indexcCs�t|j|�d��}g}g}g}xxt|�D]l}|�d|��}	t|j|	�}
|j|
||d�}|j|�|j|j�|�d|��}|j|||d�}
|j|
�q(Wt|||dd�S)Nrr)r}r~rT)rr�rr�)	r*r�rr�rrOrFrr')r�rkr}r~rPrr�rrrr�rrrr?r?r@rYs
zGenericFixed.read_multi_indexr8)r�r}r~r�cCs>|||�}d|jkr>tj|jj�dkr>tj|jj|jjd�}t|jj�}d}d|jkrlt|jj	�}t|�}|j
tt|jdd���}|j|�}i}	d|jkr�|jd|	d<d|jkr�t
|jdt�r�|jdjd	�|	d<n|jd|	d<|d
k�r|t|||j|jd�fdti|	��}
n|t|||j|jd�f|	�}
||
_	|
S)
Nr�r)r�rFr
r�rlrmzutf-8r)rCrur�)r+r;�prodr�rZrrAr$rGrFr�r*r�r:�bytesr=�_unconvert_indexrCrur�)r�r�r}r~r7r$rFr
�factoryr�rqr?r?r@rps<





zGenericFixed.read_index_node)rkrlcCsJtjd|j�}|jj|j||�t|j|�}t|j�|j	_
|j|j	_dS)z write a 0-len array rIN)rI)r;rZrr��create_arrayr�r*rEr�r+rr�)r�rkrlZarrr�r?r?r@�write_array_empty�s
zGenericFixed.write_array_empty)rkrlr�cCsJt|tjtf�stt|���||jkr6|jj|j|�|j	dk}d}t
|j�rVtd��|snt
|d�rn|j}d}d}|jdk	r�yt�jj|j�}Wntk
r�YnX|dk	r�|s�|jj|j|||j|jd�}||dd�<n|j||��nJ|jjtjk�r`tj|dd�}|�rn*|dk�rnt|||f}	tj|	td	d
�|jj|j|t�j��}
|
j |�n�|�rt|j||�n�t!|j��r�|jj"|j||j#d��dt$|j|�j%_&n�t'|j��r�|jj"|j||j(�t$|j|�}t)|j*�|j%_*d|j%_&nHt+|j��r$|jj"|j||j#d��d
t$|j|�j%_&n|jj"|j||�|t$|j|�j%_,dS)NrFz]Cannot store a category dtype in a HDF5 dataset that uses format="fixed". Use format="table".rT)r�)�skipnar��)r��i8r�r�)-r:r;rr!r�r�r�r�r\r�rr�r�r~rr�riZAtomZ
from_dtyper�Z
create_carrayr�r!Zobject_r�infer_dtype�performance_docr�r�rZcreate_vlarray�
ObjectAtomrOrr �viewr*r+rr�asi8rrmr r�)r�rkrlr�Zempty_arrayr�r��ca�
inferred_typer�Zvlarrr�r?r?r@r�sb






zGenericFixed.write_array)NN)NN)NN)NN)N)#rWrXrYrer$r(r�r�r�r�rrEr�r�r�r�rgr�r_r�r�rarrfrr%r	rr'r
rrrr!rr?r?r?r@r3�
s0
%2	r3csReZdZUdZdgZeedd��Zde	e
e	e
d�dd�Z�fd	d
�Z�Z
S)rNrDrFcCs,yt|jj�fSttfk
r&dSXdS)N)rQr�rr�rh)r�r?r?r@r�szSeriesFixed.shapeN)r}r~cCs<|j||�|jd||d�}|jd||d�}t|||jd�S)Nrq)r}r~r)rqrF)r�r	rr)rF)r�rRr�r}r~rqrr?r?r@r�szSeriesFixed.readcs:t�j|f|�|jd|j�|jd|j�|j|j_dS)Nrqr)r�rarrqrrrFr�)r�r[r�)r�r?r@raszSeriesFixed.write)NNNN)rWrXrYr�r�rrFrgr�rrfr�rar�r?r?)r�r@rNs
	rNcsPeZdZUddgZeedd��Zdeeeed�dd�Z	�fd	d
�Z
�ZS)�BlockManagerFixedr�nblockscCs�y�|j}d}xFt|j�D]8}t|jd|�d��}t|dd�}|dk	r||d7}qW|jj}t|dd�}|dk	r�t|d|d��}ng}|j|�|Stk
r�dSXdS)Nr�block�_itemsr�rI)	rrr-r*r�Z
block0_valuesrLrOrh)r�rr�rr�r�r?r?r@r�&s"
zBlockManagerFixed.shapeN)r}r~cCs |j||�|j�jd�}g}xJt|j�D]<}||kr>||fnd\}}	|jd|��||	d�}
|j|
�q*W|d}g}xht|j�D]Z}|jd|�d��}
|jd|�d�||	d�}||j	|
�}t
|j||dd�}|j|�q�Wt|�dk�rt
|dd	�}|j|d
d�}|St
|d|dd�S)
Nrr�)r}r~r.r/r�rI)r�rq)r�F)r�r8)NN)r�r�Z_get_block_manager_axisrrr	rOr-rrr#rrQr+r)r�rRr�r}r~Zselect_axisrrr�r��axr��dfs�	blk_itemsr�df�outr?r?r@r�As(zBlockManagerFixed.readcs�t�j|f|�|j}|j�s&|j�}|j|j_x@t|j�D]2\}}|dkr\|j	r\t
d��|jd|��|�q<Wt|j
�|j_xRt|j
�D]D\}}|jj|j�}|jd|�d�|j|d�|jd|�d�|�q�WdS)Nrz/Columns index has to be unique for fixed formatr�r.r�)r�r/)r�ra�_mgrZis_consolidatedZconsolidaterr�rrZ	is_uniquer�rrQ�blocksr-r�r�mgr_locsrr)r�r[r�r7rr0�blkr2)r�r?r@rafs
zBlockManagerFixed.write)NNNN)rWrXrYr�rfr-rgr�rr�rar�r?r?)r�r@r,!s
 r,c@seZdZdZeZdS)rOrEN)rWrXrYr�r#r�r?r?r?r@rO{srOcs�eZdZUdZdZdZeedZdZ	e
ee
e
eefe
ee
e
eddeded	��fd
d�
Zeed�d
d��Zed�dd�Zed�dd�Zdd�Zeed�dd��Zdd�Z eed�dd��Z!eed�dd��Z"edd��Z#ed d!��Z$ed"d#��Z%ed$d%��Z&ed&d'��Z'eed�d(d)��Z(eed�d*d+��Z)ed,d-��Z*eeefd�d.d/�Z+d0d1�Z,e
ed�d2d3�Z-eed4�d5d6�Z.ee/j0d7�d8d9�Z1ed:�d;d<�Z2d=d>�Z3d?d@�Z4dedAdB�Z5dCdD�Z6e7dEdF��Z8dfe9edG�dHdI�Z:dge9ee9ee
e
e;e;fdJ�dKdL�Z<e=edM�dNdO��Z>dPdQ�Z?dhe@edR�dSdT�ZAeBdUdV��ZCdidWdX�dYdZ�ZDe9eee9eeeefd[�d\d]�ZEdje9ee9ed^�d_d`�ZFdkee9ee9eda�dbdc�ZG�ZHS)lr�a�
    represent a table:
        facilitate read/write of various types of tables

    Attrs in Table Node
    -------------------
    These are attributes that are store in the main table node, they are
    necessary to recreate these tables when read back in.

    index_axes    : a list of tuples of the (original indexing axis and
        index column)
    non_index_axes: a list of tuples of the (original index axis and
        columns on a non-indexing axis)
    values_axes   : a list of the columns which comprise the data of this
        table
    data_columns  : a list of the columns that we are allowing indexing
        (these become single columns in values_axes), or True to force all
        columns
    nan_rep       : the string to use for nan representations for string
        objects
    levels        : the names of levels
    metadata      : the names of the metadata columns
    Z
wide_tabler_rITNrcr8)r�r�rucsRt�j||||d�|pg|_|p$g|_|p.g|_|p8g|_|	pDt�|_|
|_dS)N)rCru)	r�r��
index_axesr��values_axesrtrr<rv)r�r�r�rCrur9r�r:rtr<rv)r�r?r@r��s




zTable.__init__)r�cCs|jjd�dS)N�_r)rAr])r�r?r?r@�table_type_short�szTable.table_type_shortc
Cs�|j�t|j�rdj|j�nd}d|�d�}d}|jrZdjdd�|jD��}d|�d�}djd	d�|jD��}|jd
�|�d|j�d|j	�d
|j
�d|�d|�d�
S)z* return a pretty representation of myself rwr�z,dc->[r�r�css|]}t|�VqdS)N)rE)r�rJr?r?r@r�sz!Table.__repr__.<locals>.<genexpr>r�css|]}|jVqdS)N)rF)r�rjr?r?r@r�sz12.12z (typ->z,nrows->z,ncols->z,indexers->[r�)r�rQrtryr�r�r9r'r<r��ncols)r�Zjdcr�verZjverZjindex_axesr?r?r@r��szTable.__repr__)r�cCs"x|jD]}||jkr|SqWdS)z return the axis for c N)rrF)r�r�rjr?r?r@r��s
zTable.__getitem__c
Cs�|dkrdS|j|jkr2td|j�d|j�d���x�dD]�}t||d�}t||d�}||kr8x@t|�D]4\}}||}||krftd|�d	|�d
|�d���qfWtd|�d	|�d
|�d���q8WdS)z$ validate against an existing table Nz'incompatible table_type with existing [z - r�r9r�r:zinvalid combination of [z] on appending data [z] vs current table [)r9r�r:)rAr�r*rr�r�)r�r{r��sv�ovr�saxZoaxr?r?r@r��s 
zTable.validatecCst|jt�S)z@the levels attribute is 1 or a list in the case of a multi-index)r:rrL)r�r?r?r@�is_multi_index�szTable.is_multi_indexcCsTdd�t|jj�D�}y|j�|fStk
rN}ztd�|�WYdd}~XnXdS)ze
        validate that we can store the multi-index; reset and return the
        new object
        cSs&g|]\}}|dk	r|nd|���qS)NZlevel_r?)r�r�lr?r?r@r�
sz-Table.validate_multiindex.<locals>.<listcomp>zBduplicate names/columns in the multi-index when storing as a tableN)rrqrZreset_indexr�)r�r[rr�r?r?r@�validate_multiindex�szTable.validate_multiindexcCstjdd�|jD��S)z/ based on our axes, compute the expected nrows cSsg|]}|jjd�qS)r)r�r�)r�rr?r?r@r�
sz(Table.nrows_expected.<locals>.<listcomp>)r;rr9)r�r?r?r@�nrows_expected
szTable.nrows_expectedcCs
d|jkS)z has this table been created r_)r�)r�r?r?r@r_
szTable.is_existscCst|jdd�S)Nr_)r*r�)r�r?r?r@r�
szTable.storablecCs|jS)z. return the table group (this is my storable) )r�)r�r?r?r@r_
szTable.tablecCs|jjS)N)r_r�)r�r?r?r@r�
szTable.dtypecCs|jjS)N)r_r�)r�r?r?r@r�"
szTable.descriptioncCstj|j|j�S)N)r�r�r9r:)r�r?r?r@r&
sz
Table.axescCstdd�|jD��S)z0 the number of total columns in the values axes css|]}t|j�VqdS)N)rQr)r�rjr?r?r@r-
szTable.ncols.<locals>.<genexpr>)�sumr:)r�r?r?r@r=*
szTable.ncolscCsdS)NFr?)r�r?r?r@�
is_transposed/
szTable.is_transposedcCs(ttjdd�|jD�dd�|jD���S)z@return a tuple of my permutated axes, non_indexable at the frontcSsg|]}t|d��qS)r)rf)r�rjr?r?r@r�8
sz*Table.data_orientation.<locals>.<listcomp>cSsg|]}t|j��qSr?)rfr�)r�rjr?r?r@r�9
s)rMr�r�r�r9)r�r?r?r@�data_orientation3
szTable.data_orientationcsRddd��dd��jD�}�fdd��jD�}�fdd��jD�}t|||�S)z> return a dict of the kinds allowable columns for this object rqr�)rrIcSsg|]}|j|f�qSr?)ro)r�rjr?r?r@r�C
sz$Table.queryables.<locals>.<listcomp>csg|]\}}�|df�qS)Nr?)r�r�r)�
axis_namesr?r@r�D
scs&g|]}|jt�j�kr|j|f�qSr?)rFrrtro)r�r)r�r?r@r�F
s)r9r�r:r)r�Zd1Zd2Zd3r?)rIr�r@�
queryables=
s

zTable.queryablescCsdd�|jD�S)z  return a list of my index cols cSsg|]}|j|jf�qSr?)r�ro)r�rr?r?r@r�P
sz$Table.index_cols.<locals>.<listcomp>)r9)r�r?r?r@�
index_colsM
szTable.index_colscCsdd�|jD�S)z! return a list of my values cols cSsg|]
}|j�qSr?)ro)r�rr?r?r@r�T
sz%Table.values_cols.<locals>.<listcomp>)r:)r�r?r?r@�values_colsR
szTable.values_cols)rkr�cCs|jj}|�d|�d�S)z+ return the metadata pathname for this key z/meta/z/meta)r�r�)r�rkr�r?r?r@�_get_metadata_pathV
szTable._get_metadata_path)rkrcCs0t|�}|jj|j|�|d|j|j|jd�dS)z�
        Write out a metadata array to the key as a fixed-format Series.

        Parameters
        ----------
        key : str
        values : ndarray
        r_)rprCrurvN)r)r�rxrMrCrurv)r�rkrr?r?r@r�[
s	zTable.write_metadata)rkcCs0tt|jdd�|d�dk	r,|jj|j|��SdS)z) return the meta data array for this key r�N)r*r�r�r�rM)r�rkr?r?r@r�n
szTable.read_metadatacCspt|j�|j_|j�|j_|j�|j_|j|j_|j|j_|j|j_|j|j_|j	|j_	|j
|j_
|j|j_dS)z! set our table type & indexables N)rErAr�rKrLr�rtrvrCrurr<)r�r?r?r@r�t
s





zTable.set_attrscCs�t|jdd�pg|_t|jdd�p$g|_t|jdd�p:t�|_t|jdd�|_tt|jdd��|_t	t|jdd��|_
t|jd	d�p�g|_d
d�|jD�|_
dd�|jD�|_dS)
z retrieve our attributes r�Nrtr<rvrCrurcrcSsg|]}|jr|�qSr?)r�)r�rjr?r?r@r��
sz#Table.get_attrs.<locals>.<listcomp>cSsg|]}|js|�qSr?)r�)r�rjr?r?r@r��
s)r*r�r�rtrr<rvrDrCrArur�
indexablesr9r:)r�r?r?r@r��
szTable.get_attrscCs\|dk	rX|jddkrX|jddkrX|jddkrXtdjdd�|jD��}tj|t�dS)	z- are we trying to operate on an old version? NrrIr�r�r�cSsg|]}t|��qSr?)rE)r�rJr?r?r@r��
sz*Table.validate_version.<locals>.<listcomp>)r��incompatibility_docryr�r�r[)r�rRr�r?r?r@r��
s*zTable.validate_versioncCs^|dkrdSt|t�sdS|j�}x6|j�D]*\}}|dkr>q,||kr,td|�d���q,WdS)z�
        validate the min_itemsize doesn't contain items that are not in the
        axes this needs data_columns to be defined
        Nrzmin_itemsize has the key [z%] which is not an axis or data_column)r:rrJr�r�)r�rr�qr�rr?r?r@�validate_min_itemsize�
s
zTable.validate_min_itemsizecs�g}�j��jj�xzt�jj�D]j\}\}}t�|�}�j|�}|dk	rLdnd}|�d�}t�|d�}	t||||	|�j||d�}
|j|
�q Wt	�j
��t|�������fdd��|j�fdd�t�jj
�D��|S)	z1 create/cache the indexables if they don't exist Nr�ru)rFr�rqr$rpr_r�rrcs�t|t�st�t}|�krt}t�|�}t|�j�}t�|�d�d�}t�|�d�d�}t|�}�j	|�}t�|�d�d�}	|||||�||�j
|	||d�
}
|
S)Nrur�r�)
rFrorr$rqrpr_r�rrr�)r:rEr�r�r�r*�_maybe_adjust_namer�r�r�r_)rr�r�r��adj_namerr�r$�mdr�r[)�base_posr�descr��table_attrsr?r@r`�
s.

zTable.indexables.<locals>.fcsg|]\}}�||��qSr?r?)r�rr�)r`r?r@r��
sz$Table.indexables.<locals>.<listcomp>)r�r_r�rrKr*r�rkrOrrtrQrrL)r��_indexablesrr�rFr�rTr�rvr$�	index_colr?)rUrrVr`r�rWr@rN�
s0



# zTable.indexables)r$c	CsX|j�sdS|dkrdS|dks(|dkr8dd�|jD�}t|ttf�sL|g}t�}|dk	rb||d<|dk	rr||d<|j}x�|D]�}t|j|d�}|dk	�r"|j	r�|j
}|j}	|j}
|dk	r�|
|kr�|j
�n|
|d<|dk	r�|	|kr�|j
�n|	|d<|j	�sP|jjd��rtd	��|jf|�q~||jd
dkr~td|�d
|�d|�d���q~WdS)aZ
        Create a pytables index on the specified columns.

        Parameters
        ----------
        columns : None, bool, or listlike[str]
            Indicate which columns to create an index on.

            * False : Do not create any indexes.
            * True : Create indexes on all columns.
            * None : Create indexes on all columns.
            * listlike : Create indexes on the given columns.

        optlevel : int or None, default None
            Optimization level, if None, pytables defaults to 6.
        kind : str or None, default None
            Kind of index, if None, pytables defaults to "medium".

        Raises
        ------
        TypeError if trying to create an index on a complex-type column.

        Notes
        -----
        Cannot index Time64Col or ComplexCol.
        Pytables must be >= 3.0.
        NFTcSsg|]}|jr|j�qSr?)r�ro)r�rjr?r?r@r�sz&Table.create_index.<locals>.<listcomp>r#r$�complexz�Columns containing complex values can be stored but cannot be indexed when using table format. Either use fixed format, set index=False, or do not include the columns containing complex values to data_columns when initializing the table.rrIzcolumn z/ is not a data_column.
In order to read column z: you must reload the dataframe 
into HDFStore and include z  with the data_columns argument.)r�rr:rMrLrr_r*r
r5rqr#r$Zremove_indexr�r1r�r%r�rh)r�r�r#r$�kwr_r�rrqZcur_optlevelZcur_kindr?r?r@r%�
sF



zTable.create_index)r}r~r�c	Cs^t||||d�}|j�}g}x<|jD]2}|j|j�|j||j|j|jd�}|j	|�q$W|S)a
        Create the axes sniffed from the table.

        Parameters
        ----------
        where : ???
        start : int or None, default None
        stop : int or None, default None

        Returns
        -------
        List[Tuple[index_values, column_values]]
        )rRr}r~)rvrCru)
�	Selectionr�rr�r<r�rvrCrurO)	r�rRr}r~�	selectionrrjrj�resr?r?r@�
_read_axesHs
zTable._read_axes)r�cCs|S)z return the data for this obj r?)rYr[r�r?r?r@�
get_objectjszTable.get_objectcs�t|�sgS|d\}�|jj|t��}|jd�dkrN|rNtd|�d|����|dkr`t��}n|dkrlg}t|t�r�t|��t|�}|j�fdd	�|j	�D���fd
d	�|D�S)zd
        take the input data_columns and min_itemize and create a data
        columns spec
        rr�r'z"cannot use a multi-index on axis [z] with data_columns TNcs g|]}|dkr|�kr|�qS)rr?)r�r�)�existing_data_columnsr?r@r��sz/Table.validate_data_columns.<locals>.<listcomp>csg|]}|�kr|�qSr?r?)r�r�)�axis_labelsr?r@r��s)
rQr<r�rr�rLr:rrr�)r�rtrrr�r�r<r?)rbrar@�validate_data_columnsos$


zTable.validate_data_columns)r[r�c0s�t�t�s,|jj}td|�dt���d����dkr:dg��fdd��D��|j�rzd}d	d�|jD��t|j	�}|j
}nd
}|j}	|jdks�t
�t��|jdkr�td
��g}
|dkr�d}�fdd�d D�d}�j|}t|�}
|�r<t|
�}|j|d}ttj|
�tj|���s<ttjt|
��tjt|����r<|}
|	j|i�}t|j�|d<t|�j|d<|
j||
f��d}�j|}�j|�}t|||j|j�}||_|jd�|j |	�|j!|�|g}t|�}|dk�s�t
�t|
�dk�s�t
�x"|
D]}t"�|d|d���q�Wdd�}|jdk}|j#|||
�}|j$�|�j%�}|j&|||
|j'|�\}}g}�x�t(t)||��D�]�\}\}}t*}d}|�r�t|�dk�r�|d|k�r�t+}|d}|dk�p�t|t,��s�td��|�r6|�r6y|j'|} WnBt-t.fk
�r2}!z td|�d|j'�d��|!�WYdd}!~!XnXnd} |�pHd|��}"t/|"|| |||j|jd�}#t0|"|j1�}$|j2|#�}%t3|#j4j5�}&t6|#d��r�t7|#j8�nd}'d}(})}*t9|#j4��r�|#j:}*d}(tj|#j;d
d�j<�})t=|#�\}+},||$|"t|�|%||&|'|*|(|)|,|+d�}-|-j |	�|j|-�|d7}�qtWdd�|D�}.t|�|j>|j|j|j||
||.|	|d�
}/t6|d��rr|j?|/_?|/j@|�|�r�|�r�|/jA|�|/S)!a0
        Create and return the axes.

        Parameters
        ----------
        axes: list or None
            The names or numbers of the axes to create.
        obj : DataFrame
            The object to create axes on.
        validate: bool, default True
            Whether to validate the obj against an existing object already written.
        nan_rep :
            A value to use for string column nan_rep.
        data_columns : List[str], True, or None, default None
            Specify the columns that we want to create to allow indexing on.

            * True : Use all available columns.
            * None : Use no columns.
            * List[str] : Use the specified columns.

        min_itemsize: Dict[str, int] or None, default None
            The min itemsize for a column in bytes.
        z/cannot properly create the storer for: [group->z,value->r�Nrcsg|]}�j|��qSr?)�_get_axis_number)r�rj)r[r?r@r��sz&Table._create_axes.<locals>.<listcomp>TcSsg|]
}|j�qSr?)r�)r�rjr?r?r@r��sFr�rIz<currently only support ndim-1 indexers in an AppendableTable�nancsg|]}|�kr|�qSr?r?)r�rJ)rr?r@r��srr�cs�fdd�|D�S)Ncsg|]}�jj|j��qSr?)r�rr7)r�r8)�mgrr?r@r�
sz=Table._create_axes.<locals>.get_blk_items.<locals>.<listcomp>r?)rfr6r?)rfr@�
get_blk_itemssz)Table._create_axes.<locals>.get_blk_itemsz-cannot have non-object label DataIndexableColzIncompatible appended table [z]with existing table [Z
values_block_)�existing_colrrrvrCrurmr�)r8)rFrorrprqr$rmrr�rrr�r7cSsg|]}|jr|j�qSr?)r�rF)r�r�r?r?r@r�gs)
r�r�rCrur9r�r:rtr<rvr)rrI)Br:r#r�r�r�r�r�r9rLrtrvr<rr�rQr�rr�r"r;�arrayrr�rrWrOZ_get_axis_namerrCrur�rsr�r��
_reindex_axisrcr`r��_get_blocks_and_itemsr:rr�r�r�rE�
IndexErrorr��_maybe_convert_for_string_atomrRr�r�r�r�rFr~rrmrrr�r�r�r�rrQr�)0r�rr[r�rvrtrrr��table_existsZnew_info�new_non_index_axesr�rjZappend_axis�indexerZ
exist_axisr<�	axis_nameZ	new_indexZnew_index_axes�jrgr��	block_objr6r2Zvaxesr�b�b_itemsr�rFrhr��new_name�data_convertedrSrpr$rmr�rrrr7r�r�ZdcsZ	new_tabler?)rr[r@�_create_axes�s� 










 "





zTable._create_axescCsddd�}|jj}||j|�}t|�r�|d\}}	t|	�jt|��}
|j|
|d�j}t|j�}|||�}x<|D]4}|j|g|d�j}|j|j�|j|||j��qlW|�r\dd�t||�D�}
g}g}x�|D]�}t	|j
�}y&|
j|�\}}|j|�|j|�Wq�t
tfk
�rN}z,djdd	�|D��}td
|�d��|�WYdd}~Xq�Xq�W|}|}||fS)Ncs�fdd�|D�S)Ncsg|]}�jj|j��qSr?)r�rr7)r�r8)rfr?r@r��szFTable._get_blocks_and_items.<locals>.get_blk_items.<locals>.<listcomp>r?)rfr6r?)rfr@rg�sz2Table._get_blocks_and_items.<locals>.get_blk_itemsr)r�cSs"i|]\}}||ft|j���qSr?)rM�tolist)r�rtrur?r?r@r
�sz/Table._get_blocks_and_items.<locals>.<dictcomp>rwcss|]}t|�VqdS)N)r5)r��itemr?r?r@r�sz.Table._get_blocks_and_items.<locals>.<genexpr>z+cannot match existing table structure for [z] on appending data)r5r6rQr%rrrLrr�rMrrrOrlr�ryr�)rsrnror:rtrgr6r2r�rbZ
new_labelsrfr�Zby_itemsZ
new_blocksZ
new_blk_itemsZear�rtrur�Zjitemsr?r?r@rk�s@





zTable._get_blocks_and_itemsr\)r]c
s�|dk	rt|�}|dk	rR�jrRt�jt�s.t�x"�jD]}||kr6|jd|�q6Wx �jD]\}}t�|||��qZW|jdk	r�x2|jj	�D]$\}�}���fdd�}	|	||��q�W�S)z process axes filters Nrcs�x��jD]�}�j|�}�j|�}|dk	s,t�||krd�jrJ|jt�j��}�||�}�j|d�|S||krt	t
�|�j�}t	|�}t�t
�r�d|}�||�}�j|d�|SqWtd|�d���dS)N)r�rIzcannot find the field [z] for filtering!)Z_AXIS_ORDERSrd�	_get_axisr�rB�unionr%rrr2r*rr:r#r�)�field�filtrqZaxis_numberZaxis_valuesZtakersr)r[�opr�r?r@�process_filter�s"




z*Table.process_axes.<locals>.process_filter)
rLrBr:rr��insertr�rjrNrp)
r�r[r]r�r�r��labelsr}r~r�r?)r[rr�r@�process_axes�s
!zTable.process_axes)rnr�rr�cCs�|dkrt|jd�}td|d�}dd�|jD�|d<|rl|dkrJ|jpHd}t�j|||p\|jd	�}||d
<n|jdk	r�|j|d
<|S)z< create the description of the table from the axes & values Ni'r_)rFrcSsi|]}|j|j�qSr?)rpro)r�rjr?r?r@r
�sz,Table.create_description.<locals>.<dictcomp>r��	)rnror�r�)	�maxrErrr�rir�r�r�)r�rornr�rrr�r?r?r@�create_description�s	



zTable.create_description)r}r~c
Cs�|j|�|j�sdSt||||d�}|j�}|jdk	r�xR|jj�D]D\}}}|j||j�|j�dd�}	|||	j	||j�|�j
}qDWt|�S)zf
        select coordinates (row numbers) from a table; return the
        coordinates object
        F)rRr}r~NrI)r}r~)r�r�r\�
select_coordsrNrpr�rir��ilocrr%)
r�rRr}r~r]Zcoordsr}rr~r7r?r?r@r�
s

"zTable.read_coordinates)r�r}r~cCs�|j�|j�sdS|dk	r$td��x�|jD]v}||jkr,|jsPtd|�d���t|jj	|�}|j
|j�|j|||�|j
|j|jd�}tt|d|j�|d�Sq,Wtd|�d	���dS)
zj
        return a single column from the table, generally only indexables
        are interesting
        FNz4read_column does not currently accept a where clausezcolumn [z=] can not be extracted individually; it is not data indexable)rvrCrurI)rFz] not found in the table)r�r�r�rrFr�r�r*r_r
r�r<r�rvrCrur)r�rmr�)r�r�rRr}r~rjr�Z
col_valuesr?r?r@r�$s&


zTable.read_column)NrcNNNNNN)N)NNN)NN)TNNN)N)NNN)NNN)IrWrXrYrer�r�rErArr�rrkr9r	rfrr�r�r:rtrrrr<ryr�rgr<r�r�r�r�rBrDrEr_r�r_r�r�rr=rGrHrJrKrLrMr;rr�r�r�r�r�rQrrNrr%rr_r�r`rcr#rx�staticmethodrkr�r�r�r�r�r?r?)r�r@r��s�




JU +d1=r�c@s6eZdZdZdZd	eeeed�dd�Zdd�ZdS)
rVz�
    a write-once read-many table: this format DOES NOT ALLOW appending to a
    table. writing is a one-time operation the data are stored in a format
    that allows for searching the data on disk
    rLN)r}r~cCstd��dS)z[
        read the indices and the indexing array, calculate offset rows and return
        z!WORMTable needs to implement readN)r�)r�rRr�r}r~r?r?r@r�Zs
zWORMTable.readcKstd��dS)z�
        write in a format that we can search later on (but cannot append
        to): write out the indices and the values using _write_array
        (e.g. a CArray) create an indexing table so that we can search
        z"WORMTable needs to implement writeN)r�)r�r�r?r?r@rafszWORMTable.write)NNNN)	rWrXrYrerArrfr�rar?r?r?r@rVQsrVc@sveZdZdZdZddd�Zdeeed�d	d
�Z	e
jee
jee
jee
jd�dd
�Z
deeeed�dd�ZdS)r�z* support the new appendable table formats Z
appendableNFTcCs�|r|jr|jj|jd�|j||||||d�}x|jD]}|j�q:W|js�|j||||	d�}|j�|
|d<|jj	|jf|�|j
|j_
x|jD]}|j||�q�W|j
||
d�dS)Nr_)rr[r�rrrvrt)rornr�rr�)rs)r_r�r\r�rxrr�r�r�Zcreate_tabler<r�r��
write_data)r�r[rrOrornr�rrrrrsrvrtr�r_rj�optionsr?r?r@rats0
zAppendableTable.write)rrscs�|jj}|j}g}|rXx@|jD]6}t|j�jdd�}t|tj	�r|j
|jddd��qWt|�r�|d}x|dd�D]}||@}qvW|j
�}nd}dd	�|jD�}	t|	�}
|
dks�t|
��d
d	�|jD�}dd	�|D�}g}xBt|�D]6\}
}|f|j||
|
j}|j
||
j|��q�W|dk�r0d}tjt||�|jd
�}||d}x�t|�D]t}
|
|�t|
d||����k�r�P|j|��fdd	�|	D�|dk	�r�|���nd��fdd	�|D�d��q\WdS)z`
        we form the data into a 2-d including indexes,values,mask write chunk-by-chunk
        r)r��u1F)r8rINcSsg|]
}|j�qSr?)r�)r�rjr?r?r@r��sz.AppendableTable.write_data.<locals>.<listcomp>cSsg|]}|j��qSr?)r�)r�rjr?r?r@r��scSs,g|]$}|jtjtj|j�|jd���qS)rI)Z	transposer;Zrollr�r)r�rr?r?r@r��si��)r�csg|]}|����qSr?r?)r�rj)�end_i�start_ir?r@r��scsg|]}|����qSr?r?)r�r)r�r�r?r@r��s)�indexesr�r)r�rrEr:r,r7rr:r;rrOr�rQr�r9r�rr��reshaperZrir�write_data_chunk)r�rrsrr��masksrjr��mr��nindexesrZbvaluesrrZ	new_shape�rows�chunksr?)r�r�r@r��sJ


zAppendableTable.write_data)r�r�r�rcCs�x|D]}tj|j�sdSqW|djd}|t|�krHtj||jd�}|jj}t|�}x t|�D]\}	}
|
|||	<qbWx$t|�D]\}	}||||	|<q�W|dk	r�|j�j	t
dd�}|j�s�||}t|�r�|jj
|�|jj�dS)z�
        Parameters
        ----------
        rows : an empty memory space where we are putting the chunk
        indexes : an array of the indexes
        mask : an array of the masks
        values : an array of the values
        Nr)r�F)r8)r;rr�rQrZr�rrr�r�r�rr_rOr�)r�r�r�r�rrr�rr�rr�r�r?r?r@r��s&
z AppendableTable.write_data_chunk)r}r~cCsh|dkst|�rh|dkr<|dkr<|j}|jj|jdd�n(|dkrJ|j}|jj||d�}|jj�|S|j�stdS|j}t	||||d�}|j
�}t|�j�}t|�}	|	�rd|j
�}
t|
|
dkj�}t|�s�dg}|d|	kr�|j|	�|ddk�r|jdd�|j�}xLt|�D]@}
|jt|
|��}|j||jd||jddd�|
}�qW|jj�|	S)NT)r)r}r~rIrr�r�)rQr�r�r\r�r_Zremove_rowsr�r�r\r�r)Zsort_values�diffrLrqrOr�r�reversedrr)r�rRr}r~r�r_r]rZ
sorted_series�lnr�r�Zpgr�r�r?r?r@rsB

"

zAppendableTable.delete)NFNNNNNNFNNT)F)NNN)rWrXrYrerArarrfr�r�r;rrr�rr?r?r?r@r�os(
.?(r�c@speZdZUdZdZdZdZeZe	e
eefee
d�dd��Zee
d�d	d
��Zdeeeed�d
d�ZdS)rTz* support the new appendable table formats rBrJr�)r�cCs|jdjdkS)NrrI)r9r�)r�r?r?r@rG`sz"AppendableFrameTable.is_transposed)r�cCs|r
|j}|S)z these are written transposed )r)rYr[r�r?r?r@r`dszAppendableFrameTable.get_objectN)r}r~cs:�j|��j�sdS�j|||d�}t�j�rJ�jj�jddt��nt�}�fdd�t�j	�D�}t|�dksxt
�|d}||d}	g}
�x^t�j	�D�]N\}}|�jkr�q�||\}
}|jd�dkr�tj
|
�}nt|
�}|jd�}|dk	�r|j|d	d
��j�r*|}|}t|	t|	dd�d�}n|j}t|	t|	dd�d�}|}|jdk�rvt|tj��rv|jd|jdf�}t|tj��r�t|j||d
�}n,t|t��r�t|||d
�}nt|g||d
�}|j|jkj��s�t
|j|jf��|
j|�q�Wt|
�dk�r
|
d}nt|
dd�}t�|||d�}�j |||d�}|S)N)rRr}r~rcs"g|]\}}|�jdkr|�qS)r)r9)r�rr0)r�r?r@r��sz-AppendableFrameTable.read.<locals>.<listcomp>rIr�r'rT)�inplacerF)rF)r�rq)r�)r]r�)!r�r�r_rQr�r<r�rrrr�r:r'�from_tuplesr%�	set_namesrGr*rrr:r;rr�r�r#Zdtypesr�rrOr+r\r�)r�rRr�r}r~r�r<Zinds�indrq�framesrrjZ
index_valsr�r
rrZindex_Zcols_r3r]r?)r�r@r�ksV	
$


"
zAppendableFrameTable.read)NNNN)rWrXrYrer�rArr#r�r
rr)rgr�rGr�r`rrfr�r?r?r?r@rTXs
rTcsveZdZdZdZdZdZeZe	e
d�dd��Zee
d�d	d
��Z
d�fdd
�	Zdeeeeed��fdd�
Z�ZS)rRz* support the new appendable table formats rGrHr�)r�cCsdS)NFr?)r�r?r?r@rG�sz#AppendableSeriesTable.is_transposed)r�cCs|S)Nr?)rYr[r�r?r?r@r`�sz AppendableSeriesTable.get_objectNcs<t|t�s|jpd}|j|�}t�jf||jj�d�|��S)z- we are going to write this as a frame table r)r[rt)r:r#rFZto_framer�rar�ry)r�r[rtr�rF)r�r?r@ra�s


zAppendableSeriesTable.write)r}r~r�cs�|j}|dk	rF|rFt|jt�s"t�x"|jD]}||kr*|jd|�q*Wt�j||||d�}|rn|j|jdd�|j	dd�df}|j
dkr�d|_
|S)Nr)rRr�r}r~T)r�r)rBr:rrLr�r�r�r��	set_indexr�rF)r�rRr�r}r~rBr�r>)r�r?r@r��s
zAppendableSeriesTable.read)N)NNNN)rWrXrYrer�rArr)r�rgr�rGr�r`rarrfr�r�r?r?)r�r@rR�s	rRcs(eZdZdZdZdZ�fdd�Z�ZS)rSz* support the new appendable table formats rGrIcsJ|jpd}|j|�\}|_t|j�}|j|�||_t�jfd|i|��S)z- we are going to write this as a frame table rr[)rFrDrrLrOr�r�ra)r�r[r�rFr
)r�r?r@ra�s


z AppendableMultiSeriesTable.write)rWrXrYrer�rArar�r?r?)r�r@rS�srSc@sZeZdZdZdZdZdZeZe	e
d�dd��Ze	dd	��Zd
d�Z
edd
��Zdd�ZdS)rQz< a table that read/writes the generic pytables table format rBrCr�)r�cCs|jS)N)r�)r�r?r?r@r'szGenericTable.pandas_typecCst|jdd�p|jS)Nr_)r*r�)r�r?r?r@r�	szGenericTable.storablecCsLg|_d|_g|_dd�|jD�|_dd�|jD�|_dd�|jD�|_dS)z retrieve our attributes NcSsg|]}|jr|�qSr?)r�)r�rjr?r?r@r�sz*GenericTable.get_attrs.<locals>.<listcomp>cSsg|]}|js|�qSr?)r�)r�rjr?r?r@r�scSsg|]
}|j�qSr?)rF)r�rjr?r?r@r�s)r�rvrrNr9r:rt)r�r?r?r@r�
szGenericTable.get_attrsc

Cs�|j}|jd�}|dk	rdnd}tdd|j||d�}|g}xlt|j�D]^\}}t|t�s\t�t	||�}|j|�}|dk	r|dnd}t
|||g||j||d�}	|j|	�qFW|S)z2 create the indexables from the table description rqNr�r)rFr�r_r�rr)rFrqrrpr_r�rr)r�r�r�r_rZ_v_namesr:rEr�r*r�rO)
r�rrTr�rYrXrr�r�rr?r?r@rNs*


zGenericTable.indexablescKstd��dS)Nz cannot write on an generic table)r�)r�r�r?r?r@ra:szGenericTable.writeN)rWrXrYrer�rArr#r�rgrEr'r�r�rrNrar?r?r?r@rQ�s
#rQcsheZdZdZdZeZdZej	d�Z
eed�dd��Z
d�fd	d
�	Zdeeeed��fdd
�
Z�ZS)rUz a frame with a multi-index rKr�z^level_\d+$)r�cCsdS)NZappendable_multir?)r�r?r?r@r<Fsz*AppendableMultiFrameTable.table_type_shortNcsl|dkrg}n|dkr |jj�}|j|�\}|_x"|jD]}||kr8|jd|�q8Wt�jf||d�|��S)NTr)r[rt)r�ryrDrr�r�ra)r�r[rtr�r�)r�r?r@raJs
zAppendableMultiFrameTable.write)r}r~csDt�j||||d�}|j�j�}|jj�fdd�|jjD��|_|S)N)rRr�r}r~cs g|]}�jj|�rdn|�qS)N)�
_re_levels�search)r�rC)r�r?r@r�bsz2AppendableMultiFrameTable.read.<locals>.<listcomp>)r�r�r�rrqr�r)r�rRr�r}r~r3)r�)r�r@r�Us
zAppendableMultiFrameTable.read)N)NNNN)rWrXrYrerAr#r�r�re�compiler�rgrEr<rarrfr�r�r?r?)r�r@rU>s

rU)r[r�r�r�cCs�|j|�}t|�}|dk	r"t|�}|dks4|j|�rB|j|�rB|St|j��}|dk	rlt|j��j|dd�}|j|�s�tdd�g|j}|||<|jt|�}|S)NF)�sort)	r{r2�equals�uniquer�slicerrrM)r[r�r�r{r0Zslicerr?r?r@rjhs

rj)rmr�cCstj|�}|S)z- for a tz-aware type, return an encoded zone )rZget_timezone)rm�zoner?r?r@r�s
r)rrmr�r�cCszt|t�r"|jdks"|j|ks"t�|dk	rdt|dd�}|j�}t|�}t||d�}|jd�j|�}n|rvt	j
|dd�}|S)a
    coerce the values to a DatetimeIndex if tz is set
    preserve the input shape if possible

    Parameters
    ----------
    values : ndarray or Index
    tz : str or tzinfo
    coerce : if we do not have a passed timezone, coerce to M8[ns] ndarray
    NrF)rFr�zM8[ns])r�)r:r$rmr�r*r�rAr�r�r;r�)rrmr�rFr?r?r@r��s
r�)rFrqrCrur�c
Csjt|t�st�|j}t|�\}}t|�}tj|�}t|t�rbt	||||t
|dd�t
|dd�|d�St|t�rttd��t
j|dd�}	tj|�}
|	dkr�tjdd	�|
D�tjd
�}t	||dt�j�|d�S|	dk�r�t|
||�}|jj}t	||dt�j|�|d�S|	dk�rt	|||||d�St|tj��r4|jtk�s8t�|dk�sJt|��t�j�}t	|||||d�SdS)Nrlrm)rr$rprlrmrnzMultiIndex not supported here!F)r"rcSsg|]}|j��qSr?)�	toordinal)r�rr?r?r@r��sz"_convert_index.<locals>.<listcomp>)r�)rnr��integer�floating)rr$rprnr�)r�r�)r:rEr�rFr�r�r�r�r&rkr*r'r�rr%r;r�Zint32riZ	Time32Col�_convert_string_arrayr�rtr�rr�r')rFrqrCrurnr�r�r$r�r+rrtr?r?r@r�sL









r)r$rCrur�cCs�|dkrt|�}n�|dkr$t|�}n�|dkrxytjdd�|D�td�}Wq�tk
rttjdd�|D�td�}Yq�XnT|dkr�tj|�}n@|d
kr�t|d||d�}n&|dkr�tj|d
�}ntd|����|S)Nr�r�rcSsg|]}tj|��qSr?)rr�)r�rr?r?r@r��sz$_unconvert_index.<locals>.<listcomp>)r�cSsg|]}tj|��qSr?)rr�)r�rr?r?r@r��sr��floatr�)rvrCrur�rzunrecognized index type )r�r�)r$r*r;r�r�r�r�)r7r$rCrurqr?r?r@r�s$

 r)rFcCs�|js|jS|jj}tj|jdd�}|dkr6td��n(|dkrHtd��n|dkpV|dks^|jS|j|dd	�}t|t	�r~|d
}|j}	tj|	dd�}|dkr�x\t
t|jd
��D]F}
|j
|
�}tj|dd�}|dkr�|jj|
}td|�d|�d
���q�Wt|	||�j|	j�}
|
j|jk�s*t|
j|jf��|
j}t|t��r\t|j|��pX|jd��pXd
�}t|�pfd
|�}|dk	�r�|j|�}||k�r�|}|
jd|��dd�}
|
S)NF)r"rz+[date] is not implemented as a table columnr�z>too many timezones in this block, create separate data columnsr�r�)ZdowncastrzCannot serialize the column [z!] because
its data contents are [z] object dtyperz|S)r8)r�rr�rFrr%r�Zfillnar:rLrrQr��igetr7rpr�r�r�rtrrfr�r�r�r�)rFr.rhrrrvrCrur�r+r7rr�r�rwrtZecir?r?r@rm�sH


 


rm)r7rCrur�cCs\t|�r(t|j��jj||�jj|j�}t|j��}t	dt
j|��}tj
|d|��d�}|S)a
    Take a string-like that is object dtype and coerce to a fixed size string type.

    Parameters
    ----------
    data : np.ndarray[object]
    encoding : str
    errors : str
        Handler for encoding errors.

    Returns
    -------
    np.ndarray[fixed-length-string]
    rI�S)r�)rQr)r�rE�encoder�r�r�rr��
libwriters�max_len_string_arrayr;r�)r7rCru�ensuredrtr?r?r@r�?s
r�cCs�|j}tj|j�td�}t|�rvtjt|��}d|��}t	|dt
�r^t|�jj
||d�j}n|j|dd�jtdd�}|dkr�d}tj||�}|j|�S)	a*
    Inverse of _convert_string_array.

    Parameters
    ----------
    data : np.ndarray[fixed-length-string]
    nan_rep : the storage repr of NaN
    encoding : str
    errors : str
        Handler for encoding errors.

    Returns
    -------
    np.ndarray[object]
        Decoded data.
    )r��Ur)ruF)r8Nre)r�r;r�r�r�rQr�r�rr:rr)rEr=r�r�Z!string_array_replace_from_nan_repr�)r7rvrCrur�rtr�r?r?r@r�^s
r�)rr�rCrucCs6t|t�stt|���t|�r2t|||�}||�}|S)N)r:rEr�r��
_need_convert�_get_converter)rr�rCru�convr?r?r@r��s
r�)r$rCrucs8|dkrdd�S|dkr&��fdd�Std|����dS)Nr�cSstj|dd�S)NzM8[ns])r�)r;r�)rJr?r?r@rK�sz _get_converter.<locals>.<lambda>r�cst|d��d�S)N)rvrCru)r�)rJ)rCrur?r@rK�sz
invalid kind )r�)r$rCrur?)rCrur@r��s
r�)r$r�cCs|dkrdSdS)Nr�r�TF)r�r�r?)r$r?r?r@r��sr�)rFr�cCshyN|ddkrL|ddkrL|ddkrLtjd|�}|rL|j�d}d|��}Wntk
rbYnX|S)z�
    Prior to 0.10.1, we named values blocks like: values_block_0 an the
    name values_0, adjust the given name if necessary.

    Parameters
    ----------
    name : str
    version : Tuple[int, int, int]

    Returns
    -------
    str
    rrIr�r�zvalues_block_(\d+)Zvalues_)r�r�r�rl)rFr�r��grpr?r?r@rR�s$rR)�	dtype_strr�cCs�t|�}|jd�s|jd�r"d}n�|jd�r2d}n�|jd�rBd}n�|jd�sV|jd�r\d}nn|jd�rld}n^|jd	�r|d
}nN|jd�r�d}n>|jd�r�d}n.|jd
�r�d}n|dkr�d}ntd|�d���|S)zA
    Find the "kind" string describing the given dtype name.
    r�rr�rZrfr�r�r��	timedeltar�r�r�r�r�zcannot interpret dtype of [r�)rAr1r�)r�r$r?r?r@r��s.






r�)r7cCsbt|t�r|j}|jjjd�d}|jjdkr@tj|j	d��}nt|t
�rP|j}tj|�}||fS)zJ
    Convert the passed data into a storable form and a dtype string.
    r�rr��Mr$)r�r�)r:r-r�r�rFr]r$r;r�r(r(r))r7r�r?r?r@r��s


r�c@sDeZdZdZdeeeeed�dd�Zdd�Zdd	�Z	d
d�Z
dS)
r\z�
    Carries out a selection operation on a tables.Table object.

    Parameters
    ----------
    table : a Table object
    where : list of Terms (or convertible to)
    start, stop: indices to start and/or stop selection

    N)r_r}r~cCsP||_||_||_||_d|_d|_d|_d|_t|��ry�t	j
|dd�}|dksZ|dkr�tj|�}|j
tjkr�|j|j}}|dkr�d}|dkr�|jj}tj||�||_nNt|j
jtj�r�|jdk	r�||jkj�s�|jdk	r�||jkj�r�td��||_Wntk
�rYnX|jdk�rL|j|�|_|jdk	�rL|jj�\|_|_dS)NF)r"r��booleanrz3where must have index locations >= start and < stop)r_rRr}r~�	conditionrNZtermsr�rrr%r;r�r�Zbool_r�r��
issubclassr�r�r�r��generate�evaluate)r�r_rRr}r~�inferredr?r?r@r��s@


zSelection.__init__cCsv|dkrdS|jj�}yt|||jjd�Stk
rp}z,dj|j��}td|�d|�d��|�WYdd}~XnXdS)z) where can be a : dict,list,tuple,string N)rJrCrwzThe passed where expression: z�
            contains an invalid variable reference
            all of the variable references must be a reference to
            an axis (e.g. 'index' or 'columns'), or a data_column
            The currently defined references are: r�)r_rJr0rC�	NameErrorryr�r�)r�rRrPr�Zqkeysr?r?r@r�,s
zSelection.generatecCsX|jdk	r(|jjj|jj�|j|jd�S|jdk	rB|jjj|j�S|jjj|j|jd�S)z(
        generate the selection
        N)r}r~)	r�r_Z
read_whererpr}r~r�r�r�)r�r?r?r@r�Bs

zSelection.selectcCs�|j|j}}|jj}|dkr$d}n|dkr4||7}|jdkrD|}n|dkrT||7}|jdk	rz|jjj|jj�||dd�S|jdk	r�|jStj	||�S)z(
        generate the selection
        NrT)r}r~r�)
r}r~r_r�r�Zget_where_listrpr�r;r�)r�r}r~r�r?r?r@r�Ns 


zSelection.select_coords)NNN)rWrXrYrer�rrfr�r�r�r�r?r?r?r@r\�s
-r\)rjNNFNTNNNNrcr9)	Nr|rcNNNNFN)N)F)�rer8r�rrr�r�r�Ztypingrrrrrr	r
rr�Znumpyr;Zpandas._configrr
Zpandas._libsrrr�Zpandas._libs.tslibsrZpandas._typingrrrZpandas.compat._optionalrZpandas.compat.pickle_compatrZ
pandas.errorsrZpandas.util._decoratorsrZpandas.core.dtypes.commonrrrrrrrrr Zpandas.core.dtypes.genericr!Zpandas.core.dtypes.missingr"r�r#r$r%r&r'r(r)r*r+r,Zpandas.core.arraysr-r.r/Zpandas.core.common�core�commonrZ pandas.core.computation.pytablesr0r1Zpandas.core.indexes.apir2Zpandas.io.commonr3Zpandas.io.formats.printingr4r5rer6r7r8r�rBrArDrGrPrfrUr�rVrZ�Warningr[rOr\r�r]Z
duplicate_docr&r=rZ
dropna_docZ
format_docZ
config_prefixZregister_optionZis_boolZis_one_of_factoryrdrgrirEr�r{r�r�ryr�rkr�r�r�r�r�r3rNr,rOr�rVr�rTrRrSrQrUrjrrr�rrrmr�r�r�r�r�rRr�r�r\r?r?r?r@�<module>s (,0
\*$]q!3V ZXjd1A*	<A %!