ENH: allow saving wide dataframes to hdf with format table

pandas/io/pytables.py

@@ -38,6 +38,8 @@
 from pandas.core.index import ensure_index
 from pandas.core.internals import BlockManager, _block_shape, make_block
 
+from tables.exceptions import NoSuchNodeError
+
 from pandas.io.common import _stringify_path
 from pandas.io.formats.printing import adjoin, pprint_thing
 
@@ -1611,6 +1613,7 @@ def infer(self, handler):
         """infer this column from the table: create and return a new object"""
         table = handler.table
         new_self = self.copy()
+        new_self._handle = handler._handle
         new_self.set_table(table)
         new_self.get_attr()
         new_self.read_metadata(handler)
@@ -1668,6 +1671,10 @@ def cvalues(self):
         """ return my cython values """
         return self.values
 
+    @property
+    def handle(self):
+        return self._handle
+
     def __iter__(self):
         return iter(self.values)
 
@@ -1691,6 +1698,7 @@ def validate_names(self):
         pass
 
     def validate_and_set(self, handler, append):
+        self._handle = handler._handle
         self.set_table(handler.table)
         self.validate_col()
         self.validate_attr(append)
@@ -2230,15 +2238,38 @@ def convert(self, values, nan_rep, encoding, errors):
         return self
 
     def get_attr(self):
-        """ get the data for this column """
+        """ get the data for this colummn """
+        # reading tables prior to 0.x.x
         self.values = getattr(self.attrs, self.kind_attr, None)
+
+        if self.values is None:
+            try:
+                data = self.handle.get_node(self.attrs._v_node._v_parent,
+                                            self.kind_attr)[:]
+                data = np.array(data, dtype='object')
+                # check for multiindex
+                if len(data.shape) > 1 and data.shape[1] > 1:
+                    self.values = list(map(tuple, data.tolist()))
+                else:
+                    self.values = data.tolist()
+            except NoSuchNodeError:
+                pass
+
         self.dtype = getattr(self.attrs, self.dtype_attr, None)
         self.meta = getattr(self.attrs, self.meta_attr, None)
         self.set_kind()
 
     def set_attr(self):
         """ set the data for this column """
-        setattr(self.attrs, self.kind_attr, self.values)
+        group, key = self.attrs._v_node._v_parent, self.kind_attr
+        if key in group:
+            self.handle.remove_node(group, key)
+
+        vlarray = self.handle.create_vlarray(group, key,
+                                             _tables().ObjectAtom())
+        for fld in self.values:
+            vlarray.append(fld)
+
         setattr(self.attrs, self.meta_attr, self.meta)
         if self.dtype is not None:
             setattr(self.attrs, self.dtype_attr, self.dtype)
@@ -3240,12 +3271,48 @@ def set_info(self):
         """ update our table index info """
         self.attrs.info = self.info
 
+    def set_non_index_axes(self):
+        """ Write the axes to carrays """
+        group = self.attrs._v_node
+
+        def f(dim, flds):
+            key = "non_index_axes_%d" % dim
+            if key in group:
+                self.handle.remove_node(group, key)
+
+            vlarray = self._handle.create_vlarray(group, key,
+                                                  _tables().ObjectAtom())
+            for fld in flds:
+                vlarray.append(fld)
+            return dim, key
+
+        replacement = [f(dim, flds) for dim, flds in self.non_index_axes]
+        self.attrs.non_index_axes = replacement
+
+    def get_non_index_axes(self):
+        """Load the non-index axes from their carrays. This is a pass-through
+        for tables stored prior to v0.xx"""
+        def f(dim, flds):
+            if isinstance(flds, str):
+                flds = self._handle.get_node(self.attrs._v_node, flds)[:]
+                flds = np.array(flds, dtype='object')
+                if len(flds.shape) > 1 and flds.shape[1] > 1:
+                    flds = list(map(tuple, flds.tolist()))
+                else:
+                    flds = flds.tolist()
+                return dim, flds
+            else:
+                return dim, flds  # if not a string presumably pre v0.xx list
+        non_index_axes = getattr(self.attrs, 'non_index_axes', [])
+        new = [f(dim, flds) for dim, flds in non_index_axes]
+        return new
+
     def set_attrs(self):
         """ set our table type & indexables """
         self.attrs.table_type = str(self.table_type)
         self.attrs.index_cols = self.index_cols()
         self.attrs.values_cols = self.values_cols()
-        self.attrs.non_index_axes = self.non_index_axes
+        self.set_non_index_axes()
         self.attrs.data_columns = self.data_columns
         self.attrs.nan_rep = self.nan_rep
         self.attrs.encoding = self.encoding
@@ -3256,8 +3323,6 @@ def set_attrs(self):
 
     def get_attrs(self):
         """ retrieve our attributes """
-        self.non_index_axes = getattr(
-            self.attrs, 'non_index_axes', None) or []
         self.data_columns = getattr(
             self.attrs, 'data_columns', None) or []
         self.info = getattr(
@@ -3276,6 +3341,7 @@ def get_attrs(self):
         ]
         self.metadata = getattr(
             self.attrs, 'metadata', None) or []
+        self.non_index_axes = self.get_non_index_axes()
 
     def validate_version(self, where=None):
         """ are we trying to operate on an old version? """

pandas/tests/io/test_pytables.py

@@ -5148,3 +5148,25 @@ def test_dst_transitions(self):
                 store.append('df', df)
                 result = store.select('df')
                 assert_frame_equal(result, df)
+
+    def test_wide_table_format(self):
+        # test storing wide dataframes with in table format
+
+        df = DataFrame(np.random.random((10, 10000)))
+
+        with ensure_clean_path(self.path) as path:
+            df.to_hdf(path, 'df', format='table')
+            reread = read_hdf(path, 'df')
+            assert_frame_equal(df, reread)
+
+    def test_append_wide_table_format(self):
+        # test append to hdf with wide dataframe
+
+        df1 = DataFrame(np.random.random((10, 10000)))
+        df2 = DataFrame(np.random.random((10, 10000)))
+
+        with ensure_clean_path(self.path) as path:
+            df1.to_hdf(path, 'df', format='table')
+            df2.to_hdf(path, 'df', append=True)
+            reread = read_hdf(path)
+            assert_frame_equal(pd.concat([df1, df2]), reread)