BUG/ENH: allow appending of different item orders if the items are st…

…ill equal (will use the existing order)

         better error message on rejection of append

pandas/io/pytables.py

@@ -1174,15 +1174,17 @@ def copy(self):
         new_self = copy.copy(self)
         return new_self
 
-    def __eq__(self, other):
-        """ return True if we are 'equal' to this other table (in all respects that matter) """
+    def validate(self, other):
+        """ validate against an existing table """
+        if other is None: return
+
+        if other.table_type != self.table_type:
+            raise TypeError("incompatible table_type with existing [%s - %s]" %
+                            (other.table_type, self.table_type))
+
         for c in ['index_axes','non_index_axes','values_axes']:
             if getattr(self,c,None) != getattr(other,c,None):
-                return False
-        return True
-
-    def __ne__(self, other):
-        return not self.__eq__(other)
+                raise Exception("invalid combinate of [%s] on appending data [%s] vs current table [%s]" % (c,getattr(self,c,None),getattr(other,c,None)))
 
     @property
     def nrows(self):
@@ -1263,17 +1265,6 @@ def validate_version(self, where = None):
             if self.version is None or float(self.version) < 0.1:
                 warnings.warn("where criteria is being ignored as we this version is too old (or not-defined) [%s]" % self.version, IncompatibilityWarning)
 
-    def validate(self):
-        """ raise if we have an incompitable table type with the current """
-        et = getattr(self.attrs,'table_type',None)
-        if et is not None and et != self.table_type:
-                raise TypeError("incompatible table_type with existing [%s - %s]" %
-                                (et, self.table_type))
-        ic  = getattr(self.attrs,'index_cols',None)
-        if ic is not None and ic != self.index_cols():
-            raise TypeError("incompatible index cols with existing [%s - %s]" %
-                            (ic, self.index_cols()))
-
     @property
     def indexables(self):
         """ create/cache the indexables if they don't exist """
@@ -1417,13 +1408,28 @@ def create_axes(self, axes, obj, validate = True, min_itemsize = None):
                 j += 1
 
             else:
-                self.non_index_axes.append((i,list(a)))
+
+                # we might be able to change the axes on the appending data if necessary
+                append_axis = list(a)
+                if existing_table is not None:
+                    indexer = len(self.non_index_axes)
+                    exist_axis = existing_table.non_index_axes[indexer][1]
+                    if append_axis != exist_axis:
+
+                        # ahah! -> reindex
+                        if sorted(append_axis) == sorted(exist_axis):
+                            append_axis = exist_axis
+
+                self.non_index_axes.append((i,append_axis))
 
         # check for column conflicts
         if validate:
             for a in self.axes:
                 a.maybe_set_size(min_itemsize = min_itemsize)
 
+        # reindex by our non_index_axes
+        for a in self.non_index_axes:
+            obj = obj.reindex_axis(a[1], axis = a[0])
 
         blocks = self.get_data_blocks(obj)
 
@@ -1471,9 +1477,8 @@ def create_axes(self, axes, obj, validate = True, min_itemsize = None):
             self.values_axes.append(dc)
 
         # validate the axes if we have an existing table
-        if existing_table is not None:
-            if self != existing_table:
-                raise Exception("try to write axes [%s] that are invalid to an existing table [%s]!" % (axes,self.group))
+        if validate:
+            self.validate(existing_table)
 
     def create_description(self, compression = None, complevel = None):
         """ create the description of the table from the axes & values """
@@ -1670,13 +1675,8 @@ def write(self, axes, obj, append=False, compression=None,
             table = self.handle.createTable(self.group, **options)
 
         else:
-
-            # the table must already exist
             table = self.table
 
-            # validate the table
-            self.validate()
-
         # validate the axes and set the kinds
         for a in self.axes:
             a.validate_and_set(table, append)

pandas/io/tests/test_pytables.py

@@ -279,6 +279,15 @@ def test_append(self):
         self.store.append('p4d2', p4d2, axes=['items','major_axis','minor_axis'])
         tm.assert_panel4d_equal(self.store['p4d2'], p4d2)
 
+        # test using differt order of items on the non-index axes
+        self.store.remove('wp1')
+        wp_append1 = wp.ix[:,:10,:]
+        self.store.append('wp1', wp_append1)
+        wp_append2 = wp.ix[:,10:,:].reindex(items = wp.items[::-1])
+        self.store.append('wp1', wp_append2) 
+        tm.assert_panel_equal(self.store['wp1'], wp)
+
+
     def test_append_frame_column_oriented(self):
 
         # column oriented
@@ -459,6 +468,8 @@ def test_big_table(self):
             os.remove(self.scratchpath)
 
     def test_append_diff_item_order(self):
+        raise nose.SkipTest('append diff item order')
+
         wp = tm.makePanel()
         wp1 = wp.ix[:, :10, :]
         wp2 = wp.ix[['ItemC', 'ItemB', 'ItemA'], 10:, :]