ENH: DataFrame.drop_duplicates and DataFrame.duplicated to remove duplicate rows, GH #319

Author: wesm

pandas/core/frame.py

@@ -1310,6 +1310,54 @@ def dropna(self, axis=0, how='any', thresh=None, subset=None):
         new_labels = labels[mask]
         return self.reindex(**{axis_name : new_labels})
 
+    def drop_duplicates(self, col_or_columns=None, take_last=False):
+        """
+        Return DataFrame with duplicate rows removed, optionally only
+        considering certain columns
+
+        Parameters
+        ----------
+        col_or_columns : column label or sequence of labels, optional
+            Only consider certain columns for identifying duplicates, by
+            default use all of the columns
+        take_last : boolean, default False
+            Take the last observed row in a row. Defaults to the first row
+
+        Returns
+        -------
+        deduplicated : DataFrame
+        """
+        duplicated = self.duplicated(col_or_columns, take_last=take_last)
+        return self[-duplicated]
+
+    def duplicated(self, col_or_columns=None, take_last=False):
+        """
+        Return boolean Series denoting duplicate rows, optionally only
+        considering certain columns
+
+        Parameters
+        ----------
+        col_or_columns : column label or sequence of labels, optional
+            Only consider certain columns for identifying duplicates, by
+            default use all of the columns
+        take_last : boolean, default False
+            Take the last observed row in a row. Defaults to the first row
+
+        Returns
+        -------
+        duplicated : Series
+        """
+        if col_or_columns is not None:
+            if isinstance(col_or_columns, list):
+                keys = zip(*[self[x] for x in col_or_columns])
+            else:
+                keys = list(self[col_or_columns])
+        else:
+            keys = zip(*self.values.T)
+
+        duplicated = lib.duplicated(keys, take_last=take_last)
+        return Series(duplicated, index=self.index)
+
     #----------------------------------------------------------------------
     # Sorting
 

pandas/src/groupby.pyx

@@ -557,6 +557,33 @@ def count_level_2d(ndarray[uint8_t, ndim=2, cast=True] mask,
 
     return counts
 
+def duplicated(list values, take_last=False):
+    cdef:
+        Py_ssize_t i, n
+        dict seen = {}
+        object row
+
+    n = len(values)
+    cdef ndarray[uint8_t] result = np.zeros(n, dtype=np.uint8)
+
+    if take_last:
+        for i from n > i >= 0:
+            row = values[i]
+            if row in seen:
+                result[i] = 1
+            else:
+                seen[row] = None
+                result[i] = 0
+    else:
+        for i from 0 <= i < n:
+            row = values[i]
+            if row in seen:
+                result[i] = 1
+            else:
+                seen[row] = None
+                result[i] = 0
+
+    return result.view(np.bool_)
 
 '''
 

pandas/tests/test_frame.py

@@ -2001,6 +2001,44 @@ def test_dropna_corner(self):
         self.assertRaises(ValueError, self.frame.dropna, how='foo')
         self.assertRaises(ValueError, self.frame.dropna, how=None)
 
+    def test_drop_duplicates(self):
+        df = DataFrame({'A' : ['foo', 'bar', 'foo', 'bar',
+                               'foo', 'bar', 'bar', 'foo'],
+                        'B' : ['one', 'one', 'two', 'two',
+                               'two', 'two', 'one', 'two'],
+                        'C' : [1, 1, 2, 2, 2, 2, 1, 2],
+                        'D' : range(8)})
+
+        # single column
+        result = df.drop_duplicates('A')
+        expected = df[:2]
+        assert_frame_equal(result, expected)
+
+        result = df.drop_duplicates('A', take_last=True)
+        expected = df.ix[[6, 7]]
+        assert_frame_equal(result, expected)
+
+        # multi column
+        result = df.drop_duplicates(['A', 'B'])
+        expected = df.ix[[0, 1, 2, 3]]
+        assert_frame_equal(result, expected)
+
+        result = df.drop_duplicates(['A', 'B'], take_last=True)
+        expected = df.ix[[0, 5, 6, 7]]
+        assert_frame_equal(result, expected)
+
+        # consider everything
+        df2 = df.ix[:, ['A', 'B', 'C']]
+
+        result = df2.drop_duplicates()
+        # in this case only
+        expected = df2.drop_duplicates(['A', 'B'])
+        assert_frame_equal(result, expected)
+
+        result = df2.drop_duplicates(take_last=True)
+        expected = df2.drop_duplicates(['A', 'B'], take_last=True)
+        assert_frame_equal(result, expected)
+
     def test_fillna(self):
         self.tsframe['A'][:5] = nan
         self.tsframe['A'][-5:] = nan
@@ -3258,7 +3296,6 @@ def test_series_put_names(self):
         for k, v in series.iteritems():
             self.assertEqual(v.name, k)
 
-
 class TestDataFrameJoin(unittest.TestCase):
 
     def setUp(self):