ENH: Support observed keyword argument in Categorical.value_counts (#43498)

Author: AndrewEckart

pandas/core/arrays/categorical.py

@@ -1606,7 +1606,7 @@ def notna(self) -> np.ndarray:
 
     notnull = notna
 
-    def value_counts(self, dropna: bool = True):
+    def value_counts(self, dropna: bool = True, observed: bool = False):
         """
         Return a Series containing counts of each category.
 
@@ -1616,6 +1616,9 @@ def value_counts(self, dropna: bool = True):
         ----------
         dropna : bool, default True
             Don't include counts of NaN.
+        observed : bool, default False
+            If True, only include counts for observed categories.
+            If False, include counts for all categories.
 
         Returns
         -------
@@ -1644,7 +1647,8 @@ def value_counts(self, dropna: bool = True):
         ix = coerce_indexer_dtype(ix, self.dtype.categories)
         ix = self._from_backing_data(ix)
 
-        return Series(count, index=CategoricalIndex(ix), dtype="int64")
+        counts = Series(count, index=CategoricalIndex(ix), dtype="int64")
+        return counts[counts != 0] if observed else counts
 
     # error: Argument 2 of "_empty" is incompatible with supertype
     # "NDArrayBackedExtensionArray"; supertype defines the argument type as

pandas/tests/extension/test_categorical.py

@@ -163,6 +163,15 @@ class TestMethods(base.BaseMethodsTests):
     def test_value_counts(self, all_data, dropna):
         return super().test_value_counts(all_data, dropna)
 
+    def test_value_counts_observed(self, data):
+        data = data.add_categories(["#", "?"])  # Add some unobserved categories
+        series = pd.Series(data, dtype=data.categories.dtype)
+        result = data.value_counts(observed=True).sort_index()
+        expected = series.value_counts().sort_index()
+        self.assert_series_equal(
+            result, expected, check_index_type=False, check_categorical=False
+        )
+
     def test_combine_add(self, data_repeated):
         # GH 20825
         # When adding categoricals in combine, result is a string