Merge remote-tracking branch 'upstream/master' into deprecate-week-of-year

Author: mgmarino

doc/source/reference/groupby.rst

@@ -36,8 +36,10 @@ Function application
 
    GroupBy.apply
    GroupBy.agg
-   GroupBy.aggregate
-   GroupBy.transform
+   SeriesGroupBy.aggregate
+   DataFrameGroupBy.aggregate
+   SeriesGroupBy.transform
+   DataFrameGroupBy.transform
    GroupBy.pipe
 
 Computations / descriptive stats

doc/source/whatsnew/v1.1.0.rst

@@ -139,6 +139,7 @@ Other enhancements
 - The :meth:`DataFrame.to_feather` method now supports additional keyword
   arguments (e.g. to set the compression) that are added in pyarrow 0.17
   (:issue:`33422`).
+- The :func:`cut` will now accept parameter ``ordered`` with default ``ordered=True``. If ``ordered=False`` and no labels are provided, an error will be raised (:issue:`33141`)
 - :meth:`DataFrame.to_csv`, :meth:`DataFrame.to_pickle`,
   and :meth:`DataFrame.to_json` now support passing a dict of
   compression arguments when using the ``gzip`` and ``bz2`` protocols.
@@ -547,6 +548,7 @@ Datetimelike
 - Bug in :meth:`DatetimeIndex.tz_localize` incorrectly retaining ``freq`` in some cases where the original freq is no longer valid (:issue:`30511`)
 - Bug in :meth:`DatetimeIndex.intersection` losing ``freq`` and timezone in some cases (:issue:`33604`)
 - Bug in :class:`DatetimeIndex` addition and subtraction with some types of :class:`DateOffset` objects incorrectly retaining an invalid ``freq`` attribute (:issue:`33779`)
+- Bug in :class:`DatetimeIndex` where setting the ``freq`` attribute on an index could silently change the ``freq`` attribute on another index viewing the same data (:issue:`33552`)
 
 Timedelta
 ^^^^^^^^^
@@ -573,6 +575,7 @@ Numeric
 - Bug in :meth:`DataFrame.count` with ``level="foo"`` and index level ``"foo"`` containing NaNs causes segmentation fault (:issue:`21824`)
 - Bug in :meth:`DataFrame.diff` with ``axis=1`` returning incorrect results with mixed dtypes (:issue:`32995`)
 - Bug in :meth:`DataFrame.corr` and :meth:`DataFrame.cov` raising when handling nullable integer columns with ``pandas.NA`` (:issue:`33803`)
+- Bug in :class:`DataFrame` and :class:`Series` addition and subtraction between object-dtype objects and ``datetime64`` dtype objects (:issue:`33824`)
 
 Conversion
 ^^^^^^^^^^
@@ -723,6 +726,7 @@ Reshaping
 - Bug in :meth:`concat` where when passing a non-dict mapping as ``objs`` would raise a ``TypeError`` (:issue:`32863`)
 - :meth:`DataFrame.agg` now provides more descriptive ``SpecificationError`` message when attempting to aggregating non-existant column (:issue:`32755`)
 - Bug in :meth:`DataFrame.unstack` when MultiIndexed columns and MultiIndexed rows were used (:issue:`32624`, :issue:`24729` and :issue:`28306`)
+- Bug in :func:`cut` raised an error when non-unique labels (:issue:`33141`)
 
 
 Sparse

pandas/core/array_algos/transforms.py

@@ -10,9 +10,8 @@
 def shift(values: np.ndarray, periods: int, axis: int, fill_value) -> np.ndarray:
     new_values = values
 
-    if periods == 0:
-        # TODO: should we copy here?
-        return new_values
+    if periods == 0 or values.size == 0:
+        return new_values.copy()
 
     # make sure array sent to np.roll is c_contiguous
     f_ordered = values.flags.f_contiguous

pandas/core/arrays/categorical.py

@@ -1196,7 +1196,7 @@ def shift(self, periods, fill_value=None):
 
         fill_value = self._validate_fill_value(fill_value)
 
-        codes = shift(codes.copy(), periods, axis=0, fill_value=fill_value)
+        codes = shift(codes, periods, axis=0, fill_value=fill_value)
 
         return self._constructor(codes, dtype=self.dtype, fastpath=True)
 

pandas/core/arrays/datetimelike.py

@@ -699,8 +699,6 @@ def _values_for_argsort(self):
 
     @Appender(ExtensionArray.shift.__doc__)
     def shift(self, periods=1, fill_value=None, axis=0):
-        if not self.size or periods == 0:
-            return self.copy()
 
         fill_value = self._validate_shift_value(fill_value)
         new_values = shift(self._data, periods, axis, fill_value)
@@ -742,10 +740,12 @@ def _validate_fill_value(self, fill_value):
         return fill_value
 
     def _validate_shift_value(self, fill_value):
-        # TODO(2.0): once this deprecation is enforced, used _validate_fill_value
+        # TODO(2.0): once this deprecation is enforced, use _validate_fill_value
         if is_valid_nat_for_dtype(fill_value, self.dtype):
             fill_value = NaT
-        elif not isinstance(fill_value, self._recognized_scalars):
+        elif isinstance(fill_value, self._recognized_scalars):
+            fill_value = self._scalar_type(fill_value)
+        else:
             # only warn if we're not going to raise
             if self._scalar_type is Period and lib.is_integer(fill_value):
                 # kludge for #31971 since Period(integer) tries to cast to str
@@ -782,6 +782,9 @@ def _validate_searchsorted_value(self, value):
         elif isinstance(value, self._recognized_scalars):
             value = self._scalar_type(value)
 
+        elif isinstance(value, type(self)):
+            pass
+
         elif is_list_like(value) and not isinstance(value, type(self)):
             value = array(value)
 
@@ -791,7 +794,7 @@ def _validate_searchsorted_value(self, value):
                     f"not {type(value).__name__}"
                 )
 
-        if not (isinstance(value, (self._scalar_type, type(self))) or (value is NaT)):
+        else:
             raise TypeError(f"Unexpected type for 'value': {type(value)}")
 
         if isinstance(value, type(self)):
@@ -803,25 +806,41 @@ def _validate_searchsorted_value(self, value):
         return value
 
     def _validate_setitem_value(self, value):
-        if lib.is_scalar(value) and not isna(value):
-            value = com.maybe_box_datetimelike(value)
 
         if is_list_like(value):
-            value = type(self)._from_sequence(value, dtype=self.dtype)
-            self._check_compatible_with(value, setitem=True)
-            value = value.asi8
-        elif isinstance(value, self._scalar_type):
-            self._check_compatible_with(value, setitem=True)
-            value = self._unbox_scalar(value)
+            value = array(value)
+            if is_dtype_equal(value.dtype, "string"):
+                # We got a StringArray
+                try:
+                    # TODO: Could use from_sequence_of_strings if implemented
+                    # Note: passing dtype is necessary for PeriodArray tests
+                    value = type(self)._from_sequence(value, dtype=self.dtype)
+                except ValueError:
+                    pass
+
+            if not type(self)._is_recognized_dtype(value):
+                raise TypeError(
+                    "setitem requires compatible dtype or scalar, "
+                    f"not {type(value).__name__}"
+                )
+
+        elif isinstance(value, self._recognized_scalars):
+            value = self._scalar_type(value)
         elif is_valid_nat_for_dtype(value, self.dtype):
-            value = iNaT
+            value = NaT
         else:
             msg = (
                 f"'value' should be a '{self._scalar_type.__name__}', 'NaT', "
                 f"or array of those. Got '{type(value).__name__}' instead."
             )
             raise TypeError(msg)
 
+        self._check_compatible_with(value, setitem=True)
+        if isinstance(value, type(self)):
+            value = value.asi8
+        else:
+            value = self._unbox_scalar(value)
+
         return value
 
     def _validate_insert_value(self, value):

pandas/core/groupby/generic.py

@@ -63,10 +63,11 @@
 import pandas.core.common as com
 from pandas.core.construction import create_series_with_explicit_dtype
 from pandas.core.frame import DataFrame
-from pandas.core.generic import ABCDataFrame, ABCSeries, NDFrame, _shared_docs
+from pandas.core.generic import ABCDataFrame, ABCSeries, NDFrame
 from pandas.core.groupby import base
 from pandas.core.groupby.groupby import (
     GroupBy,
+    _agg_template,
     _apply_docs,
     _transform_template,
     get_groupby,
@@ -177,16 +178,6 @@ def _selection_name(self):
         else:
             return self._selection
 
-    _agg_see_also_doc = dedent(
-        """
-    See Also
-    --------
-    pandas.Series.groupby.apply
-    pandas.Series.groupby.transform
-    pandas.Series.aggregate
-    """
-    )
-
     _agg_examples_doc = dedent(
         """
     Examples
@@ -224,8 +215,7 @@ def _selection_name(self):
     ... )
        minimum  maximum
     1        1        2
-    2        3        4
-    """
+    2        3        4"""
     )
 
     @Appender(
@@ -237,13 +227,9 @@ def apply(self, func, *args, **kwargs):
         return super().apply(func, *args, **kwargs)
 
     @Substitution(
-        see_also=_agg_see_also_doc,
-        examples=_agg_examples_doc,
-        versionadded="",
-        klass="Series",
-        axis="",
+        examples=_agg_examples_doc, klass="Series",
     )
-    @Appender(_shared_docs["aggregate"])
+    @Appender(_agg_template)
     def aggregate(
         self, func=None, *args, engine="cython", engine_kwargs=None, **kwargs
     ):
@@ -476,7 +462,7 @@ def _aggregate_named(self, func, *args, **kwargs):
 
         return result
 
-    @Substitution(klass="Series", selected="A.")
+    @Substitution(klass="Series")
     @Appender(_transform_template)
     def transform(self, func, *args, engine="cython", engine_kwargs=None, **kwargs):
         func = self._get_cython_func(func) or func
@@ -854,16 +840,6 @@ class DataFrameGroupBy(GroupBy[DataFrame]):
 
     _apply_whitelist = base.dataframe_apply_whitelist
 
-    _agg_see_also_doc = dedent(
-        """
-    See Also
-    --------
-    pandas.DataFrame.groupby.apply
-    pandas.DataFrame.groupby.transform
-    pandas.DataFrame.aggregate
-    """
-    )
-
     _agg_examples_doc = dedent(
         """
     Examples
@@ -928,26 +904,20 @@ class DataFrameGroupBy(GroupBy[DataFrame]):
     1      1  0.590715
     2      3  0.704907
 
-
     - The keywords are the *output* column names
     - The values are tuples whose first element is the column to select
       and the second element is the aggregation to apply to that column.
       Pandas provides the ``pandas.NamedAgg`` namedtuple with the fields
       ``['column', 'aggfunc']`` to make it clearer what the arguments are.
       As usual, the aggregation can be a callable or a string alias.
 
-    See :ref:`groupby.aggregate.named` for more.
-    """
+    See :ref:`groupby.aggregate.named` for more."""
     )
 
     @Substitution(
-        see_also=_agg_see_also_doc,
-        examples=_agg_examples_doc,
-        versionadded="",
-        klass="DataFrame",
-        axis="",
+        examples=_agg_examples_doc, klass="DataFrame",
     )
-    @Appender(_shared_docs["aggregate"])
+    @Appender(_agg_template)
     def aggregate(
         self, func=None, *args, engine="cython", engine_kwargs=None, **kwargs
     ):
@@ -1467,7 +1437,7 @@ def _transform_general(
         concatenated = concatenated.reindex(concat_index, axis=other_axis, copy=False)
         return self._set_result_index_ordered(concatenated)
 
-    @Substitution(klass="DataFrame", selected="")
+    @Substitution(klass="DataFrame")
     @Appender(_transform_template)
     def transform(self, func, *args, engine="cython", engine_kwargs=None, **kwargs):
 

pandas/core/groupby/groupby.py

@@ -291,7 +291,9 @@ class providing the base-class of operations.
 
 See Also
 --------
-aggregate, transform
+%(klass)s.groupby.apply
+%(klass)s.groupby.aggregate
+%(klass)s.transform
 
 Notes
 -----
@@ -310,14 +312,17 @@ class providing the base-class of operations.
 * f must not mutate groups. Mutation is not supported and may
   produce unexpected results.
 
+When using ``engine='numba'``, there will be no "fall back" behavior internally.
+The group data and group index will be passed as numpy arrays to the JITed
+user defined function, and no alternative execution attempts will be tried.
+
 Examples
 --------
 
-# Same shape
 >>> df = pd.DataFrame({'A' : ['foo', 'bar', 'foo', 'bar',
 ...                           'foo', 'bar'],
 ...                    'B' : ['one', 'one', 'two', 'three',
-...                          'two', 'two'],
+...                           'two', 'two'],
 ...                    'C' : [1, 5, 5, 2, 5, 5],
 ...                    'D' : [2.0, 5., 8., 1., 2., 9.]})
 >>> grouped = df.groupby('A')
@@ -330,7 +335,8 @@ class providing the base-class of operations.
 4  0.577350 -0.577350
 5  0.577350  1.000000
 
-# Broadcastable
+Broadcast result of the transformation
+
 >>> grouped.transform(lambda x: x.max() - x.min())
    C    D
 0  4  6.0
@@ -341,6 +347,69 @@ class providing the base-class of operations.
 5  3  8.0
 """
 
+_agg_template = """
+Aggregate using one or more operations over the specified axis.
+
+Parameters
+----------
+func : function, str, list or dict
+    Function to use for aggregating the data. If a function, must either
+    work when passed a %(klass)s or when passed to %(klass)s.apply.
+
+    Accepted combinations are:
+
+    - function
+    - string function name
+    - list of functions and/or function names, e.g. ``[np.sum, 'mean']``
+    - dict of axis labels -> functions, function names or list of such.
+
+    Can also accept a Numba JIT function with
+    ``engine='numba'`` specified.
+
+    If the ``'numba'`` engine is chosen, the function must be
+    a user defined function with ``values`` and ``index`` as the
+    first and second arguments respectively in the function signature.
+    Each group's index will be passed to the user defined function
+    and optionally available for use.
+
+    .. versionchanged:: 1.1.0
+*args
+    Positional arguments to pass to func
+engine : str, default 'cython'
+    * ``'cython'`` : Runs the function through C-extensions from cython.
+    * ``'numba'`` : Runs the function through JIT compiled code from numba.
+
+    .. versionadded:: 1.1.0
+engine_kwargs : dict, default None
+    * For ``'cython'`` engine, there are no accepted ``engine_kwargs``
+    * For ``'numba'`` engine, the engine can accept ``nopython``, ``nogil``
+      and ``parallel`` dictionary keys. The values must either be ``True`` or
+      ``False``. The default ``engine_kwargs`` for the ``'numba'`` engine is
+      ``{'nopython': True, 'nogil': False, 'parallel': False}`` and will be
+      applied to the function
+
+    .. versionadded:: 1.1.0
+**kwargs
+    Keyword arguments to be passed into func.
+
+Returns
+-------
+%(klass)s
+
+See Also
+--------
+%(klass)s.groupby.apply
+%(klass)s.groupby.transform
+%(klass)s.aggregate
+
+Notes
+-----
+When using ``engine='numba'``, there will be no "fall back" behavior internally.
+The group data and group index will be passed as numpy arrays to the JITed
+user defined function, and no alternative execution attempts will be tried.
+%(examples)s
+"""
+
 
 class GroupByPlot(PandasObject):
     """