wip

Author: MarcoGorelli

doc/source/whatsnew/v1.5.3.rst

@@ -19,6 +19,7 @@ Fixed regressions
 - Enforced reversion of ``color`` as an alias for ``c`` and ``size`` as an alias for ``s`` in function :meth:`DataFrame.plot.scatter` (:issue:`49732`)
 - Fixed regression in :meth:`SeriesGroupBy.apply` setting a ``name`` attribute on the result if the result was a :class:`DataFrame` (:issue:`49907`)
 - Fixed performance regression in setting with the :meth:`~DataFrame.at` indexer (:issue:`49771`)
+- Fixed regression in :func:`to_datetime` raising ``ValueError`` when parsing array of ``float`` containing ``np.nan`` (:issue:`50237`)
 -
 
 .. ---------------------------------------------------------------------------

pandas/_libs/tslibs/parsing.pyx

@@ -870,7 +870,7 @@ def format_is_iso(f: str) -> bint:
     but must be consistent.  Leading 0s in dates and times are optional.
     """
     iso_template = "%Y{date_sep}%m{date_sep}%d{time_sep}%H:%M:%S{micro_or_tz}".format
-    excluded_formats = ["%Y%m%d", "%Y%m", "%Y"]
+    excluded_formats = ["%Y%m", "%Y"]
 
     for date_sep in [" ", "/", "\\", "-", ".", ""]:
         for time_sep in [" ", "T"]:

pandas/_libs/tslibs/strptime.pyx

@@ -34,15 +34,21 @@ from pandas._libs.tslibs.nattype cimport (
     c_nat_strings as nat_strings,
 )
 from pandas._libs.tslibs.np_datetime cimport (
+    NPY_DATETIMEUNIT,
     NPY_FR_ns,
     check_dts_bounds,
     npy_datetimestruct,
     npy_datetimestruct_to_datetime,
     pydate_to_dt64,
     pydatetime_to_dt64,
+    string_to_dts,
 )
 from pandas._libs.tslibs.timestamps cimport _Timestamp
-from pandas._libs.util cimport is_datetime64_object
+from pandas._libs.util cimport (
+    is_datetime64_object,
+    is_float_object,
+    is_integer_object,
+)
 
 cnp.import_array()
 
@@ -89,6 +95,7 @@ def array_strptime(
     exact : matches must be exact if True, search if False
     errors : string specifying error handling, {'raise', 'ignore', 'coerce'}
     """
+    from pandas._libs.tslibs.parsing import format_is_iso
 
     cdef:
         Py_ssize_t i, n = len(values)
@@ -106,6 +113,9 @@ def array_strptime(
         bint found_naive = False
         bint found_tz = False
         tzinfo tz_out = None
+        bint iso_format = fmt is not None and format_is_iso(fmt)
+        NPY_DATETIMEUNIT out_bestunit
+        int out_local = 0, out_tzoffset = 0
 
     assert is_raise or is_ignore or is_coerce
 
@@ -185,9 +195,50 @@ def array_strptime(
         elif is_datetime64_object(val):
             iresult[i] = get_datetime64_nanos(val, NPY_FR_ns)
             continue
+        elif (
+                (is_integer_object(val) or is_float_object(val))
+                and (val != val or val == NPY_NAT)
+        ):
+            iresult[i] = NPY_NAT
+            continue
         else:
             val = str(val)
 
+        if (iso_format and not (fmt == "%Y%m%d" and len(val) != 8)):
+            # There is a fast-path for ISO8601-formatted strings.
+            # BUT for %Y%m%d, it only works if the string is 8-digits long.
+            string_to_dts_failed = string_to_dts(
+                val, &dts, &out_bestunit, &out_local,
+                &out_tzoffset, False, fmt, exact
+            )
+            if string_to_dts_failed:
+                # An error at this point is a _parsing_ error
+                # specifically _not_ OutOfBoundsDatetime
+                if is_coerce:
+                    iresult[i] = NPY_NAT
+                    continue
+                raise ValueError(
+                    f"time data \"{val}\" at position {i} doesn't "
+                    f"match format \"{fmt}\""
+                )
+            # No error reported by string_to_dts, pick back up
+            # where we left off
+            value = npy_datetimestruct_to_datetime(NPY_FR_ns, &dts)
+            if out_local == 1:
+                # Store the out_tzoffset in seconds
+                # since we store the total_seconds of
+                # dateutil.tz.tzoffset objects
+                # out_tzoffset_vals.add(out_tzoffset * 60.)
+                tz = timezone(timedelta(minutes=out_tzoffset))
+                result_timezone[i] = tz
+                # value = tz_localize_to_utc_single(value, tz)
+                out_local = 0
+                out_tzoffset = 0
+            iresult[i] = value
+            check_dts_bounds(&dts)
+
+            continue
+
         # exact matching
         if exact:
             found = format_regex.match(val)

pandas/core/tools/datetimes.py

@@ -34,7 +34,6 @@
 )
 from pandas._libs.tslibs.parsing import (
     DateParseError,
-    format_is_iso,
     guess_datetime_format,
 )
 from pandas._libs.tslibs.strptime import array_strptime
@@ -417,7 +416,6 @@ def _convert_listlike_datetimes(
 
     # warn if passing timedelta64, raise for PeriodDtype
     # NB: this must come after unit transformation
-    orig_arg = arg
     try:
         arg, _ = maybe_convert_dtype(arg, copy=False, tz=libtimezones.maybe_get_tz(tz))
     except TypeError:
@@ -430,32 +428,19 @@ def _convert_listlike_datetimes(
         raise
 
     arg = ensure_object(arg)
-    require_iso8601 = False
 
     if format is None:
         format = _guess_datetime_format_for_array(arg, dayfirst=dayfirst)
 
-    # There is a special fast-path for iso8601 formatted datetime strings
-    require_iso8601 = format is not None and format_is_iso(format)
-
-    if format is not None and not require_iso8601:
-        return _to_datetime_with_format(
-            arg,
-            orig_arg,
-            name,
-            utc,
-            format,
-            exact,
-            errors,
-        )
+    if format is not None:
+        return _array_strptime_with_fallback(arg, name, utc, format, exact, errors)
 
     result, tz_parsed = objects_to_datetime64ns(
         arg,
         dayfirst=dayfirst,
         yearfirst=yearfirst,
         utc=utc,
         errors=errors,
-        require_iso8601=require_iso8601,
         allow_object=True,
         format=format,
         exact=exact,
@@ -522,22 +507,6 @@ def _to_datetime_with_format(
     """
     Try parsing with the given format.
     """
-    result = None
-
-    # shortcut formatting here
-    if fmt == "%Y%m%d":
-        # pass orig_arg as float-dtype may have been converted to
-        # datetime64[ns]
-        orig_arg = ensure_object(orig_arg)
-        try:
-            # may return None without raising
-            result = _attempt_YYYYMMDD(orig_arg, errors=errors)
-        except (ValueError, TypeError, OutOfBoundsDatetime) as err:
-            raise ValueError(
-                "cannot convert the input to '%Y%m%d' date format"
-            ) from err
-        if result is not None:
-            return _box_as_indexlike(result, utc=utc, name=name)
 
     # fallback
     res = _array_strptime_with_fallback(arg, name, utc, fmt, exact, errors)

pandas/tests/tools/test_to_datetime.py

@@ -132,7 +132,18 @@ def test_to_datetime_format_YYYYMMDD_with_nat(self, cache):
         # string with NaT
         ser2 = ser.apply(str)
         ser2[2] = "nat"
-        result = to_datetime(ser2, format="%Y%m%d", cache=cache)
+        with pytest.raises(ValueError, match="unconverted data remains: .0"):
+            to_datetime(ser2, format="%Y%m%d", cache=cache)
+
+    def test_to_datetime_format_YYYYMM_with_nat(self, cache):
+        # https://github.com/pandas-dev/pandas/issues/50237
+        ser = Series([198012, 198012] + [198101] * 5)
+        expected = Series(
+            [Timestamp("19801201"), Timestamp("19801201")] + [Timestamp("19810101")] * 5
+        )
+        expected[2] = np.nan
+        ser[2] = np.nan
+        result = to_datetime(ser, format="%Y%m", cache=cache)
         tm.assert_series_equal(result, expected)
 
     def test_to_datetime_format_YYYYMMDD_ignore(self, cache):
@@ -141,7 +152,7 @@ def test_to_datetime_format_YYYYMMDD_ignore(self, cache):
         ser = Series([20121231, 20141231, 99991231])
         result = to_datetime(ser, format="%Y%m%d", errors="ignore", cache=cache)
         expected = Series(
-            [datetime(2012, 12, 31), datetime(2014, 12, 31), datetime(9999, 12, 31)],
+            [20121231, 20141231, 99991231],
             dtype=object,
         )
         tm.assert_series_equal(result, expected)
@@ -2852,10 +2863,7 @@ def test_incorrect_value_exception(self):
     )
     def test_to_datetime_out_of_bounds_with_format_arg(self, format, warning):
         # see gh-23830
-        msg = (
-            "Out of bounds nanosecond timestamp: 2417-10-27 00:00:00 "
-            "present at position 0"
-        )
+        msg = "Out of bounds nanosecond timestamp: 2417-10-27 00:00:00"
         with pytest.raises(OutOfBoundsDatetime, match=msg):
             with tm.assert_produces_warning(warning, match="Could not infer format"):
                 to_datetime("2417-10-27 00:00:00", format=format)