Skip to content

API: to_datetime(ints, unit) give requested unit #63347

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jbrockmendel wants to merge 7 commits into
base: main
Choose a base branch
from
+56 −30
Open

API: to_datetime(ints, unit) give requested unit #63347

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
73c582c
API: to_datetime(ints, unit) give requested unit
jbrockmendel Dec 12, 2025
0eda6d9
fix json cases
jbrockmendel Dec 13, 2025
916da40
update doctests
jbrockmendel Dec 14, 2025
f9fd868
Merge branch 'main' into api-dt-ints
jbrockmendel Dec 15, 2025
a5d7e51
fix doctest
jbrockmendel Dec 15, 2025
a7955da
Merge branch 'main' into api-dt-ints
jbrockmendel Dec 18, 2025
b62cc4f
remove commented-out
jbrockmendel Dec 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 21 additions & 6 deletions pandas/_libs/tslib.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import numpy as np
cnp.import_array()

from pandas._libs.tslibs.dtypes cimport (
abbrev_to_npy_unit,
get_supported_reso,
npy_unit_to_abbrev,
)
Expand Down Expand Up @@ -312,7 +313,7 @@ cpdef array_to_datetime(
_TSObject tsobj
tzinfo tz, tz_out = None
cnp.flatiter it = cnp.PyArray_IterNew(values)
NPY_DATETIMEUNIT item_reso
NPY_DATETIMEUNIT item_reso, int_reso
bint infer_reso = creso == NPY_DATETIMEUNIT.NPY_FR_GENERIC
DatetimeParseState state = DatetimeParseState(creso)
str abbrev
Expand All @@ -325,11 +326,11 @@ cpdef array_to_datetime(
else:
abbrev = npy_unit_to_abbrev(creso)

if unit_for_numerics is not None:
# either creso or unit_for_numerics should be passed, not both
assert creso == NPY_FR_ns
else:
if unit_for_numerics is None:
unit_for_numerics = abbrev
int_reso = NPY_FR_ns
else:
int_reso = get_supported_reso(abbrev_to_npy_unit(unit_for_numerics))

result = np.empty((<object>values).shape, dtype=f"M8[{abbrev}]")
iresult = result.view("i8").ravel()
Expand Down Expand Up @@ -370,7 +371,20 @@ cpdef array_to_datetime(
iresult[i] = get_datetime64_nanos(val, creso)
state.found_other = True

elif is_integer_object(val) or is_float_object(val):
elif is_integer_object(val):
if val == NPY_NAT:
iresult[i] = NPY_NAT
else:
item_reso = int_reso
state.update_creso(item_reso)
if infer_reso:
creso = state.creso

iresult[i] = cast_from_unit(val, unit_for_numerics, out_reso=creso)

state.found_other = True

elif is_float_object(val):
# these must be ns unit by-definition

if val != val or val == NPY_NAT:
Expand Down Expand Up @@ -460,6 +474,7 @@ cpdef array_to_datetime(
dayfirst=dayfirst,
utc=utc,
creso=state.creso,
unit_for_numerics=unit_for_numerics,
)
elif state.creso == NPY_DATETIMEUNIT.NPY_FR_GENERIC:
# i.e. we never encountered anything non-NaT, default to "s". This
Expand Down
8 changes: 4 additions & 4 deletions pandas/core/tools/datetimes.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,11 @@
Timedelta,
Timestamp,
astype_overflowsafe,
get_supported_dtype,
is_supported_dtype,
timezones as libtimezones,
)
from pandas._libs.tslibs.conversion import cast_from_unit_vectorized
from pandas._libs.tslibs.dtypes import NpyDatetimeUnit
from pandas._libs.tslibs.parsing import (
DateParseError,
guess_datetime_format,
Expand Down Expand Up @@ -503,8 +503,9 @@ def _to_datetime_with_unit(arg, unit, name, utc: bool, errors: str) -> Index:
# Note we can't do "f" here because that could induce unwanted
# rounding GH#14156, GH#20445
arr = arg.astype(f"datetime64[{unit}]", copy=False)
dtype = get_supported_dtype(arr.dtype)
try:
arr = astype_overflowsafe(arr, np.dtype("M8[ns]"), copy=False)
arr = astype_overflowsafe(arr, dtype, copy=False)
except OutOfBoundsDatetime:
if errors == "raise":
raise
Expand Down Expand Up @@ -534,7 +535,6 @@ def _to_datetime_with_unit(arg, unit, name, utc: bool, errors: str) -> Index:
utc=utc,
errors=errors,
unit_for_numerics=unit,
creso=cast(int, NpyDatetimeUnit.NPY_FR_ns.value),
)

result = DatetimeIndex(arr, name=name)
Expand Down Expand Up @@ -904,7 +904,7 @@ def to_datetime(

>>> pd.to_datetime([1, 2, 3], unit="D", origin=pd.Timestamp("1960-01-01"))
DatetimeIndex(['1960-01-02', '1960-01-03', '1960-01-04'],
dtype='datetime64[ns]', freq=None)
dtype='datetime64[s]', freq=None)
Comment on lines 905 to +907
Copy link
Member

@jorisvandenbossche jorisvandenbossche Dec 18, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This makes me wonder about a potential corner case: what if the original actually has values for a lower resolution, like actual microseconds or nanoseconds?

Copy link
Member Author

@jbrockmendel jbrockmendel Dec 18, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good catch, this needs fixing:

ts = pd.Timestamp("2016-01-01") + pd.Timedelta(microseconds=1)
pd.to_datetime([1, 2, 3], unit="D", origin=ts)
DatetimeIndex(['2016-01-02', '2016-01-03', '2016-01-04'], dtype='datetime64[s]', freq=None)

Copy link
Member Author

@jbrockmendel jbrockmendel Dec 18, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looks like this actually happens on main too. The offending line is

ioffset = td_offset // Timedelta(1, unit=unit)


**Differences with strptime behavior**

Expand Down
6 changes: 5 additions & 1 deletion pandas/io/json/_json.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1312,7 +1312,11 @@ def _try_convert_to_date(self, data: Series) -> Series:
date_units = (self.date_unit,) if self.date_unit else self._STAMP_UNITS
for date_unit in date_units:
try:
return to_datetime(new_data, errors="raise", unit=date_unit)
# Without this as_unit cast, we would fail to overflow
# and get much-too-large dates
return to_datetime(new_data, errors="raise", unit=date_unit).dt.as_unit(
"ns"
Comment on lines +1315 to +1318
Copy link
Member

@jorisvandenbossche jorisvandenbossche Dec 18, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am not directly understanding that comment. new_data are integers here? Why does the return unit of this function need to be nanoseconds? (to preserve current functionality?) Why would this give (wrong?) too large dates?

Copy link
Member Author

@jbrockmendel jbrockmendel Dec 18, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is inside a block that tries large units and if they overflow then tries smaller units. This PR makes the large units not-overflow in cases where this piece of code expects them to. Without this edit, e.g. pandas/tests/io/json/test_pandas.py::TestPandasContainer::test_date_unit fails with

left = DatetimeIndex(['30004724859-08-03', '30007462766-08-06', '30010200673-08-08',
               '30012938580-08-10', '300...        '30106027418-11-06', '30108765325-11-08', '30111503232-11-10'],
              dtype='datetime64[s]', freq=None)
right = DatetimeIndex(['2000-01-03', '2000-01-04', '2000-01-05', '2000-01-06',
               '2000-01-07', '2000-01-10', '200...2000-02-08', '2000-02-09',
               '2000-02-10', '2000-02-11'],
              dtype='datetime64[ns]', freq=None)

)
except (ValueError, OverflowError, TypeError):
continue
return data
Expand Down
4 changes: 2 additions & 2 deletions pandas/tests/io/json/test_pandas.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -955,7 +955,7 @@ def test_date_format_frame_raises(self, datetime_frame):
],
)
def test_date_format_series(self, date, date_unit, datetime_series):
ts = Series(Timestamp(date).as_unit("ns"), index=datetime_series.index)
ts = Series(Timestamp(date), index=datetime_series.index)
ts.iloc[1] = pd.NaT
ts.iloc[5] = pd.NaT
if date_unit:
Expand All @@ -964,7 +964,7 @@ def test_date_format_series(self, date, date_unit, datetime_series):
json = ts.to_json(date_format="iso")

result = read_json(StringIO(json), typ="series")
expected = ts.copy()
expected = ts.copy().dt.as_unit("ns")
Copy link
Member

@jorisvandenbossche jorisvandenbossche Dec 18, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not for this PR, but so this is another case where we currently return ns unit but could change to use us by default?

tm.assert_series_equal(result, expected)

def test_date_format_series_raises(self, datetime_series):
Expand Down
2 changes: 1 addition & 1 deletion pandas/tests/resample/test_resampler_grouper.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -502,7 +502,7 @@ def test_groupby_resample_empty_sum_string(
result = gbrs.sum(min_count=min_count)

index = pd.MultiIndex(
levels=[[1, 2, 3], [pd.to_datetime("2000-01-01", unit="ns")]],
levels=[[1, 2, 3], [pd.to_datetime("2000-01-01", unit="ns").as_unit("ns")]],
Copy link
Member

@jorisvandenbossche jorisvandenbossche Dec 18, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Did this PR change that? (that this no longer returns nanoseconds)

codes=[[0, 1, 2], [0, 0, 0]],
names=["A", None],
)
Expand Down
39 changes: 23 additions & 16 deletions pandas/tests/tools/test_to_datetime.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1782,7 +1782,8 @@ class TestToDatetimeUnit:
def test_to_datetime_month_or_year_unit_int(self, cache, unit, item, request):
# GH#50870 Note we have separate tests that pd.Timestamp gets these right
ts = Timestamp(item, unit=unit)
expected = DatetimeIndex([ts], dtype="M8[ns]")
dtype = "M8[ns]" if isinstance(item, float) else "M8[s]"
expected = DatetimeIndex([ts], dtype=dtype)

result = to_datetime([item], unit=unit, cache=cache)
tm.assert_index_equal(result, expected)
Expand All @@ -1796,7 +1797,7 @@ def test_to_datetime_month_or_year_unit_int(self, cache, unit, item, request):
# with a nan!
result = to_datetime(np.array([item, np.nan]), unit=unit, cache=cache)
assert result.isna()[1]
tm.assert_index_equal(result[:1], expected)
tm.assert_index_equal(result[:1], expected.astype("M8[ns]"))

@pytest.mark.parametrize("unit", ["Y", "M"])
def test_to_datetime_month_or_year_unit_non_round_float(self, cache, unit):
Expand All @@ -1820,12 +1821,12 @@ def test_to_datetime_month_or_year_unit_non_round_float(self, cache, unit):
# In 3.0, the string "1.5" is parsed as as it would be without unit,
# which fails. With errors="coerce" this becomes NaT.
res = to_datetime(["1.5"], unit=unit, errors="coerce")
expected = to_datetime([NaT]).as_unit("ns")
expected = to_datetime([NaT])
tm.assert_index_equal(res, expected)

# round floats are OK
res = to_datetime([1.0], unit=unit)
expected = to_datetime([1], unit=unit)
expected = to_datetime([1], unit=unit).as_unit("ns")
tm.assert_index_equal(res, expected)

def test_unit(self, cache):
Expand Down Expand Up @@ -1853,7 +1854,7 @@ def test_unit_array_mixed_nans_large_int(self, cache):
values = [1420043460000000000000000, iNaT, NaT, np.nan, "NaT"]

result = to_datetime(values, errors="coerce", unit="s", cache=cache)
expected = DatetimeIndex(["NaT", "NaT", "NaT", "NaT", "NaT"], dtype="M8[ns]")
expected = DatetimeIndex(["NaT", "NaT", "NaT", "NaT", "NaT"], dtype="M8[s]")
tm.assert_index_equal(result, expected)

msg = "cannot convert input 1420043460000000000000000 with the unit 's'"
Expand Down Expand Up @@ -1950,12 +1951,13 @@ def test_to_datetime_unit(self, dtype):
epoch = 1370745748
ser = Series([epoch + t for t in range(20)]).astype(dtype)
result = to_datetime(ser, unit="s")
unit = "s" if dtype is int else "ns"
expected = Series(
[
Timestamp("2013-06-09 02:42:28") + timedelta(seconds=t)
for t in range(20)
],
dtype="M8[ns]",
dtype=f"M8[{unit}]",
)
tm.assert_series_equal(result, expected)

Expand All @@ -1964,10 +1966,13 @@ def test_to_datetime_unit_with_nulls(self, null):
epoch = 1370745748
ser = Series([epoch + t for t in range(20)] + [null])
result = to_datetime(ser, unit="s")
# With np.nan, the list gets cast to a float64 array, which always
# gets ns unit.
unit = "ns" if null is np.nan else "s"
expected = Series(
[Timestamp("2013-06-09 02:42:28") + timedelta(seconds=t) for t in range(20)]
+ [NaT],
dtype="M8[ns]",
dtype=f"M8[{unit}]",
)
tm.assert_series_equal(result, expected)

Expand All @@ -1992,25 +1997,25 @@ def test_to_datetime_unit_na_values(self):
result = to_datetime([1, 2, "NaT", NaT, np.nan], unit="D")
expected = DatetimeIndex(
[Timestamp("1970-01-02"), Timestamp("1970-01-03")] + ["NaT"] * 3,
dtype="M8[ns]",
dtype="M8[s]",
)
tm.assert_index_equal(result, expected)

@pytest.mark.parametrize("bad_val", ["foo", 111111111])
@pytest.mark.parametrize("bad_val", ["foo", 111111111111111])
def test_to_datetime_unit_invalid(self, bad_val):
if bad_val == "foo":
msg = f"Unknown datetime string format, unable to parse: {bad_val}"
else:
msg = "cannot convert input 111111111 with the unit 'D'"
msg = "cannot convert input 111111111111111 with the unit 'D'"
with pytest.raises(ValueError, match=msg):
to_datetime([1, 2, bad_val], unit="D")

@pytest.mark.parametrize("bad_val", ["foo", 111111111])
@pytest.mark.parametrize("bad_val", ["foo", 111111111111111])
def test_to_timestamp_unit_coerce(self, bad_val):
# coerce we can process
expected = DatetimeIndex(
[Timestamp("1970-01-02"), Timestamp("1970-01-03")] + ["NaT"] * 1,
dtype="M8[ns]",
dtype="M8[s]",
)
result = to_datetime([1, 2, bad_val], unit="D", errors="coerce")
tm.assert_index_equal(result, expected)
Expand Down Expand Up @@ -3223,7 +3228,7 @@ def test_unix(self):
result = Series(to_datetime([0, 1, 2], unit="D", origin="unix"))
expected = Series(
[Timestamp("1970-01-01"), Timestamp("1970-01-02"), Timestamp("1970-01-03")],
dtype="M8[ns]",
dtype="M8[s]",
)
tm.assert_series_equal(result, expected)

Expand Down Expand Up @@ -3262,8 +3267,10 @@ def test_invalid_origin(self, unit):
def test_epoch(self, units, epochs):
epoch_1960 = Timestamp(1960, 1, 1)
units_from_epochs = np.arange(5, dtype=np.int64)
exp_unit = "s" if units == "D" else units
expected = Series(
[pd.Timedelta(x, unit=units) + epoch_1960 for x in units_from_epochs]
[pd.Timedelta(x, unit=units) + epoch_1960 for x in units_from_epochs],
dtype=f"M8[{exp_unit}]",
)

result = Series(to_datetime(units_from_epochs, unit=units, origin=epochs))
Expand Down Expand Up @@ -3358,7 +3365,7 @@ def test_arg_tz_ns_unit(self, offset, utc, exp):
# GH 25546
arg = "2019-01-01T00:00:00.000" + offset
result = to_datetime([arg], unit="ns", utc=utc)
expected = to_datetime([exp]).as_unit("ns")
expected = to_datetime([exp]).as_unit("us")
tm.assert_index_equal(result, expected)


Expand Down Expand Up @@ -3458,7 +3465,7 @@ def test_empty_string_datetime_coerce__unit():
# GH13044
# coerce empty string to pd.NaT
result = to_datetime([1, ""], unit="s", errors="coerce")
expected = DatetimeIndex(["1970-01-01 00:00:01", "NaT"], dtype="datetime64[ns]")
expected = DatetimeIndex(["1970-01-01 00:00:01", "NaT"], dtype="datetime64[s]")
tm.assert_index_equal(expected, result)

# verify that no exception is raised even when errors='raise' is set
Expand Down
Loading