Skip to content

Commit

Permalink
Fix ns
Browse files Browse the repository at this point in the history
  • Loading branch information
mcrumiller committed Dec 9, 2024
1 parent b917c5c commit 062066e
Show file tree
Hide file tree
Showing 5 changed files with 76 additions and 26 deletions.
4 changes: 2 additions & 2 deletions crates/polars-plan/src/dsl/function_expr/datetime.rs
Original file line number Diff line number Diff line change
Expand Up @@ -579,7 +579,7 @@ pub(super) fn replace(s: &[Column]) -> PolarsResult<Column> {
let hour = s_hour.i8()?;
let minute = s_minute.i8()?;
let second = s_second.i8()?;
let microsecond = s_microsecond.i32()?;
let nanosecond = &(s_microsecond.i32()? * 1_000);
let s_ambiguous = &s[8].strict_cast(&DataType::String)?;
let ambiguous = s_ambiguous.str()?;

Expand All @@ -591,7 +591,7 @@ pub(super) fn replace(s: &[Column]) -> PolarsResult<Column> {
hour,
minute,
second,
microsecond,
nanosecond,
ambiguous,
);
out.map(|s| s.into_column())
Expand Down
11 changes: 6 additions & 5 deletions crates/polars-plan/src/dsl/function_expr/temporal.rs
Original file line number Diff line number Diff line change
Expand Up @@ -124,11 +124,12 @@ pub(super) fn datetime(
}
let second = second.i8()?;

let mut microsecond = microsecond.cast(&DataType::Int32)?;
if microsecond.len() < max_len {
microsecond = microsecond.new_from_index(0, max_len);
let mut nanosecond = microsecond.cast(&DataType::Int32)? * 1_000;
if nanosecond.len() < max_len {
nanosecond = nanosecond.new_from_index(0, max_len);
}
let microsecond = microsecond.i32()?;
let nanosecond = nanosecond.i32()?;

let mut _ambiguous = ambiguous.cast(&DataType::String)?;
if _ambiguous.len() < max_len {
_ambiguous = _ambiguous.new_from_index(0, max_len);
Expand All @@ -142,7 +143,7 @@ pub(super) fn datetime(
hour,
minute,
second,
microsecond,
nanosecond,
ambiguous,
time_unit,
time_zone,
Expand Down
12 changes: 6 additions & 6 deletions crates/polars-time/src/chunkedarray/datetime.rs
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@ pub trait DatetimeMethods: AsDatetime {
hour: &Int8Chunked,
minute: &Int8Chunked,
second: &Int8Chunked,
microsecond: &Int32Chunked,
nanosecond: &Int32Chunked,
ambiguous: &StringChunked,
time_unit: &TimeUnit,
time_zone: Option<&str>,
Expand All @@ -174,14 +174,14 @@ pub trait DatetimeMethods: AsDatetime {
.zip(hour)
.zip(minute)
.zip(second)
.zip(microsecond)
.map(|((((((y, m), d), h), mnt), s), us)| {
if let (Some(y), Some(m), Some(d), Some(h), Some(mnt), Some(s), Some(us)) =
(y, m, d, h, mnt, s, us)
.zip(nanosecond)
.map(|((((((y, m), d), h), mnt), s), ns)| {
if let (Some(y), Some(m), Some(d), Some(h), Some(mnt), Some(s), Some(ns)) =
(y, m, d, h, mnt, s, ns)
{
NaiveDate::from_ymd_opt(y, m as u32, d as u32)
.and_then(|nd| {
nd.and_hms_micro_opt(h as u32, mnt as u32, s as u32, us as u32)
nd.and_hms_nano_opt(h as u32, mnt as u32, s as u32, ns as u32)
})
.map(|ndt| match time_unit {
TimeUnit::Milliseconds => ndt.and_utc().timestamp_millis(),
Expand Down
27 changes: 14 additions & 13 deletions crates/polars-time/src/replace.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ pub fn replace_datetime(
hour: &Int8Chunked,
minute: &Int8Chunked,
second: &Int8Chunked,
microsecond: &Int32Chunked,
nanosecond: &Int32Chunked,
ambiguous: &StringChunked,
) -> PolarsResult<DatetimeChunked> {
let n = ca.len();
Expand All @@ -24,7 +24,7 @@ pub fn replace_datetime(
// 3. Value was supplied and is Series --> Update all elements with the non-null values
let year = if year.len() == 1 {
if let Some(value) = year.get(0) {
&Int32Chunked::full("".into(), value, n)
&Int32Chunked::full(PlSmallStr::EMPTY, value, n)
} else {
&ca.year()
}
Expand All @@ -33,7 +33,7 @@ pub fn replace_datetime(
};
let month = if month.len() == 1 {
if let Some(value) = month.get(0) {
&Int8Chunked::full("".into(), value, n)
&Int8Chunked::full(PlSmallStr::EMPTY, value, n)
} else {
&ca.month()
}
Expand All @@ -42,7 +42,7 @@ pub fn replace_datetime(
};
let day = if day.len() == 1 {
if let Some(value) = day.get(0) {
&Int8Chunked::full("".into(), value, n)
&Int8Chunked::full(PlSmallStr::EMPTY, value, n)
} else {
&ca.day()
}
Expand All @@ -51,7 +51,7 @@ pub fn replace_datetime(
};
let hour = if hour.len() == 1 {
if let Some(value) = hour.get(0) {
&Int8Chunked::full("".into(), value, n)
&Int8Chunked::full(PlSmallStr::EMPTY, value, n)
} else {
&ca.hour()
}
Expand All @@ -60,7 +60,7 @@ pub fn replace_datetime(
};
let minute = if minute.len() == 1 {
if let Some(value) = minute.get(0) {
&Int8Chunked::full("".into(), value, n)
&Int8Chunked::full(PlSmallStr::EMPTY, value, n)
} else {
&ca.minute()
}
Expand All @@ -69,31 +69,32 @@ pub fn replace_datetime(
};
let second = if second.len() == 1 {
if let Some(value) = second.get(0) {
&Int8Chunked::full("".into(), value, n)
&Int8Chunked::full(PlSmallStr::EMPTY, value, n)
} else {
&ca.second()
}
} else {
&second.zip_with(&second.is_not_null(), &ca.second())?
};
let microsecond = if microsecond.len() == 1 {
if let Some(value) = microsecond.get(0) {
&Int32Chunked::full("".into(), value, n)
let nanosecond = if nanosecond.len() == 1 {
if let Some(value) = nanosecond.get(0) {
&Int32Chunked::full(PlSmallStr::EMPTY, value, n)
} else {
&(ca.nanosecond() / 1000)
&ca.nanosecond()
}
} else {
&microsecond.zip_with(&microsecond.is_not_null(), &(ca.nanosecond() / 1000))?
&nanosecond.zip_with(&nanosecond.is_not_null(), &ca.nanosecond())?
};

println!("nanosecond: {:?}", nanosecond);
let mut out = DatetimeChunked::new_from_parts(
year,
month,
day,
hour,
minute,
second,
microsecond,
nanosecond,
ambiguous,
&ca.time_unit(),
ca.time_zone().as_deref(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -892,6 +892,54 @@ def test_replace_ambiguous() -> None:
value.dt.replace(hour=1, ambiguous="raise")


def test_replace_datetime_preserve_ns() -> None:
df = pl.DataFrame(
{
"a": pl.Series(["2020-01-01T00:00:00.123456789"] * 2).cast(
pl.Datetime("ns")
),
"year": [2021, None],
"microsecond": [50, None],
}
)

result = df.select(
year=pl.col("a").dt.replace(year="year"),
us=pl.col("a").dt.replace(microsecond="microsecond"),
)

expected = pl.DataFrame(
{
"year": pl.Series(
[
"2021-01-01T00:00:00.123456789",
"2020-01-01T00:00:00.123456789",
]
).cast(pl.Datetime("ns")),
"us": pl.Series(
[
"2020-01-01T00:00:00.000050",
"2020-01-01T00:00:00.123456789",
]
).cast(pl.Datetime("ns")),
}
)

assert_frame_equal(result, expected)


@pytest.mark.parametrize("tu", ["ms", "us", "ns"])
@pytest.mark.parametrize("tzinfo", [None, "Africa/Nairobi", "America/New_York"])
def test_replace_preserve_tu_and_tz(tu: TimeUnit, tzinfo: str) -> None:
s = pl.Series(
[datetime(2024, 1, 1), datetime(2024, 1, 2)],
dtype=pl.Datetime(time_unit=tu, time_zone=tzinfo),
)
result = s.dt.replace(year=2000)
assert result.dtype.time_unit == tu
assert result.dtype.time_zone == tzinfo


@pytest.mark.parametrize(
("tzinfo", "time_zone"),
[(None, None), (ZoneInfo("Asia/Kathmandu"), "Asia/Kathmandu")],
Expand Down

0 comments on commit 062066e

Please sign in to comment.