diff --git a/crates/polars-core/src/series/mod.rs b/crates/polars-core/src/series/mod.rs
index 567f7e66f9d7..4c63bedd9ffd 100644
--- a/crates/polars-core/src/series/mod.rs
+++ b/crates/polars-core/src/series/mod.rs
@@ -592,8 +592,16 @@ impl Series {
     pub fn to_physical_repr(&self) -> Cow<Series> {
         use DataType::*;
         match self.dtype() {
-            Date => Cow::Owned(self.cast(&Int32).unwrap()),
-            Datetime(_, _) | Duration(_) | Time => Cow::Owned(self.cast(&Int64).unwrap()),
+            // NOTE: Don't use cast here, as it might rechunk (if all nulls)
+            // which is not allowed in a phys repr.
+            #[cfg(feature = "dtype-date")]
+            Date => Cow::Owned(self.date().unwrap().0.clone().into_series()),
+            #[cfg(feature = "dtype-datetime")]
+            Datetime(_, _) => Cow::Owned(self.datetime().unwrap().0.clone().into_series()),
+            #[cfg(feature = "dtype-duration")]
+            Duration(_) => Cow::Owned(self.duration().unwrap().0.clone().into_series()),
+            #[cfg(feature = "dtype-time")]
+            Time => Cow::Owned(self.time().unwrap().0.clone().into_series()),
             #[cfg(feature = "dtype-categorical")]
             Categorical(_, _) | Enum(_, _) => {
                 let ca = self.categorical().unwrap();
diff --git a/py-polars/tests/unit/operations/test_gather.py b/py-polars/tests/unit/operations/test_gather.py
index e897a8da57a6..fab07dc71956 100644
--- a/py-polars/tests/unit/operations/test_gather.py
+++ b/py-polars/tests/unit/operations/test_gather.py
@@ -137,3 +137,14 @@ def test_list_get_null_on_oob_true() -> None:
     df = s_no_nulls.to_frame().with_columns(pl.lit(2).alias("idx"))
     out = df.select(pl.col("a").list.get("idx", null_on_oob=True)).to_series()
     assert_series_equal(out, expected)
+
+
+def test_chunked_gather_phys_repr_17446() -> None:
+    dfa = pl.DataFrame({"replace_unique_id": range(2)})
+
+    for dt in [pl.Date, pl.Time, pl.Duration]:
+        dfb = dfa.clone()
+        dfb = dfb.with_columns(ds_start_date_right=pl.lit(None).cast(dt))
+        dfb = pl.concat([dfb, dfb])
+
+        assert dfa.join(dfb, how="left", on=pl.col("replace_unique_id")).shape == (4, 2)