diff --git a/polars/polars-core/src/chunked_array/logical/time.rs b/polars/polars-core/src/chunked_array/logical/time.rs
index aacded60e99b..277759fc2814 100644
--- a/polars/polars-core/src/chunked_array/logical/time.rs
+++ b/polars/polars-core/src/chunked_array/logical/time.rs
@@ -29,6 +29,16 @@ impl LogicalType for TimeChunked {
     }
 
     fn cast(&self, dtype: &DataType) -> PolarsResult<Series> {
-        self.0.cast(dtype)
+        match dtype {
+            DataType::Duration(tu) => {
+                let out = self.0.cast(&DataType::Duration(TimeUnit::Nanoseconds));
+                if !matches!(tu, TimeUnit::Nanoseconds) {
+                    out?.cast(dtype)
+                } else {
+                    out
+                }
+            }
+            _ => self.0.cast(dtype),
+        }
     }
 }
diff --git a/py-polars/tests/unit/test_datelike.py b/py-polars/tests/unit/test_datelike.py
index 039f7c2f7494..d044c044a553 100644
--- a/py-polars/tests/unit/test_datelike.py
+++ b/py-polars/tests/unit/test_datelike.py
@@ -2300,3 +2300,9 @@ def test_round_by_week() -> None:
         "7d": [date(1998, 4, 9), date(2022, 12, 1)],
         "1w": [date(1998, 4, 13), date(2022, 11, 28)],
     }
+
+
+def test_cast_time_to_duration() -> None:
+    assert pl.Series([time(hour=0, minute=0, second=2)]).cast(
+        pl.Duration
+    ).item() == timedelta(seconds=2)