diff --git a/src/runtime/vm/memory_manager.cc b/src/runtime/vm/memory_manager.cc
index fa5edf18335c..410f6c2a042d 100644
--- a/src/runtime/vm/memory_manager.cc
+++ b/src/runtime/vm/memory_manager.cc
@@ -81,7 +81,8 @@ NDArray StorageObj::AllocNDArray(size_t offset, std::vector<int64_t> shape, DLDa
 
   // crtical zone: allocate header, cannot throw
   NDArray::Container* container =
-      new NDArray::Container(nullptr, shape, dtype, this->buffer.device);
+      new NDArray::Container(this->buffer.data, shape, dtype, this->buffer.device);
+  container->dl_tensor.byte_offset = offset;
 
   container->SetDeleter(StorageObj::Deleter);
   size_t needed_size = GetDataSize(container->dl_tensor);
@@ -94,12 +95,6 @@ NDArray StorageObj::AllocNDArray(size_t offset, std::vector<int64_t> shape, DLDa
   // buffer intact.
   container->manager_ctx = reinterpret_cast<void*>(this);
 
-  // is this UB?
-  // The only change we make w.r.t offset is modifying the data pointer
-  // of the backing tensor to point into the buffer instead of its start.
-  auto offset_ptr = reinterpret_cast<uint8_t*>(this->buffer.data) + offset;
-  container->dl_tensor.data = reinterpret_cast<void*>(offset_ptr);
-
   NDArray ret(GetObjectPtr<Object>(container));
   // RAII in effect, now run the check.