diff --git a/Modules/Core/SpatialObjects/include/itkImageMaskSpatialObject.hxx b/Modules/Core/SpatialObjects/include/itkImageMaskSpatialObject.hxx
index 85a2cbdfbe5..94b0148675a 100644
--- a/Modules/Core/SpatialObjects/include/itkImageMaskSpatialObject.hxx
+++ b/Modules/Core/SpatialObjects/include/itkImageMaskSpatialObject.hxx
@@ -40,20 +40,12 @@ template <unsigned int TDimension, typename TPixel>
 bool
 ImageMaskSpatialObject<TDimension, TPixel>::IsInsideInObjectSpace(const PointType & point) const
 {
-  typename Superclass::InterpolatorType::ContinuousIndexType index;
-  if (this->GetImage()->TransformPhysicalPointToContinuousIndex(point, index))
-  {
-    using InterpolatorOutputType = typename InterpolatorType::OutputType;
-    bool insideMask = (Math::NotExactlyEquals(DefaultConvertPixelTraits<InterpolatorOutputType>::GetScalarValue(
-                                                this->GetInterpolator()->EvaluateAtContinuousIndex(index)),
-                                              NumericTraits<PixelType>::ZeroValue()));
-    if (insideMask)
-    {
-      return true;
-    }
-  }
+  const ImageType * const image = this->GetImage();
+
+  const IndexType index = image->TransformPhysicalPointToIndex(point);
 
-  return false;
+  return image->GetBufferedRegion().IsInside(index) &&
+         Math::NotExactlyEquals(image->GetPixel(index), NumericTraits<PixelType>::ZeroValue());
 }
 
 
diff --git a/Modules/Core/SpatialObjects/test/itkImageMaskSpatialObjectGTest.cxx b/Modules/Core/SpatialObjects/test/itkImageMaskSpatialObjectGTest.cxx
index b04a86cba50..d6715fd855e 100644
--- a/Modules/Core/SpatialObjects/test/itkImageMaskSpatialObjectGTest.cxx
+++ b/Modules/Core/SpatialObjects/test/itkImageMaskSpatialObjectGTest.cxx
@@ -306,3 +306,20 @@ TEST(ImageMaskSpatialObject, IsInsideIndependentOfDistantPixels)
 
 
 #endif
+
+
+// Tests that IsInsideInObjectSpace returns false for a corner point, when the
+// mask image is filled with zero values. This test would sometimes fail on
+// ITK v5.0.1 and v5.1.0
+TEST(ImageMaskSpatialObject, CornerPointIsNotInsideMaskOfZeroValues)
+{
+  // Create a mask image, and fill the image with zero vales.
+  const auto image = itk::Image<unsigned char>::New();
+  image->SetRegions(itk::Size<>{ { 2, 2 } });
+  image->Allocate(true);
+
+  const auto imageMaskSpatialObject = itk::ImageMaskSpatialObject<2>::New();
+  imageMaskSpatialObject->SetImage(image);
+  const double cornerPoint[] = { 1.5, 1.5 };
+  ASSERT_FALSE(imageMaskSpatialObject->IsInsideInObjectSpace(cornerPoint));
+}