diff --git a/CI/physmon/reference/performance_amvf_gridseeder_seeded_hist.root b/CI/physmon/reference/performance_amvf_gridseeder_seeded_hist.root
index 1d453e5c49e..08e31b976f0 100644
Binary files a/CI/physmon/reference/performance_amvf_gridseeder_seeded_hist.root and b/CI/physmon/reference/performance_amvf_gridseeder_seeded_hist.root differ
diff --git a/Core/include/Acts/Vertexing/AdaptiveMultiVertexFinder.hpp b/Core/include/Acts/Vertexing/AdaptiveMultiVertexFinder.hpp
index 50c8ce3143c..8ba5b107e18 100644
--- a/Core/include/Acts/Vertexing/AdaptiveMultiVertexFinder.hpp
+++ b/Core/include/Acts/Vertexing/AdaptiveMultiVertexFinder.hpp
@@ -145,9 +145,9 @@ class AdaptiveMultiVertexFinder {
     // Use seed vertex as a constraint for the fit
     bool useSeedConstraint = true;
 
-    // Diagonal constraint covariance entries in case
-    // no beamspot constraint is provided
-    double looseConstrValue = 1e+8;
+    // Variances of the 4D vertex position before the vertex fit if no beamspot
+    // constraint is provided
+    Vector4 initialVariances = Vector4::Constant(1e+8);
 
     // Default fitQuality for constraint vertex in case no beamspot
     // constraint is provided
diff --git a/Core/include/Acts/Vertexing/AdaptiveMultiVertexFinder.ipp b/Core/include/Acts/Vertexing/AdaptiveMultiVertexFinder.ipp
index 84ada8c30ca..0bbaf41fcc9 100644
--- a/Core/include/Acts/Vertexing/AdaptiveMultiVertexFinder.ipp
+++ b/Core/include/Acts/Vertexing/AdaptiveMultiVertexFinder.ipp
@@ -185,8 +185,7 @@ auto Acts::AdaptiveMultiVertexFinder<vfitter_t, sfinder_t>::
     }
   } else {
     currentConstraint.setFullPosition(seedVertex.fullPosition());
-    currentConstraint.setFullCovariance(SquareMatrix4::Identity() *
-                                        m_cfg.looseConstrValue);
+    currentConstraint.setFullCovariance(m_cfg.initialVariances.asDiagonal());
     currentConstraint.setFitQuality(m_cfg.defaultConstrFitQuality);
   }
 }
diff --git a/Examples/Algorithms/Vertexing/src/AdaptiveMultiVertexFinderAlgorithm.cpp b/Examples/Algorithms/Vertexing/src/AdaptiveMultiVertexFinderAlgorithm.cpp
index 838228fb160..a21c39c3f92 100644
--- a/Examples/Algorithms/Vertexing/src/AdaptiveMultiVertexFinderAlgorithm.cpp
+++ b/Examples/Algorithms/Vertexing/src/AdaptiveMultiVertexFinderAlgorithm.cpp
@@ -125,7 +125,10 @@ ActsExamples::AdaptiveMultiVertexFinderAlgorithm::executeAfterSeederChoice(
   typename Finder::Config finderConfig(std::move(fitter), std::move(seedFinder),
                                        ipEstimator, std::move(linearizer),
                                        m_cfg.bField);
-  finderConfig.looseConstrValue = 1e2;
+  // Set the initial variance of the 4D vertex position. Since time is on a
+  // numerical scale, we have to provide a greater value in the corresponding
+  // dimension.
+  finderConfig.initialVariances << 1e+2, 1e+2, 1e+2, 1e+7;
   finderConfig.tracksMaxZinterval = 1. * Acts::UnitConstants::mm;
   finderConfig.maxIterations = 200;
   finderConfig.useTime = m_cfg.useTime;