diff --git a/vendor/k8s.io/kubernetes/pkg/controller/volume/attachdetach/reconciler/reconciler_test.go b/vendor/k8s.io/kubernetes/pkg/controller/volume/attachdetach/reconciler/reconciler_test.go index b9d65ac11cb3..2ee9a4bbb110 100644 --- a/vendor/k8s.io/kubernetes/pkg/controller/volume/attachdetach/reconciler/reconciler_test.go +++ b/vendor/k8s.io/kubernetes/pkg/controller/volume/attachdetach/reconciler/reconciler_test.go @@ -364,11 +364,7 @@ func Test_Run_OneVolumeAttachAndDetachMultipleNodesWithReadWriteMany(t *testing. waitForTotalAttachCallCount(t, 2 /* expectedAttachCallCount */, fakePlugin) verifyNewDetacherCallCount(t, true /* expectZeroNewDetacherCallCount */, fakePlugin) waitForDetachCallCount(t, 0 /* expectedDetachCallCount */, fakePlugin) - - nodesForVolume := asw.GetNodesForVolume(generatedVolumeName) - if len(nodesForVolume) != 2 { - t.Fatal("Volume was not attached to both nodes") - } + waitForAttachedToNodesCount(t, 2 /* expectedNodeCount */, generatedVolumeName, asw) // Act dsw.DeletePod(types.UniquePodName(podName1), generatedVolumeName, nodeName1) @@ -455,24 +451,14 @@ func Test_Run_OneVolumeAttachAndDetachMultipleNodesWithReadWriteOnce(t *testing. waitForTotalAttachCallCount(t, 1 /* expectedAttachCallCount */, fakePlugin) verifyNewDetacherCallCount(t, true /* expectZeroNewDetacherCallCount */, fakePlugin) waitForDetachCallCount(t, 0 /* expectedDetachCallCount */, fakePlugin) + waitForAttachedToNodesCount(t, 1 /* expectedNodeCount */, generatedVolumeName, asw) nodesForVolume := asw.GetNodesForVolume(generatedVolumeName) - if len(nodesForVolume) == 0 { - t.Fatal("Volume was not attached to any node") - } else if len(nodesForVolume) != 1 { - t.Fatal("Volume was attached to multiple nodes") - } // check if multiattach is marked - // at least one volume should be marked with multiattach error + // at least one volume+node should be marked with multiattach error nodeAttachedTo := nodesForVolume[0] - for _, volumeToAttach := range dsw.GetVolumesToAttach() { - if volumeToAttach.NodeName != nodeAttachedTo { - if !volumeToAttach.MultiAttachErrorReported { - t.Fatalf("Expected volume %q on node %q to have multiattach error", volumeToAttach.VolumeName, volumeToAttach.NodeName) - } - } - } + waitForMultiAttachErrorOnNode(t, nodeAttachedTo, dsw) // Act podToDelete := "" @@ -503,6 +489,28 @@ func Test_Run_OneVolumeAttachAndDetachMultipleNodesWithReadWriteOnce(t *testing. waitForTotalAttachCallCount(t, 2 /* expectedAttachCallCount */, fakePlugin) } +func waitForMultiAttachErrorOnNode( + t *testing.T, + attachedNode k8stypes.NodeName, + dsow cache.DesiredStateOfWorld) { + multAttachCheckFunc := func() (bool, error) { + for _, volumeToAttach := range dsow.GetVolumesToAttach() { + if volumeToAttach.NodeName != attachedNode { + if volumeToAttach.MultiAttachErrorReported { + return true, nil + } + } + } + t.Logf("Warning: MultiAttach error not yet set on Node. Will retry.") + return false, nil + } + + err := retryWithExponentialBackOff(100*time.Millisecond, multAttachCheckFunc) + if err != nil { + t.Fatalf("Timed out waiting for MultiAttach Error to be set on non-attached node") + } +} + func waitForNewAttacherCallCount( t *testing.T, expectedCallCount int, @@ -699,6 +707,39 @@ func waitForTotalDetachCallCount( } } +func waitForAttachedToNodesCount( + t *testing.T, + expectedNodeCount int, + volumeName v1.UniqueVolumeName, + asw cache.ActualStateOfWorld) { + + err := retryWithExponentialBackOff( + time.Duration(5*time.Millisecond), + func() (bool, error) { + count := len(asw.GetNodesForVolume(volumeName)) + if count == expectedNodeCount { + return true, nil + } + t.Logf( + "Warning: Wrong number of nodes having <%v> attached. Expected: <%v> Actual: <%v>. Will retry.", + volumeName, + expectedNodeCount, + count) + + return false, nil + }, + ) + + if err != nil { + count := len(asw.GetNodesForVolume(volumeName)) + t.Fatalf( + "Wrong number of nodes having <%v> attached. Expected: <%v> Actual: <%v>", + volumeName, + expectedNodeCount, + count) + } +} + func verifyNewAttacherCallCount( t *testing.T, expectZeroNewAttacherCallCount bool,