From 61684d3d9c07172a687b90bd4b1542f9d6d6a707 Mon Sep 17 00:00:00 2001 From: Alexander Wels Date: Fri, 24 Sep 2021 06:05:28 -0500 Subject: [PATCH] Retry clone kill which is flaky (#1950) * Retry clone kill which is flaky [test_id:4000] Create a data volume and then clone it while killing the container and verify retry count was pretty flaky lately. It was failing on attempting to connect to the upload server pod to kill it. This PR causes a retry. Signed-off-by: Alexander Wels * Retrying didn't work, try to slow down the clone so we have time to kill the process Signed-off-by: Alexander Wels * Add shorter poll interval wait for pod ready Signed-off-by: Alexander Wels --- tests/cloner_test.go | 13 ++++++++----- tests/utils/pod.go | 15 ++++++++++++--- tests/utils/pvc.go | 5 +++-- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/tests/cloner_test.go b/tests/cloner_test.go index f5d6a2b029..eb03b57f6d 100644 --- a/tests/cloner_test.go +++ b/tests/cloner_test.go @@ -1286,9 +1286,12 @@ var _ = Describe("all clone tests", func() { It("[test_id:4000] Create a data volume and then clone it while killing the container and verify retry count", func() { By("Prepare source PVC") - pvcDef := utils.NewPVCDefinition(sourcePVCName, "1Gi", nil, nil) - pvcDef.Namespace = f.Namespace.Name - sourcePvc = f.CreateAndPopulateSourcePVC(pvcDef, sourcePodFillerName, fillCommand+testFile+"; chmod 660 "+testBaseDir+testFile) + sourceDV := utils.NewDataVolumeWithHTTPImport(dataVolumeName, "1Gi", fmt.Sprintf(utils.TinyCoreIsoURL, f.CdiInstallNs)) + sourceDV, err := utils.CreateDataVolumeFromDefinition(f.CdiClient, f.Namespace.Name, sourceDV) + Expect(err).ToNot(HaveOccurred()) + + f.ForceBindPvcIfDvIsWaitForFirstConsumer(sourceDV) + sourcePvc, err = f.K8sClient.CoreV1().PersistentVolumeClaims(sourceDV.Namespace).Get(context.TODO(), sourceDV.Name, metav1.GetOptions{}) By("Create clone DV") targetNs, err := f.CreateNamespace(f.NsPrefix, map[string]string{ @@ -1296,7 +1299,7 @@ var _ = Describe("all clone tests", func() { }) Expect(err).NotTo(HaveOccurred()) f.AddNamespaceToDelete(targetNs) - targetDV := utils.NewCloningDataVolume("target-dv", "1Gi", pvcDef) + targetDV := utils.NewCloningDataVolume("target-dv", "1Gi", sourcePvc) dataVolume, err := utils.CreateDataVolumeFromDefinition(f.CdiClient, targetNs.Name, targetDV) Expect(err).ToNot(HaveOccurred()) @@ -1313,7 +1316,7 @@ var _ = Describe("all clone tests", func() { utils.WaitForPersistentVolumeClaimPhase(f.K8sClient, targetNs.Name, v1.ClaimBound, targetPvc.Name) By("Wait for upload pod") - err = utils.WaitTimeoutForPodReady(f.K8sClient, utils.UploadPodName(targetPvc), targetNs.Name, utils.PodWaitForTime) + err = utils.WaitTimeoutForPodReadyPollPeriod(f.K8sClient, utils.UploadPodName(targetPvc), targetNs.Name, utils.PodWaitIntervalFast, utils.PodWaitForTime) Expect(err).ToNot(HaveOccurred()) By("Kill upload pod to force error") diff --git a/tests/utils/pod.go b/tests/utils/pod.go index 123f383aef..cb5db78b2d 100644 --- a/tests/utils/pod.go +++ b/tests/utils/pod.go @@ -19,6 +19,10 @@ import ( const ( // PodWaitForTime is the time to wait for Pod operations to complete PodWaitForTime = defaultPollPeriod + // PodWaitForTimeFast is the fast time to wait for Pod operations to complete (30 sec) + PodWaitForTimeFast = defaultPollPeriodFast + // PodWaitIntervalFast is the fast polling interval (250ms) + PodWaitIntervalFast = 250 * time.Millisecond podCreateTime = defaultPollPeriod podDeleteTime = defaultPollPeriod @@ -131,7 +135,12 @@ func findPodByCompFuncOnce(clientSet *kubernetes.Clientset, namespace, prefix, l // WaitTimeoutForPodReady waits for the given pod to be created and ready func WaitTimeoutForPodReady(clientSet *kubernetes.Clientset, podName, namespace string, timeout time.Duration) error { - return WaitTimeoutForPodCondition(clientSet, podName, namespace, k8sv1.PodReady, timeout) + return WaitTimeoutForPodCondition(clientSet, podName, namespace, k8sv1.PodReady, 2*time.Second, timeout) +} + +// WaitTimeoutForPodReadyPollPeriod waits for the given pod to be created and ready using the passed in poll period +func WaitTimeoutForPodReadyPollPeriod(clientSet *kubernetes.Clientset, podName, namespace string, pollperiod, timeout time.Duration) error { + return WaitTimeoutForPodCondition(clientSet, podName, namespace, k8sv1.PodReady, pollperiod, timeout) } // WaitTimeoutForPodSucceeded waits for pod to succeed @@ -150,8 +159,8 @@ func WaitTimeoutForPodStatus(clientSet *kubernetes.Clientset, podName, namespace } // WaitTimeoutForPodCondition waits for the given pod to be created and have an expected condition -func WaitTimeoutForPodCondition(clientSet *kubernetes.Clientset, podName, namespace string, conditionType k8sv1.PodConditionType, timeout time.Duration) error { - return wait.PollImmediate(2*time.Second, timeout, podCondition(clientSet, podName, namespace, conditionType)) +func WaitTimeoutForPodCondition(clientSet *kubernetes.Clientset, podName, namespace string, conditionType k8sv1.PodConditionType, pollperiod, timeout time.Duration) error { + return wait.PollImmediate(pollperiod, timeout, podCondition(clientSet, podName, namespace, conditionType)) } func podStatus(clientSet *kubernetes.Clientset, podName, namespace string, status k8sv1.PodPhase) wait.ConditionFunc { diff --git a/tests/utils/pvc.go b/tests/utils/pvc.go index ea9c96c8a3..1f37c6864d 100644 --- a/tests/utils/pvc.go +++ b/tests/utils/pvc.go @@ -21,8 +21,9 @@ import ( ) const ( - defaultPollInterval = 2 * time.Second - defaultPollPeriod = 270 * time.Second + defaultPollInterval = 2 * time.Second + defaultPollPeriod = 270 * time.Second + defaultPollPeriodFast = 30 * time.Second // DefaultPvcMountPath is the default mount path used DefaultPvcMountPath = "/pvc"