Retry clone kill which is flaky (#1950)

* Retry clone kill which is flaky

[test_id:4000] Create a data volume and then clone it while killing the
container and verify retry count was pretty flaky lately. It was failing
on attempting to connect to the upload server pod to kill it. This PR
causes a retry.

Signed-off-by: Alexander Wels <awels@redhat.com>

* Retrying didn't work, try to slow down the clone so we have time
to kill the process

Signed-off-by: Alexander Wels <awels@redhat.com>

* Add shorter poll interval wait for pod ready

Signed-off-by: Alexander Wels <awels@redhat.com>

tests/cloner_test.go

@@ -1286,17 +1286,20 @@ var _ = Describe("all clone tests", func() {
 
 		It("[test_id:4000] Create a data volume and then clone it while killing the container and verify retry count", func() {
 			By("Prepare source PVC")
-			pvcDef := utils.NewPVCDefinition(sourcePVCName, "1Gi", nil, nil)
-			pvcDef.Namespace = f.Namespace.Name
-			sourcePvc = f.CreateAndPopulateSourcePVC(pvcDef, sourcePodFillerName, fillCommand+testFile+"; chmod 660 "+testBaseDir+testFile)
+			sourceDV := utils.NewDataVolumeWithHTTPImport(dataVolumeName, "1Gi", fmt.Sprintf(utils.TinyCoreIsoURL, f.CdiInstallNs))
+			sourceDV, err := utils.CreateDataVolumeFromDefinition(f.CdiClient, f.Namespace.Name, sourceDV)
+			Expect(err).ToNot(HaveOccurred())
+
+			f.ForceBindPvcIfDvIsWaitForFirstConsumer(sourceDV)
+			sourcePvc, err = f.K8sClient.CoreV1().PersistentVolumeClaims(sourceDV.Namespace).Get(context.TODO(), sourceDV.Name, metav1.GetOptions{})
 
 			By("Create clone DV")
 			targetNs, err := f.CreateNamespace(f.NsPrefix, map[string]string{
 				framework.NsPrefixLabel: f.NsPrefix,
 			})
 			Expect(err).NotTo(HaveOccurred())
 			f.AddNamespaceToDelete(targetNs)
-			targetDV := utils.NewCloningDataVolume("target-dv", "1Gi", pvcDef)
+			targetDV := utils.NewCloningDataVolume("target-dv", "1Gi", sourcePvc)
 			dataVolume, err := utils.CreateDataVolumeFromDefinition(f.CdiClient, targetNs.Name, targetDV)
 			Expect(err).ToNot(HaveOccurred())
 
@@ -1313,7 +1316,7 @@ var _ = Describe("all clone tests", func() {
 			utils.WaitForPersistentVolumeClaimPhase(f.K8sClient, targetNs.Name, v1.ClaimBound, targetPvc.Name)
 
 			By("Wait for upload pod")
-			err = utils.WaitTimeoutForPodReady(f.K8sClient, utils.UploadPodName(targetPvc), targetNs.Name, utils.PodWaitForTime)
+			err = utils.WaitTimeoutForPodReadyPollPeriod(f.K8sClient, utils.UploadPodName(targetPvc), targetNs.Name, utils.PodWaitIntervalFast, utils.PodWaitForTime)
 			Expect(err).ToNot(HaveOccurred())
 
 			By("Kill upload pod to force error")

tests/utils/pod.go

@@ -19,6 +19,10 @@ import (
 const (
 	// PodWaitForTime is the time to wait for Pod operations to complete
 	PodWaitForTime = defaultPollPeriod
+	// PodWaitForTimeFast is the fast time to wait for Pod operations to complete (30 sec)
+	PodWaitForTimeFast = defaultPollPeriodFast
+	// PodWaitIntervalFast is the fast polling interval (250ms)
+	PodWaitIntervalFast = 250 * time.Millisecond
 
 	podCreateTime = defaultPollPeriod
 	podDeleteTime = defaultPollPeriod
@@ -131,7 +135,12 @@ func findPodByCompFuncOnce(clientSet *kubernetes.Clientset, namespace, prefix, l
 
 // WaitTimeoutForPodReady waits for the given pod to be created and ready
 func WaitTimeoutForPodReady(clientSet *kubernetes.Clientset, podName, namespace string, timeout time.Duration) error {
-	return WaitTimeoutForPodCondition(clientSet, podName, namespace, k8sv1.PodReady, timeout)
+	return WaitTimeoutForPodCondition(clientSet, podName, namespace, k8sv1.PodReady, 2*time.Second, timeout)
+}
+
+// WaitTimeoutForPodReadyPollPeriod waits for the given pod to be created and ready using the passed in poll period
+func WaitTimeoutForPodReadyPollPeriod(clientSet *kubernetes.Clientset, podName, namespace string, pollperiod, timeout time.Duration) error {
+	return WaitTimeoutForPodCondition(clientSet, podName, namespace, k8sv1.PodReady, pollperiod, timeout)
 }
 
 // WaitTimeoutForPodSucceeded waits for pod to succeed
@@ -150,8 +159,8 @@ func WaitTimeoutForPodStatus(clientSet *kubernetes.Clientset, podName, namespace
 }
 
 // WaitTimeoutForPodCondition waits for the given pod to be created and have an expected condition
-func WaitTimeoutForPodCondition(clientSet *kubernetes.Clientset, podName, namespace string, conditionType k8sv1.PodConditionType, timeout time.Duration) error {
-	return wait.PollImmediate(2*time.Second, timeout, podCondition(clientSet, podName, namespace, conditionType))
+func WaitTimeoutForPodCondition(clientSet *kubernetes.Clientset, podName, namespace string, conditionType k8sv1.PodConditionType, pollperiod, timeout time.Duration) error {
+	return wait.PollImmediate(pollperiod, timeout, podCondition(clientSet, podName, namespace, conditionType))
 }
 
 func podStatus(clientSet *kubernetes.Clientset, podName, namespace string, status k8sv1.PodPhase) wait.ConditionFunc {

tests/utils/pvc.go

@@ -21,8 +21,9 @@ import (
 )
 
 const (
-	defaultPollInterval = 2 * time.Second
-	defaultPollPeriod   = 270 * time.Second
+	defaultPollInterval   = 2 * time.Second
+	defaultPollPeriod     = 270 * time.Second
+	defaultPollPeriodFast = 30 * time.Second
 
 	// DefaultPvcMountPath is the default mount path used
 	DefaultPvcMountPath = "/pvc"