Attempt to handle flakes in upload/annotations tests (#1995)

- Upload tests:
We only care that the operation will succeed eventually, were fine with one call failing.
- Annotation test:
```bash
Operation cannot be fulfilled on pods "cdi-upload-test-dv": the object has been modified; please apply your changes to the latest version and try again
```
Loop over the .Update call here as well, as were fine with it succeeding at some point.

Signed-off-by: Alex Kalenyuk <akalenyu@redhat.com>

tests/datavolume_test.go

@@ -1204,9 +1204,15 @@ var _ = Describe("[vendor:cnv-qe@redhat.com][level:component]DataVolume tests",
 			}, timeout, pollingInterval).Should(BeTrue())
 			verifyAnnotations(uploadPod)
 			// Remove non existent network so upload pod succeeds and clone can continue (some envs like OpenShift check network validity)
-			delete(uploadPod.Annotations, controller.AnnPodNetwork)
-			_, err = f.K8sClient.CoreV1().Pods(dataVolume.Namespace).Update(context.TODO(), uploadPod, metav1.UpdateOptions{})
-			Expect(err).ToNot(HaveOccurred())
+			Eventually(func() bool {
+				uploadPod, err = utils.FindPodByPrefix(f.K8sClient, dataVolume.Namespace, "cdi-upload", common.CDILabelSelector)
+				if err != nil {
+					return false
+				}
+				delete(uploadPod.Annotations, controller.AnnPodNetwork)
+				_, err = f.K8sClient.CoreV1().Pods(dataVolume.Namespace).Update(context.TODO(), uploadPod, metav1.UpdateOptions{})
+				return err == nil
+			}, 60*time.Second, 2*time.Second).Should(BeTrue())
 			Eventually(func() bool {
 				sourcePod, err = utils.FindPodBySuffix(f.K8sClient, dataVolume.Namespace, "source-pod", common.CDILabelSelector)
 				return err == nil

tests/upload_test.go

@@ -1159,8 +1159,9 @@ var _ = Describe("Preallocation", func() {
 		Expect(token).ToNot(BeEmpty())
 
 		By("Do upload")
-		err = uploadImage(uploadProxyURL, token, http.StatusOK)
-		Expect(err).ToNot(HaveOccurred())
+		Eventually(func() error {
+			return uploadImage(uploadProxyURL, token, http.StatusOK)
+		}, timeout, pollingInterval).Should(BeNil(), "Upload should eventually succeed, even if initially pod is not ready")
 
 		phase = cdiv1.Succeeded
 		By(fmt.Sprintf("Waiting for datavolume to match phase %s", string(phase)))
@@ -1264,8 +1265,9 @@ var _ = Describe("Preallocation", func() {
 		Expect(token).ToNot(BeEmpty())
 
 		By("Do upload")
-		err = uploader(uploadProxyURL, token, http.StatusOK)
-		Expect(err).ToNot(HaveOccurred())
+		Eventually(func() error {
+			return uploader(uploadProxyURL, token, http.StatusOK)
+		}, timeout, pollingInterval).Should(BeNil(), "Upload should eventually succeed, even if initially pod is not ready")
 
 		phase = cdiv1.Succeeded
 		By(fmt.Sprintf("Waiting for datavolume to match phase %s", string(phase)))