Merge branch 'kubevirt:main' into add_imagePullSecrets

WORKSPACE

@@ -692,37 +692,25 @@ rpm(
 rpm(
     name = "device-mapper-9__1.02.183-4.el9.aarch64",
     sha256 = "b080a59493ad1fa71bdffdd1ade971feca815d0ba7fd3da627c5f710f91c1470",
-    urls = [
-        "http://mirror.stream.centos.org/9-stream/BaseOS/aarch64/os/Packages/device-mapper-1.02.183-4.el9.aarch64.rpm",
-        "https://storage.googleapis.com/builddeps/b080a59493ad1fa71bdffdd1ade971feca815d0ba7fd3da627c5f710f91c1470",
-    ],
+    urls = ["https://storage.googleapis.com/builddeps/b080a59493ad1fa71bdffdd1ade971feca815d0ba7fd3da627c5f710f91c1470"],
 )
 
 rpm(
     name = "device-mapper-9__1.02.183-4.el9.x86_64",
     sha256 = "eb8323d62250dfabb5d3b1f832e785acbb30bbb82bb5e3f28e74ce7b0bef63ae",
-    urls = [
-        "http://mirror.stream.centos.org/9-stream/BaseOS/x86_64/os/Packages/device-mapper-1.02.183-4.el9.x86_64.rpm",
-        "https://storage.googleapis.com/builddeps/eb8323d62250dfabb5d3b1f832e785acbb30bbb82bb5e3f28e74ce7b0bef63ae",
-    ],
+    urls = ["https://storage.googleapis.com/builddeps/eb8323d62250dfabb5d3b1f832e785acbb30bbb82bb5e3f28e74ce7b0bef63ae"],
 )
 
 rpm(
     name = "device-mapper-libs-9__1.02.183-4.el9.aarch64",
     sha256 = "04954340e373279c3451b2854f13644a49c1242704dd85b7b6866cae90bdca51",
-    urls = [
-        "http://mirror.stream.centos.org/9-stream/BaseOS/aarch64/os/Packages/device-mapper-libs-1.02.183-4.el9.aarch64.rpm",
-        "https://storage.googleapis.com/builddeps/04954340e373279c3451b2854f13644a49c1242704dd85b7b6866cae90bdca51",
-    ],
+    urls = ["https://storage.googleapis.com/builddeps/04954340e373279c3451b2854f13644a49c1242704dd85b7b6866cae90bdca51"],
 )
 
 rpm(
     name = "device-mapper-libs-9__1.02.183-4.el9.x86_64",
     sha256 = "c0e3b8af40f1b996ab06e0bc7ae77c7007b993415540c0f927e3c4957d73c698",
-    urls = [
-        "http://mirror.stream.centos.org/9-stream/BaseOS/x86_64/os/Packages/device-mapper-libs-1.02.183-4.el9.x86_64.rpm",
-        "https://storage.googleapis.com/builddeps/c0e3b8af40f1b996ab06e0bc7ae77c7007b993415540c0f927e3c4957d73c698",
-    ],
+    urls = ["https://storage.googleapis.com/builddeps/c0e3b8af40f1b996ab06e0bc7ae77c7007b993415540c0f927e3c4957d73c698"],
 )
 
 rpm(
@@ -2465,17 +2453,11 @@ rpm(
 rpm(
     name = "zlib-0__1.2.11-32.el9.aarch64",
     sha256 = "1b99ee6c18e92f2a727c39668941273c67f25eef18f7e9fe4febd484e9a80dbd",
-    urls = [
-        "http://mirror.stream.centos.org/9-stream/BaseOS/aarch64/os/Packages/zlib-1.2.11-32.el9.aarch64.rpm",
-        "https://storage.googleapis.com/builddeps/1b99ee6c18e92f2a727c39668941273c67f25eef18f7e9fe4febd484e9a80dbd",
-    ],
+    urls = ["https://storage.googleapis.com/builddeps/1b99ee6c18e92f2a727c39668941273c67f25eef18f7e9fe4febd484e9a80dbd"],
 )
 
 rpm(
     name = "zlib-0__1.2.11-32.el9.x86_64",
     sha256 = "59b0101c691ea180b992d338b372852c1d1607931c472c6ee22056e2fb099505",
-    urls = [
-        "http://mirror.stream.centos.org/9-stream/BaseOS/x86_64/os/Packages/zlib-1.2.11-32.el9.x86_64.rpm",
-        "https://storage.googleapis.com/builddeps/59b0101c691ea180b992d338b372852c1d1607931c472c6ee22056e2fb099505",
-    ],
+    urls = ["https://storage.googleapis.com/builddeps/59b0101c691ea180b992d338b372852c1d1607931c472c6ee22056e2fb099505"],
 )

cmd/cdi-importer/importer.go

@@ -111,8 +111,6 @@ func main() {
 	volumeMode := v1.PersistentVolumeBlock
 	if _, err := os.Stat(common.WriteBlockPath); os.IsNotExist(err) {
 		volumeMode = v1.PersistentVolumeFilesystem
-	} else {
-		preallocation = true
 	}
 
 	// With writeback cache mode it's possible that the process will exit before all writes have been commited to storage.

hack/build/docker/builder/Dockerfile

@@ -33,7 +33,7 @@ RUN pip3 install --upgrade j2cli operator-courier==2.1.11 && \
 	ln -s /opt/gradle/gradle-6.6/bin/gradle /usr/local/bin/gradle && \
 	rm gradle-6.6-bin.zip
 
-ENV GIMME_GO_VERSION=1.18.8 GOPATH="/go" GO111MODULE="on"
+ENV GIMME_GO_VERSION=1.19.5 GOPATH="/go" GO111MODULE="on"
 
 RUN mkdir -p /gimme && curl -sL https://raw.githubusercontent.com/travis-ci/gimme/master/gimme | HOME=/gimme bash >> /etc/profile.d/gimme.sh
 
@@ -46,8 +46,8 @@ RUN \
 	go install github.com/mattn/goveralls@latest && \
 	go install golang.org/x/lint/golint@latest && \
 	go install github.com/rmohr/go-swagger-utils/swagger-doc@latest && \
-	go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.9.2 && \
-	go install github.com/securego/gosec/v2/cmd/gosec@0ce48a5 && \
+	go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.11.3 && \
+	go install github.com/securego/gosec/v2/cmd/gosec@latest && \
 	rm -rf "${GOPATH}/pkg"
 
 ENV BAZEL_VERSION 5.2.0

pkg/image/nbdkit.go

@@ -23,10 +23,6 @@ const (
 	defaultUserAgent      = "cdi-nbdkit-importer"
 )
 
-type nbdkitOperations struct {
-	nbdkit *Nbdkit
-}
-
 // NbdkitPlugin represents a plugin for nbdkit
 type NbdkitPlugin string
 
@@ -104,18 +100,22 @@ func NewNbdkitCurl(nbdkitPidFile, user, password, certDir, socket string, extraH
 	for _, header := range extraHeaders {
 		pluginArgs = append(pluginArgs, fmt.Sprintf("header=%s", header))
 	}
+	// Don't do exponential retry, the container restart will be exponential
+	pluginArgs = append(pluginArgs, "retry-exponential=no")
 	for _, header := range secretExtraHeaders {
 		redactArgs = append(redactArgs, fmt.Sprintf("header=%s", header))
 	}
-
-	return &Nbdkit{
+	n := &Nbdkit{
 		NbdPidFile: nbdkitPidFile,
 		plugin:     NbdkitCurlPlugin,
 		nbdkitArgs: args,
 		pluginArgs: pluginArgs,
 		redactArgs: redactArgs,
 		Socket:     socket,
 	}
+	// Should be last filter
+	n.AddFilter(NbdkitRetryFilter)
+	return n
 }
 
 // NewNbdkitVddk creates a new Nbdkit instance with the vddk plugin
@@ -211,9 +211,8 @@ func (n *Nbdkit) StartNbdkit(source string) error {
 		argsNbdkit = append(argsNbdkit, fmt.Sprintf("--filter=%s", f))
 	}
 	// set additional arguments
-	for _, a := range n.nbdkitArgs {
-		argsNbdkit = append(argsNbdkit, a)
-	}
+	argsNbdkit = append(argsNbdkit, n.nbdkitArgs...)
+
 	// append nbdkit plugin arguments
 	argsNbdkit = append(argsNbdkit, string(n.plugin))
 	argsNbdkit = append(argsNbdkit, n.pluginArgs...)

pkg/image/qemu.go

@@ -289,6 +289,7 @@ func (o *qemuOperations) CreateBlankImage(dest string, size resource.Quantity, p
 	err = os.Chmod(dest, 0660)
 	if err != nil {
 		err = errors.Wrap(err, "Unable to change permissions of target file")
+		return err
 	}
 
 	return nil

pkg/image/qemu_test.go

@@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 
-    http://www.apache.org/licenses/LICENSE-2.0
+	http://www.apache.org/licenses/LICENSE-2.0
 
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
@@ -295,22 +295,37 @@ var _ = Describe("quantity to qemu", func() {
 })
 
 var _ = Describe("Create blank image", func() {
+	var tmpDir, destPath string
+
+	BeforeEach(func() {
+		tmpDir, err := os.MkdirTemp(os.TempDir(), "qemutestdest")
+		Expect(err).NotTo(HaveOccurred())
+		By("tmpDir: " + tmpDir)
+		destPath = filepath.Join(tmpDir, "dest")
+		_, err = os.Create(destPath)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	AfterEach(func() {
+		os.RemoveAll(tmpDir)
+	})
+
 	It("Should complete successfully if qemu-img resize succeeds", func() {
 		quantity, err := resource.ParseQuantity("10Gi")
 		Expect(err).NotTo(HaveOccurred())
 		size := convertQuantityToQemuSize(quantity)
-		replaceExecFunction(mockExecFunction("", "", nil, "create", "-f", "raw", "image", size), func() {
-			err = CreateBlankImage("image", quantity, false)
-			Expect(err).NotTo(HaveOccurred())
+		replaceExecFunction(mockExecFunction("", "", nil, "create", "-f", "raw", destPath, size), func() {
+			err = CreateBlankImage(destPath, quantity, false)
+			Expect(err).ToNot(HaveOccurred())
 		})
 	})
 
 	It("Should fail if qemu-img resize fails", func() {
 		quantity, err := resource.ParseQuantity("10Gi")
 		Expect(err).NotTo(HaveOccurred())
 		size := convertQuantityToQemuSize(quantity)
-		replaceExecFunction(mockExecFunction("", "exit 1", nil, "create", "-f", "raw", "image", size), func() {
-			err = CreateBlankImage("image", quantity, false)
+		replaceExecFunction(mockExecFunction("", "exit 1", nil, "create", "-f", "raw", destPath, size), func() {
+			err = CreateBlankImage(destPath, quantity, false)
 			Expect(err).To(HaveOccurred())
 			Expect(strings.Contains(err.Error(), "could not create raw image with size ")).To(BeTrue())
 		})
@@ -320,19 +335,19 @@ var _ = Describe("Create blank image", func() {
 		quantity, err := resource.ParseQuantity("10Gi")
 		Expect(err).NotTo(HaveOccurred())
 		size := convertQuantityToQemuSize(quantity)
-		replaceExecFunction(mockExecFunctionStrict("", "", nil, "create", "-f", "raw", "image", size, "-o", "preallocation=falloc"), func() {
-			err = CreateBlankImage("image", quantity, true)
-			Expect(err).NotTo(HaveOccurred())
+		replaceExecFunction(mockExecFunctionStrict("", "", nil, "create", "-f", "raw", destPath, size, "-o", "preallocation=falloc"), func() {
+			err = CreateBlankImage(destPath, quantity, true)
+			Expect(err).ToNot(HaveOccurred())
 		})
 	})
 
 	It("should not add preallocation if not requested", func() {
 		quantity, err := resource.ParseQuantity("10Gi")
 		Expect(err).NotTo(HaveOccurred())
 		size := convertQuantityToQemuSize(quantity)
-		replaceExecFunction(mockExecFunctionStrict("", "", nil, "create", "-f", "raw", "image", size), func() {
-			err = CreateBlankImage("image", quantity, false)
-			Expect(err).NotTo(HaveOccurred())
+		replaceExecFunction(mockExecFunctionStrict("", "", nil, "create", "-f", "raw", destPath, size), func() {
+			err = CreateBlankImage(destPath, quantity, false)
+			Expect(err).ToNot(HaveOccurred())
 		})
 	})
 })

tests/datasource_test.go

@@ -9,7 +9,6 @@ import (
 	. "github.com/onsi/gomega"
 
 	corev1 "k8s.io/api/core/v1"
-	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	cdiv1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1"
@@ -88,9 +87,9 @@ var _ = Describe("DataSource", func() {
 
 		By("Verify DV conditions")
 		utils.WaitForConditions(f, dv.Name, dv.Namespace, time.Minute, pollingInterval,
-			&cdiv1.DataVolumeCondition{Type: cdiv1.DataVolumeBound, Status: v1.ConditionUnknown, Message: "No PVC found", Reason: dvc.CloneWithoutSource},
-			&cdiv1.DataVolumeCondition{Type: cdiv1.DataVolumeReady, Status: v1.ConditionFalse, Reason: dvc.MessageCloneWithoutSource},
-			&cdiv1.DataVolumeCondition{Type: cdiv1.DataVolumeRunning, Status: v1.ConditionFalse})
+			&cdiv1.DataVolumeCondition{Type: cdiv1.DataVolumeBound, Status: corev1.ConditionUnknown, Message: "No PVC found", Reason: dvc.CloneWithoutSource},
+			&cdiv1.DataVolumeCondition{Type: cdiv1.DataVolumeReady, Status: corev1.ConditionFalse, Reason: dvc.CloneWithoutSource},
+			&cdiv1.DataVolumeCondition{Type: cdiv1.DataVolumeRunning, Status: corev1.ConditionFalse})
 		f.ExpectEvent(dv.Namespace).Should(ContainSubstring(dvc.CloneWithoutSource))
 
 		By("Create import DV so the missing DataSource source PVC will be ready")

tests/operator_test.go

@@ -795,6 +795,9 @@ var _ = Describe("ALL Operator tests", func() {
 				cdiPods                *corev1.PodList
 				numAddedStorageClasses int
 				originalMetricVal      int
+
+				metricPollingInterval = 5 * time.Second
+				metricPollingTimeout  = 5 * time.Minute
 			)
 
 			f := framework.NewFramework("alert-tests")
@@ -866,7 +869,7 @@ var _ = Describe("ALL Operator tests", func() {
 
 				Eventually(func() int {
 					return getMetricValue("kubevirt_cdi_incomplete_storageprofiles_total")
-				}, 2*time.Minute, 1*time.Second).Should(BeNumerically("==", originalMetricVal))
+				}, metricPollingTimeout, metricPollingInterval).Should(BeNumerically("==", originalMetricVal))
 			})
 
 			createUnknownStorageClass := func(name, provisioner string) *storagev1.StorageClass {
@@ -1043,7 +1046,7 @@ var _ = Describe("ALL Operator tests", func() {
 				expectedIncomplete := originalMetricVal + numAddedStorageClasses
 				Eventually(func() int {
 					return getMetricValue("kubevirt_cdi_incomplete_storageprofiles_total")
-				}, 2*time.Minute, 1*time.Second).Should(BeNumerically("==", expectedIncomplete))
+				}, metricPollingTimeout, metricPollingInterval).Should(BeNumerically("==", expectedIncomplete))
 
 				By("Fix profiles to be complete and test metric value equals original")
 				for i := 0; i < numAddedStorageClasses; i++ {

tests/utils/datavolume.go

@@ -873,25 +873,27 @@ func WaitForConditions(ci ClientsIface, dataVolumeName, namespace string, timeou
 	}, timeout, pollingInterval).Should(gomega.BeTrue())
 }
 
-// verifyConditions checks if the conditions match testConditions
+// verifyConditions checks if all the conditions exist and match testConditions
 func verifyConditions(actualConditions []cdiv1.DataVolumeCondition, testConditions []*cdiv1.DataVolumeCondition) bool {
 	for _, condition := range testConditions {
-		if condition != nil {
-			actualCondition := dvc.FindConditionByType(condition.Type, actualConditions)
-			if actualCondition != nil {
-				if actualCondition.Status != condition.Status {
-					fmt.Fprintf(ginkgo.GinkgoWriter, "INFO: Condition.Status does not match for type: %s, status expected: [%s], status found: [%s]\n", condition.Type, condition.Status, actualCondition.Status)
-					return false
-				}
-				if strings.Compare(actualCondition.Reason, condition.Reason) != 0 {
-					fmt.Fprintf(ginkgo.GinkgoWriter, "INFO: Condition.Reason does not match for type: %s, reason expected [%s], reason found: [%s]\n", condition.Type, condition.Reason, actualCondition.Reason)
-					return false
-				}
-				if !strings.Contains(actualCondition.Message, condition.Message) {
-					fmt.Fprintf(ginkgo.GinkgoWriter, "INFO: Condition.Message does not match for type: %s, message expected: [%s],  message found: [%s]\n", condition.Type, condition.Message, actualCondition.Message)
-					return false
-				}
-			}
+		if condition == nil {
+			continue
+		}
+		actualCondition := dvc.FindConditionByType(condition.Type, actualConditions)
+		if actualCondition == nil {
+			return false
+		}
+		if actualCondition.Status != condition.Status {
+			fmt.Fprintf(ginkgo.GinkgoWriter, "INFO: Condition.Status does not match for type: %s, status expected: [%s], status found: [%s]\n", condition.Type, condition.Status, actualCondition.Status)
+			return false
+		}
+		if strings.Compare(actualCondition.Reason, condition.Reason) != 0 {
+			fmt.Fprintf(ginkgo.GinkgoWriter, "INFO: Condition.Reason does not match for type: %s, reason expected [%s], reason found: [%s]\n", condition.Type, condition.Reason, actualCondition.Reason)
+			return false
+		}
+		if !strings.Contains(actualCondition.Message, condition.Message) {
+			fmt.Fprintf(ginkgo.GinkgoWriter, "INFO: Condition.Message does not match for type: %s, message expected: [%s],  message found: [%s]\n", condition.Type, condition.Message, actualCondition.Message)
+			return false
 		}
 	}
 	return true