Merge branch 'main' into main

Signed-off-by: Marcelo Feitoza Parisi <41125128+feitnomore@users.noreply.github.com>

Makefile

@@ -47,7 +47,7 @@ update-codegen:
 
 generate: update-codegen bazel-generate generate-doc
 
-generate-verify: generate
+generate-verify: generate bootstrap-ginkgo
 	git difftool -y --trust-exit-code --extcmd=./hack/diff-csv.sh
 
 gomod-update:
@@ -157,6 +157,9 @@ openshift-ci-image-push:
 generate-doc: build-docgen
 	_out/tools/metricsdocs/metricsdocs > doc/metrics.md
 
+bootstrap-ginkgo:
+	${DO_BAZ} ./hack/build/bootstrap-ginkgo.sh
+
 build-docgen:
 	${DO_BAZ} "BUILD_ARCH=${BUILD_ARCH} ./hack/build/bazel-build-metricsdocs.sh"
 

WORKSPACE

@@ -692,37 +692,25 @@ rpm(
 rpm(
     name = "device-mapper-9__1.02.183-4.el9.aarch64",
     sha256 = "b080a59493ad1fa71bdffdd1ade971feca815d0ba7fd3da627c5f710f91c1470",
-    urls = [
-        "http://mirror.stream.centos.org/9-stream/BaseOS/aarch64/os/Packages/device-mapper-1.02.183-4.el9.aarch64.rpm",
-        "https://storage.googleapis.com/builddeps/b080a59493ad1fa71bdffdd1ade971feca815d0ba7fd3da627c5f710f91c1470",
-    ],
+    urls = ["https://storage.googleapis.com/builddeps/b080a59493ad1fa71bdffdd1ade971feca815d0ba7fd3da627c5f710f91c1470"],
 )
 
 rpm(
     name = "device-mapper-9__1.02.183-4.el9.x86_64",
     sha256 = "eb8323d62250dfabb5d3b1f832e785acbb30bbb82bb5e3f28e74ce7b0bef63ae",
-    urls = [
-        "http://mirror.stream.centos.org/9-stream/BaseOS/x86_64/os/Packages/device-mapper-1.02.183-4.el9.x86_64.rpm",
-        "https://storage.googleapis.com/builddeps/eb8323d62250dfabb5d3b1f832e785acbb30bbb82bb5e3f28e74ce7b0bef63ae",
-    ],
+    urls = ["https://storage.googleapis.com/builddeps/eb8323d62250dfabb5d3b1f832e785acbb30bbb82bb5e3f28e74ce7b0bef63ae"],
 )
 
 rpm(
     name = "device-mapper-libs-9__1.02.183-4.el9.aarch64",
     sha256 = "04954340e373279c3451b2854f13644a49c1242704dd85b7b6866cae90bdca51",
-    urls = [
-        "http://mirror.stream.centos.org/9-stream/BaseOS/aarch64/os/Packages/device-mapper-libs-1.02.183-4.el9.aarch64.rpm",
-        "https://storage.googleapis.com/builddeps/04954340e373279c3451b2854f13644a49c1242704dd85b7b6866cae90bdca51",
-    ],
+    urls = ["https://storage.googleapis.com/builddeps/04954340e373279c3451b2854f13644a49c1242704dd85b7b6866cae90bdca51"],
 )
 
 rpm(
     name = "device-mapper-libs-9__1.02.183-4.el9.x86_64",
     sha256 = "c0e3b8af40f1b996ab06e0bc7ae77c7007b993415540c0f927e3c4957d73c698",
-    urls = [
-        "http://mirror.stream.centos.org/9-stream/BaseOS/x86_64/os/Packages/device-mapper-libs-1.02.183-4.el9.x86_64.rpm",
-        "https://storage.googleapis.com/builddeps/c0e3b8af40f1b996ab06e0bc7ae77c7007b993415540c0f927e3c4957d73c698",
-    ],
+    urls = ["https://storage.googleapis.com/builddeps/c0e3b8af40f1b996ab06e0bc7ae77c7007b993415540c0f927e3c4957d73c698"],
 )
 
 rpm(
@@ -2465,17 +2453,11 @@ rpm(
 rpm(
     name = "zlib-0__1.2.11-32.el9.aarch64",
     sha256 = "1b99ee6c18e92f2a727c39668941273c67f25eef18f7e9fe4febd484e9a80dbd",
-    urls = [
-        "http://mirror.stream.centos.org/9-stream/BaseOS/aarch64/os/Packages/zlib-1.2.11-32.el9.aarch64.rpm",
-        "https://storage.googleapis.com/builddeps/1b99ee6c18e92f2a727c39668941273c67f25eef18f7e9fe4febd484e9a80dbd",
-    ],
+    urls = ["https://storage.googleapis.com/builddeps/1b99ee6c18e92f2a727c39668941273c67f25eef18f7e9fe4febd484e9a80dbd"],
 )
 
 rpm(
     name = "zlib-0__1.2.11-32.el9.x86_64",
     sha256 = "59b0101c691ea180b992d338b372852c1d1607931c472c6ee22056e2fb099505",
-    urls = [
-        "http://mirror.stream.centos.org/9-stream/BaseOS/x86_64/os/Packages/zlib-1.2.11-32.el9.x86_64.rpm",
-        "https://storage.googleapis.com/builddeps/59b0101c691ea180b992d338b372852c1d1607931c472c6ee22056e2fb099505",
-    ],
+    urls = ["https://storage.googleapis.com/builddeps/59b0101c691ea180b992d338b372852c1d1607931c472c6ee22056e2fb099505"],
 )

cmd/cdi-controller/controller.go

@@ -204,6 +204,11 @@ func start(ctx context.Context, cfg *rest.Config) {
 		os.Exit(1)
 	}
 
+	if err := dvc.CreateCommonIndexes(mgr); err != nil {
+		klog.Errorf("Unable to create shared indexes: %v", err)
+		os.Exit(1)
+	}
+
 	// TODO: Current DV controller had threadiness 3, should we do the same here, defaults to one thread.
 	if _, err := dvc.NewImportController(ctx, mgr, log, installerLabels); err != nil {
 		klog.Errorf("Unable to setup datavolume import controller: %v", err)

cmd/cdi-importer/importer.go

@@ -111,8 +111,6 @@ func main() {
 	volumeMode := v1.PersistentVolumeBlock
 	if _, err := os.Stat(common.WriteBlockPath); os.IsNotExist(err) {
 		volumeMode = v1.PersistentVolumeFilesystem
-	} else {
-		preallocation = true
 	}
 
 	// With writeback cache mode it's possible that the process will exit before all writes have been commited to storage.

go.mod

@@ -48,6 +48,7 @@ require (
 	k8s.io/client-go v12.0.0+incompatible
 	k8s.io/cluster-bootstrap v0.0.0
 	k8s.io/code-generator v0.23.3
+	k8s.io/component-helpers v0.23.0
 	k8s.io/klog/v2 v2.70.1
 	k8s.io/kube-aggregator v0.23.0
 	k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1

go.sum

@@ -2029,6 +2029,8 @@ k8s.io/code-generator v0.23.0 h1:lhyd2KJVCEmpjaCpuoooGs+e3xhPwpYvupnNRidO0Ds=
 k8s.io/code-generator v0.23.0/go.mod h1:vQvOhDXhuzqiVfM/YHp+dmg10WDZCchJVObc9MvowsE=
 k8s.io/component-base v0.23.0 h1:UAnyzjvVZ2ZR1lF35YwtNY6VMN94WtOnArcXBu34es8=
 k8s.io/component-base v0.23.0/go.mod h1:DHH5uiFvLC1edCpvcTDV++NKULdYYU6pR9Tt3HIKMKI=
+k8s.io/component-helpers v0.23.0 h1:qNbqN10QTefiWcCOPkHL/0nn81sdKVv6ZgEXcSyot/U=
+k8s.io/component-helpers v0.23.0/go.mod h1:liXMh6FZS4qamKtMJQ7uLHnFe3tlC86RX5mJEk/aerg=
 k8s.io/cri-api v0.23.0/go.mod h1:2edENu3/mkyW3c6fVPPPaVGEFbLRacJizBbSp7ZOLOo=
 k8s.io/gengo v0.0.0-20181113154421-fd15ee9cc2f7/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=

hack/build/bootstrap-ginkgo.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -e
+
+source hack/build/common.sh
+
+# Find every folder containing tests
+for dir in $(find ${CDI_DIR}/pkg/ -type f -name '*_test.go' -printf '%h\n' | sort -u); do
+    # If there is no file ending with _suite_test.go, bootstrap ginkgo
+    SUITE_FILE=$(find $dir -maxdepth 1 -type f -name '*_suite_test.go')
+    if [ -z "$SUITE_FILE" ]; then
+        (cd $dir && ginkgo bootstrap || :)
+    fi
+done

hack/build/docker/builder/Dockerfile

@@ -33,7 +33,7 @@ RUN pip3 install --upgrade j2cli operator-courier==2.1.11 && \
 	ln -s /opt/gradle/gradle-6.6/bin/gradle /usr/local/bin/gradle && \
 	rm gradle-6.6-bin.zip
 
-ENV GIMME_GO_VERSION=1.18.8 GOPATH="/go" GO111MODULE="on"
+ENV GIMME_GO_VERSION=1.19.5 GOPATH="/go" GO111MODULE="on"
 
 RUN mkdir -p /gimme && curl -sL https://raw.githubusercontent.com/travis-ci/gimme/master/gimme | HOME=/gimme bash >> /etc/profile.d/gimme.sh
 
@@ -46,8 +46,8 @@ RUN \
 	go install github.com/mattn/goveralls@latest && \
 	go install golang.org/x/lint/golint@latest && \
 	go install github.com/rmohr/go-swagger-utils/swagger-doc@latest && \
-	go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.9.2 && \
-	go install github.com/securego/gosec/v2/cmd/gosec@0ce48a5 && \
+	go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.11.3 && \
+	go install github.com/securego/gosec/v2/cmd/gosec@latest && \
 	rm -rf "${GOPATH}/pkg"
 
 ENV BAZEL_VERSION 5.2.0

pkg/apiserver/webhooks/datavolume-mutate.go

@@ -26,6 +26,7 @@ import (
 	snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1"
 	admissionv1 "k8s.io/api/admission/v1"
 	authv1 "k8s.io/api/authorization/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	k8sfield "k8s.io/apimachinery/pkg/util/validation/field"
 	"k8s.io/client-go/kubernetes"
@@ -85,7 +86,7 @@ func (p *sarProxy) Create(sar *authv1.SubjectAccessReview) (*authv1.SubjectAcces
 }
 
 func (wh *dataVolumeMutatingWebhook) Admit(ar admissionv1.AdmissionReview) *admissionv1.AdmissionResponse {
-	var dataVolume, oldDataVolume cdiv1.DataVolume
+	dataVolume := &cdiv1.DataVolume{}
 
 	klog.V(3).Infof("Got AdmissionReview %+v", ar)
 
@@ -102,22 +103,7 @@ func (wh *dataVolumeMutatingWebhook) Admit(ar admissionv1.AdmissionReview) *admi
 		return allowedAdmissionResponse()
 	}
 
-	cloneSourceHandler, err := newCloneSourceHandler(&dataVolume, wh.cdiClient)
-	if err != nil {
-		return toAdmissionResponseError(err)
-	}
-
-	targetNamespace, targetName := dataVolume.Namespace, dataVolume.Name
-	if targetNamespace == "" {
-		targetNamespace = ar.Request.Namespace
-	}
-
-	if targetName == "" {
-		targetName = ar.Request.Name
-	}
-
 	modifiedDataVolume := dataVolume.DeepCopy()
-	modified := false
 
 	if ar.Request.Operation == admissionv1.Create {
 		config, err := wh.cdiClient.CdiV1beta1().CDIConfigs().Get(context.TODO(), common.ConfigName, metav1.GetOptions{})
@@ -130,17 +116,41 @@ func (wh *dataVolumeMutatingWebhook) Admit(ar admissionv1.AdmissionReview) *admi
 			}
 			if modifiedDataVolume.Annotations[cc.AnnDeleteAfterCompletion] != "false" {
 				modifiedDataVolume.Annotations[cc.AnnDeleteAfterCompletion] = "true"
-				modified = true
 			}
 		}
 	}
 
-	if cloneSourceHandler.cloneType == noClone {
-		klog.V(3).Infof("DataVolume %s/%s not cloning", targetNamespace, targetName)
-		if modified {
+	_, prePopulated := dataVolume.Annotations[cc.AnnPrePopulated]
+	_, checkStaticVolume := dataVolume.Annotations[cc.AnnCheckStaticVolume]
+	noTokenOkay := prePopulated || checkStaticVolume
+
+	targetNamespace, targetName := dataVolume.Namespace, dataVolume.Name
+	if targetNamespace == "" {
+		targetNamespace = ar.Request.Namespace
+	}
+
+	if targetName == "" {
+		targetName = ar.Request.Name
+	}
+
+	cloneSourceHandler, err := newCloneSourceHandler(dataVolume, wh.cdiClient)
+	if err != nil {
+		if k8serrors.IsNotFound(err) && noTokenOkay {
+			// no token needed, likely since no datasource
+			klog.V(3).Infof("DataVolume %s/%s is pre/static populated, not adding token, no datasource", targetNamespace, targetName)
 			return toPatchResponse(dataVolume, modifiedDataVolume)
 		}
-		return allowedAdmissionResponse()
+		return toAdmissionResponseError(err)
+	}
+
+	if cloneSourceHandler.cloneType == noClone {
+		klog.V(3).Infof("DataVolume %s/%s not cloning", targetNamespace, targetName)
+		return toPatchResponse(dataVolume, modifiedDataVolume)
+	}
+
+	// only add token at create time
+	if ar.Request.Operation != admissionv1.Create {
+		return toPatchResponse(dataVolume, modifiedDataVolume)
 	}
 
 	sourceName, sourceNamespace := cloneSourceHandler.sourceName, cloneSourceHandler.sourceNamespace
@@ -150,19 +160,12 @@ func (wh *dataVolumeMutatingWebhook) Admit(ar admissionv1.AdmissionReview) *admi
 
 	_, err = wh.k8sClient.CoreV1().Namespaces().Get(context.TODO(), sourceNamespace, metav1.GetOptions{})
 	if err != nil {
-		return toAdmissionResponseError(err)
-	}
-
-	if ar.Request.Operation == admissionv1.Update {
-		if err := json.Unmarshal(ar.Request.OldObject.Raw, &oldDataVolume); err != nil {
-			return toAdmissionResponseError(err)
-		}
-
-		_, ok := oldDataVolume.Annotations[cc.AnnCloneToken]
-		if ok {
-			klog.V(3).Infof("DataVolume %s/%s already has clone token", targetNamespace, targetName)
-			return allowedAdmissionResponse()
+		if k8serrors.IsNotFound(err) && noTokenOkay {
+			// no token needed, likely since no source namespace
+			klog.V(3).Infof("DataVolume %s/%s is pre/static populated, not adding token, no source namespace", targetNamespace, targetName)
+			return toPatchResponse(dataVolume, modifiedDataVolume)
 		}
+		return toAdmissionResponseError(err)
 	}
 
 	ok, reason, err := cloneSourceHandler.cloneAuthFunc(wh.proxy, sourceNamespace, sourceName, targetNamespace, ar.Request.UserInfo)
@@ -171,6 +174,11 @@ func (wh *dataVolumeMutatingWebhook) Admit(ar admissionv1.AdmissionReview) *admi
 	}
 
 	if !ok {
+		if noTokenOkay {
+			klog.V(3).Infof("DataVolume %s/%s is pre/static populated, not adding token, auth failed", targetNamespace, targetName)
+			return toPatchResponse(dataVolume, modifiedDataVolume)
+		}
+
 		causes := []metav1.StatusCause{
 			{
 				Type:    metav1.CauseTypeFieldValueInvalid,