From 83a4ef15dfda6403862437f8acf3606f255e888a Mon Sep 17 00:00:00 2001 From: Peeyush Gupta Date: Tue, 19 Nov 2019 08:58:58 -0500 Subject: [PATCH 01/56] Adding build for ppc64le --- build.make | 1 + 1 file changed, 1 insertion(+) diff --git a/build.make b/build.make index 1b6f35fe1..7075a37ee 100644 --- a/build.make +++ b/build.make @@ -70,6 +70,7 @@ build-%: check-go-version-go CGO_ENABLED=0 GOOS=linux go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$* ./cmd/$* if [ "$$ARCH" = "amd64" ]; then \ CGO_ENABLED=0 GOOS=windows go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*.exe ./cmd/$* ; \ + CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*-ppc64le ./cmd/$* fi container-%: build-% From 003c14b2d4ae3b1463db5e5b3ff91f39b03f5ba8 Mon Sep 17 00:00:00 2001 From: Grant Griffiths Date: Mon, 11 Nov 2019 23:49:42 -0800 Subject: [PATCH 02/56] Add snapshotter CRDs after cluster setup Signed-off-by: Grant Griffiths --- prow.sh | 80 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/prow.sh b/prow.sh index 3b9621a21..bc9c9f03b 100755 --- a/prow.sh +++ b/prow.sh @@ -322,6 +322,9 @@ configvar CSI_PROW_E2E_ALPHA_GATES_1_16 'VolumeSnapshotDataSource=true' "alpha f configvar CSI_PROW_E2E_ALPHA_GATES_LATEST 'VolumeSnapshotDataSource=true' "alpha feature gates for latest Kubernetes" configvar CSI_PROW_E2E_ALPHA_GATES "$(get_versioned_variable CSI_PROW_E2E_ALPHA_GATES "${csi_prow_kubernetes_version_suffix}")" "alpha E2E feature gates" +# Which external-snapshotter tag to use for the snapshotter CRD and snapshot-controller deployment +configvar CSI_SNAPSHOTTER_VERSION 'v2.0.0-rc4' "external-snapshotter version tag" + # Some tests are known to be unusable in a KinD cluster. For example, # stopping kubelet with "ssh systemctl stop kubelet" simply # doesn't work. Such tests should be written in a way that they verify @@ -657,6 +660,59 @@ install_hostpath () { fi } +# Installs all nessesary snapshotter CRDs +install_snapshot_crds() { + # Wait until volumesnapshot CRDs are in place. + CRD_BASE_DIR="https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/config/crd" + kubectl apply -f "${CRD_BASE_DIR}/snapshot.storage.k8s.io_volumesnapshotclasses.yaml" --validate=false + kubectl apply -f "${CRD_BASE_DIR}/snapshot.storage.k8s.io_volumesnapshots.yaml" --validate=false + kubectl apply -f "${CRD_BASE_DIR}/snapshot.storage.k8s.io_volumesnapshotcontents.yaml" --validate=false + cnt=0 + until kubectl get volumesnapshotclasses.snapshot.storage.k8s.io \ + && kubectl get volumesnapshots.snapshot.storage.k8s.io \ + && kubectl get volumesnapshotcontents.snapshot.storage.k8s.io; do + if [ $cnt -gt 30 ]; then + echo >&2 "ERROR: snapshot CRDs not ready after over 1 min" + exit 1 + fi + echo "$(date +%H:%M:%S)" "waiting for snapshot CRDs, attempt #$cnt" + cnt=$((cnt + 1)) + sleep 2 + done +} + +# Install snapshot controller and associated RBAC, retrying until the pod is running. +install_snapshot_controller() { + kubectl apply -f "https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml" + cnt=0 + until kubectl get clusterrolebinding snapshot-controller-role; do + if [ $cnt -gt 30 ]; then + echo "Cluster role bindings:" + kubectl describe clusterrolebinding + echo >&2 "ERROR: snapshot controller RBAC not ready after over 5 min" + exit 1 + fi + echo "$(date +%H:%M:%S)" "waiting for snapshot RBAC setup complete, attempt #$cnt" + cnt=$((cnt + 1)) + sleep 10 + done + + + kubectl apply -f "https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml" + cnt=0 + until kubectl get statefulset snapshot-controller | grep snapshot-controller | grep "1/1"; do + if [ $cnt -gt 30 ]; then + echo "Running statefulsets:" + kubectl describe statefulsets + echo >&2 "ERROR: snapshot controller not ready after over 5 min" + exit 1 + fi + echo "$(date +%H:%M:%S)" "waiting for snapshot controller deployment to complete, attempt #$cnt" + cnt=$((cnt + 1)) + sleep 10 + done +} + # collect logs and cluster status (like the version of all components, Kubernetes version, test version) collect_cluster_info () { cat < Date: Sat, 30 Nov 2019 00:29:00 +0530 Subject: [PATCH 03/56] Use kind v0.6.0 kind v0.6.0 appends the kubeconfig with the default config at ~/.kube/config. --- prow.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/prow.sh b/prow.sh index d719e9808..b88230629 100755 --- a/prow.sh +++ b/prow.sh @@ -107,8 +107,7 @@ configvar CSI_PROW_GO_VERSION_GINKGO "${CSI_PROW_GO_VERSION_BUILD}" "Go version # kind version to use. If the pre-installed version is different, # the desired version is downloaded from https://github.com/kubernetes-sigs/kind/releases/download/ # (if available), otherwise it is built from source. -# TODO: https://github.com/kubernetes-csi/csi-release-tools/issues/39 -configvar CSI_PROW_KIND_VERSION "86bc23d84ac12dcb56a0528890736e2c347c2dc3" "kind" +configvar CSI_PROW_KIND_VERSION "v0.6.0" "kind" # ginkgo test runner version to use. If the pre-installed version is # different, the desired version is built from source. @@ -579,8 +578,7 @@ EOF die "Cluster creation failed again, giving up. See the 'kind-cluster' artifact directory for additional logs." fi fi - KUBECONFIG="$(kind get kubeconfig-path --name=csi-prow)" - export KUBECONFIG + export KUBECONFIG="${HOME}/.kube/config" } # Deletes kind cluster inside a prow job From 9a7a685ee169d669a0182532315aeecd8a8715dc Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Mon, 2 Dec 2019 17:45:57 -0800 Subject: [PATCH 04/56] Create a kind cluster with two worker nodes so that the topology feature can be tested. Test cases that test accessing volumes from multiple nodes need to be skipped --- prow.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/prow.sh b/prow.sh index b88230629..815bc9d8a 100755 --- a/prow.sh +++ b/prow.sh @@ -330,7 +330,11 @@ configvar CSI_SNAPSHOTTER_VERSION 'v2.0.0-rc4' "external-snapshotter version tag # whether they can run with the current cluster provider, but until # they are, we filter them out by name. Like the other test selection # variables, this is again a space separated list of regular expressions. -configvar CSI_PROW_E2E_SKIP 'Disruptive' "tests that need to be skipped" +# +# "different node" test skips can be removed once +# https://github.com/kubernetes/kubernetes/pull/82678 has been backported +# to all the K8s versions we test against +configvar CSI_PROW_E2E_SKIP 'Disruptive|different\s+node' "tests that need to be skipped" # This is the directory for additional result files. Usually set by Prow, but # if not (for example, when invoking manually) it defaults to the work directory. @@ -526,6 +530,7 @@ apiVersion: kind.sigs.k8s.io/v1alpha3 nodes: - role: control-plane - role: worker +- role: worker EOF # kubeadm has API dependencies between apiVersion and Kubernetes version @@ -840,10 +845,6 @@ run_e2e () ( install_e2e || die "building e2e.test failed" install_ginkgo || die "installing ginkgo failed" - # TODO (?): multi-node cluster (depends on https://github.com/kubernetes-csi/csi-driver-host-path/pull/14). - # When running on a multi-node cluster, we need to figure out where the - # hostpath driver was deployed and set ClientNodeName accordingly. - generate_test_driver >"${CSI_PROW_WORK}/test-driver.yaml" || die "generating test-driver.yaml failed" # Rename, merge and filter JUnit files. Necessary in case that we run the E2E suite again From 53888ae7d59b862bfded46b883570c98f5fc5fff Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Tue, 3 Dec 2019 18:18:38 -0800 Subject: [PATCH 05/56] Improve README by adding an explicit Kubernetes dependency section --- README.md | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index cc40f161c..08c828846 100644 --- a/README.md +++ b/README.md @@ -141,17 +141,6 @@ The `vendor` directory is optional. It is still present in projects because it avoids downloading sources during CI builds. If this is no longer deemed necessary, then a project can also remove the directory. -When using packages that are part of the Kubernetes source code, the -commands above are not enough because the [lack of semantic -versioning](https://github.com/kubernetes/kubernetes/issues/72638) -prevents `go mod` from finding newer releases. Importing directly from -`kubernetes/kubernetes` also needs `replace` statements to override -the fake `v0.0.0` versions -(https://github.com/kubernetes/kubernetes/issues/79384). The -`go-get-kubernetes.sh` script can be used to update all packages in -lockstep to a different Kubernetes version. It takes a single version -number like "1.16.0". - Conversion of a repository that uses `dep` to `go mod` can be done with: GO111MODULE=on go mod init @@ -160,3 +149,18 @@ Conversion of a repository that uses `dep` to `go mod` can be done with: GO111MODULE=on go mod vendor git rm -f Gopkg.toml Gopkg.lock git add go.mod go.sum vendor + +### Updating Kubernetes dependencies + +When using packages that are part of the Kubernetes source code, the +commands above are not enough because the [lack of semantic +versioning](https://github.com/kubernetes/kubernetes/issues/72638) +prevents `go mod` from finding newer releases. Importing directly from +`kubernetes/kubernetes` also needs `replace` statements to override +the fake `v0.0.0` versions +(https://github.com/kubernetes/kubernetes/issues/79384). The +`go-get-kubernetes.sh` script can be used to update all packages in +lockstep to a different Kubernetes version. Example usage: +``` +$ ./release-tools/go-get-kubernetes.sh 1.16.4 +``` From 4ad69492c97834b52d74d0b67b6f47b0590e4be1 Mon Sep 17 00:00:00 2001 From: Grant Griffiths Date: Tue, 3 Dec 2019 23:48:29 -0800 Subject: [PATCH 06/56] Improve snapshot pod running checks and improve version_gt Signed-off-by: Grant Griffiths --- prow.sh | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/prow.sh b/prow.sh index d719e9808..27c705129 100755 --- a/prow.sh +++ b/prow.sh @@ -713,10 +713,11 @@ install_snapshot_controller() { kubectl apply -f "https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml" cnt=0 - until kubectl get statefulset snapshot-controller | grep snapshot-controller | grep "1/1"; do + expected_running_pods=$(curl https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${CSI_SNAPSHOTTER_VERSION}"/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml | grep replicas | cut -d ':' -f 2-) + while [ "$(kubectl get pods -l app=snapshot-controller | grep 'Running' -c)" -lt "$expected_running_pods" ]; do if [ $cnt -gt 30 ]; then - echo "Running statefulsets:" - kubectl describe statefulsets + echo "snapshot-controller pod status:" + kubectl describe pods -l app=snapshot-controller echo >&2 "ERROR: snapshot controller not ready after over 5 min" exit 1 fi @@ -996,8 +997,30 @@ make_test_to_junit () { fi } +# version_gt returns true if arg1 is greater than arg2. +# +# This function expects versions to be one of the following formats: +# X.Y.Z, release-X.Y.Z, vX.Y.Z +# +# where X,Y, and Z are any number. +# +# Partial versions (1.2, release-1.2) work as well. +# The follow substrings are stripped before version comparison: +# - "v" +# - "release-" +# +# Usage: +# version_gt release-1.3 v1.2.0 (returns true) +# version_gt v1.1.1 v1.2.0 (returns false) +# version_gt 1.1.1 v1.2.0 (returns false) +# version_gt 1.3.1 v1.2.0 (returns true) +# version_gt 1.1.1 release-1.2.0 (returns false) +# version_gt 1.2.0 1.2.2 (returns false) function version_gt() { - test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1"; + versions=$(for ver in "$@"; do ver=${ver#release-}; echo "${ver#v}"; done) + greaterVersion=${1#"release-"}; + greaterVersion=${greaterVersion#"v"}; + test "$(printf '%s' "$versions" | sort -V | head -n 1)" != "$greaterVersion" } main () { From a4e629966848d0097461f49f8aab0dfeb68d7a68 Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Wed, 4 Dec 2019 14:12:50 -0800 Subject: [PATCH 07/56] fix syntax for ppc64le build --- build.make | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.make b/build.make index 7075a37ee..a9b9d25dc 100644 --- a/build.make +++ b/build.make @@ -70,7 +70,7 @@ build-%: check-go-version-go CGO_ENABLED=0 GOOS=linux go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$* ./cmd/$* if [ "$$ARCH" = "amd64" ]; then \ CGO_ENABLED=0 GOOS=windows go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*.exe ./cmd/$* ; \ - CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*-ppc64le ./cmd/$* + CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*-ppc64le ./cmd/$* ; \ fi container-%: build-% From b98b2aed087f4e768db673a9d43dff4c7abd1482 Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Mon, 16 Dec 2019 19:13:38 -0800 Subject: [PATCH 08/56] Enable snapshot tests in 1.17 to be run in non-alpha jobs. This requires adding one more parallel e2e test run with a special focus flag because snapshot tests are still guarded with a "[Feature:VolumeSnapshotDataSource]" tag. The setting that skips all tests with "[Feature:.*]" has to be removed because it overrides the focus. We don't have serial snapshot tests yet. This needs to be modified again if we add any in the future. --- prow.sh | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/prow.sh b/prow.sh index 9bfdc2e28..c160f886c 100755 --- a/prow.sh +++ b/prow.sh @@ -132,7 +132,7 @@ configvar CSI_PROW_BUILD_JOB true "building code in repo enabled" # use the same settings as for "latest" Kubernetes. This works # as long as there are no breaking changes in Kubernetes, like # deprecating or changing the implementation of an alpha feature. -configvar CSI_PROW_KUBERNETES_VERSION 1.15.3 "Kubernetes" +configvar CSI_PROW_KUBERNETES_VERSION 1.17.0 "Kubernetes" # This is a hack to workaround the issue that each version # of kind currently only supports specific patch versions of @@ -142,7 +142,6 @@ configvar CSI_PROW_KUBERNETES_VERSION 1.15.3 "Kubernetes" # # If the version is prefixed with "release-", then nothing # is overridden. -override_k8s_version "1.14.6" override_k8s_version "1.15.3" # CSI_PROW_KUBERNETES_VERSION reduced to first two version numbers and @@ -206,9 +205,9 @@ configvar CSI_PROW_HOSTPATH_CANARY "" "hostpath image" # # CSI_PROW_E2E_REPO=none disables E2E testing. # TOOO: remove versioned variables and make e2e version match k8s version -configvar CSI_PROW_E2E_VERSION_1_14 v1.14.0 "E2E version for Kubernetes 1.14.x" configvar CSI_PROW_E2E_VERSION_1_15 v1.15.0 "E2E version for Kubernetes 1.15.x" configvar CSI_PROW_E2E_VERSION_1_16 v1.16.0 "E2E version for Kubernetes 1.16.x" +configvar CSI_PROW_E2E_VERSION_1_17 v1.17.0 "E2E version for Kubernetes 1.17.x" # TODO: add new CSI_PROW_E2E_VERSION entry for future Kubernetes releases configvar CSI_PROW_E2E_VERSION_LATEST master "E2E version for Kubernetes master" # testing against Kubernetes master is already tracking a moving target, so we might as well use a moving E2E version configvar CSI_PROW_E2E_REPO_LATEST https://github.com/kubernetes/kubernetes "E2E repo for Kubernetes >= 1.13.x" # currently the same for all versions @@ -278,6 +277,14 @@ tests_need_alpha_cluster () { tests_enabled "parallel-alpha" "serial-alpha" } +# Regex for non-alpha, feature-tagged tests that should be run. +# +# Starting with 1.17, snapshots is beta, but the E2E tests still have the +# [Feature:] tag. They need to be explicitly enabled. +configvar CSI_PROW_E2E_FOCUS_1_15 '^' "non-alpha, feature-tagged tests for Kubernetes = 1.15" # no tests to run, match nothing +configvar CSI_PROW_E2E_FOCUS_1_16 '^' "non-alpha, feature-tagged tests for Kubernetes = 1.16" # no tests to run, match nothing +configvar CSI_PROW_E2E_FOCUS_LATEST '\[Feature:VolumeSnapshotDataSource\]' "non-alpha, feature-tagged tests for Kubernetes >= 1.17" +configvar CSI_PROW_E2E_FOCUS "$(get_versioned_variable CSI_PROW_E2E_FOCUS "${csi_prow_kubernetes_version_suffix}")" "non-alpha, feature-tagged tests" # Serial vs. parallel is always determined by these regular expressions. # Individual regular expressions are seperated by spaces for readability @@ -313,12 +320,11 @@ configvar CSI_PROW_E2E_ALPHA "$(get_versioned_variable CSI_PROW_E2E_ALPHA "${csi # kubernetes-csi components must be updated, either by disabling # the failing test for "latest" or by updating the test and not running # it anymore for older releases. -configvar CSI_PROW_E2E_ALPHA_GATES_1_14 'VolumeSnapshotDataSource=true,ExpandCSIVolumes=true' "alpha feature gates for Kubernetes 1.14" configvar CSI_PROW_E2E_ALPHA_GATES_1_15 'VolumeSnapshotDataSource=true,ExpandCSIVolumes=true' "alpha feature gates for Kubernetes 1.15" configvar CSI_PROW_E2E_ALPHA_GATES_1_16 'VolumeSnapshotDataSource=true' "alpha feature gates for Kubernetes 1.16" # TODO: add new CSI_PROW_ALPHA_GATES_xxx entry for future Kubernetes releases and # add new gates to CSI_PROW_E2E_ALPHA_GATES_LATEST. -configvar CSI_PROW_E2E_ALPHA_GATES_LATEST 'VolumeSnapshotDataSource=true' "alpha feature gates for latest Kubernetes" +configvar CSI_PROW_E2E_ALPHA_GATES_LATEST '' "alpha feature gates for latest Kubernetes" configvar CSI_PROW_E2E_ALPHA_GATES "$(get_versioned_variable CSI_PROW_E2E_ALPHA_GATES "${csi_prow_kubernetes_version_suffix}")" "alpha E2E feature gates" # Which external-snapshotter tag to use for the snapshotter CRD and snapshot-controller deployment @@ -1111,6 +1117,16 @@ main () { warn "E2E parallel failed" ret=1 fi + + # Run tests that are feature tagged, but non-alpha + # Ignore: Double quote to prevent globbing and word splitting. + # shellcheck disable=SC2086 + if ! run_e2e parallel ${CSI_PROW_GINKO_PARALLEL} \ + -focus="External.Storage.*($(regex_join "${CSI_PROW_E2E_FOCUS}"))" \ + -skip="$(regex_join "${CSI_PROW_E2E_SERIAL}")"; then + warn "E2E parallel features failed" + ret=1 + fi fi if tests_enabled "serial"; then From fc80975954a9720a1611fa9a5195cbfc48b64167 Mon Sep 17 00:00:00 2001 From: Grant Griffiths Date: Fri, 20 Dec 2019 16:30:25 -0800 Subject: [PATCH 09/56] Fix version_gt to work with kubernetes prefix Signed-off-by: Grant Griffiths --- prow.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index c160f886c..832da1ee5 100755 --- a/prow.sh +++ b/prow.sh @@ -1013,6 +1013,7 @@ make_test_to_junit () { # The follow substrings are stripped before version comparison: # - "v" # - "release-" +# - "kubernetes-" # # Usage: # version_gt release-1.3 v1.2.0 (returns true) @@ -1022,8 +1023,9 @@ make_test_to_junit () { # version_gt 1.1.1 release-1.2.0 (returns false) # version_gt 1.2.0 1.2.2 (returns false) function version_gt() { - versions=$(for ver in "$@"; do ver=${ver#release-}; echo "${ver#v}"; done) + versions=$(for ver in "$@"; do ver=${ver#release-}; ver=${ver#kubernetes-}; echo "${ver#v}"; done) greaterVersion=${1#"release-"}; + greaterVersion=${greaterVersion#"kubernetes-"}; greaterVersion=${greaterVersion#"v"}; test "$(printf '%s' "$versions" | sort -V | head -n 1)" != "$greaterVersion" } From af9549b5a11c4849ff7b4a1a42a7b8aea59137c3 Mon Sep 17 00:00:00 2001 From: saad-ali Date: Thu, 2 Jan 2020 14:29:40 -0800 Subject: [PATCH 10/56] Update prow hostpath driver version to 1.3.0-rc2 --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index 832da1ee5..ae51947de 100755 --- a/prow.sh +++ b/prow.sh @@ -187,7 +187,7 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip # # When no deploy script is found (nothing in `deploy` directory, # CSI_PROW_HOSTPATH_REPO=none), nothing gets deployed. -configvar CSI_PROW_HOSTPATH_VERSION "v1.2.0" "hostpath driver" +configvar CSI_PROW_HOSTPATH_VERSION "v1.3.0-rc2" "hostpath driver" configvar CSI_PROW_HOSTPATH_REPO https://github.com/kubernetes-csi/csi-driver-host-path "hostpath repo" configvar CSI_PROW_DEPLOYMENT "" "deployment" configvar CSI_PROW_HOSTPATH_DRIVER_NAME "hostpath.csi.k8s.io" "the hostpath driver name" From 8b0316c7e4d94a3206cdab0ea2e8d7f7b3be637b Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Thu, 2 Jan 2020 14:33:46 -0800 Subject: [PATCH 11/56] Fix overriding of junit results by using unique names for each e2e run --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index ae51947de..e21a7ba9e 100755 --- a/prow.sh +++ b/prow.sh @@ -1123,7 +1123,7 @@ main () { # Run tests that are feature tagged, but non-alpha # Ignore: Double quote to prevent globbing and word splitting. # shellcheck disable=SC2086 - if ! run_e2e parallel ${CSI_PROW_GINKO_PARALLEL} \ + if ! run_e2e parallel-features ${CSI_PROW_GINKO_PARALLEL} \ -focus="External.Storage.*($(regex_join "${CSI_PROW_E2E_FOCUS}"))" \ -skip="$(regex_join "${CSI_PROW_E2E_SERIAL}")"; then warn "E2E parallel features failed" From 8191eab6ffe694d11bf1e0d254330287c961dc77 Mon Sep 17 00:00:00 2001 From: Grant Griffiths Date: Tue, 7 Jan 2020 18:09:31 -0800 Subject: [PATCH 12/56] Update snapshotter to version v2.0.0 Signed-off-by: Grant Griffiths --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index e21a7ba9e..eb451ef1b 100755 --- a/prow.sh +++ b/prow.sh @@ -328,7 +328,7 @@ configvar CSI_PROW_E2E_ALPHA_GATES_LATEST '' "alpha feature gates for latest Kub configvar CSI_PROW_E2E_ALPHA_GATES "$(get_versioned_variable CSI_PROW_E2E_ALPHA_GATES "${csi_prow_kubernetes_version_suffix}")" "alpha E2E feature gates" # Which external-snapshotter tag to use for the snapshotter CRD and snapshot-controller deployment -configvar CSI_SNAPSHOTTER_VERSION 'v2.0.0-rc4' "external-snapshotter version tag" +configvar CSI_SNAPSHOTTER_VERSION 'v2.0.0' "external-snapshotter version tag" # Some tests are known to be unusable in a KinD cluster. For example, # stopping kubelet with "ssh systemctl stop kubelet" simply From 6582f2ff3bd584e662035b7da2a90efc52184b1b Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Thu, 9 Jan 2020 17:25:38 -0800 Subject: [PATCH 13/56] Update hostpath driver version to get fix for connection-timeout --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index e21a7ba9e..27309a796 100755 --- a/prow.sh +++ b/prow.sh @@ -187,7 +187,7 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip # # When no deploy script is found (nothing in `deploy` directory, # CSI_PROW_HOSTPATH_REPO=none), nothing gets deployed. -configvar CSI_PROW_HOSTPATH_VERSION "v1.3.0-rc2" "hostpath driver" +configvar CSI_PROW_HOSTPATH_VERSION "v1.3.0-rc3" "hostpath driver" configvar CSI_PROW_HOSTPATH_REPO https://github.com/kubernetes-csi/csi-driver-host-path "hostpath repo" configvar CSI_PROW_DEPLOYMENT "" "deployment" configvar CSI_PROW_HOSTPATH_DRIVER_NAME "hostpath.csi.k8s.io" "the hostpath driver name" From ac8a0212b93cad817689029be503839ad1959e21 Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Wed, 11 Dec 2019 17:56:08 -0800 Subject: [PATCH 14/56] Document the process for releasing a new sidecar --- SIDECAR_RELEASE_PROCESS.md | 90 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 SIDECAR_RELEASE_PROCESS.md diff --git a/SIDECAR_RELEASE_PROCESS.md b/SIDECAR_RELEASE_PROCESS.md new file mode 100644 index 000000000..91a1e1b82 --- /dev/null +++ b/SIDECAR_RELEASE_PROCESS.md @@ -0,0 +1,90 @@ +# Sidecar Release Process + +This page describes the process for releasing a kubernetes-csi sidecar. + +## Prerequisites + +The release manager must: + +* Be a member of the kubernetes-csi organization. Open an + [issue](https://github.com/kubernetes/org/issues/new?assignees=&labels=area%2Fgithub-membership&template=membership.md&title=REQUEST%3A+New+membership+for+%3Cyour-GH-handle%3E) in + kubernetes/org to request membership +* Be a top level approver for the repository. To become a top level approver, + the candidate must demonstrate ownership and deep knowledge of the repository + through active maintainence, responding to and fixing issues, reviewing PRs, + test triage. +* Be part of the maintainers or admin group for the repository. admin is a + superset of maintainers, only maintainers level is required for cutting a + release. Membership can be requested by submitting a PR to kubernetes/org. + [Example](https://github.com/kubernetes/org/pull/1467) + +## Updating CI Jobs +Whenever a new Kubernetes minor version is released, our kubernetes-csi CI jobs +must be updated. + +[Our CI jobs](https://k8s-testgrid.appspot.com/sig-storage-csi-ci) have the +naming convention `-on-`. + +1. Jobs should be actively monitored to find and fix failures in sidecars and + infrastructure changes early in the development cycle. Test failures are sent + to kubernetes-sig-storage-test-failures@googlegroups.com. +1. "-on-master" jobs are the closest reflection to the new Kubernetes version. +1. Fixes to our prow.sh CI script can be tested in the [CSI hostpath + repo](https://github.com/kubernetes-csi/csi-driver-host-path) by modifying + [prow.sh](https://github.com/kubernetes-csi/csi-driver-host-path/blob/master/release-tools/prow.sh) + along with any overrides in + [.prow.sh](https://github.com/kubernetes-csi/csi-driver-host-path/blob/master/.prow.sh) + to mirror the failing environment. Once e2e tests are passing (verify-unit tests + will fail), then the prow.sh changes can be submitted to [csi-release-tools](https://github.com/kubernetes-csi/csi-release-tools). +1. Changes can then be updated in all the sidecar repos and hostpath driver repo + by following the [update + instructions](https://github.com/kubernetes-csi/csi-release-tools/blob/master/README.md#sharing-and-updating). +1. New pull and CI jobs are configured by + [gen-jobs.sh](https://github.com/kubernetes/test-infra/blob/master/config/jobs/kubernetes-csi/gen-jobs.sh). + New pull jobs that have been unverified should be initially made optional. + [Example](https://github.com/kubernetes/test-infra/pull/15055) +1. Once new pull and CI jobs have been verified, and the new Kubernetes version + is released, we can make the optional jobs required, and also remove the + Kubernetes versions that are no longer supported. + +## Release Process +1. Identify all issues and ongoing PRs that should go into the release, and + drive them to resolution. +1. Download [K8s release notes + generator](https://github.com/kubernetes/release/tree/master/cmd/release-notes) +1. Generate release notes for the release. Replace arguments with the relevant + information. + ``` + GITHUB_TOKEN= ./release-notes --start-sha=0ed6978fd199e3ca10326b82b4b8b8e916211c9b --end-sha=3cb3d2f18ed8cb40371c6d8886edcabd1f27e7b9 \ + --github-org=kubernetes-csi --github-repo=external-attacher -branch=master -output out.md + ``` + * `--start-sha` should point to the last release from the same branch. For + example: + * `1.X-1.0` tag when releasing `1.X.0` + * `1.X.Y-1` tag when releasing `1.X.Y` +1. Compare the generated output to the new commits for the release to check if + any notable change missed a release note. +1. Reword release notes as needed. Make sure to check notes for breaking + changes and deprecations. +1. If release is a new major/minor version, create a new `CHANGELOG-..md` + file. Otherwise, add the release notes to the top of the existing CHANGELOG + file for that minor version. +1. Submit a PR for the CHANGELOG changes. +1. Submit a PR for README changes, in particular, Compatibility, Feature status, + and any other sections that may need updating. +1. Check that all [canary CI + jobs](https://k8s-testgrid.appspot.com/sig-storage-csi-ci) are passing, + and that test coverage is adequate for the changes that are going into the release. +1. Make sure that no new PRs have merged in the meantime, and no PRs are in + flight and soon to be merged. +1. Create a new release following a previous release as a template. Be sure to select the correct + branch. This requires Github release permissions as required by the prerequisites. + [external-provisioner example](https://github.com/kubernetes-csi/external-provisioner/releases/new) +1. If release was a new major/minor version, create a new `release-` + branch at that commit. +1. Update [kubernetes-csi/docs](https://github.com/kubernetes-csi/docs) sidecar + and feature pages with the new released version. +1. After all the sidecars have been released, update + CSI hostpath driver with the new sidecars in the [CSI repo](https://github.com/kubernetes-csi/csi-driver-host-path/tree/master/deploy) + and [k/k + in-tree](https://github.com/kubernetes/kubernetes/tree/master/test/e2e/testing-manifests/storage-csi/hostpath/hostpath) From fa90abd07862e6fa0a7b66ea2435b9db364ce6c9 Mon Sep 17 00:00:00 2001 From: wangzheng03 Date: Sun, 19 Jan 2020 09:42:26 +0800 Subject: [PATCH 15/56] fix incorrect link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 08c828846..60eab2a98 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ on what is enabled in Prow, see https://github.com/kubernetes/test-infra/tree/master/config/jobs/kubernetes-csi Test results for periodic jobs are visible in -https://testgrid.k8s.io/sig-storage-csi +https://testgrid.k8s.io/sig-storage-csi-ci It is possible to reproduce the Prow testing locally on a suitable machine: - Linux host From a2979dfd15f80572370602c460b7a4f277fac7df Mon Sep 17 00:00:00 2001 From: wangzheng03 Date: Mon, 20 Jan 2020 09:46:45 +0800 Subject: [PATCH 16/56] fix link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5578d35ff..325b08e8f 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ The `VolumeSnapshotDataSource` feature gate was introduced in Kubernetes 1.12 an ## Design -Both the snapshot controller and CSI external-snapshotter sidecar follow [controller](https://github.com/kubernetes/community/blob/master/contributors/devel/controllers.md) pattern and uses informers to watch for events. The snapshot controller watches for `VolumeSnapshot` and `VolumeSnapshotContent` create/update/delete events. +Both the snapshot controller and CSI external-snapshotter sidecar follow [controller](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-api-machinery/controllers.md) pattern and uses informers to watch for events. The snapshot controller watches for `VolumeSnapshot` and `VolumeSnapshotContent` create/update/delete events. The CSI external-snapshotter sidecar only watches for `VolumeSnapshotContent` create/update/delete events. It filters out these objects with `Driver==` specified in the associated VolumeSnapshotClass object and then processes these events in workqueues with exponential backoff. From fb6225dc417022f30826ffac8def9ea20b073cec Mon Sep 17 00:00:00 2001 From: Alex Szakaly Date: Wed, 22 Jan 2020 09:50:01 +0100 Subject: [PATCH 17/56] fix: go module requirements for semantic versioning This commit also adds the compatibility with dep (Go dependency management tool) via symlink creation (same method applied in kubernetes-csi/csi-test) TODO: fix hack/update-generated-code.sh Fixes issue #239. Signed-off-by: Alex Szakaly --- cmd/csi-snapshotter/main.go | 10 +++++----- cmd/snapshot-controller/main.go | 8 ++++---- go.mod | 2 +- pkg/client/clientset/versioned/clientset.go | 2 +- .../versioned/fake/clientset_generated.go | 6 +++--- .../clientset/versioned/fake/register.go | 2 +- .../clientset/versioned/scheme/register.go | 2 +- .../v1beta1/fake/fake_volumesnapshot.go | 2 +- .../v1beta1/fake/fake_volumesnapshot_client.go | 2 +- .../v1beta1/fake/fake_volumesnapshotclass.go | 2 +- .../v1beta1/fake/fake_volumesnapshotcontent.go | 2 +- .../volumesnapshot/v1beta1/volumesnapshot.go | 4 ++-- .../v1beta1/volumesnapshot_client.go | 4 ++-- .../v1beta1/volumesnapshotclass.go | 4 ++-- .../v1beta1/volumesnapshotcontent.go | 4 ++-- .../informers/externalversions/factory.go | 6 +++--- .../informers/externalversions/generic.go | 2 +- .../internalinterfaces/factory_interfaces.go | 2 +- .../volumesnapshot/interface.go | 4 ++-- .../volumesnapshot/v1beta1/interface.go | 2 +- .../volumesnapshot/v1beta1/volumesnapshot.go | 8 ++++---- .../v1beta1/volumesnapshotclass.go | 8 ++++---- .../v1beta1/volumesnapshotcontent.go | 8 ++++---- .../volumesnapshot/v1beta1/volumesnapshot.go | 2 +- .../v1beta1/volumesnapshotclass.go | 2 +- .../v1beta1/volumesnapshotcontent.go | 2 +- pkg/common-controller/framework_test.go | 14 +++++++------- pkg/common-controller/snapshot_controller.go | 4 ++-- .../snapshot_controller_base.go | 10 +++++----- .../snapshot_controller_test.go | 4 ++-- pkg/common-controller/snapshot_create_test.go | 2 +- pkg/common-controller/snapshot_delete_test.go | 4 ++-- pkg/sidecar-controller/csi_handler.go | 6 +++--- pkg/sidecar-controller/framework_test.go | 14 +++++++------- pkg/sidecar-controller/snapshot_controller.go | 4 ++-- .../snapshot_controller_base.go | 10 +++++----- .../snapshot_controller_test.go | 4 ++-- pkg/sidecar-controller/snapshot_delete_test.go | 4 ++-- pkg/utils/util.go | 2 +- pkg/utils/util_test.go | 2 +- v2/README.md | 18 ++++++++++++++++++ v2/cmd/csi-snapshotter/main.go | 1 + v2/cmd/csi-snapshotter/main_test.go | 1 + v2/cmd/snapshot-controller/main.go | 1 + v2/pkg/apis/volumesnapshot/v1beta1/doc.go | 1 + v2/pkg/apis/volumesnapshot/v1beta1/register.go | 1 + v2/pkg/apis/volumesnapshot/v1beta1/types.go | 1 + .../v1beta1/zz_generated.deepcopy.go | 1 + v2/pkg/client/clientset/versioned/clientset.go | 1 + v2/pkg/client/clientset/versioned/doc.go | 1 + .../versioned/fake/clientset_generated.go | 1 + v2/pkg/client/clientset/versioned/fake/doc.go | 1 + .../clientset/versioned/fake/register.go | 1 + .../client/clientset/versioned/scheme/doc.go | 1 + .../clientset/versioned/scheme/register.go | 1 + .../typed/volumesnapshot/v1beta1/doc.go | 1 + .../typed/volumesnapshot/v1beta1/fake/doc.go | 1 + .../v1beta1/fake/fake_volumesnapshot.go | 1 + .../v1beta1/fake/fake_volumesnapshot_client.go | 1 + .../v1beta1/fake/fake_volumesnapshotclass.go | 1 + .../v1beta1/fake/fake_volumesnapshotcontent.go | 1 + .../v1beta1/generated_expansion.go | 1 + .../volumesnapshot/v1beta1/volumesnapshot.go | 1 + .../v1beta1/volumesnapshot_client.go | 1 + .../v1beta1/volumesnapshotclass.go | 1 + .../v1beta1/volumesnapshotcontent.go | 1 + .../informers/externalversions/factory.go | 1 + .../informers/externalversions/generic.go | 1 + .../internalinterfaces/factory_interfaces.go | 1 + .../volumesnapshot/interface.go | 1 + .../volumesnapshot/v1beta1/interface.go | 1 + .../volumesnapshot/v1beta1/volumesnapshot.go | 1 + .../v1beta1/volumesnapshotclass.go | 1 + .../v1beta1/volumesnapshotcontent.go | 1 + .../v1beta1/expansion_generated.go | 1 + .../volumesnapshot/v1beta1/volumesnapshot.go | 1 + .../v1beta1/volumesnapshotclass.go | 1 + .../v1beta1/volumesnapshotcontent.go | 1 + v2/pkg/common-controller/framework_test.go | 1 + .../common-controller/snapshot_controller.go | 1 + .../snapshot_controller_base.go | 1 + .../snapshot_controller_test.go | 1 + .../common-controller/snapshot_create_test.go | 1 + .../common-controller/snapshot_delete_test.go | 1 + .../snapshot_finalizer_test.go | 1 + .../common-controller/snapshot_update_test.go | 1 + v2/pkg/common-controller/snapshotclass_test.go | 1 + .../sidecar-controller/content_create_test.go | 1 + v2/pkg/sidecar-controller/csi_handler.go | 1 + v2/pkg/sidecar-controller/framework_test.go | 1 + .../sidecar-controller/snapshot_controller.go | 1 + .../snapshot_controller_base.go | 1 + .../snapshot_controller_test.go | 1 + .../sidecar-controller/snapshot_delete_test.go | 1 + .../snapshot_finalizer_test.go | 1 + v2/pkg/snapshotter/snapshotter.go | 1 + v2/pkg/snapshotter/snapshotter_test.go | 1 + v2/pkg/utils/util.go | 1 + v2/pkg/utils/util_test.go | 1 + 99 files changed, 169 insertions(+), 93 deletions(-) create mode 100644 v2/README.md create mode 120000 v2/cmd/csi-snapshotter/main.go create mode 120000 v2/cmd/csi-snapshotter/main_test.go create mode 120000 v2/cmd/snapshot-controller/main.go create mode 120000 v2/pkg/apis/volumesnapshot/v1beta1/doc.go create mode 120000 v2/pkg/apis/volumesnapshot/v1beta1/register.go create mode 120000 v2/pkg/apis/volumesnapshot/v1beta1/types.go create mode 120000 v2/pkg/apis/volumesnapshot/v1beta1/zz_generated.deepcopy.go create mode 120000 v2/pkg/client/clientset/versioned/clientset.go create mode 120000 v2/pkg/client/clientset/versioned/doc.go create mode 120000 v2/pkg/client/clientset/versioned/fake/clientset_generated.go create mode 120000 v2/pkg/client/clientset/versioned/fake/doc.go create mode 120000 v2/pkg/client/clientset/versioned/fake/register.go create mode 120000 v2/pkg/client/clientset/versioned/scheme/doc.go create mode 120000 v2/pkg/client/clientset/versioned/scheme/register.go create mode 120000 v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/doc.go create mode 120000 v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/fake/doc.go create mode 120000 v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshot.go create mode 120000 v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshot_client.go create mode 120000 v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshotclass.go create mode 120000 v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshotcontent.go create mode 120000 v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/generated_expansion.go create mode 120000 v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshot.go create mode 120000 v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshot_client.go create mode 120000 v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshotclass.go create mode 120000 v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshotcontent.go create mode 120000 v2/pkg/client/informers/externalversions/factory.go create mode 120000 v2/pkg/client/informers/externalversions/generic.go create mode 120000 v2/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go create mode 120000 v2/pkg/client/informers/externalversions/volumesnapshot/interface.go create mode 120000 v2/pkg/client/informers/externalversions/volumesnapshot/v1beta1/interface.go create mode 120000 v2/pkg/client/informers/externalversions/volumesnapshot/v1beta1/volumesnapshot.go create mode 120000 v2/pkg/client/informers/externalversions/volumesnapshot/v1beta1/volumesnapshotclass.go create mode 120000 v2/pkg/client/informers/externalversions/volumesnapshot/v1beta1/volumesnapshotcontent.go create mode 120000 v2/pkg/client/listers/volumesnapshot/v1beta1/expansion_generated.go create mode 120000 v2/pkg/client/listers/volumesnapshot/v1beta1/volumesnapshot.go create mode 120000 v2/pkg/client/listers/volumesnapshot/v1beta1/volumesnapshotclass.go create mode 120000 v2/pkg/client/listers/volumesnapshot/v1beta1/volumesnapshotcontent.go create mode 120000 v2/pkg/common-controller/framework_test.go create mode 120000 v2/pkg/common-controller/snapshot_controller.go create mode 120000 v2/pkg/common-controller/snapshot_controller_base.go create mode 120000 v2/pkg/common-controller/snapshot_controller_test.go create mode 120000 v2/pkg/common-controller/snapshot_create_test.go create mode 120000 v2/pkg/common-controller/snapshot_delete_test.go create mode 120000 v2/pkg/common-controller/snapshot_finalizer_test.go create mode 120000 v2/pkg/common-controller/snapshot_update_test.go create mode 120000 v2/pkg/common-controller/snapshotclass_test.go create mode 120000 v2/pkg/sidecar-controller/content_create_test.go create mode 120000 v2/pkg/sidecar-controller/csi_handler.go create mode 120000 v2/pkg/sidecar-controller/framework_test.go create mode 120000 v2/pkg/sidecar-controller/snapshot_controller.go create mode 120000 v2/pkg/sidecar-controller/snapshot_controller_base.go create mode 120000 v2/pkg/sidecar-controller/snapshot_controller_test.go create mode 120000 v2/pkg/sidecar-controller/snapshot_delete_test.go create mode 120000 v2/pkg/sidecar-controller/snapshot_finalizer_test.go create mode 120000 v2/pkg/snapshotter/snapshotter.go create mode 120000 v2/pkg/snapshotter/snapshotter_test.go create mode 120000 v2/pkg/utils/util.go create mode 120000 v2/pkg/utils/util_test.go diff --git a/cmd/csi-snapshotter/main.go b/cmd/csi-snapshotter/main.go index 1c557d368..82f584361 100644 --- a/cmd/csi-snapshotter/main.go +++ b/cmd/csi-snapshotter/main.go @@ -38,12 +38,12 @@ import ( "github.com/kubernetes-csi/csi-lib-utils/leaderelection" "github.com/kubernetes-csi/csi-lib-utils/metrics" csirpc "github.com/kubernetes-csi/csi-lib-utils/rpc" - controller "github.com/kubernetes-csi/external-snapshotter/pkg/sidecar-controller" - "github.com/kubernetes-csi/external-snapshotter/pkg/snapshotter" + controller "github.com/kubernetes-csi/external-snapshotter/v2/pkg/sidecar-controller" + "github.com/kubernetes-csi/external-snapshotter/v2/pkg/snapshotter" - clientset "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned" - snapshotscheme "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned/scheme" - informers "github.com/kubernetes-csi/external-snapshotter/pkg/client/informers/externalversions" + clientset "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/clientset/versioned" + snapshotscheme "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/clientset/versioned/scheme" + informers "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/informers/externalversions" coreinformers "k8s.io/client-go/informers" ) diff --git a/cmd/snapshot-controller/main.go b/cmd/snapshot-controller/main.go index cbe92f0fe..0831ae659 100644 --- a/cmd/snapshot-controller/main.go +++ b/cmd/snapshot-controller/main.go @@ -31,11 +31,11 @@ import ( "k8s.io/klog" "github.com/kubernetes-csi/csi-lib-utils/leaderelection" - controller "github.com/kubernetes-csi/external-snapshotter/pkg/common-controller" + controller "github.com/kubernetes-csi/external-snapshotter/v2/pkg/common-controller" - clientset "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned" - snapshotscheme "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned/scheme" - informers "github.com/kubernetes-csi/external-snapshotter/pkg/client/informers/externalversions" + clientset "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/clientset/versioned" + snapshotscheme "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/clientset/versioned/scheme" + informers "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/informers/externalversions" coreinformers "k8s.io/client-go/informers" ) diff --git a/go.mod b/go.mod index bbd8f7396..c2d7df746 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/kubernetes-csi/external-snapshotter +module github.com/kubernetes-csi/external-snapshotter/v2 go 1.12 diff --git a/pkg/client/clientset/versioned/clientset.go b/pkg/client/clientset/versioned/clientset.go index 99b304b66..e7d62a9db 100644 --- a/pkg/client/clientset/versioned/clientset.go +++ b/pkg/client/clientset/versioned/clientset.go @@ -21,7 +21,7 @@ package versioned import ( "fmt" - snapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1" + snapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1" discovery "k8s.io/client-go/discovery" rest "k8s.io/client-go/rest" flowcontrol "k8s.io/client-go/util/flowcontrol" diff --git a/pkg/client/clientset/versioned/fake/clientset_generated.go b/pkg/client/clientset/versioned/fake/clientset_generated.go index 0f39d5b2c..e0c9eaa3d 100644 --- a/pkg/client/clientset/versioned/fake/clientset_generated.go +++ b/pkg/client/clientset/versioned/fake/clientset_generated.go @@ -19,9 +19,9 @@ limitations under the License. package fake import ( - clientset "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned" - snapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1" - fakesnapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/fake" + clientset "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/clientset/versioned" + snapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1" + fakesnapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/fake" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/discovery" diff --git a/pkg/client/clientset/versioned/fake/register.go b/pkg/client/clientset/versioned/fake/register.go index 4c92bf112..da4d8c602 100644 --- a/pkg/client/clientset/versioned/fake/register.go +++ b/pkg/client/clientset/versioned/fake/register.go @@ -19,7 +19,7 @@ limitations under the License. package fake import ( - snapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" + snapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" diff --git a/pkg/client/clientset/versioned/scheme/register.go b/pkg/client/clientset/versioned/scheme/register.go index 296f7fa70..61c23a76b 100644 --- a/pkg/client/clientset/versioned/scheme/register.go +++ b/pkg/client/clientset/versioned/scheme/register.go @@ -19,7 +19,7 @@ limitations under the License. package scheme import ( - snapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" + snapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" diff --git a/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshot.go b/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshot.go index 72faad7d8..431f0f59c 100644 --- a/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshot.go +++ b/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshot.go @@ -19,7 +19,7 @@ limitations under the License. package fake import ( - v1beta1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" + v1beta1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" labels "k8s.io/apimachinery/pkg/labels" schema "k8s.io/apimachinery/pkg/runtime/schema" diff --git a/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshot_client.go b/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshot_client.go index 56f7a0dd1..231201de6 100644 --- a/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshot_client.go +++ b/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshot_client.go @@ -19,7 +19,7 @@ limitations under the License. package fake import ( - v1beta1 "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1" + v1beta1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1" rest "k8s.io/client-go/rest" testing "k8s.io/client-go/testing" ) diff --git a/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshotclass.go b/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshotclass.go index bd4013c39..78bb2b4e5 100644 --- a/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshotclass.go +++ b/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshotclass.go @@ -19,7 +19,7 @@ limitations under the License. package fake import ( - v1beta1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" + v1beta1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" labels "k8s.io/apimachinery/pkg/labels" schema "k8s.io/apimachinery/pkg/runtime/schema" diff --git a/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshotcontent.go b/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshotcontent.go index 36cfc70fa..7558cb2e4 100644 --- a/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshotcontent.go +++ b/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshotcontent.go @@ -19,7 +19,7 @@ limitations under the License. package fake import ( - v1beta1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" + v1beta1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" labels "k8s.io/apimachinery/pkg/labels" schema "k8s.io/apimachinery/pkg/runtime/schema" diff --git a/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshot.go b/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshot.go index f9ef41512..d46c1583f 100644 --- a/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshot.go +++ b/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshot.go @@ -21,8 +21,8 @@ package v1beta1 import ( "time" - v1beta1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" - scheme "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned/scheme" + v1beta1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" + scheme "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/clientset/versioned/scheme" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" diff --git a/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshot_client.go b/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshot_client.go index ef3651723..81b62687d 100644 --- a/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshot_client.go +++ b/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshot_client.go @@ -19,8 +19,8 @@ limitations under the License. package v1beta1 import ( - v1beta1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" - "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned/scheme" + v1beta1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" + "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/clientset/versioned/scheme" rest "k8s.io/client-go/rest" ) diff --git a/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshotclass.go b/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshotclass.go index 4a8e1d324..c26130b08 100644 --- a/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshotclass.go +++ b/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshotclass.go @@ -21,8 +21,8 @@ package v1beta1 import ( "time" - v1beta1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" - scheme "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned/scheme" + v1beta1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" + scheme "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/clientset/versioned/scheme" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" diff --git a/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshotcontent.go b/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshotcontent.go index 02f672f5e..c99152a13 100644 --- a/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshotcontent.go +++ b/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshotcontent.go @@ -21,8 +21,8 @@ package v1beta1 import ( "time" - v1beta1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" - scheme "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned/scheme" + v1beta1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" + scheme "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/clientset/versioned/scheme" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" diff --git a/pkg/client/informers/externalversions/factory.go b/pkg/client/informers/externalversions/factory.go index f01d7993b..32eabdb34 100644 --- a/pkg/client/informers/externalversions/factory.go +++ b/pkg/client/informers/externalversions/factory.go @@ -23,9 +23,9 @@ import ( sync "sync" time "time" - versioned "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned" - internalinterfaces "github.com/kubernetes-csi/external-snapshotter/pkg/client/informers/externalversions/internalinterfaces" - volumesnapshot "github.com/kubernetes-csi/external-snapshotter/pkg/client/informers/externalversions/volumesnapshot" + versioned "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/clientset/versioned" + internalinterfaces "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/informers/externalversions/internalinterfaces" + volumesnapshot "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/informers/externalversions/volumesnapshot" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" diff --git a/pkg/client/informers/externalversions/generic.go b/pkg/client/informers/externalversions/generic.go index abf7d5fa8..bddedbfff 100644 --- a/pkg/client/informers/externalversions/generic.go +++ b/pkg/client/informers/externalversions/generic.go @@ -21,7 +21,7 @@ package externalversions import ( "fmt" - v1beta1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" + v1beta1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" schema "k8s.io/apimachinery/pkg/runtime/schema" cache "k8s.io/client-go/tools/cache" ) diff --git a/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go b/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go index daf0bfe89..8d879afcb 100644 --- a/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go +++ b/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go @@ -21,7 +21,7 @@ package internalinterfaces import ( time "time" - versioned "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned" + versioned "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/clientset/versioned" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" cache "k8s.io/client-go/tools/cache" diff --git a/pkg/client/informers/externalversions/volumesnapshot/interface.go b/pkg/client/informers/externalversions/volumesnapshot/interface.go index f95385e93..0ab1fb091 100644 --- a/pkg/client/informers/externalversions/volumesnapshot/interface.go +++ b/pkg/client/informers/externalversions/volumesnapshot/interface.go @@ -19,8 +19,8 @@ limitations under the License. package volumesnapshot import ( - internalinterfaces "github.com/kubernetes-csi/external-snapshotter/pkg/client/informers/externalversions/internalinterfaces" - v1beta1 "github.com/kubernetes-csi/external-snapshotter/pkg/client/informers/externalversions/volumesnapshot/v1beta1" + internalinterfaces "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/informers/externalversions/internalinterfaces" + v1beta1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/informers/externalversions/volumesnapshot/v1beta1" ) // Interface provides access to each of this group's versions. diff --git a/pkg/client/informers/externalversions/volumesnapshot/v1beta1/interface.go b/pkg/client/informers/externalversions/volumesnapshot/v1beta1/interface.go index ecccf563e..36f657971 100644 --- a/pkg/client/informers/externalversions/volumesnapshot/v1beta1/interface.go +++ b/pkg/client/informers/externalversions/volumesnapshot/v1beta1/interface.go @@ -19,7 +19,7 @@ limitations under the License. package v1beta1 import ( - internalinterfaces "github.com/kubernetes-csi/external-snapshotter/pkg/client/informers/externalversions/internalinterfaces" + internalinterfaces "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/informers/externalversions/internalinterfaces" ) // Interface provides access to all the informers in this group version. diff --git a/pkg/client/informers/externalversions/volumesnapshot/v1beta1/volumesnapshot.go b/pkg/client/informers/externalversions/volumesnapshot/v1beta1/volumesnapshot.go index 211b6f5f1..17648cf9f 100644 --- a/pkg/client/informers/externalversions/volumesnapshot/v1beta1/volumesnapshot.go +++ b/pkg/client/informers/externalversions/volumesnapshot/v1beta1/volumesnapshot.go @@ -21,10 +21,10 @@ package v1beta1 import ( time "time" - volumesnapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" - versioned "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned" - internalinterfaces "github.com/kubernetes-csi/external-snapshotter/pkg/client/informers/externalversions/internalinterfaces" - v1beta1 "github.com/kubernetes-csi/external-snapshotter/pkg/client/listers/volumesnapshot/v1beta1" + volumesnapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" + versioned "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/clientset/versioned" + internalinterfaces "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/informers/externalversions/internalinterfaces" + v1beta1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/listers/volumesnapshot/v1beta1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" watch "k8s.io/apimachinery/pkg/watch" diff --git a/pkg/client/informers/externalversions/volumesnapshot/v1beta1/volumesnapshotclass.go b/pkg/client/informers/externalversions/volumesnapshot/v1beta1/volumesnapshotclass.go index 800e12428..94d4263b7 100644 --- a/pkg/client/informers/externalversions/volumesnapshot/v1beta1/volumesnapshotclass.go +++ b/pkg/client/informers/externalversions/volumesnapshot/v1beta1/volumesnapshotclass.go @@ -21,10 +21,10 @@ package v1beta1 import ( time "time" - volumesnapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" - versioned "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned" - internalinterfaces "github.com/kubernetes-csi/external-snapshotter/pkg/client/informers/externalversions/internalinterfaces" - v1beta1 "github.com/kubernetes-csi/external-snapshotter/pkg/client/listers/volumesnapshot/v1beta1" + volumesnapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" + versioned "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/clientset/versioned" + internalinterfaces "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/informers/externalversions/internalinterfaces" + v1beta1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/listers/volumesnapshot/v1beta1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" watch "k8s.io/apimachinery/pkg/watch" diff --git a/pkg/client/informers/externalversions/volumesnapshot/v1beta1/volumesnapshotcontent.go b/pkg/client/informers/externalversions/volumesnapshot/v1beta1/volumesnapshotcontent.go index 835292145..d34fbe183 100644 --- a/pkg/client/informers/externalversions/volumesnapshot/v1beta1/volumesnapshotcontent.go +++ b/pkg/client/informers/externalversions/volumesnapshot/v1beta1/volumesnapshotcontent.go @@ -21,10 +21,10 @@ package v1beta1 import ( time "time" - volumesnapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" - versioned "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned" - internalinterfaces "github.com/kubernetes-csi/external-snapshotter/pkg/client/informers/externalversions/internalinterfaces" - v1beta1 "github.com/kubernetes-csi/external-snapshotter/pkg/client/listers/volumesnapshot/v1beta1" + volumesnapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" + versioned "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/clientset/versioned" + internalinterfaces "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/informers/externalversions/internalinterfaces" + v1beta1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/listers/volumesnapshot/v1beta1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" watch "k8s.io/apimachinery/pkg/watch" diff --git a/pkg/client/listers/volumesnapshot/v1beta1/volumesnapshot.go b/pkg/client/listers/volumesnapshot/v1beta1/volumesnapshot.go index 4309a30e6..ed4457826 100644 --- a/pkg/client/listers/volumesnapshot/v1beta1/volumesnapshot.go +++ b/pkg/client/listers/volumesnapshot/v1beta1/volumesnapshot.go @@ -19,7 +19,7 @@ limitations under the License. package v1beta1 import ( - v1beta1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" + v1beta1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/tools/cache" diff --git a/pkg/client/listers/volumesnapshot/v1beta1/volumesnapshotclass.go b/pkg/client/listers/volumesnapshot/v1beta1/volumesnapshotclass.go index 2d56d5d54..253e83fac 100644 --- a/pkg/client/listers/volumesnapshot/v1beta1/volumesnapshotclass.go +++ b/pkg/client/listers/volumesnapshot/v1beta1/volumesnapshotclass.go @@ -19,7 +19,7 @@ limitations under the License. package v1beta1 import ( - v1beta1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" + v1beta1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/tools/cache" diff --git a/pkg/client/listers/volumesnapshot/v1beta1/volumesnapshotcontent.go b/pkg/client/listers/volumesnapshot/v1beta1/volumesnapshotcontent.go index 87edbc8fc..8fb6be3a3 100644 --- a/pkg/client/listers/volumesnapshot/v1beta1/volumesnapshotcontent.go +++ b/pkg/client/listers/volumesnapshot/v1beta1/volumesnapshotcontent.go @@ -19,7 +19,7 @@ limitations under the License. package v1beta1 import ( - v1beta1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" + v1beta1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/tools/cache" diff --git a/pkg/common-controller/framework_test.go b/pkg/common-controller/framework_test.go index e4b141a59..e815aa13a 100644 --- a/pkg/common-controller/framework_test.go +++ b/pkg/common-controller/framework_test.go @@ -28,13 +28,13 @@ import ( "testing" "time" - crdv1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" - clientset "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned" - "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned/fake" - snapshotscheme "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned/scheme" - informers "github.com/kubernetes-csi/external-snapshotter/pkg/client/informers/externalversions" - storagelisters "github.com/kubernetes-csi/external-snapshotter/pkg/client/listers/volumesnapshot/v1beta1" - "github.com/kubernetes-csi/external-snapshotter/pkg/utils" + crdv1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" + clientset "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/clientset/versioned" + "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/clientset/versioned/fake" + snapshotscheme "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/clientset/versioned/scheme" + informers "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/informers/externalversions" + storagelisters "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/listers/volumesnapshot/v1beta1" + "github.com/kubernetes-csi/external-snapshotter/v2/pkg/utils" v1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/resource" diff --git a/pkg/common-controller/snapshot_controller.go b/pkg/common-controller/snapshot_controller.go index fa4cdc1c9..a68e5f429 100644 --- a/pkg/common-controller/snapshot_controller.go +++ b/pkg/common-controller/snapshot_controller.go @@ -21,8 +21,8 @@ import ( "strings" "time" - crdv1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" - "github.com/kubernetes-csi/external-snapshotter/pkg/utils" + crdv1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" + "github.com/kubernetes-csi/external-snapshotter/v2/pkg/utils" "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" diff --git a/pkg/common-controller/snapshot_controller_base.go b/pkg/common-controller/snapshot_controller_base.go index 2518dad85..97659d73b 100644 --- a/pkg/common-controller/snapshot_controller_base.go +++ b/pkg/common-controller/snapshot_controller_base.go @@ -20,11 +20,11 @@ import ( "fmt" "time" - crdv1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" - clientset "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned" - storageinformers "github.com/kubernetes-csi/external-snapshotter/pkg/client/informers/externalversions/volumesnapshot/v1beta1" - storagelisters "github.com/kubernetes-csi/external-snapshotter/pkg/client/listers/volumesnapshot/v1beta1" - "github.com/kubernetes-csi/external-snapshotter/pkg/utils" + crdv1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" + clientset "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/clientset/versioned" + storageinformers "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/informers/externalversions/volumesnapshot/v1beta1" + storagelisters "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/listers/volumesnapshot/v1beta1" + "github.com/kubernetes-csi/external-snapshotter/v2/pkg/utils" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" diff --git a/pkg/common-controller/snapshot_controller_test.go b/pkg/common-controller/snapshot_controller_test.go index e1785208f..12a206c0a 100644 --- a/pkg/common-controller/snapshot_controller_test.go +++ b/pkg/common-controller/snapshot_controller_test.go @@ -19,8 +19,8 @@ package common_controller import ( "testing" - crdv1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" - "github.com/kubernetes-csi/external-snapshotter/pkg/utils" + crdv1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" + "github.com/kubernetes-csi/external-snapshotter/v2/pkg/utils" "k8s.io/client-go/tools/cache" ) diff --git a/pkg/common-controller/snapshot_create_test.go b/pkg/common-controller/snapshot_create_test.go index ffaa370a2..2a952200c 100644 --- a/pkg/common-controller/snapshot_create_test.go +++ b/pkg/common-controller/snapshot_create_test.go @@ -21,7 +21,7 @@ import ( "testing" "time" - crdv1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" + crdv1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" v1 "k8s.io/api/core/v1" storage "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/common-controller/snapshot_delete_test.go b/pkg/common-controller/snapshot_delete_test.go index ca2ea2e6f..be80e01cd 100644 --- a/pkg/common-controller/snapshot_delete_test.go +++ b/pkg/common-controller/snapshot_delete_test.go @@ -20,8 +20,8 @@ import ( //"errors" "testing" - crdv1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" - "github.com/kubernetes-csi/external-snapshotter/pkg/utils" + crdv1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" + "github.com/kubernetes-csi/external-snapshotter/v2/pkg/utils" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) diff --git a/pkg/sidecar-controller/csi_handler.go b/pkg/sidecar-controller/csi_handler.go index f261c8c97..2a7d289ae 100644 --- a/pkg/sidecar-controller/csi_handler.go +++ b/pkg/sidecar-controller/csi_handler.go @@ -22,9 +22,9 @@ import ( "strings" "time" - crdv1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" - "github.com/kubernetes-csi/external-snapshotter/pkg/snapshotter" - "github.com/kubernetes-csi/external-snapshotter/pkg/utils" + crdv1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" + "github.com/kubernetes-csi/external-snapshotter/v2/pkg/snapshotter" + "github.com/kubernetes-csi/external-snapshotter/v2/pkg/utils" ) // Handler is responsible for handling VolumeSnapshot events from informer. diff --git a/pkg/sidecar-controller/framework_test.go b/pkg/sidecar-controller/framework_test.go index 009fea6e8..1d803d465 100644 --- a/pkg/sidecar-controller/framework_test.go +++ b/pkg/sidecar-controller/framework_test.go @@ -25,13 +25,13 @@ import ( "testing" "time" - crdv1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" - clientset "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned" - "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned/fake" - snapshotscheme "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned/scheme" - informers "github.com/kubernetes-csi/external-snapshotter/pkg/client/informers/externalversions" - storagelisters "github.com/kubernetes-csi/external-snapshotter/pkg/client/listers/volumesnapshot/v1beta1" - "github.com/kubernetes-csi/external-snapshotter/pkg/utils" + crdv1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" + clientset "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/clientset/versioned" + "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/clientset/versioned/fake" + snapshotscheme "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/clientset/versioned/scheme" + informers "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/informers/externalversions" + storagelisters "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/listers/volumesnapshot/v1beta1" + "github.com/kubernetes-csi/external-snapshotter/v2/pkg/utils" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/sidecar-controller/snapshot_controller.go b/pkg/sidecar-controller/snapshot_controller.go index 72fd52308..db15a63e8 100644 --- a/pkg/sidecar-controller/snapshot_controller.go +++ b/pkg/sidecar-controller/snapshot_controller.go @@ -21,8 +21,8 @@ import ( "strings" "time" - crdv1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" - "github.com/kubernetes-csi/external-snapshotter/pkg/utils" + crdv1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" + "github.com/kubernetes-csi/external-snapshotter/v2/pkg/utils" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog" diff --git a/pkg/sidecar-controller/snapshot_controller_base.go b/pkg/sidecar-controller/snapshot_controller_base.go index 7612b23cb..94493cd4e 100644 --- a/pkg/sidecar-controller/snapshot_controller_base.go +++ b/pkg/sidecar-controller/snapshot_controller_base.go @@ -20,11 +20,11 @@ import ( "fmt" "time" - crdv1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" - clientset "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned" - storageinformers "github.com/kubernetes-csi/external-snapshotter/pkg/client/informers/externalversions/volumesnapshot/v1beta1" - storagelisters "github.com/kubernetes-csi/external-snapshotter/pkg/client/listers/volumesnapshot/v1beta1" - "github.com/kubernetes-csi/external-snapshotter/pkg/snapshotter" + crdv1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" + clientset "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/clientset/versioned" + storageinformers "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/informers/externalversions/volumesnapshot/v1beta1" + storagelisters "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/listers/volumesnapshot/v1beta1" + "github.com/kubernetes-csi/external-snapshotter/v2/pkg/snapshotter" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" diff --git a/pkg/sidecar-controller/snapshot_controller_test.go b/pkg/sidecar-controller/snapshot_controller_test.go index c3303e3bc..71e86eb45 100644 --- a/pkg/sidecar-controller/snapshot_controller_test.go +++ b/pkg/sidecar-controller/snapshot_controller_test.go @@ -16,8 +16,8 @@ package sidecar_controller import ( "testing" - crdv1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" - "github.com/kubernetes-csi/external-snapshotter/pkg/utils" + crdv1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" + "github.com/kubernetes-csi/external-snapshotter/v2/pkg/utils" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/cache" ) diff --git a/pkg/sidecar-controller/snapshot_delete_test.go b/pkg/sidecar-controller/snapshot_delete_test.go index 8a8bb607d..9a27f8b50 100644 --- a/pkg/sidecar-controller/snapshot_delete_test.go +++ b/pkg/sidecar-controller/snapshot_delete_test.go @@ -23,8 +23,8 @@ import ( "errors" - crdv1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" - "github.com/kubernetes-csi/external-snapshotter/pkg/utils" + crdv1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" + "github.com/kubernetes-csi/external-snapshotter/v2/pkg/utils" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) diff --git a/pkg/utils/util.go b/pkg/utils/util.go index 78866481a..3c249889f 100644 --- a/pkg/utils/util.go +++ b/pkg/utils/util.go @@ -24,7 +24,7 @@ import ( "strconv" "time" - crdv1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" + crdv1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/utils/util_test.go b/pkg/utils/util_test.go index d7159a8c9..b9e3bf8e2 100644 --- a/pkg/utils/util_test.go +++ b/pkg/utils/util_test.go @@ -17,7 +17,7 @@ limitations under the License. package utils import ( - crdv1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" + crdv1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "reflect" diff --git a/v2/README.md b/v2/README.md new file mode 100644 index 000000000..1888d0069 --- /dev/null +++ b/v2/README.md @@ -0,0 +1,18 @@ +This directory mirrors the source code via symlinks. +This makes it possible to vendor v2.x releases of +external-snapshotter with `dep` versions that do not +support semantic imports. Support for that is currently +[pending in dep](https://github.com/golang/dep/pull/1963). + +If users of dep have enabled pruning, they must disable if +for external-snapshotter in their Gopk.toml, like this: + +```toml +[prune] + go-tests = true + unused-packages = true + + [[prune.project]] + name = "github.com/kubernetes-csi/external-snapshotter" + unused-packages = false +``` diff --git a/v2/cmd/csi-snapshotter/main.go b/v2/cmd/csi-snapshotter/main.go new file mode 120000 index 000000000..cd776b4c7 --- /dev/null +++ b/v2/cmd/csi-snapshotter/main.go @@ -0,0 +1 @@ +../../../cmd/csi-snapshotter/main.go \ No newline at end of file diff --git a/v2/cmd/csi-snapshotter/main_test.go b/v2/cmd/csi-snapshotter/main_test.go new file mode 120000 index 000000000..3fedc316b --- /dev/null +++ b/v2/cmd/csi-snapshotter/main_test.go @@ -0,0 +1 @@ +../../../cmd/csi-snapshotter/main_test.go \ No newline at end of file diff --git a/v2/cmd/snapshot-controller/main.go b/v2/cmd/snapshot-controller/main.go new file mode 120000 index 000000000..4d9ee57d1 --- /dev/null +++ b/v2/cmd/snapshot-controller/main.go @@ -0,0 +1 @@ +../../../cmd/snapshot-controller/main.go \ No newline at end of file diff --git a/v2/pkg/apis/volumesnapshot/v1beta1/doc.go b/v2/pkg/apis/volumesnapshot/v1beta1/doc.go new file mode 120000 index 000000000..10a7247f9 --- /dev/null +++ b/v2/pkg/apis/volumesnapshot/v1beta1/doc.go @@ -0,0 +1 @@ +../../../../../pkg/apis/volumesnapshot/v1beta1/doc.go \ No newline at end of file diff --git a/v2/pkg/apis/volumesnapshot/v1beta1/register.go b/v2/pkg/apis/volumesnapshot/v1beta1/register.go new file mode 120000 index 000000000..f1348d2b7 --- /dev/null +++ b/v2/pkg/apis/volumesnapshot/v1beta1/register.go @@ -0,0 +1 @@ +../../../../../pkg/apis/volumesnapshot/v1beta1/register.go \ No newline at end of file diff --git a/v2/pkg/apis/volumesnapshot/v1beta1/types.go b/v2/pkg/apis/volumesnapshot/v1beta1/types.go new file mode 120000 index 000000000..b249fc62c --- /dev/null +++ b/v2/pkg/apis/volumesnapshot/v1beta1/types.go @@ -0,0 +1 @@ +../../../../../pkg/apis/volumesnapshot/v1beta1/types.go \ No newline at end of file diff --git a/v2/pkg/apis/volumesnapshot/v1beta1/zz_generated.deepcopy.go b/v2/pkg/apis/volumesnapshot/v1beta1/zz_generated.deepcopy.go new file mode 120000 index 000000000..30372a306 --- /dev/null +++ b/v2/pkg/apis/volumesnapshot/v1beta1/zz_generated.deepcopy.go @@ -0,0 +1 @@ +../../../../../pkg/apis/volumesnapshot/v1beta1/zz_generated.deepcopy.go \ No newline at end of file diff --git a/v2/pkg/client/clientset/versioned/clientset.go b/v2/pkg/client/clientset/versioned/clientset.go new file mode 120000 index 000000000..5c44ba912 --- /dev/null +++ b/v2/pkg/client/clientset/versioned/clientset.go @@ -0,0 +1 @@ +../../../../../pkg/client/clientset/versioned/clientset.go \ No newline at end of file diff --git a/v2/pkg/client/clientset/versioned/doc.go b/v2/pkg/client/clientset/versioned/doc.go new file mode 120000 index 000000000..e8e682eea --- /dev/null +++ b/v2/pkg/client/clientset/versioned/doc.go @@ -0,0 +1 @@ +../../../../../pkg/client/clientset/versioned/doc.go \ No newline at end of file diff --git a/v2/pkg/client/clientset/versioned/fake/clientset_generated.go b/v2/pkg/client/clientset/versioned/fake/clientset_generated.go new file mode 120000 index 000000000..fc0afbeb6 --- /dev/null +++ b/v2/pkg/client/clientset/versioned/fake/clientset_generated.go @@ -0,0 +1 @@ +../../../../../../pkg/client/clientset/versioned/fake/clientset_generated.go \ No newline at end of file diff --git a/v2/pkg/client/clientset/versioned/fake/doc.go b/v2/pkg/client/clientset/versioned/fake/doc.go new file mode 120000 index 000000000..f2079c154 --- /dev/null +++ b/v2/pkg/client/clientset/versioned/fake/doc.go @@ -0,0 +1 @@ +../../../../../../pkg/client/clientset/versioned/fake/doc.go \ No newline at end of file diff --git a/v2/pkg/client/clientset/versioned/fake/register.go b/v2/pkg/client/clientset/versioned/fake/register.go new file mode 120000 index 000000000..33e497a4b --- /dev/null +++ b/v2/pkg/client/clientset/versioned/fake/register.go @@ -0,0 +1 @@ +../../../../../../pkg/client/clientset/versioned/fake/register.go \ No newline at end of file diff --git a/v2/pkg/client/clientset/versioned/scheme/doc.go b/v2/pkg/client/clientset/versioned/scheme/doc.go new file mode 120000 index 000000000..472b17558 --- /dev/null +++ b/v2/pkg/client/clientset/versioned/scheme/doc.go @@ -0,0 +1 @@ +../../../../../../pkg/client/clientset/versioned/scheme/doc.go \ No newline at end of file diff --git a/v2/pkg/client/clientset/versioned/scheme/register.go b/v2/pkg/client/clientset/versioned/scheme/register.go new file mode 120000 index 000000000..f2a9ed298 --- /dev/null +++ b/v2/pkg/client/clientset/versioned/scheme/register.go @@ -0,0 +1 @@ +../../../../../../pkg/client/clientset/versioned/scheme/register.go \ No newline at end of file diff --git a/v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/doc.go b/v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/doc.go new file mode 120000 index 000000000..ea1f3f450 --- /dev/null +++ b/v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/doc.go @@ -0,0 +1 @@ +../../../../../../../../pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/doc.go \ No newline at end of file diff --git a/v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/fake/doc.go b/v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/fake/doc.go new file mode 120000 index 000000000..436ed904c --- /dev/null +++ b/v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/fake/doc.go @@ -0,0 +1 @@ +../../../../../../../../../pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/fake/doc.go \ No newline at end of file diff --git a/v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshot.go b/v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshot.go new file mode 120000 index 000000000..56d5866d2 --- /dev/null +++ b/v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshot.go @@ -0,0 +1 @@ +../../../../../../../../../pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshot.go \ No newline at end of file diff --git a/v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshot_client.go b/v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshot_client.go new file mode 120000 index 000000000..051fc033f --- /dev/null +++ b/v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshot_client.go @@ -0,0 +1 @@ +../../../../../../../../../pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshot_client.go \ No newline at end of file diff --git a/v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshotclass.go b/v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshotclass.go new file mode 120000 index 000000000..9069924a3 --- /dev/null +++ b/v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshotclass.go @@ -0,0 +1 @@ +../../../../../../../../../pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshotclass.go \ No newline at end of file diff --git a/v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshotcontent.go b/v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshotcontent.go new file mode 120000 index 000000000..ab59e49a5 --- /dev/null +++ b/v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshotcontent.go @@ -0,0 +1 @@ +../../../../../../../../../pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshotcontent.go \ No newline at end of file diff --git a/v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/generated_expansion.go b/v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/generated_expansion.go new file mode 120000 index 000000000..dacace3b9 --- /dev/null +++ b/v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/generated_expansion.go @@ -0,0 +1 @@ +../../../../../../../../pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/generated_expansion.go \ No newline at end of file diff --git a/v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshot.go b/v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshot.go new file mode 120000 index 000000000..c4e57e87a --- /dev/null +++ b/v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshot.go @@ -0,0 +1 @@ +../../../../../../../../pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshot.go \ No newline at end of file diff --git a/v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshot_client.go b/v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshot_client.go new file mode 120000 index 000000000..2203e9c93 --- /dev/null +++ b/v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshot_client.go @@ -0,0 +1 @@ +../../../../../../../../pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshot_client.go \ No newline at end of file diff --git a/v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshotclass.go b/v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshotclass.go new file mode 120000 index 000000000..1012e7e36 --- /dev/null +++ b/v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshotclass.go @@ -0,0 +1 @@ +../../../../../../../../pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshotclass.go \ No newline at end of file diff --git a/v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshotcontent.go b/v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshotcontent.go new file mode 120000 index 000000000..c6ccc3217 --- /dev/null +++ b/v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshotcontent.go @@ -0,0 +1 @@ +../../../../../../../../pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshotcontent.go \ No newline at end of file diff --git a/v2/pkg/client/informers/externalversions/factory.go b/v2/pkg/client/informers/externalversions/factory.go new file mode 120000 index 000000000..34d515986 --- /dev/null +++ b/v2/pkg/client/informers/externalversions/factory.go @@ -0,0 +1 @@ +../../../../../pkg/client/informers/externalversions/factory.go \ No newline at end of file diff --git a/v2/pkg/client/informers/externalversions/generic.go b/v2/pkg/client/informers/externalversions/generic.go new file mode 120000 index 000000000..3e2aa68d6 --- /dev/null +++ b/v2/pkg/client/informers/externalversions/generic.go @@ -0,0 +1 @@ +../../../../../pkg/client/informers/externalversions/generic.go \ No newline at end of file diff --git a/v2/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go b/v2/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go new file mode 120000 index 000000000..b6faa161e --- /dev/null +++ b/v2/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go @@ -0,0 +1 @@ +../../../../../../pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go \ No newline at end of file diff --git a/v2/pkg/client/informers/externalversions/volumesnapshot/interface.go b/v2/pkg/client/informers/externalversions/volumesnapshot/interface.go new file mode 120000 index 000000000..781cb65c0 --- /dev/null +++ b/v2/pkg/client/informers/externalversions/volumesnapshot/interface.go @@ -0,0 +1 @@ +../../../../../../pkg/client/informers/externalversions/volumesnapshot/interface.go \ No newline at end of file diff --git a/v2/pkg/client/informers/externalversions/volumesnapshot/v1beta1/interface.go b/v2/pkg/client/informers/externalversions/volumesnapshot/v1beta1/interface.go new file mode 120000 index 000000000..33d8b66ce --- /dev/null +++ b/v2/pkg/client/informers/externalversions/volumesnapshot/v1beta1/interface.go @@ -0,0 +1 @@ +../../../../../../../pkg/client/informers/externalversions/volumesnapshot/v1beta1/interface.go \ No newline at end of file diff --git a/v2/pkg/client/informers/externalversions/volumesnapshot/v1beta1/volumesnapshot.go b/v2/pkg/client/informers/externalversions/volumesnapshot/v1beta1/volumesnapshot.go new file mode 120000 index 000000000..a0d088859 --- /dev/null +++ b/v2/pkg/client/informers/externalversions/volumesnapshot/v1beta1/volumesnapshot.go @@ -0,0 +1 @@ +../../../../../../../pkg/client/informers/externalversions/volumesnapshot/v1beta1/volumesnapshot.go \ No newline at end of file diff --git a/v2/pkg/client/informers/externalversions/volumesnapshot/v1beta1/volumesnapshotclass.go b/v2/pkg/client/informers/externalversions/volumesnapshot/v1beta1/volumesnapshotclass.go new file mode 120000 index 000000000..24668d5d2 --- /dev/null +++ b/v2/pkg/client/informers/externalversions/volumesnapshot/v1beta1/volumesnapshotclass.go @@ -0,0 +1 @@ +../../../../../../../pkg/client/informers/externalversions/volumesnapshot/v1beta1/volumesnapshotclass.go \ No newline at end of file diff --git a/v2/pkg/client/informers/externalversions/volumesnapshot/v1beta1/volumesnapshotcontent.go b/v2/pkg/client/informers/externalversions/volumesnapshot/v1beta1/volumesnapshotcontent.go new file mode 120000 index 000000000..af01ac1ce --- /dev/null +++ b/v2/pkg/client/informers/externalversions/volumesnapshot/v1beta1/volumesnapshotcontent.go @@ -0,0 +1 @@ +../../../../../../../pkg/client/informers/externalversions/volumesnapshot/v1beta1/volumesnapshotcontent.go \ No newline at end of file diff --git a/v2/pkg/client/listers/volumesnapshot/v1beta1/expansion_generated.go b/v2/pkg/client/listers/volumesnapshot/v1beta1/expansion_generated.go new file mode 120000 index 000000000..7566209c2 --- /dev/null +++ b/v2/pkg/client/listers/volumesnapshot/v1beta1/expansion_generated.go @@ -0,0 +1 @@ +../../../../../../pkg/client/listers/volumesnapshot/v1beta1/expansion_generated.go \ No newline at end of file diff --git a/v2/pkg/client/listers/volumesnapshot/v1beta1/volumesnapshot.go b/v2/pkg/client/listers/volumesnapshot/v1beta1/volumesnapshot.go new file mode 120000 index 000000000..7f49c1180 --- /dev/null +++ b/v2/pkg/client/listers/volumesnapshot/v1beta1/volumesnapshot.go @@ -0,0 +1 @@ +../../../../../../pkg/client/listers/volumesnapshot/v1beta1/volumesnapshot.go \ No newline at end of file diff --git a/v2/pkg/client/listers/volumesnapshot/v1beta1/volumesnapshotclass.go b/v2/pkg/client/listers/volumesnapshot/v1beta1/volumesnapshotclass.go new file mode 120000 index 000000000..91279ccc0 --- /dev/null +++ b/v2/pkg/client/listers/volumesnapshot/v1beta1/volumesnapshotclass.go @@ -0,0 +1 @@ +../../../../../../pkg/client/listers/volumesnapshot/v1beta1/volumesnapshotclass.go \ No newline at end of file diff --git a/v2/pkg/client/listers/volumesnapshot/v1beta1/volumesnapshotcontent.go b/v2/pkg/client/listers/volumesnapshot/v1beta1/volumesnapshotcontent.go new file mode 120000 index 000000000..0559700bf --- /dev/null +++ b/v2/pkg/client/listers/volumesnapshot/v1beta1/volumesnapshotcontent.go @@ -0,0 +1 @@ +../../../../../../pkg/client/listers/volumesnapshot/v1beta1/volumesnapshotcontent.go \ No newline at end of file diff --git a/v2/pkg/common-controller/framework_test.go b/v2/pkg/common-controller/framework_test.go new file mode 120000 index 000000000..2a2153e64 --- /dev/null +++ b/v2/pkg/common-controller/framework_test.go @@ -0,0 +1 @@ +../../../pkg/common-controller/framework_test.go \ No newline at end of file diff --git a/v2/pkg/common-controller/snapshot_controller.go b/v2/pkg/common-controller/snapshot_controller.go new file mode 120000 index 000000000..aa9a5f651 --- /dev/null +++ b/v2/pkg/common-controller/snapshot_controller.go @@ -0,0 +1 @@ +../../../pkg/common-controller/snapshot_controller.go \ No newline at end of file diff --git a/v2/pkg/common-controller/snapshot_controller_base.go b/v2/pkg/common-controller/snapshot_controller_base.go new file mode 120000 index 000000000..fe5c51069 --- /dev/null +++ b/v2/pkg/common-controller/snapshot_controller_base.go @@ -0,0 +1 @@ +../../../pkg/common-controller/snapshot_controller_base.go \ No newline at end of file diff --git a/v2/pkg/common-controller/snapshot_controller_test.go b/v2/pkg/common-controller/snapshot_controller_test.go new file mode 120000 index 000000000..7cbca01b3 --- /dev/null +++ b/v2/pkg/common-controller/snapshot_controller_test.go @@ -0,0 +1 @@ +../../../pkg/common-controller/snapshot_controller_test.go \ No newline at end of file diff --git a/v2/pkg/common-controller/snapshot_create_test.go b/v2/pkg/common-controller/snapshot_create_test.go new file mode 120000 index 000000000..b664b222f --- /dev/null +++ b/v2/pkg/common-controller/snapshot_create_test.go @@ -0,0 +1 @@ +../../../pkg/common-controller/snapshot_create_test.go \ No newline at end of file diff --git a/v2/pkg/common-controller/snapshot_delete_test.go b/v2/pkg/common-controller/snapshot_delete_test.go new file mode 120000 index 000000000..f05e3719f --- /dev/null +++ b/v2/pkg/common-controller/snapshot_delete_test.go @@ -0,0 +1 @@ +../../../pkg/common-controller/snapshot_delete_test.go \ No newline at end of file diff --git a/v2/pkg/common-controller/snapshot_finalizer_test.go b/v2/pkg/common-controller/snapshot_finalizer_test.go new file mode 120000 index 000000000..339a8b5a4 --- /dev/null +++ b/v2/pkg/common-controller/snapshot_finalizer_test.go @@ -0,0 +1 @@ +../../../pkg/common-controller/snapshot_finalizer_test.go \ No newline at end of file diff --git a/v2/pkg/common-controller/snapshot_update_test.go b/v2/pkg/common-controller/snapshot_update_test.go new file mode 120000 index 000000000..165dc2a92 --- /dev/null +++ b/v2/pkg/common-controller/snapshot_update_test.go @@ -0,0 +1 @@ +../../../pkg/common-controller/snapshot_update_test.go \ No newline at end of file diff --git a/v2/pkg/common-controller/snapshotclass_test.go b/v2/pkg/common-controller/snapshotclass_test.go new file mode 120000 index 000000000..55ea44e8d --- /dev/null +++ b/v2/pkg/common-controller/snapshotclass_test.go @@ -0,0 +1 @@ +../../../pkg/common-controller/snapshotclass_test.go \ No newline at end of file diff --git a/v2/pkg/sidecar-controller/content_create_test.go b/v2/pkg/sidecar-controller/content_create_test.go new file mode 120000 index 000000000..c4e8af087 --- /dev/null +++ b/v2/pkg/sidecar-controller/content_create_test.go @@ -0,0 +1 @@ +../../../pkg/sidecar-controller/content_create_test.go \ No newline at end of file diff --git a/v2/pkg/sidecar-controller/csi_handler.go b/v2/pkg/sidecar-controller/csi_handler.go new file mode 120000 index 000000000..5051fca58 --- /dev/null +++ b/v2/pkg/sidecar-controller/csi_handler.go @@ -0,0 +1 @@ +../../../pkg/sidecar-controller/csi_handler.go \ No newline at end of file diff --git a/v2/pkg/sidecar-controller/framework_test.go b/v2/pkg/sidecar-controller/framework_test.go new file mode 120000 index 000000000..cd7942435 --- /dev/null +++ b/v2/pkg/sidecar-controller/framework_test.go @@ -0,0 +1 @@ +../../../pkg/sidecar-controller/framework_test.go \ No newline at end of file diff --git a/v2/pkg/sidecar-controller/snapshot_controller.go b/v2/pkg/sidecar-controller/snapshot_controller.go new file mode 120000 index 000000000..dd90969cb --- /dev/null +++ b/v2/pkg/sidecar-controller/snapshot_controller.go @@ -0,0 +1 @@ +../../../pkg/sidecar-controller/snapshot_controller.go \ No newline at end of file diff --git a/v2/pkg/sidecar-controller/snapshot_controller_base.go b/v2/pkg/sidecar-controller/snapshot_controller_base.go new file mode 120000 index 000000000..c1b8d0b3e --- /dev/null +++ b/v2/pkg/sidecar-controller/snapshot_controller_base.go @@ -0,0 +1 @@ +../../../pkg/sidecar-controller/snapshot_controller_base.go \ No newline at end of file diff --git a/v2/pkg/sidecar-controller/snapshot_controller_test.go b/v2/pkg/sidecar-controller/snapshot_controller_test.go new file mode 120000 index 000000000..3f74ada09 --- /dev/null +++ b/v2/pkg/sidecar-controller/snapshot_controller_test.go @@ -0,0 +1 @@ +../../../pkg/sidecar-controller/snapshot_controller_test.go \ No newline at end of file diff --git a/v2/pkg/sidecar-controller/snapshot_delete_test.go b/v2/pkg/sidecar-controller/snapshot_delete_test.go new file mode 120000 index 000000000..cbdf5610b --- /dev/null +++ b/v2/pkg/sidecar-controller/snapshot_delete_test.go @@ -0,0 +1 @@ +../../../pkg/sidecar-controller/snapshot_delete_test.go \ No newline at end of file diff --git a/v2/pkg/sidecar-controller/snapshot_finalizer_test.go b/v2/pkg/sidecar-controller/snapshot_finalizer_test.go new file mode 120000 index 000000000..724b4c813 --- /dev/null +++ b/v2/pkg/sidecar-controller/snapshot_finalizer_test.go @@ -0,0 +1 @@ +../../../pkg/sidecar-controller/snapshot_finalizer_test.go \ No newline at end of file diff --git a/v2/pkg/snapshotter/snapshotter.go b/v2/pkg/snapshotter/snapshotter.go new file mode 120000 index 000000000..30012fbdf --- /dev/null +++ b/v2/pkg/snapshotter/snapshotter.go @@ -0,0 +1 @@ +../../../pkg/snapshotter/snapshotter.go \ No newline at end of file diff --git a/v2/pkg/snapshotter/snapshotter_test.go b/v2/pkg/snapshotter/snapshotter_test.go new file mode 120000 index 000000000..d090b6eda --- /dev/null +++ b/v2/pkg/snapshotter/snapshotter_test.go @@ -0,0 +1 @@ +../../../pkg/snapshotter/snapshotter_test.go \ No newline at end of file diff --git a/v2/pkg/utils/util.go b/v2/pkg/utils/util.go new file mode 120000 index 000000000..0b6944afb --- /dev/null +++ b/v2/pkg/utils/util.go @@ -0,0 +1 @@ +../../../pkg/utils/util.go \ No newline at end of file diff --git a/v2/pkg/utils/util_test.go b/v2/pkg/utils/util_test.go new file mode 120000 index 000000000..a616b09cc --- /dev/null +++ b/v2/pkg/utils/util_test.go @@ -0,0 +1 @@ +../../../pkg/utils/util_test.go \ No newline at end of file From 3b76de49994e43bf9b683e63a4e3dc4ef657b5a5 Mon Sep 17 00:00:00 2001 From: bells17 Date: Sun, 9 Feb 2020 01:29:36 +0900 Subject: [PATCH 18/56] Added the feature to use secrets of ListSnapshots --- go.mod | 2 +- go.sum | 2 + pkg/common-controller/snapshot_controller.go | 2 +- pkg/sidecar-controller/content_create_test.go | 2 +- pkg/sidecar-controller/csi_handler.go | 6 +- pkg/sidecar-controller/framework_test.go | 8 +- pkg/sidecar-controller/snapshot_controller.go | 25 +- .../snapshot_delete_test.go | 20 +- pkg/snapshotter/snapshotter.go | 5 +- pkg/snapshotter/snapshotter_test.go | 41 +- pkg/utils/util.go | 15 +- pkg/utils/util_test.go | 37 +- .../spec/lib/go/csi/csi.pb.go | 1581 +++++++++-------- vendor/modules.txt | 2 +- 14 files changed, 915 insertions(+), 833 deletions(-) diff --git a/go.mod b/go.mod index c2d7df746..2e7e5f0a9 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/kubernetes-csi/external-snapshotter/v2 go 1.12 require ( - github.com/container-storage-interface/spec v1.1.0 + github.com/container-storage-interface/spec v1.2.0 github.com/golang/mock v1.2.0 github.com/golang/protobuf v1.3.2 github.com/google/go-cmp v0.3.1 // indirect diff --git a/go.sum b/go.sum index c6aa7aff1..d9642079a 100644 --- a/go.sum +++ b/go.sum @@ -28,6 +28,8 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/container-storage-interface/spec v1.1.0 h1:qPsTqtR1VUPvMPeK0UnCZMtXaKGyyLPG8gj/wG6VqMs= github.com/container-storage-interface/spec v1.1.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4= +github.com/container-storage-interface/spec v1.2.0 h1:bD9KIVgaVKKkQ/UbVUY9kCaH/CJbhNxe0eeB4JeJV2s= +github.com/container-storage-interface/spec v1.2.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4= github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= diff --git a/pkg/common-controller/snapshot_controller.go b/pkg/common-controller/snapshot_controller.go index a68e5f429..433027fef 100644 --- a/pkg/common-controller/snapshot_controller.go +++ b/pkg/common-controller/snapshot_controller.go @@ -573,7 +573,7 @@ func (ctrl *csiSnapshotCommonController) getCreateSnapshotInput(snapshot *crdv1. contentName := utils.GetSnapshotContentNameForSnapshot(snapshot) // Resolve snapshotting secret credentials. - snapshotterSecretRef, err := utils.GetSecretReference(class.Parameters, contentName, snapshot) + snapshotterSecretRef, err := utils.GetSecretReference(utils.SnapshotterSecretParams, class.Parameters, contentName, snapshot) if err != nil { return nil, nil, "", nil, err } diff --git a/pkg/sidecar-controller/content_create_test.go b/pkg/sidecar-controller/content_create_test.go index c2dd23139..7358c693a 100644 --- a/pkg/sidecar-controller/content_create_test.go +++ b/pkg/sidecar-controller/content_create_test.go @@ -39,7 +39,7 @@ func TestSyncContent(t *testing.T) { readyToUse: true, }, }, - expectedListCalls: []listCall{{"sid1-1", true, time.Now(), 1, nil}}, + expectedListCalls: []listCall{{"sid1-1", map[string]string{}, true, time.Now(), 1, nil}}, errors: noerrors, test: testSyncContent, }) diff --git a/pkg/sidecar-controller/csi_handler.go b/pkg/sidecar-controller/csi_handler.go index 2a7d289ae..ac6a38f85 100644 --- a/pkg/sidecar-controller/csi_handler.go +++ b/pkg/sidecar-controller/csi_handler.go @@ -31,7 +31,7 @@ import ( type Handler interface { CreateSnapshot(content *crdv1.VolumeSnapshotContent, parameters map[string]string, snapshotterCredentials map[string]string) (string, string, time.Time, int64, bool, error) DeleteSnapshot(content *crdv1.VolumeSnapshotContent, snapshotterCredentials map[string]string) error - GetSnapshotStatus(content *crdv1.VolumeSnapshotContent) (bool, time.Time, int64, error) + GetSnapshotStatus(content *crdv1.VolumeSnapshotContent, snapshotterListCredentials map[string]string) (bool, time.Time, int64, error) } // csiHandler is a handler that calls CSI to create/delete volume snapshot. @@ -103,7 +103,7 @@ func (handler *csiHandler) DeleteSnapshot(content *crdv1.VolumeSnapshotContent, return nil } -func (handler *csiHandler) GetSnapshotStatus(content *crdv1.VolumeSnapshotContent) (bool, time.Time, int64, error) { +func (handler *csiHandler) GetSnapshotStatus(content *crdv1.VolumeSnapshotContent, snapshotterListCredentials map[string]string) (bool, time.Time, int64, error) { ctx, cancel := context.WithTimeout(context.Background(), handler.timeout) defer cancel() @@ -117,7 +117,7 @@ func (handler *csiHandler) GetSnapshotStatus(content *crdv1.VolumeSnapshotConten return false, time.Time{}, 0, fmt.Errorf("failed to list snapshot for content %s: snapshotHandle is missing", content.Name) } - csiSnapshotStatus, timestamp, size, err := handler.snapshotter.GetSnapshotStatus(ctx, snapshotHandle) + csiSnapshotStatus, timestamp, size, err := handler.snapshotter.GetSnapshotStatus(ctx, snapshotHandle, snapshotterListCredentials) if err != nil { return false, time.Time{}, 0, fmt.Errorf("failed to list snapshot for content %s: %q", content.Name, err) } diff --git a/pkg/sidecar-controller/framework_test.go b/pkg/sidecar-controller/framework_test.go index 1d803d465..5d853907e 100644 --- a/pkg/sidecar-controller/framework_test.go +++ b/pkg/sidecar-controller/framework_test.go @@ -810,6 +810,7 @@ func emptyDataSecretAnnotations() map[string]string { type listCall struct { snapshotID string + secrets map[string]string // information to return readyToUse bool createTime time.Time @@ -911,7 +912,7 @@ func (f *fakeSnapshotter) DeleteSnapshot(ctx context.Context, snapshotID string, return call.err } -func (f *fakeSnapshotter) GetSnapshotStatus(ctx context.Context, snapshotID string) (bool, time.Time, int64, error) { +func (f *fakeSnapshotter) GetSnapshotStatus(ctx context.Context, snapshotID string, snapshotterListCredentials map[string]string) (bool, time.Time, int64, error) { if f.listCallCounter >= len(f.listCalls) { f.t.Errorf("Unexpected CSI list Snapshot call: snapshotID=%s, index: %d, calls: %+v", snapshotID, f.createCallCounter, f.createCalls) return false, time.Time{}, 0, fmt.Errorf("unexpected call") @@ -925,6 +926,11 @@ func (f *fakeSnapshotter) GetSnapshotStatus(ctx context.Context, snapshotID stri err = fmt.Errorf("unexpected List snapshot call") } + if !reflect.DeepEqual(call.secrets, snapshotterListCredentials) { + f.t.Errorf("Wrong CSI List Snapshot call: snapshotID=%s, expected secrets %+v, got %+v", snapshotID, call.secrets, snapshotterListCredentials) + err = fmt.Errorf("unexpected Delete Snapshot call") + } + if err != nil { return false, time.Time{}, 0, fmt.Errorf("unexpected call") } diff --git a/pkg/sidecar-controller/snapshot_controller.go b/pkg/sidecar-controller/snapshot_controller.go index db15a63e8..136109efd 100644 --- a/pkg/sidecar-controller/snapshot_controller.go +++ b/pkg/sidecar-controller/snapshot_controller.go @@ -252,10 +252,33 @@ func (ctrl *csiSnapshotSideCarController) checkandUpdateContentStatusOperation(c var readyToUse = false var driverName string var snapshotID string + var snapshotterListCredentials map[string]string if content.Spec.Source.SnapshotHandle != nil { klog.V(5).Infof("checkandUpdateContentStatusOperation: call GetSnapshotStatus for snapshot which is pre-bound to content [%s]", content.Name) - readyToUse, creationTime, size, err = ctrl.handler.GetSnapshotStatus(content) + + if content.Spec.VolumeSnapshotClassName != nil { + class, err := ctrl.getSnapshotClass(*content.Spec.VolumeSnapshotClassName) + if err != nil { + klog.Errorf("failed to getSnapshotClass %s", err) + return nil, fmt.Errorf("cannot get snapshot class for snapshot content %s", err) + } + + snapshotterListSecretRef, err := utils.GetSecretReference(utils.SnapshotterListSecretParams, class.Parameters, content.GetObjectMeta().GetName(), nil) + if err != nil { + klog.Errorf("Failed to get secret reference for snapshot %s: %s", content.Name, err.Error()) + return nil, fmt.Errorf("cannot get secret reference for snapshot content %#v", content.Name) + } + + snapshotterListCredentials, err = utils.GetCredentials(ctrl.client, snapshotterListSecretRef) + if err != nil { + // Continue with deletion, as the secret may have already been deleted. + klog.Errorf("Failed to credentials for snapshot %s: %s", content.Name, err.Error()) + return nil, fmt.Errorf("cannot get credentials for snapshot content %#v", content.Name) + } + } + + readyToUse, creationTime, size, err = ctrl.handler.GetSnapshotStatus(content, snapshotterListCredentials) if err != nil { klog.Errorf("checkandUpdateContentStatusOperation: failed to call get snapshot status to check whether snapshot is ready to use %q", err) return nil, err diff --git a/pkg/sidecar-controller/snapshot_delete_test.go b/pkg/sidecar-controller/snapshot_delete_test.go index 9a27f8b50..85283f5e9 100644 --- a/pkg/sidecar-controller/snapshot_delete_test.go +++ b/pkg/sidecar-controller/snapshot_delete_test.go @@ -156,7 +156,7 @@ func TestDeleteSync(t *testing.T) { readyToUse: true, }, }, - expectedListCalls: []listCall{{"sid1-1", true, time.Now(), 1, nil}}, + expectedListCalls: []listCall{{"sid1-1", map[string]string{}, true, time.Now(), 1, nil}}, expectedDeleteCalls: []deleteCall{{"sid1-1", nil, nil}}, test: testSyncContent, }, @@ -178,7 +178,7 @@ func TestDeleteSync(t *testing.T) { readyToUse: true, }, }, - expectedListCalls: []listCall{{"sid1-2", true, time.Now(), 1, nil}}, + expectedListCalls: []listCall{{"sid1-2", map[string]string{}, true, time.Now(), 1, nil}}, expectedDeleteCalls: []deleteCall{{"sid1-2", nil, nil}}, test: testSyncContent, }, @@ -201,7 +201,7 @@ func TestDeleteSync(t *testing.T) { }, expectedDeleteCalls: []deleteCall{{"sid1-3", nil, fmt.Errorf("mock csi driver delete error")}}, expectedEvents: []string{"Warning SnapshotDeleteError"}, - expectedListCalls: []listCall{{"sid1-3", true, time.Now(), 1, nil}}, + expectedListCalls: []listCall{{"sid1-3", map[string]string{}, true, time.Now(), 1, nil}}, test: testSyncContent, }, { @@ -216,7 +216,7 @@ func TestDeleteSync(t *testing.T) { name: "1-5 - csi driver delete snapshot returns error, bound finalizer should remain", initialContents: newContentArrayWithDeletionTimestamp("content1-5", "sid1-5", "snap1-5", "sid1-5", validSecretClass, "", "snap1-5-volumehandle", deletionPolicy, nil, &defaultSize, true, &timeNowMetav1), expectedContents: newContentArrayWithDeletionTimestamp("content1-5", "sid1-5", "snap1-5", "sid1-5", validSecretClass, "", "snap1-5-volumehandle", deletionPolicy, nil, &defaultSize, true, &timeNowMetav1), - expectedListCalls: []listCall{{"sid1-5", true, time.Now(), 1000, nil}}, + expectedListCalls: []listCall{{"sid1-5", map[string]string{}, true, time.Now(), 1000, nil}}, expectedDeleteCalls: []deleteCall{{"sid1-5", nil, errors.New("mock csi driver delete error")}}, expectedEvents: []string{"Warning SnapshotDeleteError"}, errors: noerrors, @@ -227,7 +227,7 @@ func TestDeleteSync(t *testing.T) { name: "1-6 - content is deleted before deleting", initialContents: newContentArray("content1-6", "sid1-6", "snap1-6", "sid1-6", classGold, "sid1-6", "", deletionPolicy, nil, nil, true), expectedContents: nocontents, - expectedListCalls: []listCall{{"sid1-6", false, time.Now(), 0, nil}}, + expectedListCalls: []listCall{{"sid1-6", nil, false, time.Now(), 0, nil}}, expectedDeleteCalls: []deleteCall{{"sid1-6", map[string]string{"foo": "bar"}, nil}}, expectedEvents: noevents, errors: noerrors, @@ -243,7 +243,7 @@ func TestDeleteSync(t *testing.T) { initialContents: newContentArrayWithReadyToUse("content1-7", "", "snap1-7", "sid1-7", validSecretClass, "sid1-7", "", deletePolicy, nil, &defaultSize, &True, true), expectedContents: newContentArrayWithReadyToUse("content1-7", "", "snap1-7", "sid1-7", validSecretClass, "sid1-7", "", deletePolicy, nil, &defaultSize, &True, true), expectedEvents: noevents, - expectedListCalls: []listCall{{"sid1-7", true, time.Now(), 1000, nil}}, + expectedListCalls: []listCall{{"sid1-7", map[string]string{}, true, time.Now(), 1000, nil}}, initialSecrets: []*v1.Secret{secret()}, errors: noerrors, test: testSyncContent, @@ -253,7 +253,7 @@ func TestDeleteSync(t *testing.T) { initialContents: newContentArrayWithReadyToUse("content1-8", "sid1-8", "none-existed-snapshot", "sid1-8", validSecretClass, "sid1-8", "", retainPolicy, nil, &defaultSize, &True, true), expectedContents: newContentArrayWithReadyToUse("content1-8", "sid1-8", "none-existed-snapshot", "sid1-8", validSecretClass, "sid1-8", "", retainPolicy, nil, &defaultSize, &True, true), expectedEvents: noevents, - expectedListCalls: []listCall{{"sid1-8", true, time.Now(), 0, nil}}, + expectedListCalls: []listCall{{"sid1-8", map[string]string{}, true, time.Now(), 0, nil}}, errors: noerrors, test: testSyncContent, }, @@ -262,7 +262,7 @@ func TestDeleteSync(t *testing.T) { initialContents: newContentArrayWithDeletionTimestamp("content1-9", "sid1-9", "snap1-9", "sid1-9", emptySecretClass, "", "snap1-9-volumehandle", deletePolicy, nil, &defaultSize, true, &timeNowMetav1), expectedContents: newContentArrayWithDeletionTimestamp("content1-9", "sid1-9", "snap1-9", "", emptySecretClass, "", "snap1-9-volumehandle", deletePolicy, nil, &defaultSize, false, &timeNowMetav1), expectedEvents: noevents, - expectedListCalls: []listCall{{"sid1-9", true, time.Now(), 0, nil}}, + expectedListCalls: []listCall{{"sid1-9", map[string]string{}, true, time.Now(), 0, nil}}, errors: noerrors, initialSecrets: []*v1.Secret{}, // secret does not exist expectedDeleteCalls: []deleteCall{{"sid1-9", nil, nil}}, @@ -273,7 +273,7 @@ func TestDeleteSync(t *testing.T) { initialContents: newContentArrayWithDeletionTimestamp("content1-10", "sid1-10", "snap1-10", "sid1-10", emptySecretClass, "", "snap1-10-volumehandle", retainPolicy, nil, &defaultSize, true, &timeNowMetav1), expectedContents: newContentArrayWithDeletionTimestamp("content1-10", "sid1-10", "snap1-10", "sid1-10", emptySecretClass, "", "snap1-10-volumehandle", retainPolicy, nil, &defaultSize, false, &timeNowMetav1), expectedEvents: noevents, - expectedListCalls: []listCall{{"sid1-10", true, time.Now(), 0, nil}}, + expectedListCalls: []listCall{{"sid1-10", map[string]string{}, true, time.Now(), 0, nil}}, errors: noerrors, initialSecrets: []*v1.Secret{}, test: testSyncContent, @@ -292,7 +292,7 @@ func TestDeleteSync(t *testing.T) { initialContents: newContentArrayWithDeletionTimestamp("content1-12", "sid1-12", "snap1-12", "sid1-12", emptySecretClass, "sid1-12", "", retainPolicy, nil, &defaultSize, true, &timeNowMetav1), expectedContents: newContentArrayWithDeletionTimestamp("content1-12", "sid1-12", "snap1-12", "sid1-12", emptySecretClass, "sid1-12", "", retainPolicy, nil, &defaultSize, false, &timeNowMetav1), expectedEvents: noevents, - expectedListCalls: []listCall{{"sid1-12", true, time.Now(), 0, nil}}, + expectedListCalls: []listCall{{"sid1-12", map[string]string{}, true, time.Now(), 0, nil}}, errors: noerrors, initialSecrets: []*v1.Secret{}, test: testSyncContent, diff --git a/pkg/snapshotter/snapshotter.go b/pkg/snapshotter/snapshotter.go index 068142126..379b6c000 100644 --- a/pkg/snapshotter/snapshotter.go +++ b/pkg/snapshotter/snapshotter.go @@ -39,7 +39,7 @@ type Snapshotter interface { DeleteSnapshot(ctx context.Context, snapshotID string, snapshotterCredentials map[string]string) (err error) // GetSnapshotStatus returns if a snapshot is ready to use, creation time, and restore size. - GetSnapshotStatus(ctx context.Context, snapshotID string) (bool, time.Time, int64, error) + GetSnapshotStatus(ctx context.Context, snapshotID string, snapshotterListCredentials map[string]string) (bool, time.Time, int64, error) } type snapshot struct { @@ -112,7 +112,7 @@ func (s *snapshot) isListSnapshotsSupported(ctx context.Context) (bool, error) { return false, nil } -func (s *snapshot) GetSnapshotStatus(ctx context.Context, snapshotID string) (bool, time.Time, int64, error) { +func (s *snapshot) GetSnapshotStatus(ctx context.Context, snapshotID string, snapshotterListCredentials map[string]string) (bool, time.Time, int64, error) { klog.V(5).Infof("GetSnapshotStatus: %s", snapshotID) client := csi.NewControllerClient(s.conn) @@ -127,6 +127,7 @@ func (s *snapshot) GetSnapshotStatus(ctx context.Context, snapshotID string) (bo } req := csi.ListSnapshotsRequest{ SnapshotId: snapshotID, + Secrets: snapshotterListCredentials, } rsp, err := client.ListSnapshots(ctx, &req) if err != nil { diff --git a/pkg/snapshotter/snapshotter_test.go b/pkg/snapshotter/snapshotter_test.go index 67cec6bef..25c14cb5d 100644 --- a/pkg/snapshotter/snapshotter_test.go +++ b/pkg/snapshotter/snapshotter_test.go @@ -362,17 +362,24 @@ func TestGetSnapshotStatus(t *testing.T) { }, } + secret := map[string]string{"foo": "bar"} + secretRequest := &csi.ListSnapshotsRequest{ + SnapshotId: defaultID, + Secrets: secret, + } + tests := []struct { - name string - snapshotID string - listSnapshotsSupported bool - input *csi.ListSnapshotsRequest - output *csi.ListSnapshotsResponse - injectError codes.Code - expectError bool - expectReady bool - expectCreateAt time.Time - expectSize int64 + name string + snapshotID string + snapshotterListCredentials map[string]string + listSnapshotsSupported bool + input *csi.ListSnapshotsRequest + output *csi.ListSnapshotsResponse + injectError codes.Code + expectError bool + expectReady bool + expectCreateAt time.Time + expectSize int64 }{ { name: "success", @@ -385,6 +392,18 @@ func TestGetSnapshotStatus(t *testing.T) { expectCreateAt: createTime, expectSize: size, }, + { + name: "secret", + snapshotID: defaultID, + snapshotterListCredentials: secret, + listSnapshotsSupported: true, + input: secretRequest, + output: defaultResponse, + expectError: false, + expectReady: true, + expectCreateAt: createTime, + expectSize: size, + }, { name: "ListSnapshots not supported", snapshotID: defaultID, @@ -455,7 +474,7 @@ func TestGetSnapshotStatus(t *testing.T) { } s := NewSnapshotter(csiConn) - ready, createTime, size, err := s.GetSnapshotStatus(context.Background(), test.snapshotID) + ready, createTime, size, err := s.GetSnapshotStatus(context.Background(), test.snapshotID, test.snapshotterListCredentials) if test.expectError && err == nil { t.Errorf("test %q: Expected error, got none", test.name) } diff --git a/pkg/utils/util.go b/pkg/utils/util.go index 3c249889f..c2a155abe 100644 --- a/pkg/utils/util.go +++ b/pkg/utils/util.go @@ -56,6 +56,9 @@ const ( prefixedSnapshotterSecretNameKey = csiParameterPrefix + "snapshotter-secret-name" prefixedSnapshotterSecretNamespaceKey = csiParameterPrefix + "snapshotter-secret-namespace" + prefixedSnapshotterListSecretNameKey = csiParameterPrefix + "snapshotter-list-secret-name" + prefixedSnapshotterListSecretNamespaceKey = csiParameterPrefix + "snapshotter-list-secret-namespace" + // Name of finalizer on VolumeSnapshotContents that are bound by VolumeSnapshots VolumeSnapshotContentFinalizer = "snapshot.storage.kubernetes.io/volumesnapshotcontent-bound-protection" // Name of finalizer on VolumeSnapshot that is being used as a source to create a PVC @@ -81,12 +84,18 @@ const ( AnnDeletionSecretRefNamespace = "snapshot.storage.kubernetes.io/deletion-secret-namespace" ) -var snapshotterSecretParams = secretParamsMap{ +var SnapshotterSecretParams = secretParamsMap{ name: "Snapshotter", secretNameKey: prefixedSnapshotterSecretNameKey, secretNamespaceKey: prefixedSnapshotterSecretNamespaceKey, } +var SnapshotterListSecretParams = secretParamsMap{ + name: "SnapshotterList", + secretNameKey: prefixedSnapshotterListSecretNameKey, + secretNamespaceKey: prefixedSnapshotterListSecretNamespaceKey, +} + func SnapshotKey(vs *crdv1.VolumeSnapshot) string { return fmt.Sprintf("%s/%s", vs.Namespace, vs.Name) } @@ -222,8 +231,8 @@ func verifyAndGetSecretNameAndNamespaceTemplate(secret secretParamsMap, snapshot // - the nameTemplate or namespaceTemplate contains a token that cannot be resolved // - the resolved name is not a valid secret name // - the resolved namespace is not a valid namespace name -func GetSecretReference(snapshotClassParams map[string]string, snapContentName string, snapshot *crdv1.VolumeSnapshot) (*v1.SecretReference, error) { - nameTemplate, namespaceTemplate, err := verifyAndGetSecretNameAndNamespaceTemplate(snapshotterSecretParams, snapshotClassParams) +func GetSecretReference(secretParams secretParamsMap, snapshotClassParams map[string]string, snapContentName string, snapshot *crdv1.VolumeSnapshot) (*v1.SecretReference, error) { + nameTemplate, namespaceTemplate, err := verifyAndGetSecretNameAndNamespaceTemplate(secretParams, snapshotClassParams) if err != nil { return nil, fmt.Errorf("failed to get name and namespace template from params: %v", err) } diff --git a/pkg/utils/util_test.go b/pkg/utils/util_test.go index b9e3bf8e2..d8e853b59 100644 --- a/pkg/utils/util_test.go +++ b/pkg/utils/util_test.go @@ -17,15 +17,17 @@ limitations under the License. package utils import ( - crdv1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" - "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "reflect" "testing" + + crdv1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func TestGetSecretReference(t *testing.T) { testcases := map[string]struct { + secretParams secretParamsMap params map[string]string snapContentName string snapshot *crdv1.VolumeSnapshot @@ -33,25 +35,30 @@ func TestGetSecretReference(t *testing.T) { expectErr bool }{ "no params": { - params: nil, - expectRef: nil, + secretParams: SnapshotterSecretParams, + params: nil, + expectRef: nil, }, "namespace, no name": { - params: map[string]string{prefixedSnapshotterSecretNamespaceKey: "foo"}, - expectErr: true, + secretParams: SnapshotterSecretParams, + params: map[string]string{prefixedSnapshotterSecretNamespaceKey: "foo"}, + expectErr: true, }, "simple - valid": { - params: map[string]string{prefixedSnapshotterSecretNameKey: "name", prefixedSnapshotterSecretNamespaceKey: "ns"}, - snapshot: &crdv1.VolumeSnapshot{}, - expectRef: &v1.SecretReference{Name: "name", Namespace: "ns"}, + secretParams: SnapshotterSecretParams, + params: map[string]string{prefixedSnapshotterSecretNameKey: "name", prefixedSnapshotterSecretNamespaceKey: "ns"}, + snapshot: &crdv1.VolumeSnapshot{}, + expectRef: &v1.SecretReference{Name: "name", Namespace: "ns"}, }, "simple - invalid name": { - params: map[string]string{prefixedSnapshotterSecretNameKey: "bad name", prefixedSnapshotterSecretNamespaceKey: "ns"}, - snapshot: &crdv1.VolumeSnapshot{}, - expectRef: nil, - expectErr: true, + secretParams: SnapshotterSecretParams, + params: map[string]string{prefixedSnapshotterSecretNameKey: "bad name", prefixedSnapshotterSecretNamespaceKey: "ns"}, + snapshot: &crdv1.VolumeSnapshot{}, + expectRef: nil, + expectErr: true, }, "template - invalid": { + secretParams: SnapshotterSecretParams, params: map[string]string{ prefixedSnapshotterSecretNameKey: "static-${volumesnapshotcontent.name}-${volumesnapshot.namespace}-${volumesnapshot.name}-${volumesnapshot.annotations['akey']}", prefixedSnapshotterSecretNamespaceKey: "static-${volumesnapshotcontent.name}-${volumesnapshot.namespace}", @@ -71,7 +78,7 @@ func TestGetSecretReference(t *testing.T) { for k, tc := range testcases { t.Run(k, func(t *testing.T) { - ref, err := GetSecretReference(tc.params, tc.snapContentName, tc.snapshot) + ref, err := GetSecretReference(tc.secretParams, tc.params, tc.snapContentName, tc.snapshot) if err != nil { if tc.expectErr { return diff --git a/vendor/github.com/container-storage-interface/spec/lib/go/csi/csi.pb.go b/vendor/github.com/container-storage-interface/spec/lib/go/csi/csi.pb.go index d84db2fca..e2e02d808 100644 --- a/vendor/github.com/container-storage-interface/spec/lib/go/csi/csi.pb.go +++ b/vendor/github.com/container-storage-interface/spec/lib/go/csi/csi.pb.go @@ -3,16 +3,17 @@ package csi -import proto "github.com/golang/protobuf/proto" -import fmt "fmt" -import math "math" -import descriptor "github.com/golang/protobuf/protoc-gen-go/descriptor" -import timestamp "github.com/golang/protobuf/ptypes/timestamp" -import wrappers "github.com/golang/protobuf/ptypes/wrappers" - import ( - context "golang.org/x/net/context" + context "context" + fmt "fmt" + proto "github.com/golang/protobuf/proto" + descriptor "github.com/golang/protobuf/protoc-gen-go/descriptor" + timestamp "github.com/golang/protobuf/ptypes/timestamp" + wrappers "github.com/golang/protobuf/ptypes/wrappers" grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" + math "math" ) // Reference imports to suppress errors if they are not otherwise used. @@ -24,7 +25,7 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package type PluginCapability_Service_Type int32 @@ -53,6 +54,7 @@ var PluginCapability_Service_Type_name = map[int32]string{ 1: "CONTROLLER_SERVICE", 2: "VOLUME_ACCESSIBILITY_CONSTRAINTS", } + var PluginCapability_Service_Type_value = map[string]int32{ "UNKNOWN": 0, "CONTROLLER_SERVICE": 1, @@ -62,8 +64,9 @@ var PluginCapability_Service_Type_value = map[string]int32{ func (x PluginCapability_Service_Type) String() string { return proto.EnumName(PluginCapability_Service_Type_name, int32(x)) } + func (PluginCapability_Service_Type) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{4, 0, 0} + return fileDescriptor_9cdb00adce470e01, []int{4, 0, 0} } type PluginCapability_VolumeExpansion_Type int32 @@ -114,6 +117,7 @@ var PluginCapability_VolumeExpansion_Type_name = map[int32]string{ 1: "ONLINE", 2: "OFFLINE", } + var PluginCapability_VolumeExpansion_Type_value = map[string]int32{ "UNKNOWN": 0, "ONLINE": 1, @@ -123,8 +127,9 @@ var PluginCapability_VolumeExpansion_Type_value = map[string]int32{ func (x PluginCapability_VolumeExpansion_Type) String() string { return proto.EnumName(PluginCapability_VolumeExpansion_Type_name, int32(x)) } + func (PluginCapability_VolumeExpansion_Type) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{4, 1, 0} + return fileDescriptor_9cdb00adce470e01, []int{4, 1, 0} } type VolumeCapability_AccessMode_Mode int32 @@ -155,6 +160,7 @@ var VolumeCapability_AccessMode_Mode_name = map[int32]string{ 4: "MULTI_NODE_SINGLE_WRITER", 5: "MULTI_NODE_MULTI_WRITER", } + var VolumeCapability_AccessMode_Mode_value = map[string]int32{ "UNKNOWN": 0, "SINGLE_NODE_WRITER": 1, @@ -167,8 +173,9 @@ var VolumeCapability_AccessMode_Mode_value = map[string]int32{ func (x VolumeCapability_AccessMode_Mode) String() string { return proto.EnumName(VolumeCapability_AccessMode_Mode_name, int32(x)) } + func (VolumeCapability_AccessMode_Mode) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{10, 2, 0} + return fileDescriptor_9cdb00adce470e01, []int{10, 2, 0} } type ControllerServiceCapability_RPC_Type int32 @@ -195,38 +202,45 @@ const ( ControllerServiceCapability_RPC_PUBLISH_READONLY ControllerServiceCapability_RPC_Type = 8 // See VolumeExpansion for details. ControllerServiceCapability_RPC_EXPAND_VOLUME ControllerServiceCapability_RPC_Type = 9 + // Indicates the SP supports the + // ListVolumesResponse.entry.published_nodes field + ControllerServiceCapability_RPC_LIST_VOLUMES_PUBLISHED_NODES ControllerServiceCapability_RPC_Type = 10 ) var ControllerServiceCapability_RPC_Type_name = map[int32]string{ - 0: "UNKNOWN", - 1: "CREATE_DELETE_VOLUME", - 2: "PUBLISH_UNPUBLISH_VOLUME", - 3: "LIST_VOLUMES", - 4: "GET_CAPACITY", - 5: "CREATE_DELETE_SNAPSHOT", - 6: "LIST_SNAPSHOTS", - 7: "CLONE_VOLUME", - 8: "PUBLISH_READONLY", - 9: "EXPAND_VOLUME", + 0: "UNKNOWN", + 1: "CREATE_DELETE_VOLUME", + 2: "PUBLISH_UNPUBLISH_VOLUME", + 3: "LIST_VOLUMES", + 4: "GET_CAPACITY", + 5: "CREATE_DELETE_SNAPSHOT", + 6: "LIST_SNAPSHOTS", + 7: "CLONE_VOLUME", + 8: "PUBLISH_READONLY", + 9: "EXPAND_VOLUME", + 10: "LIST_VOLUMES_PUBLISHED_NODES", } + var ControllerServiceCapability_RPC_Type_value = map[string]int32{ - "UNKNOWN": 0, - "CREATE_DELETE_VOLUME": 1, - "PUBLISH_UNPUBLISH_VOLUME": 2, - "LIST_VOLUMES": 3, - "GET_CAPACITY": 4, - "CREATE_DELETE_SNAPSHOT": 5, - "LIST_SNAPSHOTS": 6, - "CLONE_VOLUME": 7, - "PUBLISH_READONLY": 8, - "EXPAND_VOLUME": 9, + "UNKNOWN": 0, + "CREATE_DELETE_VOLUME": 1, + "PUBLISH_UNPUBLISH_VOLUME": 2, + "LIST_VOLUMES": 3, + "GET_CAPACITY": 4, + "CREATE_DELETE_SNAPSHOT": 5, + "LIST_SNAPSHOTS": 6, + "CLONE_VOLUME": 7, + "PUBLISH_READONLY": 8, + "EXPAND_VOLUME": 9, + "LIST_VOLUMES_PUBLISHED_NODES": 10, } func (x ControllerServiceCapability_RPC_Type) String() string { return proto.EnumName(ControllerServiceCapability_RPC_Type_name, int32(x)) } + func (ControllerServiceCapability_RPC_Type) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{29, 0, 0} + return fileDescriptor_9cdb00adce470e01, []int{29, 0, 0} } type VolumeUsage_Unit int32 @@ -242,6 +256,7 @@ var VolumeUsage_Unit_name = map[int32]string{ 1: "BYTES", 2: "INODES", } + var VolumeUsage_Unit_value = map[string]int32{ "UNKNOWN": 0, "BYTES": 1, @@ -251,8 +266,9 @@ var VolumeUsage_Unit_value = map[string]int32{ func (x VolumeUsage_Unit) String() string { return proto.EnumName(VolumeUsage_Unit_name, int32(x)) } + func (VolumeUsage_Unit) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{49, 0} + return fileDescriptor_9cdb00adce470e01, []int{49, 0} } type NodeServiceCapability_RPC_Type int32 @@ -274,6 +290,7 @@ var NodeServiceCapability_RPC_Type_name = map[int32]string{ 2: "GET_VOLUME_STATS", 3: "EXPAND_VOLUME", } + var NodeServiceCapability_RPC_Type_value = map[string]int32{ "UNKNOWN": 0, "STAGE_UNSTAGE_VOLUME": 1, @@ -284,8 +301,9 @@ var NodeServiceCapability_RPC_Type_value = map[string]int32{ func (x NodeServiceCapability_RPC_Type) String() string { return proto.EnumName(NodeServiceCapability_RPC_Type_name, int32(x)) } + func (NodeServiceCapability_RPC_Type) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{52, 0, 0} + return fileDescriptor_9cdb00adce470e01, []int{52, 0, 0} } type GetPluginInfoRequest struct { @@ -298,16 +316,17 @@ func (m *GetPluginInfoRequest) Reset() { *m = GetPluginInfoRequest{} } func (m *GetPluginInfoRequest) String() string { return proto.CompactTextString(m) } func (*GetPluginInfoRequest) ProtoMessage() {} func (*GetPluginInfoRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{0} + return fileDescriptor_9cdb00adce470e01, []int{0} } + func (m *GetPluginInfoRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_GetPluginInfoRequest.Unmarshal(m, b) } func (m *GetPluginInfoRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_GetPluginInfoRequest.Marshal(b, m, deterministic) } -func (dst *GetPluginInfoRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_GetPluginInfoRequest.Merge(dst, src) +func (m *GetPluginInfoRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetPluginInfoRequest.Merge(m, src) } func (m *GetPluginInfoRequest) XXX_Size() int { return xxx_messageInfo_GetPluginInfoRequest.Size(m) @@ -340,16 +359,17 @@ func (m *GetPluginInfoResponse) Reset() { *m = GetPluginInfoResponse{} } func (m *GetPluginInfoResponse) String() string { return proto.CompactTextString(m) } func (*GetPluginInfoResponse) ProtoMessage() {} func (*GetPluginInfoResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{1} + return fileDescriptor_9cdb00adce470e01, []int{1} } + func (m *GetPluginInfoResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_GetPluginInfoResponse.Unmarshal(m, b) } func (m *GetPluginInfoResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_GetPluginInfoResponse.Marshal(b, m, deterministic) } -func (dst *GetPluginInfoResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_GetPluginInfoResponse.Merge(dst, src) +func (m *GetPluginInfoResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetPluginInfoResponse.Merge(m, src) } func (m *GetPluginInfoResponse) XXX_Size() int { return xxx_messageInfo_GetPluginInfoResponse.Size(m) @@ -391,16 +411,17 @@ func (m *GetPluginCapabilitiesRequest) Reset() { *m = GetPluginCapabilit func (m *GetPluginCapabilitiesRequest) String() string { return proto.CompactTextString(m) } func (*GetPluginCapabilitiesRequest) ProtoMessage() {} func (*GetPluginCapabilitiesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{2} + return fileDescriptor_9cdb00adce470e01, []int{2} } + func (m *GetPluginCapabilitiesRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_GetPluginCapabilitiesRequest.Unmarshal(m, b) } func (m *GetPluginCapabilitiesRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_GetPluginCapabilitiesRequest.Marshal(b, m, deterministic) } -func (dst *GetPluginCapabilitiesRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_GetPluginCapabilitiesRequest.Merge(dst, src) +func (m *GetPluginCapabilitiesRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetPluginCapabilitiesRequest.Merge(m, src) } func (m *GetPluginCapabilitiesRequest) XXX_Size() int { return xxx_messageInfo_GetPluginCapabilitiesRequest.Size(m) @@ -424,16 +445,17 @@ func (m *GetPluginCapabilitiesResponse) Reset() { *m = GetPluginCapabili func (m *GetPluginCapabilitiesResponse) String() string { return proto.CompactTextString(m) } func (*GetPluginCapabilitiesResponse) ProtoMessage() {} func (*GetPluginCapabilitiesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{3} + return fileDescriptor_9cdb00adce470e01, []int{3} } + func (m *GetPluginCapabilitiesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_GetPluginCapabilitiesResponse.Unmarshal(m, b) } func (m *GetPluginCapabilitiesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_GetPluginCapabilitiesResponse.Marshal(b, m, deterministic) } -func (dst *GetPluginCapabilitiesResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_GetPluginCapabilitiesResponse.Merge(dst, src) +func (m *GetPluginCapabilitiesResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetPluginCapabilitiesResponse.Merge(m, src) } func (m *GetPluginCapabilitiesResponse) XXX_Size() int { return xxx_messageInfo_GetPluginCapabilitiesResponse.Size(m) @@ -466,16 +488,17 @@ func (m *PluginCapability) Reset() { *m = PluginCapability{} } func (m *PluginCapability) String() string { return proto.CompactTextString(m) } func (*PluginCapability) ProtoMessage() {} func (*PluginCapability) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{4} + return fileDescriptor_9cdb00adce470e01, []int{4} } + func (m *PluginCapability) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_PluginCapability.Unmarshal(m, b) } func (m *PluginCapability) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_PluginCapability.Marshal(b, m, deterministic) } -func (dst *PluginCapability) XXX_Merge(src proto.Message) { - xxx_messageInfo_PluginCapability.Merge(dst, src) +func (m *PluginCapability) XXX_Merge(src proto.Message) { + xxx_messageInfo_PluginCapability.Merge(m, src) } func (m *PluginCapability) XXX_Size() int { return xxx_messageInfo_PluginCapability.Size(m) @@ -523,80 +546,14 @@ func (m *PluginCapability) GetVolumeExpansion() *PluginCapability_VolumeExpansio return nil } -// XXX_OneofFuncs is for the internal use of the proto package. -func (*PluginCapability) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { - return _PluginCapability_OneofMarshaler, _PluginCapability_OneofUnmarshaler, _PluginCapability_OneofSizer, []interface{}{ +// XXX_OneofWrappers is for the internal use of the proto package. +func (*PluginCapability) XXX_OneofWrappers() []interface{} { + return []interface{}{ (*PluginCapability_Service_)(nil), (*PluginCapability_VolumeExpansion_)(nil), } } -func _PluginCapability_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { - m := msg.(*PluginCapability) - // type - switch x := m.Type.(type) { - case *PluginCapability_Service_: - b.EncodeVarint(1<<3 | proto.WireBytes) - if err := b.EncodeMessage(x.Service); err != nil { - return err - } - case *PluginCapability_VolumeExpansion_: - b.EncodeVarint(2<<3 | proto.WireBytes) - if err := b.EncodeMessage(x.VolumeExpansion); err != nil { - return err - } - case nil: - default: - return fmt.Errorf("PluginCapability.Type has unexpected type %T", x) - } - return nil -} - -func _PluginCapability_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { - m := msg.(*PluginCapability) - switch tag { - case 1: // type.service - if wire != proto.WireBytes { - return true, proto.ErrInternalBadWireType - } - msg := new(PluginCapability_Service) - err := b.DecodeMessage(msg) - m.Type = &PluginCapability_Service_{msg} - return true, err - case 2: // type.volume_expansion - if wire != proto.WireBytes { - return true, proto.ErrInternalBadWireType - } - msg := new(PluginCapability_VolumeExpansion) - err := b.DecodeMessage(msg) - m.Type = &PluginCapability_VolumeExpansion_{msg} - return true, err - default: - return false, nil - } -} - -func _PluginCapability_OneofSizer(msg proto.Message) (n int) { - m := msg.(*PluginCapability) - // type - switch x := m.Type.(type) { - case *PluginCapability_Service_: - s := proto.Size(x.Service) - n += 1 // tag and wire - n += proto.SizeVarint(uint64(s)) - n += s - case *PluginCapability_VolumeExpansion_: - s := proto.Size(x.VolumeExpansion) - n += 1 // tag and wire - n += proto.SizeVarint(uint64(s)) - n += s - case nil: - default: - panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) - } - return n -} - type PluginCapability_Service struct { Type PluginCapability_Service_Type `protobuf:"varint,1,opt,name=type,proto3,enum=csi.v1.PluginCapability_Service_Type" json:"type,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -608,16 +565,17 @@ func (m *PluginCapability_Service) Reset() { *m = PluginCapability_Servi func (m *PluginCapability_Service) String() string { return proto.CompactTextString(m) } func (*PluginCapability_Service) ProtoMessage() {} func (*PluginCapability_Service) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{4, 0} + return fileDescriptor_9cdb00adce470e01, []int{4, 0} } + func (m *PluginCapability_Service) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_PluginCapability_Service.Unmarshal(m, b) } func (m *PluginCapability_Service) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_PluginCapability_Service.Marshal(b, m, deterministic) } -func (dst *PluginCapability_Service) XXX_Merge(src proto.Message) { - xxx_messageInfo_PluginCapability_Service.Merge(dst, src) +func (m *PluginCapability_Service) XXX_Merge(src proto.Message) { + xxx_messageInfo_PluginCapability_Service.Merge(m, src) } func (m *PluginCapability_Service) XXX_Size() int { return xxx_messageInfo_PluginCapability_Service.Size(m) @@ -646,16 +604,17 @@ func (m *PluginCapability_VolumeExpansion) Reset() { *m = PluginCapabili func (m *PluginCapability_VolumeExpansion) String() string { return proto.CompactTextString(m) } func (*PluginCapability_VolumeExpansion) ProtoMessage() {} func (*PluginCapability_VolumeExpansion) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{4, 1} + return fileDescriptor_9cdb00adce470e01, []int{4, 1} } + func (m *PluginCapability_VolumeExpansion) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_PluginCapability_VolumeExpansion.Unmarshal(m, b) } func (m *PluginCapability_VolumeExpansion) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_PluginCapability_VolumeExpansion.Marshal(b, m, deterministic) } -func (dst *PluginCapability_VolumeExpansion) XXX_Merge(src proto.Message) { - xxx_messageInfo_PluginCapability_VolumeExpansion.Merge(dst, src) +func (m *PluginCapability_VolumeExpansion) XXX_Merge(src proto.Message) { + xxx_messageInfo_PluginCapability_VolumeExpansion.Merge(m, src) } func (m *PluginCapability_VolumeExpansion) XXX_Size() int { return xxx_messageInfo_PluginCapability_VolumeExpansion.Size(m) @@ -683,16 +642,17 @@ func (m *ProbeRequest) Reset() { *m = ProbeRequest{} } func (m *ProbeRequest) String() string { return proto.CompactTextString(m) } func (*ProbeRequest) ProtoMessage() {} func (*ProbeRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{5} + return fileDescriptor_9cdb00adce470e01, []int{5} } + func (m *ProbeRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ProbeRequest.Unmarshal(m, b) } func (m *ProbeRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_ProbeRequest.Marshal(b, m, deterministic) } -func (dst *ProbeRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_ProbeRequest.Merge(dst, src) +func (m *ProbeRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ProbeRequest.Merge(m, src) } func (m *ProbeRequest) XXX_Size() int { return xxx_messageInfo_ProbeRequest.Size(m) @@ -734,16 +694,17 @@ func (m *ProbeResponse) Reset() { *m = ProbeResponse{} } func (m *ProbeResponse) String() string { return proto.CompactTextString(m) } func (*ProbeResponse) ProtoMessage() {} func (*ProbeResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{6} + return fileDescriptor_9cdb00adce470e01, []int{6} } + func (m *ProbeResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ProbeResponse.Unmarshal(m, b) } func (m *ProbeResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_ProbeResponse.Marshal(b, m, deterministic) } -func (dst *ProbeResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_ProbeResponse.Merge(dst, src) +func (m *ProbeResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_ProbeResponse.Merge(m, src) } func (m *ProbeResponse) XXX_Size() int { return xxx_messageInfo_ProbeResponse.Size(m) @@ -843,16 +804,17 @@ func (m *CreateVolumeRequest) Reset() { *m = CreateVolumeRequest{} } func (m *CreateVolumeRequest) String() string { return proto.CompactTextString(m) } func (*CreateVolumeRequest) ProtoMessage() {} func (*CreateVolumeRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{7} + return fileDescriptor_9cdb00adce470e01, []int{7} } + func (m *CreateVolumeRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CreateVolumeRequest.Unmarshal(m, b) } func (m *CreateVolumeRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CreateVolumeRequest.Marshal(b, m, deterministic) } -func (dst *CreateVolumeRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_CreateVolumeRequest.Merge(dst, src) +func (m *CreateVolumeRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_CreateVolumeRequest.Merge(m, src) } func (m *CreateVolumeRequest) XXX_Size() int { return xxx_messageInfo_CreateVolumeRequest.Size(m) @@ -928,16 +890,17 @@ func (m *VolumeContentSource) Reset() { *m = VolumeContentSource{} } func (m *VolumeContentSource) String() string { return proto.CompactTextString(m) } func (*VolumeContentSource) ProtoMessage() {} func (*VolumeContentSource) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{8} + return fileDescriptor_9cdb00adce470e01, []int{8} } + func (m *VolumeContentSource) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_VolumeContentSource.Unmarshal(m, b) } func (m *VolumeContentSource) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_VolumeContentSource.Marshal(b, m, deterministic) } -func (dst *VolumeContentSource) XXX_Merge(src proto.Message) { - xxx_messageInfo_VolumeContentSource.Merge(dst, src) +func (m *VolumeContentSource) XXX_Merge(src proto.Message) { + xxx_messageInfo_VolumeContentSource.Merge(m, src) } func (m *VolumeContentSource) XXX_Size() int { return xxx_messageInfo_VolumeContentSource.Size(m) @@ -985,80 +948,14 @@ func (m *VolumeContentSource) GetVolume() *VolumeContentSource_VolumeSource { return nil } -// XXX_OneofFuncs is for the internal use of the proto package. -func (*VolumeContentSource) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { - return _VolumeContentSource_OneofMarshaler, _VolumeContentSource_OneofUnmarshaler, _VolumeContentSource_OneofSizer, []interface{}{ +// XXX_OneofWrappers is for the internal use of the proto package. +func (*VolumeContentSource) XXX_OneofWrappers() []interface{} { + return []interface{}{ (*VolumeContentSource_Snapshot)(nil), (*VolumeContentSource_Volume)(nil), } } -func _VolumeContentSource_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { - m := msg.(*VolumeContentSource) - // type - switch x := m.Type.(type) { - case *VolumeContentSource_Snapshot: - b.EncodeVarint(1<<3 | proto.WireBytes) - if err := b.EncodeMessage(x.Snapshot); err != nil { - return err - } - case *VolumeContentSource_Volume: - b.EncodeVarint(2<<3 | proto.WireBytes) - if err := b.EncodeMessage(x.Volume); err != nil { - return err - } - case nil: - default: - return fmt.Errorf("VolumeContentSource.Type has unexpected type %T", x) - } - return nil -} - -func _VolumeContentSource_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { - m := msg.(*VolumeContentSource) - switch tag { - case 1: // type.snapshot - if wire != proto.WireBytes { - return true, proto.ErrInternalBadWireType - } - msg := new(VolumeContentSource_SnapshotSource) - err := b.DecodeMessage(msg) - m.Type = &VolumeContentSource_Snapshot{msg} - return true, err - case 2: // type.volume - if wire != proto.WireBytes { - return true, proto.ErrInternalBadWireType - } - msg := new(VolumeContentSource_VolumeSource) - err := b.DecodeMessage(msg) - m.Type = &VolumeContentSource_Volume{msg} - return true, err - default: - return false, nil - } -} - -func _VolumeContentSource_OneofSizer(msg proto.Message) (n int) { - m := msg.(*VolumeContentSource) - // type - switch x := m.Type.(type) { - case *VolumeContentSource_Snapshot: - s := proto.Size(x.Snapshot) - n += 1 // tag and wire - n += proto.SizeVarint(uint64(s)) - n += s - case *VolumeContentSource_Volume: - s := proto.Size(x.Volume) - n += 1 // tag and wire - n += proto.SizeVarint(uint64(s)) - n += s - case nil: - default: - panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) - } - return n -} - type VolumeContentSource_SnapshotSource struct { // Contains identity information for the existing source snapshot. // This field is REQUIRED. Plugin is REQUIRED to support creating @@ -1074,16 +971,17 @@ func (m *VolumeContentSource_SnapshotSource) Reset() { *m = VolumeConten func (m *VolumeContentSource_SnapshotSource) String() string { return proto.CompactTextString(m) } func (*VolumeContentSource_SnapshotSource) ProtoMessage() {} func (*VolumeContentSource_SnapshotSource) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{8, 0} + return fileDescriptor_9cdb00adce470e01, []int{8, 0} } + func (m *VolumeContentSource_SnapshotSource) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_VolumeContentSource_SnapshotSource.Unmarshal(m, b) } func (m *VolumeContentSource_SnapshotSource) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_VolumeContentSource_SnapshotSource.Marshal(b, m, deterministic) } -func (dst *VolumeContentSource_SnapshotSource) XXX_Merge(src proto.Message) { - xxx_messageInfo_VolumeContentSource_SnapshotSource.Merge(dst, src) +func (m *VolumeContentSource_SnapshotSource) XXX_Merge(src proto.Message) { + xxx_messageInfo_VolumeContentSource_SnapshotSource.Merge(m, src) } func (m *VolumeContentSource_SnapshotSource) XXX_Size() int { return xxx_messageInfo_VolumeContentSource_SnapshotSource.Size(m) @@ -1115,16 +1013,17 @@ func (m *VolumeContentSource_VolumeSource) Reset() { *m = VolumeContentS func (m *VolumeContentSource_VolumeSource) String() string { return proto.CompactTextString(m) } func (*VolumeContentSource_VolumeSource) ProtoMessage() {} func (*VolumeContentSource_VolumeSource) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{8, 1} + return fileDescriptor_9cdb00adce470e01, []int{8, 1} } + func (m *VolumeContentSource_VolumeSource) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_VolumeContentSource_VolumeSource.Unmarshal(m, b) } func (m *VolumeContentSource_VolumeSource) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_VolumeContentSource_VolumeSource.Marshal(b, m, deterministic) } -func (dst *VolumeContentSource_VolumeSource) XXX_Merge(src proto.Message) { - xxx_messageInfo_VolumeContentSource_VolumeSource.Merge(dst, src) +func (m *VolumeContentSource_VolumeSource) XXX_Merge(src proto.Message) { + xxx_messageInfo_VolumeContentSource_VolumeSource.Merge(m, src) } func (m *VolumeContentSource_VolumeSource) XXX_Size() int { return xxx_messageInfo_VolumeContentSource_VolumeSource.Size(m) @@ -1156,16 +1055,17 @@ func (m *CreateVolumeResponse) Reset() { *m = CreateVolumeResponse{} } func (m *CreateVolumeResponse) String() string { return proto.CompactTextString(m) } func (*CreateVolumeResponse) ProtoMessage() {} func (*CreateVolumeResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{9} + return fileDescriptor_9cdb00adce470e01, []int{9} } + func (m *CreateVolumeResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CreateVolumeResponse.Unmarshal(m, b) } func (m *CreateVolumeResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CreateVolumeResponse.Marshal(b, m, deterministic) } -func (dst *CreateVolumeResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CreateVolumeResponse.Merge(dst, src) +func (m *CreateVolumeResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CreateVolumeResponse.Merge(m, src) } func (m *CreateVolumeResponse) XXX_Size() int { return xxx_messageInfo_CreateVolumeResponse.Size(m) @@ -1203,16 +1103,17 @@ func (m *VolumeCapability) Reset() { *m = VolumeCapability{} } func (m *VolumeCapability) String() string { return proto.CompactTextString(m) } func (*VolumeCapability) ProtoMessage() {} func (*VolumeCapability) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{10} + return fileDescriptor_9cdb00adce470e01, []int{10} } + func (m *VolumeCapability) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_VolumeCapability.Unmarshal(m, b) } func (m *VolumeCapability) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_VolumeCapability.Marshal(b, m, deterministic) } -func (dst *VolumeCapability) XXX_Merge(src proto.Message) { - xxx_messageInfo_VolumeCapability.Merge(dst, src) +func (m *VolumeCapability) XXX_Merge(src proto.Message) { + xxx_messageInfo_VolumeCapability.Merge(m, src) } func (m *VolumeCapability) XXX_Size() int { return xxx_messageInfo_VolumeCapability.Size(m) @@ -1267,80 +1168,14 @@ func (m *VolumeCapability) GetAccessMode() *VolumeCapability_AccessMode { return nil } -// XXX_OneofFuncs is for the internal use of the proto package. -func (*VolumeCapability) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { - return _VolumeCapability_OneofMarshaler, _VolumeCapability_OneofUnmarshaler, _VolumeCapability_OneofSizer, []interface{}{ +// XXX_OneofWrappers is for the internal use of the proto package. +func (*VolumeCapability) XXX_OneofWrappers() []interface{} { + return []interface{}{ (*VolumeCapability_Block)(nil), (*VolumeCapability_Mount)(nil), } } -func _VolumeCapability_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { - m := msg.(*VolumeCapability) - // access_type - switch x := m.AccessType.(type) { - case *VolumeCapability_Block: - b.EncodeVarint(1<<3 | proto.WireBytes) - if err := b.EncodeMessage(x.Block); err != nil { - return err - } - case *VolumeCapability_Mount: - b.EncodeVarint(2<<3 | proto.WireBytes) - if err := b.EncodeMessage(x.Mount); err != nil { - return err - } - case nil: - default: - return fmt.Errorf("VolumeCapability.AccessType has unexpected type %T", x) - } - return nil -} - -func _VolumeCapability_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { - m := msg.(*VolumeCapability) - switch tag { - case 1: // access_type.block - if wire != proto.WireBytes { - return true, proto.ErrInternalBadWireType - } - msg := new(VolumeCapability_BlockVolume) - err := b.DecodeMessage(msg) - m.AccessType = &VolumeCapability_Block{msg} - return true, err - case 2: // access_type.mount - if wire != proto.WireBytes { - return true, proto.ErrInternalBadWireType - } - msg := new(VolumeCapability_MountVolume) - err := b.DecodeMessage(msg) - m.AccessType = &VolumeCapability_Mount{msg} - return true, err - default: - return false, nil - } -} - -func _VolumeCapability_OneofSizer(msg proto.Message) (n int) { - m := msg.(*VolumeCapability) - // access_type - switch x := m.AccessType.(type) { - case *VolumeCapability_Block: - s := proto.Size(x.Block) - n += 1 // tag and wire - n += proto.SizeVarint(uint64(s)) - n += s - case *VolumeCapability_Mount: - s := proto.Size(x.Mount) - n += 1 // tag and wire - n += proto.SizeVarint(uint64(s)) - n += s - case nil: - default: - panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) - } - return n -} - // Indicate that the volume will be accessed via the block device API. type VolumeCapability_BlockVolume struct { XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -1352,16 +1187,17 @@ func (m *VolumeCapability_BlockVolume) Reset() { *m = VolumeCapability_B func (m *VolumeCapability_BlockVolume) String() string { return proto.CompactTextString(m) } func (*VolumeCapability_BlockVolume) ProtoMessage() {} func (*VolumeCapability_BlockVolume) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{10, 0} + return fileDescriptor_9cdb00adce470e01, []int{10, 0} } + func (m *VolumeCapability_BlockVolume) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_VolumeCapability_BlockVolume.Unmarshal(m, b) } func (m *VolumeCapability_BlockVolume) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_VolumeCapability_BlockVolume.Marshal(b, m, deterministic) } -func (dst *VolumeCapability_BlockVolume) XXX_Merge(src proto.Message) { - xxx_messageInfo_VolumeCapability_BlockVolume.Merge(dst, src) +func (m *VolumeCapability_BlockVolume) XXX_Merge(src proto.Message) { + xxx_messageInfo_VolumeCapability_BlockVolume.Merge(m, src) } func (m *VolumeCapability_BlockVolume) XXX_Size() int { return xxx_messageInfo_VolumeCapability_BlockVolume.Size(m) @@ -1392,16 +1228,17 @@ func (m *VolumeCapability_MountVolume) Reset() { *m = VolumeCapability_M func (m *VolumeCapability_MountVolume) String() string { return proto.CompactTextString(m) } func (*VolumeCapability_MountVolume) ProtoMessage() {} func (*VolumeCapability_MountVolume) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{10, 1} + return fileDescriptor_9cdb00adce470e01, []int{10, 1} } + func (m *VolumeCapability_MountVolume) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_VolumeCapability_MountVolume.Unmarshal(m, b) } func (m *VolumeCapability_MountVolume) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_VolumeCapability_MountVolume.Marshal(b, m, deterministic) } -func (dst *VolumeCapability_MountVolume) XXX_Merge(src proto.Message) { - xxx_messageInfo_VolumeCapability_MountVolume.Merge(dst, src) +func (m *VolumeCapability_MountVolume) XXX_Merge(src proto.Message) { + xxx_messageInfo_VolumeCapability_MountVolume.Merge(m, src) } func (m *VolumeCapability_MountVolume) XXX_Size() int { return xxx_messageInfo_VolumeCapability_MountVolume.Size(m) @@ -1439,16 +1276,17 @@ func (m *VolumeCapability_AccessMode) Reset() { *m = VolumeCapability_Ac func (m *VolumeCapability_AccessMode) String() string { return proto.CompactTextString(m) } func (*VolumeCapability_AccessMode) ProtoMessage() {} func (*VolumeCapability_AccessMode) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{10, 2} + return fileDescriptor_9cdb00adce470e01, []int{10, 2} } + func (m *VolumeCapability_AccessMode) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_VolumeCapability_AccessMode.Unmarshal(m, b) } func (m *VolumeCapability_AccessMode) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_VolumeCapability_AccessMode.Marshal(b, m, deterministic) } -func (dst *VolumeCapability_AccessMode) XXX_Merge(src proto.Message) { - xxx_messageInfo_VolumeCapability_AccessMode.Merge(dst, src) +func (m *VolumeCapability_AccessMode) XXX_Merge(src proto.Message) { + xxx_messageInfo_VolumeCapability_AccessMode.Merge(m, src) } func (m *VolumeCapability_AccessMode) XXX_Size() int { return xxx_messageInfo_VolumeCapability_AccessMode.Size(m) @@ -1487,16 +1325,17 @@ func (m *CapacityRange) Reset() { *m = CapacityRange{} } func (m *CapacityRange) String() string { return proto.CompactTextString(m) } func (*CapacityRange) ProtoMessage() {} func (*CapacityRange) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{11} + return fileDescriptor_9cdb00adce470e01, []int{11} } + func (m *CapacityRange) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CapacityRange.Unmarshal(m, b) } func (m *CapacityRange) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CapacityRange.Marshal(b, m, deterministic) } -func (dst *CapacityRange) XXX_Merge(src proto.Message) { - xxx_messageInfo_CapacityRange.Merge(dst, src) +func (m *CapacityRange) XXX_Merge(src proto.Message) { + xxx_messageInfo_CapacityRange.Merge(m, src) } func (m *CapacityRange) XXX_Size() int { return xxx_messageInfo_CapacityRange.Size(m) @@ -1592,16 +1431,17 @@ func (m *Volume) Reset() { *m = Volume{} } func (m *Volume) String() string { return proto.CompactTextString(m) } func (*Volume) ProtoMessage() {} func (*Volume) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{12} + return fileDescriptor_9cdb00adce470e01, []int{12} } + func (m *Volume) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Volume.Unmarshal(m, b) } func (m *Volume) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Volume.Marshal(b, m, deterministic) } -func (dst *Volume) XXX_Merge(src proto.Message) { - xxx_messageInfo_Volume.Merge(dst, src) +func (m *Volume) XXX_Merge(src proto.Message) { + xxx_messageInfo_Volume.Merge(m, src) } func (m *Volume) XXX_Size() int { return xxx_messageInfo_Volume.Size(m) @@ -1784,16 +1624,17 @@ func (m *TopologyRequirement) Reset() { *m = TopologyRequirement{} } func (m *TopologyRequirement) String() string { return proto.CompactTextString(m) } func (*TopologyRequirement) ProtoMessage() {} func (*TopologyRequirement) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{13} + return fileDescriptor_9cdb00adce470e01, []int{13} } + func (m *TopologyRequirement) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_TopologyRequirement.Unmarshal(m, b) } func (m *TopologyRequirement) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_TopologyRequirement.Marshal(b, m, deterministic) } -func (dst *TopologyRequirement) XXX_Merge(src proto.Message) { - xxx_messageInfo_TopologyRequirement.Merge(dst, src) +func (m *TopologyRequirement) XXX_Merge(src proto.Message) { + xxx_messageInfo_TopologyRequirement.Merge(m, src) } func (m *TopologyRequirement) XXX_Size() int { return xxx_messageInfo_TopologyRequirement.Size(m) @@ -1858,16 +1699,17 @@ func (m *Topology) Reset() { *m = Topology{} } func (m *Topology) String() string { return proto.CompactTextString(m) } func (*Topology) ProtoMessage() {} func (*Topology) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{14} + return fileDescriptor_9cdb00adce470e01, []int{14} } + func (m *Topology) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Topology.Unmarshal(m, b) } func (m *Topology) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Topology.Marshal(b, m, deterministic) } -func (dst *Topology) XXX_Merge(src proto.Message) { - xxx_messageInfo_Topology.Merge(dst, src) +func (m *Topology) XXX_Merge(src proto.Message) { + xxx_messageInfo_Topology.Merge(m, src) } func (m *Topology) XXX_Size() int { return xxx_messageInfo_Topology.Size(m) @@ -1902,16 +1744,17 @@ func (m *DeleteVolumeRequest) Reset() { *m = DeleteVolumeRequest{} } func (m *DeleteVolumeRequest) String() string { return proto.CompactTextString(m) } func (*DeleteVolumeRequest) ProtoMessage() {} func (*DeleteVolumeRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{15} + return fileDescriptor_9cdb00adce470e01, []int{15} } + func (m *DeleteVolumeRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DeleteVolumeRequest.Unmarshal(m, b) } func (m *DeleteVolumeRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_DeleteVolumeRequest.Marshal(b, m, deterministic) } -func (dst *DeleteVolumeRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_DeleteVolumeRequest.Merge(dst, src) +func (m *DeleteVolumeRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_DeleteVolumeRequest.Merge(m, src) } func (m *DeleteVolumeRequest) XXX_Size() int { return xxx_messageInfo_DeleteVolumeRequest.Size(m) @@ -1946,16 +1789,17 @@ func (m *DeleteVolumeResponse) Reset() { *m = DeleteVolumeResponse{} } func (m *DeleteVolumeResponse) String() string { return proto.CompactTextString(m) } func (*DeleteVolumeResponse) ProtoMessage() {} func (*DeleteVolumeResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{16} + return fileDescriptor_9cdb00adce470e01, []int{16} } + func (m *DeleteVolumeResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DeleteVolumeResponse.Unmarshal(m, b) } func (m *DeleteVolumeResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_DeleteVolumeResponse.Marshal(b, m, deterministic) } -func (dst *DeleteVolumeResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_DeleteVolumeResponse.Merge(dst, src) +func (m *DeleteVolumeResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_DeleteVolumeResponse.Merge(m, src) } func (m *DeleteVolumeResponse) XXX_Size() int { return xxx_messageInfo_DeleteVolumeResponse.Size(m) @@ -2000,16 +1844,17 @@ func (m *ControllerPublishVolumeRequest) Reset() { *m = ControllerPublis func (m *ControllerPublishVolumeRequest) String() string { return proto.CompactTextString(m) } func (*ControllerPublishVolumeRequest) ProtoMessage() {} func (*ControllerPublishVolumeRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{17} + return fileDescriptor_9cdb00adce470e01, []int{17} } + func (m *ControllerPublishVolumeRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ControllerPublishVolumeRequest.Unmarshal(m, b) } func (m *ControllerPublishVolumeRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_ControllerPublishVolumeRequest.Marshal(b, m, deterministic) } -func (dst *ControllerPublishVolumeRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_ControllerPublishVolumeRequest.Merge(dst, src) +func (m *ControllerPublishVolumeRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ControllerPublishVolumeRequest.Merge(m, src) } func (m *ControllerPublishVolumeRequest) XXX_Size() int { return xxx_messageInfo_ControllerPublishVolumeRequest.Size(m) @@ -2086,16 +1931,17 @@ func (m *ControllerPublishVolumeResponse) Reset() { *m = ControllerPubli func (m *ControllerPublishVolumeResponse) String() string { return proto.CompactTextString(m) } func (*ControllerPublishVolumeResponse) ProtoMessage() {} func (*ControllerPublishVolumeResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{18} + return fileDescriptor_9cdb00adce470e01, []int{18} } + func (m *ControllerPublishVolumeResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ControllerPublishVolumeResponse.Unmarshal(m, b) } func (m *ControllerPublishVolumeResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_ControllerPublishVolumeResponse.Marshal(b, m, deterministic) } -func (dst *ControllerPublishVolumeResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_ControllerPublishVolumeResponse.Merge(dst, src) +func (m *ControllerPublishVolumeResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_ControllerPublishVolumeResponse.Merge(m, src) } func (m *ControllerPublishVolumeResponse) XXX_Size() int { return xxx_messageInfo_ControllerPublishVolumeResponse.Size(m) @@ -2137,16 +1983,17 @@ func (m *ControllerUnpublishVolumeRequest) Reset() { *m = ControllerUnpu func (m *ControllerUnpublishVolumeRequest) String() string { return proto.CompactTextString(m) } func (*ControllerUnpublishVolumeRequest) ProtoMessage() {} func (*ControllerUnpublishVolumeRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{19} + return fileDescriptor_9cdb00adce470e01, []int{19} } + func (m *ControllerUnpublishVolumeRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ControllerUnpublishVolumeRequest.Unmarshal(m, b) } func (m *ControllerUnpublishVolumeRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_ControllerUnpublishVolumeRequest.Marshal(b, m, deterministic) } -func (dst *ControllerUnpublishVolumeRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_ControllerUnpublishVolumeRequest.Merge(dst, src) +func (m *ControllerUnpublishVolumeRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ControllerUnpublishVolumeRequest.Merge(m, src) } func (m *ControllerUnpublishVolumeRequest) XXX_Size() int { return xxx_messageInfo_ControllerUnpublishVolumeRequest.Size(m) @@ -2188,16 +2035,17 @@ func (m *ControllerUnpublishVolumeResponse) Reset() { *m = ControllerUnp func (m *ControllerUnpublishVolumeResponse) String() string { return proto.CompactTextString(m) } func (*ControllerUnpublishVolumeResponse) ProtoMessage() {} func (*ControllerUnpublishVolumeResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{20} + return fileDescriptor_9cdb00adce470e01, []int{20} } + func (m *ControllerUnpublishVolumeResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ControllerUnpublishVolumeResponse.Unmarshal(m, b) } func (m *ControllerUnpublishVolumeResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_ControllerUnpublishVolumeResponse.Marshal(b, m, deterministic) } -func (dst *ControllerUnpublishVolumeResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_ControllerUnpublishVolumeResponse.Merge(dst, src) +func (m *ControllerUnpublishVolumeResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_ControllerUnpublishVolumeResponse.Merge(m, src) } func (m *ControllerUnpublishVolumeResponse) XXX_Size() int { return xxx_messageInfo_ControllerUnpublishVolumeResponse.Size(m) @@ -2235,16 +2083,17 @@ func (m *ValidateVolumeCapabilitiesRequest) Reset() { *m = ValidateVolum func (m *ValidateVolumeCapabilitiesRequest) String() string { return proto.CompactTextString(m) } func (*ValidateVolumeCapabilitiesRequest) ProtoMessage() {} func (*ValidateVolumeCapabilitiesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{21} + return fileDescriptor_9cdb00adce470e01, []int{21} } + func (m *ValidateVolumeCapabilitiesRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ValidateVolumeCapabilitiesRequest.Unmarshal(m, b) } func (m *ValidateVolumeCapabilitiesRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_ValidateVolumeCapabilitiesRequest.Marshal(b, m, deterministic) } -func (dst *ValidateVolumeCapabilitiesRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_ValidateVolumeCapabilitiesRequest.Merge(dst, src) +func (m *ValidateVolumeCapabilitiesRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ValidateVolumeCapabilitiesRequest.Merge(m, src) } func (m *ValidateVolumeCapabilitiesRequest) XXX_Size() int { return xxx_messageInfo_ValidateVolumeCapabilitiesRequest.Size(m) @@ -2313,16 +2162,17 @@ func (m *ValidateVolumeCapabilitiesResponse) Reset() { *m = ValidateVolu func (m *ValidateVolumeCapabilitiesResponse) String() string { return proto.CompactTextString(m) } func (*ValidateVolumeCapabilitiesResponse) ProtoMessage() {} func (*ValidateVolumeCapabilitiesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{22} + return fileDescriptor_9cdb00adce470e01, []int{22} } + func (m *ValidateVolumeCapabilitiesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ValidateVolumeCapabilitiesResponse.Unmarshal(m, b) } func (m *ValidateVolumeCapabilitiesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_ValidateVolumeCapabilitiesResponse.Marshal(b, m, deterministic) } -func (dst *ValidateVolumeCapabilitiesResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_ValidateVolumeCapabilitiesResponse.Merge(dst, src) +func (m *ValidateVolumeCapabilitiesResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_ValidateVolumeCapabilitiesResponse.Merge(m, src) } func (m *ValidateVolumeCapabilitiesResponse) XXX_Size() int { return xxx_messageInfo_ValidateVolumeCapabilitiesResponse.Size(m) @@ -2370,16 +2220,17 @@ func (m *ValidateVolumeCapabilitiesResponse_Confirmed) String() string { } func (*ValidateVolumeCapabilitiesResponse_Confirmed) ProtoMessage() {} func (*ValidateVolumeCapabilitiesResponse_Confirmed) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{22, 0} + return fileDescriptor_9cdb00adce470e01, []int{22, 0} } + func (m *ValidateVolumeCapabilitiesResponse_Confirmed) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ValidateVolumeCapabilitiesResponse_Confirmed.Unmarshal(m, b) } func (m *ValidateVolumeCapabilitiesResponse_Confirmed) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_ValidateVolumeCapabilitiesResponse_Confirmed.Marshal(b, m, deterministic) } -func (dst *ValidateVolumeCapabilitiesResponse_Confirmed) XXX_Merge(src proto.Message) { - xxx_messageInfo_ValidateVolumeCapabilitiesResponse_Confirmed.Merge(dst, src) +func (m *ValidateVolumeCapabilitiesResponse_Confirmed) XXX_Merge(src proto.Message) { + xxx_messageInfo_ValidateVolumeCapabilitiesResponse_Confirmed.Merge(m, src) } func (m *ValidateVolumeCapabilitiesResponse_Confirmed) XXX_Size() int { return xxx_messageInfo_ValidateVolumeCapabilitiesResponse_Confirmed.Size(m) @@ -2435,16 +2286,17 @@ func (m *ListVolumesRequest) Reset() { *m = ListVolumesRequest{} } func (m *ListVolumesRequest) String() string { return proto.CompactTextString(m) } func (*ListVolumesRequest) ProtoMessage() {} func (*ListVolumesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{23} + return fileDescriptor_9cdb00adce470e01, []int{23} } + func (m *ListVolumesRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ListVolumesRequest.Unmarshal(m, b) } func (m *ListVolumesRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_ListVolumesRequest.Marshal(b, m, deterministic) } -func (dst *ListVolumesRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_ListVolumesRequest.Merge(dst, src) +func (m *ListVolumesRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ListVolumesRequest.Merge(m, src) } func (m *ListVolumesRequest) XXX_Size() int { return xxx_messageInfo_ListVolumesRequest.Size(m) @@ -2487,16 +2339,17 @@ func (m *ListVolumesResponse) Reset() { *m = ListVolumesResponse{} } func (m *ListVolumesResponse) String() string { return proto.CompactTextString(m) } func (*ListVolumesResponse) ProtoMessage() {} func (*ListVolumesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{24} + return fileDescriptor_9cdb00adce470e01, []int{24} } + func (m *ListVolumesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ListVolumesResponse.Unmarshal(m, b) } func (m *ListVolumesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_ListVolumesResponse.Marshal(b, m, deterministic) } -func (dst *ListVolumesResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_ListVolumesResponse.Merge(dst, src) +func (m *ListVolumesResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_ListVolumesResponse.Merge(m, src) } func (m *ListVolumesResponse) XXX_Size() int { return xxx_messageInfo_ListVolumesResponse.Size(m) @@ -2521,27 +2374,82 @@ func (m *ListVolumesResponse) GetNextToken() string { return "" } -type ListVolumesResponse_Entry struct { - Volume *Volume `protobuf:"bytes,1,opt,name=volume,proto3" json:"volume,omitempty"` +type ListVolumesResponse_VolumeStatus struct { + // A list of all `node_id` of nodes that the volume in this entry + // is controller published on. + // This field is OPTIONAL. If it is not specified and the SP has + // the LIST_VOLUMES_PUBLISHED_NODES controller capability, the CO + // MAY assume the volume is not controller published to any nodes. + // If the field is not specified and the SP does not have the + // LIST_VOLUMES_PUBLISHED_NODES controller capability, the CO MUST + // not interpret this field. + // published_node_ids MAY include nodes not published to or + // reported by the SP. The CO MUST be resilient to that. + PublishedNodeIds []string `protobuf:"bytes,1,rep,name=published_node_ids,json=publishedNodeIds,proto3" json:"published_node_ids,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } +func (m *ListVolumesResponse_VolumeStatus) Reset() { *m = ListVolumesResponse_VolumeStatus{} } +func (m *ListVolumesResponse_VolumeStatus) String() string { return proto.CompactTextString(m) } +func (*ListVolumesResponse_VolumeStatus) ProtoMessage() {} +func (*ListVolumesResponse_VolumeStatus) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{24, 0} +} + +func (m *ListVolumesResponse_VolumeStatus) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ListVolumesResponse_VolumeStatus.Unmarshal(m, b) +} +func (m *ListVolumesResponse_VolumeStatus) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ListVolumesResponse_VolumeStatus.Marshal(b, m, deterministic) +} +func (m *ListVolumesResponse_VolumeStatus) XXX_Merge(src proto.Message) { + xxx_messageInfo_ListVolumesResponse_VolumeStatus.Merge(m, src) +} +func (m *ListVolumesResponse_VolumeStatus) XXX_Size() int { + return xxx_messageInfo_ListVolumesResponse_VolumeStatus.Size(m) +} +func (m *ListVolumesResponse_VolumeStatus) XXX_DiscardUnknown() { + xxx_messageInfo_ListVolumesResponse_VolumeStatus.DiscardUnknown(m) +} + +var xxx_messageInfo_ListVolumesResponse_VolumeStatus proto.InternalMessageInfo + +func (m *ListVolumesResponse_VolumeStatus) GetPublishedNodeIds() []string { + if m != nil { + return m.PublishedNodeIds + } + return nil +} + +type ListVolumesResponse_Entry struct { + // This field is REQUIRED + Volume *Volume `protobuf:"bytes,1,opt,name=volume,proto3" json:"volume,omitempty"` + // This field is OPTIONAL. This field MUST be specified if the + // LIST_VOLUMES_PUBLISHED_NODES controller capability is + // supported. + Status *ListVolumesResponse_VolumeStatus `protobuf:"bytes,2,opt,name=status,proto3" json:"status,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + func (m *ListVolumesResponse_Entry) Reset() { *m = ListVolumesResponse_Entry{} } func (m *ListVolumesResponse_Entry) String() string { return proto.CompactTextString(m) } func (*ListVolumesResponse_Entry) ProtoMessage() {} func (*ListVolumesResponse_Entry) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{24, 0} + return fileDescriptor_9cdb00adce470e01, []int{24, 1} } + func (m *ListVolumesResponse_Entry) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ListVolumesResponse_Entry.Unmarshal(m, b) } func (m *ListVolumesResponse_Entry) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_ListVolumesResponse_Entry.Marshal(b, m, deterministic) } -func (dst *ListVolumesResponse_Entry) XXX_Merge(src proto.Message) { - xxx_messageInfo_ListVolumesResponse_Entry.Merge(dst, src) +func (m *ListVolumesResponse_Entry) XXX_Merge(src proto.Message) { + xxx_messageInfo_ListVolumesResponse_Entry.Merge(m, src) } func (m *ListVolumesResponse_Entry) XXX_Size() int { return xxx_messageInfo_ListVolumesResponse_Entry.Size(m) @@ -2559,6 +2467,13 @@ func (m *ListVolumesResponse_Entry) GetVolume() *Volume { return nil } +func (m *ListVolumesResponse_Entry) GetStatus() *ListVolumesResponse_VolumeStatus { + if m != nil { + return m.Status + } + return nil +} + type GetCapacityRequest struct { // If specified, the Plugin SHALL report the capacity of the storage // that can be used to provision volumes that satisfy ALL of the @@ -2587,16 +2502,17 @@ func (m *GetCapacityRequest) Reset() { *m = GetCapacityRequest{} } func (m *GetCapacityRequest) String() string { return proto.CompactTextString(m) } func (*GetCapacityRequest) ProtoMessage() {} func (*GetCapacityRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{25} + return fileDescriptor_9cdb00adce470e01, []int{25} } + func (m *GetCapacityRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_GetCapacityRequest.Unmarshal(m, b) } func (m *GetCapacityRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_GetCapacityRequest.Marshal(b, m, deterministic) } -func (dst *GetCapacityRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_GetCapacityRequest.Merge(dst, src) +func (m *GetCapacityRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetCapacityRequest.Merge(m, src) } func (m *GetCapacityRequest) XXX_Size() int { return xxx_messageInfo_GetCapacityRequest.Size(m) @@ -2645,16 +2561,17 @@ func (m *GetCapacityResponse) Reset() { *m = GetCapacityResponse{} } func (m *GetCapacityResponse) String() string { return proto.CompactTextString(m) } func (*GetCapacityResponse) ProtoMessage() {} func (*GetCapacityResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{26} + return fileDescriptor_9cdb00adce470e01, []int{26} } + func (m *GetCapacityResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_GetCapacityResponse.Unmarshal(m, b) } func (m *GetCapacityResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_GetCapacityResponse.Marshal(b, m, deterministic) } -func (dst *GetCapacityResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_GetCapacityResponse.Merge(dst, src) +func (m *GetCapacityResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetCapacityResponse.Merge(m, src) } func (m *GetCapacityResponse) XXX_Size() int { return xxx_messageInfo_GetCapacityResponse.Size(m) @@ -2682,16 +2599,17 @@ func (m *ControllerGetCapabilitiesRequest) Reset() { *m = ControllerGetC func (m *ControllerGetCapabilitiesRequest) String() string { return proto.CompactTextString(m) } func (*ControllerGetCapabilitiesRequest) ProtoMessage() {} func (*ControllerGetCapabilitiesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{27} + return fileDescriptor_9cdb00adce470e01, []int{27} } + func (m *ControllerGetCapabilitiesRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ControllerGetCapabilitiesRequest.Unmarshal(m, b) } func (m *ControllerGetCapabilitiesRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_ControllerGetCapabilitiesRequest.Marshal(b, m, deterministic) } -func (dst *ControllerGetCapabilitiesRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_ControllerGetCapabilitiesRequest.Merge(dst, src) +func (m *ControllerGetCapabilitiesRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ControllerGetCapabilitiesRequest.Merge(m, src) } func (m *ControllerGetCapabilitiesRequest) XXX_Size() int { return xxx_messageInfo_ControllerGetCapabilitiesRequest.Size(m) @@ -2715,16 +2633,17 @@ func (m *ControllerGetCapabilitiesResponse) Reset() { *m = ControllerGet func (m *ControllerGetCapabilitiesResponse) String() string { return proto.CompactTextString(m) } func (*ControllerGetCapabilitiesResponse) ProtoMessage() {} func (*ControllerGetCapabilitiesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{28} + return fileDescriptor_9cdb00adce470e01, []int{28} } + func (m *ControllerGetCapabilitiesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ControllerGetCapabilitiesResponse.Unmarshal(m, b) } func (m *ControllerGetCapabilitiesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_ControllerGetCapabilitiesResponse.Marshal(b, m, deterministic) } -func (dst *ControllerGetCapabilitiesResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_ControllerGetCapabilitiesResponse.Merge(dst, src) +func (m *ControllerGetCapabilitiesResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_ControllerGetCapabilitiesResponse.Merge(m, src) } func (m *ControllerGetCapabilitiesResponse) XXX_Size() int { return xxx_messageInfo_ControllerGetCapabilitiesResponse.Size(m) @@ -2756,16 +2675,17 @@ func (m *ControllerServiceCapability) Reset() { *m = ControllerServiceCa func (m *ControllerServiceCapability) String() string { return proto.CompactTextString(m) } func (*ControllerServiceCapability) ProtoMessage() {} func (*ControllerServiceCapability) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{29} + return fileDescriptor_9cdb00adce470e01, []int{29} } + func (m *ControllerServiceCapability) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ControllerServiceCapability.Unmarshal(m, b) } func (m *ControllerServiceCapability) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_ControllerServiceCapability.Marshal(b, m, deterministic) } -func (dst *ControllerServiceCapability) XXX_Merge(src proto.Message) { - xxx_messageInfo_ControllerServiceCapability.Merge(dst, src) +func (m *ControllerServiceCapability) XXX_Merge(src proto.Message) { + xxx_messageInfo_ControllerServiceCapability.Merge(m, src) } func (m *ControllerServiceCapability) XXX_Size() int { return xxx_messageInfo_ControllerServiceCapability.Size(m) @@ -2800,61 +2720,13 @@ func (m *ControllerServiceCapability) GetRpc() *ControllerServiceCapability_RPC return nil } -// XXX_OneofFuncs is for the internal use of the proto package. -func (*ControllerServiceCapability) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { - return _ControllerServiceCapability_OneofMarshaler, _ControllerServiceCapability_OneofUnmarshaler, _ControllerServiceCapability_OneofSizer, []interface{}{ +// XXX_OneofWrappers is for the internal use of the proto package. +func (*ControllerServiceCapability) XXX_OneofWrappers() []interface{} { + return []interface{}{ (*ControllerServiceCapability_Rpc)(nil), } } -func _ControllerServiceCapability_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { - m := msg.(*ControllerServiceCapability) - // type - switch x := m.Type.(type) { - case *ControllerServiceCapability_Rpc: - b.EncodeVarint(1<<3 | proto.WireBytes) - if err := b.EncodeMessage(x.Rpc); err != nil { - return err - } - case nil: - default: - return fmt.Errorf("ControllerServiceCapability.Type has unexpected type %T", x) - } - return nil -} - -func _ControllerServiceCapability_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { - m := msg.(*ControllerServiceCapability) - switch tag { - case 1: // type.rpc - if wire != proto.WireBytes { - return true, proto.ErrInternalBadWireType - } - msg := new(ControllerServiceCapability_RPC) - err := b.DecodeMessage(msg) - m.Type = &ControllerServiceCapability_Rpc{msg} - return true, err - default: - return false, nil - } -} - -func _ControllerServiceCapability_OneofSizer(msg proto.Message) (n int) { - m := msg.(*ControllerServiceCapability) - // type - switch x := m.Type.(type) { - case *ControllerServiceCapability_Rpc: - s := proto.Size(x.Rpc) - n += 1 // tag and wire - n += proto.SizeVarint(uint64(s)) - n += s - case nil: - default: - panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) - } - return n -} - type ControllerServiceCapability_RPC struct { Type ControllerServiceCapability_RPC_Type `protobuf:"varint,1,opt,name=type,proto3,enum=csi.v1.ControllerServiceCapability_RPC_Type" json:"type,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -2866,16 +2738,17 @@ func (m *ControllerServiceCapability_RPC) Reset() { *m = ControllerServi func (m *ControllerServiceCapability_RPC) String() string { return proto.CompactTextString(m) } func (*ControllerServiceCapability_RPC) ProtoMessage() {} func (*ControllerServiceCapability_RPC) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{29, 0} + return fileDescriptor_9cdb00adce470e01, []int{29, 0} } + func (m *ControllerServiceCapability_RPC) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ControllerServiceCapability_RPC.Unmarshal(m, b) } func (m *ControllerServiceCapability_RPC) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_ControllerServiceCapability_RPC.Marshal(b, m, deterministic) } -func (dst *ControllerServiceCapability_RPC) XXX_Merge(src proto.Message) { - xxx_messageInfo_ControllerServiceCapability_RPC.Merge(dst, src) +func (m *ControllerServiceCapability_RPC) XXX_Merge(src proto.Message) { + xxx_messageInfo_ControllerServiceCapability_RPC.Merge(m, src) } func (m *ControllerServiceCapability_RPC) XXX_Size() int { return xxx_messageInfo_ControllerServiceCapability_RPC.Size(m) @@ -2928,16 +2801,17 @@ func (m *CreateSnapshotRequest) Reset() { *m = CreateSnapshotRequest{} } func (m *CreateSnapshotRequest) String() string { return proto.CompactTextString(m) } func (*CreateSnapshotRequest) ProtoMessage() {} func (*CreateSnapshotRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{30} + return fileDescriptor_9cdb00adce470e01, []int{30} } + func (m *CreateSnapshotRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CreateSnapshotRequest.Unmarshal(m, b) } func (m *CreateSnapshotRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CreateSnapshotRequest.Marshal(b, m, deterministic) } -func (dst *CreateSnapshotRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_CreateSnapshotRequest.Merge(dst, src) +func (m *CreateSnapshotRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_CreateSnapshotRequest.Merge(m, src) } func (m *CreateSnapshotRequest) XXX_Size() int { return xxx_messageInfo_CreateSnapshotRequest.Size(m) @@ -2990,16 +2864,17 @@ func (m *CreateSnapshotResponse) Reset() { *m = CreateSnapshotResponse{} func (m *CreateSnapshotResponse) String() string { return proto.CompactTextString(m) } func (*CreateSnapshotResponse) ProtoMessage() {} func (*CreateSnapshotResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{31} + return fileDescriptor_9cdb00adce470e01, []int{31} } + func (m *CreateSnapshotResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CreateSnapshotResponse.Unmarshal(m, b) } func (m *CreateSnapshotResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CreateSnapshotResponse.Marshal(b, m, deterministic) } -func (dst *CreateSnapshotResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CreateSnapshotResponse.Merge(dst, src) +func (m *CreateSnapshotResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CreateSnapshotResponse.Merge(m, src) } func (m *CreateSnapshotResponse) XXX_Size() int { return xxx_messageInfo_CreateSnapshotResponse.Size(m) @@ -3057,16 +2932,17 @@ func (m *Snapshot) Reset() { *m = Snapshot{} } func (m *Snapshot) String() string { return proto.CompactTextString(m) } func (*Snapshot) ProtoMessage() {} func (*Snapshot) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{32} + return fileDescriptor_9cdb00adce470e01, []int{32} } + func (m *Snapshot) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Snapshot.Unmarshal(m, b) } func (m *Snapshot) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Snapshot.Marshal(b, m, deterministic) } -func (dst *Snapshot) XXX_Merge(src proto.Message) { - xxx_messageInfo_Snapshot.Merge(dst, src) +func (m *Snapshot) XXX_Merge(src proto.Message) { + xxx_messageInfo_Snapshot.Merge(m, src) } func (m *Snapshot) XXX_Size() int { return xxx_messageInfo_Snapshot.Size(m) @@ -3129,16 +3005,17 @@ func (m *DeleteSnapshotRequest) Reset() { *m = DeleteSnapshotRequest{} } func (m *DeleteSnapshotRequest) String() string { return proto.CompactTextString(m) } func (*DeleteSnapshotRequest) ProtoMessage() {} func (*DeleteSnapshotRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{33} + return fileDescriptor_9cdb00adce470e01, []int{33} } + func (m *DeleteSnapshotRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DeleteSnapshotRequest.Unmarshal(m, b) } func (m *DeleteSnapshotRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_DeleteSnapshotRequest.Marshal(b, m, deterministic) } -func (dst *DeleteSnapshotRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_DeleteSnapshotRequest.Merge(dst, src) +func (m *DeleteSnapshotRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_DeleteSnapshotRequest.Merge(m, src) } func (m *DeleteSnapshotRequest) XXX_Size() int { return xxx_messageInfo_DeleteSnapshotRequest.Size(m) @@ -3173,16 +3050,17 @@ func (m *DeleteSnapshotResponse) Reset() { *m = DeleteSnapshotResponse{} func (m *DeleteSnapshotResponse) String() string { return proto.CompactTextString(m) } func (*DeleteSnapshotResponse) ProtoMessage() {} func (*DeleteSnapshotResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{34} + return fileDescriptor_9cdb00adce470e01, []int{34} } + func (m *DeleteSnapshotResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DeleteSnapshotResponse.Unmarshal(m, b) } func (m *DeleteSnapshotResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_DeleteSnapshotResponse.Marshal(b, m, deterministic) } -func (dst *DeleteSnapshotResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_DeleteSnapshotResponse.Merge(dst, src) +func (m *DeleteSnapshotResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_DeleteSnapshotResponse.Merge(m, src) } func (m *DeleteSnapshotResponse) XXX_Size() int { return xxx_messageInfo_DeleteSnapshotResponse.Size(m) @@ -3218,26 +3096,31 @@ type ListSnapshotsRequest struct { // ListSnapshots will return with current snapshot information // and will not block if the snapshot is being processed after // it is cut. - SnapshotId string `protobuf:"bytes,4,opt,name=snapshot_id,json=snapshotId,proto3" json:"snapshot_id,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + SnapshotId string `protobuf:"bytes,4,opt,name=snapshot_id,json=snapshotId,proto3" json:"snapshot_id,omitempty"` + // Secrets required by plugin to complete ListSnapshot request. + // This field is OPTIONAL. Refer to the `Secrets Requirements` + // section on how to use this field. + Secrets map[string]string `protobuf:"bytes,5,rep,name=secrets,proto3" json:"secrets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *ListSnapshotsRequest) Reset() { *m = ListSnapshotsRequest{} } func (m *ListSnapshotsRequest) String() string { return proto.CompactTextString(m) } func (*ListSnapshotsRequest) ProtoMessage() {} func (*ListSnapshotsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{35} + return fileDescriptor_9cdb00adce470e01, []int{35} } + func (m *ListSnapshotsRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ListSnapshotsRequest.Unmarshal(m, b) } func (m *ListSnapshotsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_ListSnapshotsRequest.Marshal(b, m, deterministic) } -func (dst *ListSnapshotsRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_ListSnapshotsRequest.Merge(dst, src) +func (m *ListSnapshotsRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ListSnapshotsRequest.Merge(m, src) } func (m *ListSnapshotsRequest) XXX_Size() int { return xxx_messageInfo_ListSnapshotsRequest.Size(m) @@ -3276,6 +3159,13 @@ func (m *ListSnapshotsRequest) GetSnapshotId() string { return "" } +func (m *ListSnapshotsRequest) GetSecrets() map[string]string { + if m != nil { + return m.Secrets + } + return nil +} + type ListSnapshotsResponse struct { Entries []*ListSnapshotsResponse_Entry `protobuf:"bytes,1,rep,name=entries,proto3" json:"entries,omitempty"` // This token allows you to get the next page of entries for @@ -3294,16 +3184,17 @@ func (m *ListSnapshotsResponse) Reset() { *m = ListSnapshotsResponse{} } func (m *ListSnapshotsResponse) String() string { return proto.CompactTextString(m) } func (*ListSnapshotsResponse) ProtoMessage() {} func (*ListSnapshotsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{36} + return fileDescriptor_9cdb00adce470e01, []int{36} } + func (m *ListSnapshotsResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ListSnapshotsResponse.Unmarshal(m, b) } func (m *ListSnapshotsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_ListSnapshotsResponse.Marshal(b, m, deterministic) } -func (dst *ListSnapshotsResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_ListSnapshotsResponse.Merge(dst, src) +func (m *ListSnapshotsResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_ListSnapshotsResponse.Merge(m, src) } func (m *ListSnapshotsResponse) XXX_Size() int { return xxx_messageInfo_ListSnapshotsResponse.Size(m) @@ -3339,16 +3230,17 @@ func (m *ListSnapshotsResponse_Entry) Reset() { *m = ListSnapshotsRespon func (m *ListSnapshotsResponse_Entry) String() string { return proto.CompactTextString(m) } func (*ListSnapshotsResponse_Entry) ProtoMessage() {} func (*ListSnapshotsResponse_Entry) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{36, 0} + return fileDescriptor_9cdb00adce470e01, []int{36, 0} } + func (m *ListSnapshotsResponse_Entry) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ListSnapshotsResponse_Entry.Unmarshal(m, b) } func (m *ListSnapshotsResponse_Entry) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_ListSnapshotsResponse_Entry.Marshal(b, m, deterministic) } -func (dst *ListSnapshotsResponse_Entry) XXX_Merge(src proto.Message) { - xxx_messageInfo_ListSnapshotsResponse_Entry.Merge(dst, src) +func (m *ListSnapshotsResponse_Entry) XXX_Merge(src proto.Message) { + xxx_messageInfo_ListSnapshotsResponse_Entry.Merge(m, src) } func (m *ListSnapshotsResponse_Entry) XXX_Size() int { return xxx_messageInfo_ListSnapshotsResponse_Entry.Size(m) @@ -3374,7 +3266,15 @@ type ControllerExpandVolumeRequest struct { CapacityRange *CapacityRange `protobuf:"bytes,2,opt,name=capacity_range,json=capacityRange,proto3" json:"capacity_range,omitempty"` // Secrets required by the plugin for expanding the volume. // This field is OPTIONAL. - Secrets map[string]string `protobuf:"bytes,3,rep,name=secrets,proto3" json:"secrets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + Secrets map[string]string `protobuf:"bytes,3,rep,name=secrets,proto3" json:"secrets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // Volume capability describing how the CO intends to use this volume. + // This allows SP to determine if volume is being used as a block + // device or mounted file system. For example - if volume is + // being used as a block device - the SP MAY set + // node_expansion_required to false in ControllerExpandVolumeResponse + // to skip invocation of NodeExpandVolume on the node by the CO. + // This is an OPTIONAL field. + VolumeCapability *VolumeCapability `protobuf:"bytes,4,opt,name=volume_capability,json=volumeCapability,proto3" json:"volume_capability,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -3384,16 +3284,17 @@ func (m *ControllerExpandVolumeRequest) Reset() { *m = ControllerExpandV func (m *ControllerExpandVolumeRequest) String() string { return proto.CompactTextString(m) } func (*ControllerExpandVolumeRequest) ProtoMessage() {} func (*ControllerExpandVolumeRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{37} + return fileDescriptor_9cdb00adce470e01, []int{37} } + func (m *ControllerExpandVolumeRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ControllerExpandVolumeRequest.Unmarshal(m, b) } func (m *ControllerExpandVolumeRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_ControllerExpandVolumeRequest.Marshal(b, m, deterministic) } -func (dst *ControllerExpandVolumeRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_ControllerExpandVolumeRequest.Merge(dst, src) +func (m *ControllerExpandVolumeRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ControllerExpandVolumeRequest.Merge(m, src) } func (m *ControllerExpandVolumeRequest) XXX_Size() int { return xxx_messageInfo_ControllerExpandVolumeRequest.Size(m) @@ -3425,6 +3326,13 @@ func (m *ControllerExpandVolumeRequest) GetSecrets() map[string]string { return nil } +func (m *ControllerExpandVolumeRequest) GetVolumeCapability() *VolumeCapability { + if m != nil { + return m.VolumeCapability + } + return nil +} + type ControllerExpandVolumeResponse struct { // Capacity of volume after expansion. This field is REQUIRED. CapacityBytes int64 `protobuf:"varint,1,opt,name=capacity_bytes,json=capacityBytes,proto3" json:"capacity_bytes,omitempty"` @@ -3441,16 +3349,17 @@ func (m *ControllerExpandVolumeResponse) Reset() { *m = ControllerExpand func (m *ControllerExpandVolumeResponse) String() string { return proto.CompactTextString(m) } func (*ControllerExpandVolumeResponse) ProtoMessage() {} func (*ControllerExpandVolumeResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{38} + return fileDescriptor_9cdb00adce470e01, []int{38} } + func (m *ControllerExpandVolumeResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ControllerExpandVolumeResponse.Unmarshal(m, b) } func (m *ControllerExpandVolumeResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_ControllerExpandVolumeResponse.Marshal(b, m, deterministic) } -func (dst *ControllerExpandVolumeResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_ControllerExpandVolumeResponse.Merge(dst, src) +func (m *ControllerExpandVolumeResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_ControllerExpandVolumeResponse.Merge(m, src) } func (m *ControllerExpandVolumeResponse) XXX_Size() int { return xxx_messageInfo_ControllerExpandVolumeResponse.Size(m) @@ -3516,16 +3425,17 @@ func (m *NodeStageVolumeRequest) Reset() { *m = NodeStageVolumeRequest{} func (m *NodeStageVolumeRequest) String() string { return proto.CompactTextString(m) } func (*NodeStageVolumeRequest) ProtoMessage() {} func (*NodeStageVolumeRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{39} + return fileDescriptor_9cdb00adce470e01, []int{39} } + func (m *NodeStageVolumeRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodeStageVolumeRequest.Unmarshal(m, b) } func (m *NodeStageVolumeRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_NodeStageVolumeRequest.Marshal(b, m, deterministic) } -func (dst *NodeStageVolumeRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_NodeStageVolumeRequest.Merge(dst, src) +func (m *NodeStageVolumeRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodeStageVolumeRequest.Merge(m, src) } func (m *NodeStageVolumeRequest) XXX_Size() int { return xxx_messageInfo_NodeStageVolumeRequest.Size(m) @@ -3588,16 +3498,17 @@ func (m *NodeStageVolumeResponse) Reset() { *m = NodeStageVolumeResponse func (m *NodeStageVolumeResponse) String() string { return proto.CompactTextString(m) } func (*NodeStageVolumeResponse) ProtoMessage() {} func (*NodeStageVolumeResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{40} + return fileDescriptor_9cdb00adce470e01, []int{40} } + func (m *NodeStageVolumeResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodeStageVolumeResponse.Unmarshal(m, b) } func (m *NodeStageVolumeResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_NodeStageVolumeResponse.Marshal(b, m, deterministic) } -func (dst *NodeStageVolumeResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_NodeStageVolumeResponse.Merge(dst, src) +func (m *NodeStageVolumeResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodeStageVolumeResponse.Merge(m, src) } func (m *NodeStageVolumeResponse) XXX_Size() int { return xxx_messageInfo_NodeStageVolumeResponse.Size(m) @@ -3624,16 +3535,17 @@ func (m *NodeUnstageVolumeRequest) Reset() { *m = NodeUnstageVolumeReque func (m *NodeUnstageVolumeRequest) String() string { return proto.CompactTextString(m) } func (*NodeUnstageVolumeRequest) ProtoMessage() {} func (*NodeUnstageVolumeRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{41} + return fileDescriptor_9cdb00adce470e01, []int{41} } + func (m *NodeUnstageVolumeRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodeUnstageVolumeRequest.Unmarshal(m, b) } func (m *NodeUnstageVolumeRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_NodeUnstageVolumeRequest.Marshal(b, m, deterministic) } -func (dst *NodeUnstageVolumeRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_NodeUnstageVolumeRequest.Merge(dst, src) +func (m *NodeUnstageVolumeRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodeUnstageVolumeRequest.Merge(m, src) } func (m *NodeUnstageVolumeRequest) XXX_Size() int { return xxx_messageInfo_NodeUnstageVolumeRequest.Size(m) @@ -3668,16 +3580,17 @@ func (m *NodeUnstageVolumeResponse) Reset() { *m = NodeUnstageVolumeResp func (m *NodeUnstageVolumeResponse) String() string { return proto.CompactTextString(m) } func (*NodeUnstageVolumeResponse) ProtoMessage() {} func (*NodeUnstageVolumeResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{42} + return fileDescriptor_9cdb00adce470e01, []int{42} } + func (m *NodeUnstageVolumeResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodeUnstageVolumeResponse.Unmarshal(m, b) } func (m *NodeUnstageVolumeResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_NodeUnstageVolumeResponse.Marshal(b, m, deterministic) } -func (dst *NodeUnstageVolumeResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_NodeUnstageVolumeResponse.Merge(dst, src) +func (m *NodeUnstageVolumeResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodeUnstageVolumeResponse.Merge(m, src) } func (m *NodeUnstageVolumeResponse) XXX_Size() int { return xxx_messageInfo_NodeUnstageVolumeResponse.Size(m) @@ -3742,16 +3655,17 @@ func (m *NodePublishVolumeRequest) Reset() { *m = NodePublishVolumeReque func (m *NodePublishVolumeRequest) String() string { return proto.CompactTextString(m) } func (*NodePublishVolumeRequest) ProtoMessage() {} func (*NodePublishVolumeRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{43} + return fileDescriptor_9cdb00adce470e01, []int{43} } + func (m *NodePublishVolumeRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodePublishVolumeRequest.Unmarshal(m, b) } func (m *NodePublishVolumeRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_NodePublishVolumeRequest.Marshal(b, m, deterministic) } -func (dst *NodePublishVolumeRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_NodePublishVolumeRequest.Merge(dst, src) +func (m *NodePublishVolumeRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodePublishVolumeRequest.Merge(m, src) } func (m *NodePublishVolumeRequest) XXX_Size() int { return xxx_messageInfo_NodePublishVolumeRequest.Size(m) @@ -3828,16 +3742,17 @@ func (m *NodePublishVolumeResponse) Reset() { *m = NodePublishVolumeResp func (m *NodePublishVolumeResponse) String() string { return proto.CompactTextString(m) } func (*NodePublishVolumeResponse) ProtoMessage() {} func (*NodePublishVolumeResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{44} + return fileDescriptor_9cdb00adce470e01, []int{44} } + func (m *NodePublishVolumeResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodePublishVolumeResponse.Unmarshal(m, b) } func (m *NodePublishVolumeResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_NodePublishVolumeResponse.Marshal(b, m, deterministic) } -func (dst *NodePublishVolumeResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_NodePublishVolumeResponse.Merge(dst, src) +func (m *NodePublishVolumeResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodePublishVolumeResponse.Merge(m, src) } func (m *NodePublishVolumeResponse) XXX_Size() int { return xxx_messageInfo_NodePublishVolumeResponse.Size(m) @@ -3865,16 +3780,17 @@ func (m *NodeUnpublishVolumeRequest) Reset() { *m = NodeUnpublishVolumeR func (m *NodeUnpublishVolumeRequest) String() string { return proto.CompactTextString(m) } func (*NodeUnpublishVolumeRequest) ProtoMessage() {} func (*NodeUnpublishVolumeRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{45} + return fileDescriptor_9cdb00adce470e01, []int{45} } + func (m *NodeUnpublishVolumeRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodeUnpublishVolumeRequest.Unmarshal(m, b) } func (m *NodeUnpublishVolumeRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_NodeUnpublishVolumeRequest.Marshal(b, m, deterministic) } -func (dst *NodeUnpublishVolumeRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_NodeUnpublishVolumeRequest.Merge(dst, src) +func (m *NodeUnpublishVolumeRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodeUnpublishVolumeRequest.Merge(m, src) } func (m *NodeUnpublishVolumeRequest) XXX_Size() int { return xxx_messageInfo_NodeUnpublishVolumeRequest.Size(m) @@ -3909,16 +3825,17 @@ func (m *NodeUnpublishVolumeResponse) Reset() { *m = NodeUnpublishVolume func (m *NodeUnpublishVolumeResponse) String() string { return proto.CompactTextString(m) } func (*NodeUnpublishVolumeResponse) ProtoMessage() {} func (*NodeUnpublishVolumeResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{46} + return fileDescriptor_9cdb00adce470e01, []int{46} } + func (m *NodeUnpublishVolumeResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodeUnpublishVolumeResponse.Unmarshal(m, b) } func (m *NodeUnpublishVolumeResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_NodeUnpublishVolumeResponse.Marshal(b, m, deterministic) } -func (dst *NodeUnpublishVolumeResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_NodeUnpublishVolumeResponse.Merge(dst, src) +func (m *NodeUnpublishVolumeResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodeUnpublishVolumeResponse.Merge(m, src) } func (m *NodeUnpublishVolumeResponse) XXX_Size() int { return xxx_messageInfo_NodeUnpublishVolumeResponse.Size(m) @@ -3937,7 +3854,13 @@ type NodeGetVolumeStatsRequest struct { // It MUST be an absolute path in the root filesystem of // the process serving this request. // This is a REQUIRED field. - VolumePath string `protobuf:"bytes,2,opt,name=volume_path,json=volumePath,proto3" json:"volume_path,omitempty"` + VolumePath string `protobuf:"bytes,2,opt,name=volume_path,json=volumePath,proto3" json:"volume_path,omitempty"` + // The path where the volume is staged, if the plugin has the + // STAGE_UNSTAGE_VOLUME capability, otherwise empty. + // If not empty, it MUST be an absolute path in the root + // filesystem of the process serving this request. + // This field is OPTIONAL. + StagingTargetPath string `protobuf:"bytes,3,opt,name=staging_target_path,json=stagingTargetPath,proto3" json:"staging_target_path,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -3947,16 +3870,17 @@ func (m *NodeGetVolumeStatsRequest) Reset() { *m = NodeGetVolumeStatsReq func (m *NodeGetVolumeStatsRequest) String() string { return proto.CompactTextString(m) } func (*NodeGetVolumeStatsRequest) ProtoMessage() {} func (*NodeGetVolumeStatsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{47} + return fileDescriptor_9cdb00adce470e01, []int{47} } + func (m *NodeGetVolumeStatsRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodeGetVolumeStatsRequest.Unmarshal(m, b) } func (m *NodeGetVolumeStatsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_NodeGetVolumeStatsRequest.Marshal(b, m, deterministic) } -func (dst *NodeGetVolumeStatsRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_NodeGetVolumeStatsRequest.Merge(dst, src) +func (m *NodeGetVolumeStatsRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodeGetVolumeStatsRequest.Merge(m, src) } func (m *NodeGetVolumeStatsRequest) XXX_Size() int { return xxx_messageInfo_NodeGetVolumeStatsRequest.Size(m) @@ -3981,6 +3905,13 @@ func (m *NodeGetVolumeStatsRequest) GetVolumePath() string { return "" } +func (m *NodeGetVolumeStatsRequest) GetStagingTargetPath() string { + if m != nil { + return m.StagingTargetPath + } + return "" +} + type NodeGetVolumeStatsResponse struct { // This field is OPTIONAL. Usage []*VolumeUsage `protobuf:"bytes,1,rep,name=usage,proto3" json:"usage,omitempty"` @@ -3993,16 +3924,17 @@ func (m *NodeGetVolumeStatsResponse) Reset() { *m = NodeGetVolumeStatsRe func (m *NodeGetVolumeStatsResponse) String() string { return proto.CompactTextString(m) } func (*NodeGetVolumeStatsResponse) ProtoMessage() {} func (*NodeGetVolumeStatsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{48} + return fileDescriptor_9cdb00adce470e01, []int{48} } + func (m *NodeGetVolumeStatsResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodeGetVolumeStatsResponse.Unmarshal(m, b) } func (m *NodeGetVolumeStatsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_NodeGetVolumeStatsResponse.Marshal(b, m, deterministic) } -func (dst *NodeGetVolumeStatsResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_NodeGetVolumeStatsResponse.Merge(dst, src) +func (m *NodeGetVolumeStatsResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodeGetVolumeStatsResponse.Merge(m, src) } func (m *NodeGetVolumeStatsResponse) XXX_Size() int { return xxx_messageInfo_NodeGetVolumeStatsResponse.Size(m) @@ -4041,16 +3973,17 @@ func (m *VolumeUsage) Reset() { *m = VolumeUsage{} } func (m *VolumeUsage) String() string { return proto.CompactTextString(m) } func (*VolumeUsage) ProtoMessage() {} func (*VolumeUsage) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{49} + return fileDescriptor_9cdb00adce470e01, []int{49} } + func (m *VolumeUsage) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_VolumeUsage.Unmarshal(m, b) } func (m *VolumeUsage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_VolumeUsage.Marshal(b, m, deterministic) } -func (dst *VolumeUsage) XXX_Merge(src proto.Message) { - xxx_messageInfo_VolumeUsage.Merge(dst, src) +func (m *VolumeUsage) XXX_Merge(src proto.Message) { + xxx_messageInfo_VolumeUsage.Merge(m, src) } func (m *VolumeUsage) XXX_Size() int { return xxx_messageInfo_VolumeUsage.Size(m) @@ -4099,16 +4032,17 @@ func (m *NodeGetCapabilitiesRequest) Reset() { *m = NodeGetCapabilitiesR func (m *NodeGetCapabilitiesRequest) String() string { return proto.CompactTextString(m) } func (*NodeGetCapabilitiesRequest) ProtoMessage() {} func (*NodeGetCapabilitiesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{50} + return fileDescriptor_9cdb00adce470e01, []int{50} } + func (m *NodeGetCapabilitiesRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodeGetCapabilitiesRequest.Unmarshal(m, b) } func (m *NodeGetCapabilitiesRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_NodeGetCapabilitiesRequest.Marshal(b, m, deterministic) } -func (dst *NodeGetCapabilitiesRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_NodeGetCapabilitiesRequest.Merge(dst, src) +func (m *NodeGetCapabilitiesRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodeGetCapabilitiesRequest.Merge(m, src) } func (m *NodeGetCapabilitiesRequest) XXX_Size() int { return xxx_messageInfo_NodeGetCapabilitiesRequest.Size(m) @@ -4132,16 +4066,17 @@ func (m *NodeGetCapabilitiesResponse) Reset() { *m = NodeGetCapabilities func (m *NodeGetCapabilitiesResponse) String() string { return proto.CompactTextString(m) } func (*NodeGetCapabilitiesResponse) ProtoMessage() {} func (*NodeGetCapabilitiesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{51} + return fileDescriptor_9cdb00adce470e01, []int{51} } + func (m *NodeGetCapabilitiesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodeGetCapabilitiesResponse.Unmarshal(m, b) } func (m *NodeGetCapabilitiesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_NodeGetCapabilitiesResponse.Marshal(b, m, deterministic) } -func (dst *NodeGetCapabilitiesResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_NodeGetCapabilitiesResponse.Merge(dst, src) +func (m *NodeGetCapabilitiesResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodeGetCapabilitiesResponse.Merge(m, src) } func (m *NodeGetCapabilitiesResponse) XXX_Size() int { return xxx_messageInfo_NodeGetCapabilitiesResponse.Size(m) @@ -4173,16 +4108,17 @@ func (m *NodeServiceCapability) Reset() { *m = NodeServiceCapability{} } func (m *NodeServiceCapability) String() string { return proto.CompactTextString(m) } func (*NodeServiceCapability) ProtoMessage() {} func (*NodeServiceCapability) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{52} + return fileDescriptor_9cdb00adce470e01, []int{52} } + func (m *NodeServiceCapability) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodeServiceCapability.Unmarshal(m, b) } func (m *NodeServiceCapability) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_NodeServiceCapability.Marshal(b, m, deterministic) } -func (dst *NodeServiceCapability) XXX_Merge(src proto.Message) { - xxx_messageInfo_NodeServiceCapability.Merge(dst, src) +func (m *NodeServiceCapability) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodeServiceCapability.Merge(m, src) } func (m *NodeServiceCapability) XXX_Size() int { return xxx_messageInfo_NodeServiceCapability.Size(m) @@ -4217,61 +4153,13 @@ func (m *NodeServiceCapability) GetRpc() *NodeServiceCapability_RPC { return nil } -// XXX_OneofFuncs is for the internal use of the proto package. -func (*NodeServiceCapability) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { - return _NodeServiceCapability_OneofMarshaler, _NodeServiceCapability_OneofUnmarshaler, _NodeServiceCapability_OneofSizer, []interface{}{ +// XXX_OneofWrappers is for the internal use of the proto package. +func (*NodeServiceCapability) XXX_OneofWrappers() []interface{} { + return []interface{}{ (*NodeServiceCapability_Rpc)(nil), } } -func _NodeServiceCapability_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { - m := msg.(*NodeServiceCapability) - // type - switch x := m.Type.(type) { - case *NodeServiceCapability_Rpc: - b.EncodeVarint(1<<3 | proto.WireBytes) - if err := b.EncodeMessage(x.Rpc); err != nil { - return err - } - case nil: - default: - return fmt.Errorf("NodeServiceCapability.Type has unexpected type %T", x) - } - return nil -} - -func _NodeServiceCapability_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { - m := msg.(*NodeServiceCapability) - switch tag { - case 1: // type.rpc - if wire != proto.WireBytes { - return true, proto.ErrInternalBadWireType - } - msg := new(NodeServiceCapability_RPC) - err := b.DecodeMessage(msg) - m.Type = &NodeServiceCapability_Rpc{msg} - return true, err - default: - return false, nil - } -} - -func _NodeServiceCapability_OneofSizer(msg proto.Message) (n int) { - m := msg.(*NodeServiceCapability) - // type - switch x := m.Type.(type) { - case *NodeServiceCapability_Rpc: - s := proto.Size(x.Rpc) - n += 1 // tag and wire - n += proto.SizeVarint(uint64(s)) - n += s - case nil: - default: - panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) - } - return n -} - type NodeServiceCapability_RPC struct { Type NodeServiceCapability_RPC_Type `protobuf:"varint,1,opt,name=type,proto3,enum=csi.v1.NodeServiceCapability_RPC_Type" json:"type,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -4283,16 +4171,17 @@ func (m *NodeServiceCapability_RPC) Reset() { *m = NodeServiceCapability func (m *NodeServiceCapability_RPC) String() string { return proto.CompactTextString(m) } func (*NodeServiceCapability_RPC) ProtoMessage() {} func (*NodeServiceCapability_RPC) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{52, 0} + return fileDescriptor_9cdb00adce470e01, []int{52, 0} } + func (m *NodeServiceCapability_RPC) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodeServiceCapability_RPC.Unmarshal(m, b) } func (m *NodeServiceCapability_RPC) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_NodeServiceCapability_RPC.Marshal(b, m, deterministic) } -func (dst *NodeServiceCapability_RPC) XXX_Merge(src proto.Message) { - xxx_messageInfo_NodeServiceCapability_RPC.Merge(dst, src) +func (m *NodeServiceCapability_RPC) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodeServiceCapability_RPC.Merge(m, src) } func (m *NodeServiceCapability_RPC) XXX_Size() int { return xxx_messageInfo_NodeServiceCapability_RPC.Size(m) @@ -4320,16 +4209,17 @@ func (m *NodeGetInfoRequest) Reset() { *m = NodeGetInfoRequest{} } func (m *NodeGetInfoRequest) String() string { return proto.CompactTextString(m) } func (*NodeGetInfoRequest) ProtoMessage() {} func (*NodeGetInfoRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{53} + return fileDescriptor_9cdb00adce470e01, []int{53} } + func (m *NodeGetInfoRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodeGetInfoRequest.Unmarshal(m, b) } func (m *NodeGetInfoRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_NodeGetInfoRequest.Marshal(b, m, deterministic) } -func (dst *NodeGetInfoRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_NodeGetInfoRequest.Merge(dst, src) +func (m *NodeGetInfoRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodeGetInfoRequest.Merge(m, src) } func (m *NodeGetInfoRequest) XXX_Size() int { return xxx_messageInfo_NodeGetInfoRequest.Size(m) @@ -4383,16 +4273,17 @@ func (m *NodeGetInfoResponse) Reset() { *m = NodeGetInfoResponse{} } func (m *NodeGetInfoResponse) String() string { return proto.CompactTextString(m) } func (*NodeGetInfoResponse) ProtoMessage() {} func (*NodeGetInfoResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{54} + return fileDescriptor_9cdb00adce470e01, []int{54} } + func (m *NodeGetInfoResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodeGetInfoResponse.Unmarshal(m, b) } func (m *NodeGetInfoResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_NodeGetInfoResponse.Marshal(b, m, deterministic) } -func (dst *NodeGetInfoResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_NodeGetInfoResponse.Merge(dst, src) +func (m *NodeGetInfoResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodeGetInfoResponse.Merge(m, src) } func (m *NodeGetInfoResponse) XXX_Size() int { return xxx_messageInfo_NodeGetInfoResponse.Size(m) @@ -4435,26 +4326,43 @@ type NodeExpandVolumeRequest struct { // capacity to which the volume can be expanded. In such cases a // plugin MAY expand the volume to its maximum capacity. // This field is OPTIONAL. - CapacityRange *CapacityRange `protobuf:"bytes,3,opt,name=capacity_range,json=capacityRange,proto3" json:"capacity_range,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + CapacityRange *CapacityRange `protobuf:"bytes,3,opt,name=capacity_range,json=capacityRange,proto3" json:"capacity_range,omitempty"` + // The path where the volume is staged, if the plugin has the + // STAGE_UNSTAGE_VOLUME capability, otherwise empty. + // If not empty, it MUST be an absolute path in the root + // filesystem of the process serving this request. + // This field is OPTIONAL. + StagingTargetPath string `protobuf:"bytes,4,opt,name=staging_target_path,json=stagingTargetPath,proto3" json:"staging_target_path,omitempty"` + // Volume capability describing how the CO intends to use this volume. + // This allows SP to determine if volume is being used as a block + // device or mounted file system. For example - if volume is being + // used as a block device the SP MAY choose to skip expanding the + // filesystem in NodeExpandVolume implementation but still perform + // rest of the housekeeping needed for expanding the volume. If + // volume_capability is omitted the SP MAY determine + // access_type from given volume_path for the volume and perform + // node expansion. This is an OPTIONAL field. + VolumeCapability *VolumeCapability `protobuf:"bytes,5,opt,name=volume_capability,json=volumeCapability,proto3" json:"volume_capability,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *NodeExpandVolumeRequest) Reset() { *m = NodeExpandVolumeRequest{} } func (m *NodeExpandVolumeRequest) String() string { return proto.CompactTextString(m) } func (*NodeExpandVolumeRequest) ProtoMessage() {} func (*NodeExpandVolumeRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{55} + return fileDescriptor_9cdb00adce470e01, []int{55} } + func (m *NodeExpandVolumeRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodeExpandVolumeRequest.Unmarshal(m, b) } func (m *NodeExpandVolumeRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_NodeExpandVolumeRequest.Marshal(b, m, deterministic) } -func (dst *NodeExpandVolumeRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_NodeExpandVolumeRequest.Merge(dst, src) +func (m *NodeExpandVolumeRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodeExpandVolumeRequest.Merge(m, src) } func (m *NodeExpandVolumeRequest) XXX_Size() int { return xxx_messageInfo_NodeExpandVolumeRequest.Size(m) @@ -4486,6 +4394,20 @@ func (m *NodeExpandVolumeRequest) GetCapacityRange() *CapacityRange { return nil } +func (m *NodeExpandVolumeRequest) GetStagingTargetPath() string { + if m != nil { + return m.StagingTargetPath + } + return "" +} + +func (m *NodeExpandVolumeRequest) GetVolumeCapability() *VolumeCapability { + if m != nil { + return m.VolumeCapability + } + return nil +} + type NodeExpandVolumeResponse struct { // The capacity of the volume in bytes. This field is OPTIONAL. CapacityBytes int64 `protobuf:"varint,1,opt,name=capacity_bytes,json=capacityBytes,proto3" json:"capacity_bytes,omitempty"` @@ -4498,16 +4420,17 @@ func (m *NodeExpandVolumeResponse) Reset() { *m = NodeExpandVolumeRespon func (m *NodeExpandVolumeResponse) String() string { return proto.CompactTextString(m) } func (*NodeExpandVolumeResponse) ProtoMessage() {} func (*NodeExpandVolumeResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_csi_2c5455657a82ae49, []int{56} + return fileDescriptor_9cdb00adce470e01, []int{56} } + func (m *NodeExpandVolumeResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NodeExpandVolumeResponse.Unmarshal(m, b) } func (m *NodeExpandVolumeResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_NodeExpandVolumeResponse.Marshal(b, m, deterministic) } -func (dst *NodeExpandVolumeResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_NodeExpandVolumeResponse.Merge(dst, src) +func (m *NodeExpandVolumeResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodeExpandVolumeResponse.Merge(m, src) } func (m *NodeExpandVolumeResponse) XXX_Size() int { return xxx_messageInfo_NodeExpandVolumeResponse.Size(m) @@ -4530,11 +4453,17 @@ var E_CsiSecret = &proto.ExtensionDesc{ ExtensionType: (*bool)(nil), Field: 1059, Name: "csi.v1.csi_secret", - Tag: "varint,1059,opt,name=csi_secret,json=csiSecret", + Tag: "varint,1059,opt,name=csi_secret", Filename: "github.com/container-storage-interface/spec/csi.proto", } func init() { + proto.RegisterEnum("csi.v1.PluginCapability_Service_Type", PluginCapability_Service_Type_name, PluginCapability_Service_Type_value) + proto.RegisterEnum("csi.v1.PluginCapability_VolumeExpansion_Type", PluginCapability_VolumeExpansion_Type_name, PluginCapability_VolumeExpansion_Type_value) + proto.RegisterEnum("csi.v1.VolumeCapability_AccessMode_Mode", VolumeCapability_AccessMode_Mode_name, VolumeCapability_AccessMode_Mode_value) + proto.RegisterEnum("csi.v1.ControllerServiceCapability_RPC_Type", ControllerServiceCapability_RPC_Type_name, ControllerServiceCapability_RPC_Type_value) + proto.RegisterEnum("csi.v1.VolumeUsage_Unit", VolumeUsage_Unit_name, VolumeUsage_Unit_value) + proto.RegisterEnum("csi.v1.NodeServiceCapability_RPC_Type", NodeServiceCapability_RPC_Type_name, NodeServiceCapability_RPC_Type_value) proto.RegisterType((*GetPluginInfoRequest)(nil), "csi.v1.GetPluginInfoRequest") proto.RegisterType((*GetPluginInfoResponse)(nil), "csi.v1.GetPluginInfoResponse") proto.RegisterMapType((map[string]string)(nil), "csi.v1.GetPluginInfoResponse.ManifestEntry") @@ -4583,6 +4512,7 @@ func init() { proto.RegisterMapType((map[string]string)(nil), "csi.v1.ValidateVolumeCapabilitiesResponse.Confirmed.VolumeContextEntry") proto.RegisterType((*ListVolumesRequest)(nil), "csi.v1.ListVolumesRequest") proto.RegisterType((*ListVolumesResponse)(nil), "csi.v1.ListVolumesResponse") + proto.RegisterType((*ListVolumesResponse_VolumeStatus)(nil), "csi.v1.ListVolumesResponse.VolumeStatus") proto.RegisterType((*ListVolumesResponse_Entry)(nil), "csi.v1.ListVolumesResponse.Entry") proto.RegisterType((*GetCapacityRequest)(nil), "csi.v1.GetCapacityRequest") proto.RegisterMapType((map[string]string)(nil), "csi.v1.GetCapacityRequest.ParametersEntry") @@ -4600,6 +4530,7 @@ func init() { proto.RegisterMapType((map[string]string)(nil), "csi.v1.DeleteSnapshotRequest.SecretsEntry") proto.RegisterType((*DeleteSnapshotResponse)(nil), "csi.v1.DeleteSnapshotResponse") proto.RegisterType((*ListSnapshotsRequest)(nil), "csi.v1.ListSnapshotsRequest") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.ListSnapshotsRequest.SecretsEntry") proto.RegisterType((*ListSnapshotsResponse)(nil), "csi.v1.ListSnapshotsResponse") proto.RegisterType((*ListSnapshotsResponse_Entry)(nil), "csi.v1.ListSnapshotsResponse.Entry") proto.RegisterType((*ControllerExpandVolumeRequest)(nil), "csi.v1.ControllerExpandVolumeRequest") @@ -4630,15 +4561,228 @@ func init() { proto.RegisterType((*NodeGetInfoResponse)(nil), "csi.v1.NodeGetInfoResponse") proto.RegisterType((*NodeExpandVolumeRequest)(nil), "csi.v1.NodeExpandVolumeRequest") proto.RegisterType((*NodeExpandVolumeResponse)(nil), "csi.v1.NodeExpandVolumeResponse") - proto.RegisterEnum("csi.v1.PluginCapability_Service_Type", PluginCapability_Service_Type_name, PluginCapability_Service_Type_value) - proto.RegisterEnum("csi.v1.PluginCapability_VolumeExpansion_Type", PluginCapability_VolumeExpansion_Type_name, PluginCapability_VolumeExpansion_Type_value) - proto.RegisterEnum("csi.v1.VolumeCapability_AccessMode_Mode", VolumeCapability_AccessMode_Mode_name, VolumeCapability_AccessMode_Mode_value) - proto.RegisterEnum("csi.v1.ControllerServiceCapability_RPC_Type", ControllerServiceCapability_RPC_Type_name, ControllerServiceCapability_RPC_Type_value) - proto.RegisterEnum("csi.v1.VolumeUsage_Unit", VolumeUsage_Unit_name, VolumeUsage_Unit_value) - proto.RegisterEnum("csi.v1.NodeServiceCapability_RPC_Type", NodeServiceCapability_RPC_Type_name, NodeServiceCapability_RPC_Type_value) proto.RegisterExtension(E_CsiSecret) } +func init() { + proto.RegisterFile("github.com/container-storage-interface/spec/csi.proto", fileDescriptor_9cdb00adce470e01) +} + +var fileDescriptor_9cdb00adce470e01 = []byte{ + // 3366 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe4, 0x5a, 0x4d, 0x70, 0xdb, 0xc6, + 0xf5, 0x27, 0xf8, 0x25, 0xea, 0xe9, 0xc3, 0xf4, 0xea, 0xc3, 0x34, 0x24, 0xd9, 0x32, 0x1c, 0x3b, + 0xb2, 0x63, 0xd3, 0xff, 0x28, 0x71, 0xe6, 0x1f, 0x5b, 0x69, 0x43, 0x51, 0xb4, 0xc4, 0x98, 0xa6, + 0x14, 0x90, 0x92, 0x63, 0xb7, 0x19, 0x04, 0x22, 0x57, 0x34, 0x26, 0x24, 0xc0, 0x00, 0xa0, 0x2a, + 0xf5, 0xd2, 0x99, 0xf6, 0xd4, 0x69, 0xef, 0x6d, 0x4f, 0x9d, 0x49, 0x6f, 0x6d, 0x33, 0x39, 0x75, + 0x7a, 0xec, 0x4c, 0x0f, 0x3d, 0xf4, 0xd0, 0xe9, 0xad, 0x9d, 0x5e, 0x72, 0xed, 0x64, 0xda, 0x99, + 0x4c, 0x8f, 0x9d, 0x1e, 0x3a, 0xc0, 0x2e, 0x40, 0x2c, 0x08, 0x80, 0xa4, 0x65, 0x4f, 0x0e, 0x3d, + 0x49, 0x7c, 0xfb, 0xf6, 0xed, 0xdb, 0xb7, 0xef, 0xbd, 0x7d, 0xef, 0xb7, 0x80, 0xbb, 0x2d, 0xc5, + 0x7c, 0xd6, 0x3b, 0xcc, 0x37, 0xb4, 0xce, 0x9d, 0x86, 0xa6, 0x9a, 0xb2, 0xa2, 0x62, 0xfd, 0xb6, + 0x61, 0x6a, 0xba, 0xdc, 0xc2, 0xb7, 0x15, 0xd5, 0xc4, 0xfa, 0x91, 0xdc, 0xc0, 0x77, 0x8c, 0x2e, + 0x6e, 0xdc, 0x69, 0x18, 0x4a, 0xbe, 0xab, 0x6b, 0xa6, 0x86, 0xd2, 0xd6, 0xbf, 0xc7, 0xaf, 0xf3, + 0xab, 0x2d, 0x4d, 0x6b, 0xb5, 0xf1, 0x1d, 0x9b, 0x7a, 0xd8, 0x3b, 0xba, 0xd3, 0xc4, 0x46, 0x43, + 0x57, 0xba, 0xa6, 0xa6, 0x13, 0x4e, 0xfe, 0xb2, 0x9f, 0xc3, 0x54, 0x3a, 0xd8, 0x30, 0xe5, 0x4e, + 0x97, 0x32, 0x5c, 0xf2, 0x33, 0x7c, 0x47, 0x97, 0xbb, 0x5d, 0xac, 0x1b, 0x64, 0x5c, 0x58, 0x84, + 0xf9, 0x6d, 0x6c, 0xee, 0xb5, 0x7b, 0x2d, 0x45, 0x2d, 0xab, 0x47, 0x9a, 0x88, 0x3f, 0xe9, 0x61, + 0xc3, 0x14, 0xfe, 0xca, 0xc1, 0x82, 0x6f, 0xc0, 0xe8, 0x6a, 0xaa, 0x81, 0x11, 0x82, 0xa4, 0x2a, + 0x77, 0x70, 0x8e, 0x5b, 0xe5, 0xd6, 0x26, 0x45, 0xfb, 0x7f, 0x74, 0x0d, 0x66, 0x8f, 0xb1, 0xda, + 0xd4, 0x74, 0xe9, 0x18, 0xeb, 0x86, 0xa2, 0xa9, 0xb9, 0xb8, 0x3d, 0x3a, 0x43, 0xa8, 0x07, 0x84, + 0x88, 0xb6, 0x21, 0xd3, 0x91, 0x55, 0xe5, 0x08, 0x1b, 0x66, 0x2e, 0xb1, 0x9a, 0x58, 0x9b, 0x5a, + 0x7f, 0x2d, 0x4f, 0xb6, 0x9a, 0x0f, 0x5c, 0x2b, 0xff, 0x88, 0x72, 0x97, 0x54, 0x53, 0x3f, 0x15, + 0xdd, 0xc9, 0xfc, 0x7d, 0x98, 0x61, 0x86, 0x50, 0x16, 0x12, 0x1f, 0xe3, 0x53, 0xaa, 0x93, 0xf5, + 0x2f, 0x9a, 0x87, 0xd4, 0xb1, 0xdc, 0xee, 0x61, 0xaa, 0x09, 0xf9, 0x71, 0x2f, 0xfe, 0xff, 0x9c, + 0x70, 0x09, 0x96, 0xdd, 0xd5, 0x8a, 0x72, 0x57, 0x3e, 0x54, 0xda, 0x8a, 0xa9, 0x60, 0xc3, 0xd9, + 0xfa, 0x87, 0xb0, 0x12, 0x32, 0x4e, 0x2d, 0xb0, 0x01, 0xd3, 0x0d, 0x0f, 0x3d, 0xc7, 0xd9, 0x5b, + 0xc9, 0x39, 0x5b, 0xf1, 0xcd, 0x3c, 0x15, 0x19, 0x6e, 0xe1, 0x4f, 0x09, 0xc8, 0xfa, 0x59, 0xd0, + 0x06, 0x4c, 0x18, 0x58, 0x3f, 0x56, 0x1a, 0xc4, 0xae, 0x53, 0xeb, 0xab, 0x61, 0xd2, 0xf2, 0x35, + 0xc2, 0xb7, 0x13, 0x13, 0x9d, 0x29, 0x68, 0x1f, 0xb2, 0xc7, 0x5a, 0xbb, 0xd7, 0xc1, 0x12, 0x3e, + 0xe9, 0xca, 0xaa, 0x7b, 0x00, 0x53, 0xeb, 0x6b, 0xa1, 0x62, 0x0e, 0xec, 0x09, 0x25, 0x87, 0x7f, + 0x27, 0x26, 0x9e, 0x3b, 0x66, 0x49, 0xfc, 0x4f, 0x38, 0x98, 0xa0, 0xab, 0xa1, 0xb7, 0x21, 0x69, + 0x9e, 0x76, 0x89, 0x76, 0xb3, 0xeb, 0xd7, 0x86, 0x69, 0x97, 0xaf, 0x9f, 0x76, 0xb1, 0x68, 0x4f, + 0x11, 0xde, 0x87, 0xa4, 0xf5, 0x0b, 0x4d, 0xc1, 0xc4, 0x7e, 0xf5, 0x61, 0x75, 0xf7, 0x71, 0x35, + 0x1b, 0x43, 0x8b, 0x80, 0x8a, 0xbb, 0xd5, 0xba, 0xb8, 0x5b, 0xa9, 0x94, 0x44, 0xa9, 0x56, 0x12, + 0x0f, 0xca, 0xc5, 0x52, 0x96, 0x43, 0xaf, 0xc0, 0xea, 0xc1, 0x6e, 0x65, 0xff, 0x51, 0x49, 0x2a, + 0x14, 0x8b, 0xa5, 0x5a, 0xad, 0xbc, 0x59, 0xae, 0x94, 0xeb, 0x4f, 0xa4, 0xe2, 0x6e, 0xb5, 0x56, + 0x17, 0x0b, 0xe5, 0x6a, 0xbd, 0x96, 0x8d, 0xf3, 0xdf, 0xe7, 0xe0, 0x9c, 0x6f, 0x03, 0xa8, 0xc0, + 0x68, 0x78, 0x7b, 0xd4, 0x8d, 0x7b, 0x35, 0xbd, 0x15, 0xa4, 0x29, 0x40, 0x7a, 0xb7, 0x5a, 0x29, + 0x57, 0x2d, 0xed, 0xa6, 0x60, 0x62, 0xf7, 0xc1, 0x03, 0xfb, 0x47, 0x7c, 0x33, 0x4d, 0x16, 0x14, + 0x66, 0x61, 0x7a, 0x4f, 0xd7, 0x0e, 0xb1, 0xe3, 0x3f, 0x05, 0x98, 0xa1, 0xbf, 0xa9, 0xbf, 0xfc, + 0x1f, 0xa4, 0x74, 0x2c, 0x37, 0x4f, 0xe9, 0xd1, 0xf2, 0x79, 0x12, 0x93, 0x79, 0x27, 0x26, 0xf3, + 0x9b, 0x9a, 0xd6, 0x3e, 0xb0, 0xfc, 0x53, 0x24, 0x8c, 0xc2, 0x57, 0x49, 0x98, 0x2b, 0xea, 0x58, + 0x36, 0x31, 0xd1, 0x96, 0x8a, 0x0e, 0x8c, 0xbd, 0x0d, 0x98, 0xb5, 0xfc, 0xab, 0xa1, 0x98, 0xa7, + 0x92, 0x2e, 0xab, 0x2d, 0x4c, 0x8f, 0x7e, 0xc1, 0xb1, 0x40, 0x91, 0x8e, 0x8a, 0xd6, 0xa0, 0x38, + 0xd3, 0xf0, 0xfe, 0x44, 0x65, 0x98, 0xa3, 0xae, 0xc3, 0xb8, 0x74, 0x82, 0x75, 0x69, 0xa2, 0x85, + 0xc7, 0xa5, 0xd1, 0x31, 0x4b, 0x51, 0xb0, 0x81, 0x1e, 0x02, 0x74, 0x65, 0x5d, 0xee, 0x60, 0x13, + 0xeb, 0x46, 0x2e, 0xc9, 0xc6, 0x77, 0xc0, 0x6e, 0xf2, 0x7b, 0x2e, 0x37, 0x89, 0x6f, 0xcf, 0x74, + 0xb4, 0x6d, 0x05, 0x44, 0x43, 0xc7, 0xa6, 0x91, 0x4b, 0xd9, 0x92, 0xd6, 0xa2, 0x24, 0xd5, 0x08, + 0xab, 0x2d, 0x66, 0x33, 0xf1, 0xd3, 0x4d, 0x4e, 0x74, 0x66, 0xa3, 0x5d, 0x58, 0x70, 0x36, 0xa8, + 0xa9, 0x26, 0x56, 0x4d, 0xc9, 0xd0, 0x7a, 0x7a, 0x03, 0xe7, 0xd2, 0xb6, 0x95, 0x96, 0x7c, 0x5b, + 0x24, 0x3c, 0x35, 0x9b, 0x45, 0xa4, 0xa6, 0x61, 0x88, 0xe8, 0x29, 0xf0, 0x72, 0xa3, 0x81, 0x0d, + 0x43, 0x21, 0xb6, 0x90, 0x74, 0xfc, 0x49, 0x4f, 0xd1, 0x71, 0x07, 0xab, 0xa6, 0x91, 0x9b, 0x60, + 0xa5, 0xd6, 0xb5, 0xae, 0xd6, 0xd6, 0x5a, 0xa7, 0x62, 0x9f, 0x47, 0xbc, 0xc8, 0x4c, 0xf7, 0x8c, + 0x18, 0xfc, 0x3b, 0x70, 0xce, 0x67, 0x94, 0x71, 0x32, 0x1b, 0x7f, 0x0f, 0xa6, 0xbd, 0x96, 0x18, + 0x2b, 0x2b, 0xfe, 0x28, 0x0e, 0x73, 0x01, 0x36, 0x40, 0x3b, 0x90, 0x31, 0x54, 0xb9, 0x6b, 0x3c, + 0xd3, 0x4c, 0xea, 0xbf, 0x37, 0x23, 0x4c, 0x96, 0xaf, 0x51, 0x5e, 0xf2, 0x73, 0x27, 0x26, 0xba, + 0xb3, 0xd1, 0x26, 0xa4, 0x89, 0x3d, 0xfd, 0xb9, 0x29, 0x48, 0x0e, 0xa1, 0xb9, 0x52, 0xe8, 0x4c, + 0xfe, 0x75, 0x98, 0x65, 0x57, 0x40, 0x97, 0x61, 0xca, 0x59, 0x41, 0x52, 0x9a, 0x74, 0xaf, 0xe0, + 0x90, 0xca, 0x4d, 0xfe, 0x35, 0x98, 0xf6, 0x0a, 0x43, 0x4b, 0x30, 0x49, 0x1d, 0xc2, 0x65, 0xcf, + 0x10, 0x42, 0xb9, 0xe9, 0xc6, 0xf4, 0x37, 0x60, 0x9e, 0xf5, 0x33, 0x1a, 0xca, 0xd7, 0xdd, 0x3d, + 0x10, 0x5b, 0xcc, 0xb2, 0x7b, 0x70, 0xf4, 0x14, 0x7e, 0x99, 0x84, 0xac, 0x3f, 0x68, 0xd0, 0x06, + 0xa4, 0x0e, 0xdb, 0x5a, 0xe3, 0x63, 0x3a, 0xf7, 0x95, 0xb0, 0xe8, 0xca, 0x6f, 0x5a, 0x5c, 0x84, + 0xba, 0x13, 0x13, 0xc9, 0x24, 0x6b, 0x76, 0x47, 0xeb, 0xa9, 0x26, 0xb5, 0x5e, 0xf8, 0xec, 0x47, + 0x16, 0x57, 0x7f, 0xb6, 0x3d, 0x09, 0x6d, 0xc1, 0x14, 0x71, 0x3b, 0xa9, 0xa3, 0x35, 0x71, 0x2e, + 0x61, 0xcb, 0xb8, 0x1a, 0x2a, 0xa3, 0x60, 0xf3, 0x3e, 0xd2, 0x9a, 0x58, 0x04, 0xd9, 0xfd, 0x9f, + 0x9f, 0x81, 0x29, 0x8f, 0x6e, 0xfc, 0x36, 0x4c, 0x79, 0x16, 0x43, 0x17, 0x60, 0xe2, 0xc8, 0x90, + 0xdc, 0x24, 0x3c, 0x29, 0xa6, 0x8f, 0x0c, 0x3b, 0x9f, 0x5e, 0x86, 0x29, 0x5b, 0x0b, 0xe9, 0xa8, + 0x2d, 0xb7, 0x8c, 0x5c, 0x7c, 0x35, 0x61, 0x9d, 0x91, 0x4d, 0x7a, 0x60, 0x51, 0xf8, 0x7f, 0x70, + 0x00, 0xfd, 0x25, 0xd1, 0x06, 0x24, 0x6d, 0x2d, 0x49, 0x2a, 0x5f, 0x1b, 0x41, 0xcb, 0xbc, 0xad, + 0xaa, 0x3d, 0x4b, 0xf8, 0x39, 0x07, 0x49, 0x5b, 0x8c, 0xff, 0xc2, 0xa9, 0x95, 0xab, 0xdb, 0x95, + 0x92, 0x54, 0xdd, 0xdd, 0x2a, 0x49, 0x8f, 0xc5, 0x72, 0xbd, 0x24, 0x66, 0x39, 0xb4, 0x04, 0x17, + 0xbc, 0x74, 0xb1, 0x54, 0xd8, 0x2a, 0x89, 0xd2, 0x6e, 0xb5, 0xf2, 0x24, 0x1b, 0x47, 0x3c, 0x2c, + 0x3e, 0xda, 0xaf, 0xd4, 0xcb, 0x83, 0x63, 0x09, 0xb4, 0x0c, 0x39, 0xcf, 0x18, 0x95, 0x41, 0xc5, + 0x26, 0x2d, 0xb1, 0x9e, 0x51, 0xf2, 0x2f, 0x1d, 0x4c, 0x6d, 0xce, 0xb8, 0x87, 0x61, 0x3b, 0xdb, + 0x63, 0x98, 0x61, 0x72, 0xb4, 0x55, 0x4e, 0xd1, 0xa4, 0xd2, 0x94, 0x0e, 0x4f, 0x4d, 0xbb, 0xc4, + 0xe0, 0xd6, 0x12, 0xe2, 0x8c, 0x43, 0xdd, 0xb4, 0x88, 0x96, 0x59, 0xdb, 0x4a, 0x47, 0x31, 0x29, + 0x4f, 0xdc, 0xe6, 0x01, 0x9b, 0x64, 0x33, 0x08, 0x5f, 0xc4, 0x21, 0x4d, 0xcf, 0xe6, 0x9a, 0xe7, + 0x96, 0x60, 0x44, 0x3a, 0x54, 0x22, 0x92, 0x09, 0x8e, 0x38, 0x1b, 0x1c, 0x68, 0x07, 0x66, 0xbd, + 0xa9, 0xf4, 0xc4, 0x29, 0xe2, 0xae, 0xb0, 0x07, 0xe4, 0x8d, 0xe7, 0x13, 0x5a, 0xba, 0xcd, 0x1c, + 0x7b, 0x69, 0x68, 0x13, 0x66, 0x7d, 0xd9, 0x38, 0x39, 0x3c, 0x1b, 0xcf, 0x34, 0x98, 0xc4, 0x54, + 0x80, 0x39, 0x27, 0x91, 0xb6, 0xb1, 0x64, 0xd2, 0x44, 0x4b, 0x6f, 0x8b, 0xec, 0x40, 0x02, 0x46, + 0x7d, 0x66, 0x87, 0xc6, 0xbf, 0x0b, 0x68, 0x50, 0xd7, 0xb1, 0xb2, 0x66, 0x0f, 0xe6, 0x02, 0x52, + 0x3c, 0xca, 0xc3, 0xa4, 0x7d, 0x54, 0x86, 0x62, 0x62, 0x5a, 0x1e, 0x0e, 0x6a, 0xd4, 0x67, 0xb1, + 0xf8, 0xbb, 0x3a, 0x3e, 0xc2, 0xba, 0x8e, 0x9b, 0x76, 0x78, 0x04, 0xf2, 0xbb, 0x2c, 0xc2, 0x0f, + 0x38, 0xc8, 0x38, 0x74, 0x74, 0x0f, 0x32, 0x06, 0x6e, 0x91, 0xeb, 0x87, 0xac, 0x75, 0xc9, 0x3f, + 0x37, 0x5f, 0xa3, 0x0c, 0xb4, 0x90, 0x76, 0xf8, 0xad, 0x42, 0x9a, 0x19, 0x1a, 0x6b, 0xf3, 0xbf, + 0xe5, 0x60, 0x6e, 0x0b, 0xb7, 0xb1, 0xbf, 0x4a, 0x89, 0xca, 0xb0, 0xde, 0x8b, 0x3d, 0xce, 0x5e, + 0xec, 0x01, 0xa2, 0x22, 0x2e, 0xf6, 0x33, 0x5d, 0x76, 0x8b, 0x30, 0xcf, 0xae, 0x46, 0xd2, 0xbb, + 0xf0, 0xcf, 0x04, 0x5c, 0xb2, 0x7c, 0x41, 0xd7, 0xda, 0x6d, 0xac, 0xef, 0xf5, 0x0e, 0xdb, 0x8a, + 0xf1, 0x6c, 0x8c, 0xcd, 0x5d, 0x80, 0x09, 0x55, 0x6b, 0x7a, 0x82, 0x27, 0x6d, 0xfd, 0x2c, 0x37, + 0x51, 0x09, 0xce, 0xfb, 0xcb, 0xac, 0x53, 0x9a, 0x84, 0xc3, 0x8b, 0xac, 0xec, 0xb1, 0xff, 0x06, + 0xe1, 0x21, 0x63, 0x15, 0x88, 0x9a, 0xda, 0x3e, 0xb5, 0x23, 0x26, 0x23, 0xba, 0xbf, 0x91, 0xe8, + 0xaf, 0x98, 0xde, 0x70, 0x2b, 0xa6, 0xc8, 0x1d, 0x45, 0x15, 0x4f, 0x1f, 0x0d, 0x44, 0x7c, 0xda, + 0x16, 0xfd, 0xf6, 0x88, 0xa2, 0x87, 0x66, 0x82, 0xb3, 0x9c, 0xe2, 0x0b, 0x08, 0xdf, 0x3f, 0x72, + 0x70, 0x39, 0x74, 0x0b, 0xf4, 0xca, 0x6f, 0xc2, 0xb9, 0x2e, 0x19, 0x70, 0x8d, 0x40, 0xa2, 0xec, + 0xfe, 0x50, 0x23, 0xd0, 0x2e, 0x96, 0x52, 0x19, 0x33, 0xcc, 0x76, 0x19, 0x22, 0x5f, 0x80, 0xb9, + 0x00, 0xb6, 0xb1, 0x36, 0xf3, 0x25, 0x07, 0xab, 0x7d, 0x55, 0xf6, 0xd5, 0xee, 0x8b, 0x73, 0xdf, + 0x7a, 0xdf, 0xb7, 0x48, 0xca, 0xbf, 0x3b, 0xb8, 0xf7, 0xe0, 0x05, 0x5f, 0x56, 0x04, 0x5f, 0x85, + 0x2b, 0x11, 0x4b, 0xd3, 0x70, 0xfe, 0x22, 0x09, 0x57, 0x0e, 0xe4, 0xb6, 0xd2, 0x74, 0x0b, 0xb9, + 0x80, 0x7e, 0x3f, 0xda, 0x24, 0x8d, 0x81, 0x08, 0x20, 0x59, 0x6b, 0xc3, 0x8d, 0xda, 0x61, 0xf2, + 0x47, 0xb8, 0x0e, 0x5f, 0x60, 0x13, 0xf6, 0x24, 0xa0, 0x09, 0x7b, 0x7b, 0x74, 0x5d, 0xa3, 0x5a, + 0xb2, 0x7d, 0x7f, 0x82, 0x79, 0x6b, 0x74, 0xb9, 0x11, 0x5e, 0x70, 0xe6, 0x28, 0xfe, 0x3a, 0xbb, + 0xa6, 0xdf, 0x27, 0x41, 0x88, 0xda, 0x3d, 0xcd, 0x21, 0x22, 0x4c, 0x36, 0x34, 0xf5, 0x48, 0xd1, + 0x3b, 0xb8, 0x49, 0xab, 0xff, 0x37, 0x47, 0x31, 0x1e, 0x4d, 0x20, 0x45, 0x67, 0xae, 0xd8, 0x17, + 0x83, 0x72, 0x30, 0xd1, 0xc1, 0x86, 0x21, 0xb7, 0x1c, 0xb5, 0x9c, 0x9f, 0xfc, 0x67, 0x09, 0x98, + 0x74, 0xa7, 0x20, 0x75, 0xc0, 0x83, 0x49, 0xfa, 0xda, 0x7e, 0x1e, 0x05, 0x9e, 0xdf, 0x99, 0xe3, + 0xcf, 0xe1, 0xcc, 0x4d, 0xc6, 0x99, 0x49, 0x38, 0x6c, 0x3d, 0x97, 0xda, 0x11, 0x7e, 0xfd, 0xb5, + 0x3b, 0xa0, 0xf0, 0x6d, 0x40, 0x15, 0xc5, 0xa0, 0x5d, 0x94, 0x9b, 0x96, 0xac, 0xa6, 0x49, 0x3e, + 0x91, 0xb0, 0x6a, 0xea, 0x0a, 0x2d, 0xd7, 0x53, 0x22, 0x74, 0xe4, 0x93, 0x12, 0xa1, 0x58, 0x25, + 0xbd, 0x61, 0xca, 0xba, 0xa9, 0xa8, 0x2d, 0xc9, 0xd4, 0x3e, 0xc6, 0x2e, 0xe8, 0xea, 0x50, 0xeb, + 0x16, 0x51, 0xf8, 0x34, 0x0e, 0x73, 0x8c, 0x78, 0xea, 0x93, 0xf7, 0x61, 0xa2, 0x2f, 0x9b, 0x29, + 0xe3, 0x03, 0xb8, 0xf3, 0xc4, 0x6c, 0xce, 0x0c, 0xb4, 0x02, 0xa0, 0xe2, 0x13, 0x93, 0x59, 0x77, + 0xd2, 0xa2, 0xd8, 0x6b, 0xf2, 0x1b, 0x6e, 0xcf, 0x6d, 0xca, 0x66, 0xcf, 0x40, 0xb7, 0x00, 0xd1, + 0x0c, 0x8d, 0x9b, 0x12, 0xbd, 0x62, 0xc8, 0xb2, 0x93, 0x62, 0xd6, 0x1d, 0xa9, 0xda, 0x97, 0x8d, + 0xc1, 0x7f, 0x02, 0x29, 0x62, 0xc4, 0x11, 0xbb, 0x6d, 0xf4, 0x2e, 0xa4, 0x0d, 0x7b, 0x21, 0x3f, + 0xb2, 0x10, 0xb4, 0x13, 0xaf, 0x62, 0x22, 0x9d, 0x27, 0x7c, 0x16, 0x07, 0xb4, 0x8d, 0x4d, 0xb7, + 0x0d, 0xa3, 0x67, 0x10, 0xe2, 0xcb, 0xdc, 0x73, 0xf8, 0xf2, 0x7b, 0x8c, 0x2f, 0x93, 0x68, 0xb8, + 0xe9, 0x41, 0xbf, 0x7d, 0x4b, 0x47, 0x66, 0xe2, 0x90, 0xd6, 0x87, 0xd4, 0x93, 0xa3, 0xb5, 0x3e, + 0x67, 0x74, 0xd9, 0x2d, 0x98, 0x63, 0x74, 0xa6, 0x3e, 0x75, 0x1b, 0x90, 0x7c, 0x2c, 0x2b, 0x6d, + 0xd9, 0xd2, 0xcb, 0xe9, 0x2c, 0x69, 0xa7, 0x79, 0xde, 0x1d, 0x71, 0xa6, 0x09, 0x82, 0xb7, 0x60, + 0xa1, 0xf2, 0xfc, 0x68, 0x7c, 0xdb, 0x7b, 0xd1, 0x0f, 0xf0, 0xd0, 0x75, 0xb7, 0x03, 0x11, 0xf9, + 0xab, 0x83, 0x45, 0x0a, 0x85, 0xa7, 0x43, 0xc1, 0xf9, 0x5f, 0x25, 0x60, 0x29, 0x82, 0x1b, 0xdd, + 0x87, 0x84, 0xde, 0x6d, 0x50, 0x77, 0x7c, 0x75, 0x04, 0xf9, 0x79, 0x71, 0xaf, 0xb8, 0x13, 0x13, + 0xad, 0x59, 0xfc, 0x1f, 0xe2, 0x90, 0x10, 0xf7, 0x8a, 0xe8, 0x5d, 0x06, 0xa9, 0xbe, 0x35, 0xa2, + 0x14, 0x2f, 0x50, 0xfd, 0x1f, 0x2e, 0x08, 0xa9, 0xce, 0xc1, 0x7c, 0x51, 0x2c, 0x15, 0xea, 0x25, + 0x69, 0xab, 0x54, 0x29, 0xd5, 0x4b, 0x12, 0x41, 0xd2, 0xb3, 0x1c, 0x5a, 0x86, 0xdc, 0xde, 0xfe, + 0x66, 0xa5, 0x5c, 0xdb, 0x91, 0xf6, 0xab, 0xce, 0x7f, 0x74, 0x34, 0x8e, 0xb2, 0x30, 0x5d, 0x29, + 0xd7, 0xea, 0x94, 0x50, 0xcb, 0x26, 0x2c, 0xca, 0x76, 0xa9, 0x2e, 0x15, 0x0b, 0x7b, 0x85, 0x62, + 0xb9, 0xfe, 0x24, 0x9b, 0x44, 0x3c, 0x2c, 0xb2, 0xb2, 0x6b, 0xd5, 0xc2, 0x5e, 0x6d, 0x67, 0xb7, + 0x9e, 0x4d, 0x21, 0x04, 0xb3, 0xf6, 0x7c, 0x87, 0x54, 0xcb, 0xa6, 0x2d, 0x09, 0xc5, 0xca, 0x6e, + 0xd5, 0xd5, 0x61, 0x02, 0xcd, 0x43, 0xd6, 0x59, 0x59, 0x2c, 0x15, 0xb6, 0x6c, 0x14, 0x25, 0x83, + 0xce, 0xc3, 0x4c, 0xe9, 0x83, 0xbd, 0x42, 0x75, 0xcb, 0x61, 0x9c, 0x44, 0xab, 0xb0, 0xec, 0x55, + 0x47, 0xa2, 0xb3, 0x4a, 0x5b, 0x36, 0x96, 0x52, 0xcb, 0x82, 0x8b, 0xd2, 0x7d, 0x19, 0x87, 0x05, + 0x02, 0xd3, 0x39, 0xa0, 0xa0, 0x13, 0xb8, 0x6b, 0x90, 0x25, 0xc0, 0x82, 0xe4, 0x2f, 0xed, 0x66, + 0x09, 0xfd, 0xc0, 0x29, 0xf0, 0x1c, 0x48, 0x3d, 0xee, 0x81, 0xd4, 0xcb, 0xfe, 0x72, 0xf7, 0x26, + 0x0b, 0x3e, 0xfb, 0x56, 0x8b, 0xea, 0xa0, 0x1e, 0x05, 0xd4, 0x63, 0xb7, 0xa3, 0xa5, 0x45, 0xdd, + 0x55, 0x67, 0x69, 0x97, 0xce, 0x18, 0xf2, 0x0f, 0x60, 0xd1, 0xaf, 0x2f, 0x8d, 0xbe, 0x5b, 0x03, + 0x10, 0xb1, 0x9b, 0x83, 0x5c, 0x5e, 0x97, 0x43, 0xf8, 0x0b, 0x07, 0x19, 0x87, 0x6c, 0xdd, 0x23, + 0x86, 0xf2, 0x5d, 0xcc, 0x40, 0x52, 0x93, 0x16, 0xc5, 0x45, 0xb8, 0xbc, 0xe0, 0x6e, 0xdc, 0x0f, + 0xee, 0x06, 0x9e, 0x73, 0x22, 0xf0, 0x9c, 0xbf, 0x09, 0x33, 0x0d, 0x4b, 0x7d, 0x45, 0x53, 0x25, + 0x53, 0xe9, 0x38, 0x88, 0xd3, 0xe0, 0x63, 0x4c, 0xdd, 0x79, 0x41, 0x15, 0xa7, 0x9d, 0x09, 0x16, + 0x09, 0xad, 0xc2, 0xb4, 0xfd, 0x38, 0x23, 0x99, 0x9a, 0xd4, 0x33, 0x70, 0x2e, 0x65, 0xf7, 0xdf, + 0x60, 0xd3, 0xea, 0xda, 0xbe, 0x81, 0x85, 0xdf, 0x71, 0xb0, 0x40, 0x60, 0x05, 0xbf, 0x3b, 0x0e, + 0x03, 0xa9, 0xbd, 0x1e, 0xe7, 0xbb, 0x1a, 0x02, 0x05, 0xbe, 0xac, 0xae, 0x2a, 0x07, 0x8b, 0xfe, + 0xf5, 0x68, 0x2b, 0xf5, 0x79, 0x1c, 0xe6, 0xad, 0xdb, 0xd4, 0x19, 0x78, 0xd1, 0x65, 0xca, 0x18, + 0x27, 0xe9, 0x33, 0x66, 0x72, 0xc0, 0x98, 0x3b, 0xfe, 0x46, 0xe5, 0x86, 0xb7, 0x1e, 0xf0, 0xef, + 0xe0, 0x65, 0xd9, 0xf2, 0xd7, 0x1c, 0x2c, 0xf8, 0xd6, 0xa3, 0xf1, 0xf2, 0x8e, 0xbf, 0xf2, 0xba, + 0x1a, 0xa2, 0xdf, 0x73, 0xd5, 0x5e, 0x77, 0x9d, 0xea, 0x69, 0xbc, 0xb0, 0xfc, 0x73, 0x1c, 0x56, + 0xfa, 0x37, 0x90, 0xfd, 0x3c, 0xda, 0x1c, 0x03, 0x3a, 0x38, 0xdb, 0x2b, 0xe4, 0xfb, 0xfe, 0x84, + 0xbb, 0x3e, 0x78, 0x29, 0x06, 0xa8, 0x14, 0x95, 0x78, 0x03, 0x11, 0xb7, 0xe4, 0xb8, 0x88, 0xdb, + 0x99, 0x3c, 0xe0, 0x7b, 0x5e, 0x30, 0x91, 0x55, 0x9f, 0x7a, 0xc2, 0x88, 0xa8, 0xfc, 0x5b, 0x70, + 0xc1, 0x2e, 0x9a, 0xdd, 0xd7, 0x7d, 0xe7, 0xcd, 0x91, 0xa4, 0xc4, 0x8c, 0xb8, 0x60, 0x0d, 0xbb, + 0x4f, 0xda, 0x14, 0x89, 0x6e, 0x0a, 0x5f, 0x25, 0x61, 0xd1, 0x2a, 0xaa, 0x6b, 0xa6, 0xdc, 0x1a, + 0x07, 0xa3, 0xfd, 0xd6, 0x20, 0xe4, 0x15, 0x67, 0x8f, 0x25, 0x58, 0xea, 0x28, 0x48, 0x17, 0xca, + 0xc3, 0x9c, 0x61, 0xca, 0x2d, 0x3b, 0x1d, 0xc8, 0x7a, 0x0b, 0x9b, 0x52, 0x57, 0x36, 0x9f, 0xd1, + 0x58, 0x3f, 0x4f, 0x87, 0xea, 0xf6, 0xc8, 0x9e, 0x6c, 0x3e, 0x7b, 0x41, 0x07, 0x89, 0xde, 0xf3, + 0x27, 0x85, 0xd7, 0x86, 0xec, 0x25, 0xc2, 0xb7, 0x3e, 0x08, 0x81, 0x45, 0x5f, 0x1f, 0x22, 0x72, + 0x38, 0x1c, 0x7a, 0x76, 0x18, 0xf0, 0x6b, 0x46, 0x54, 0x2f, 0xc2, 0x85, 0x81, 0xcd, 0xd3, 0x2b, + 0xa4, 0x05, 0x39, 0x6b, 0x68, 0x5f, 0x35, 0xc6, 0x74, 0xc7, 0x10, 0x8f, 0x89, 0x87, 0x78, 0x8c, + 0xb0, 0x04, 0x17, 0x03, 0x16, 0xa2, 0x5a, 0xfc, 0x26, 0x45, 0xd4, 0x18, 0x1f, 0xdc, 0xff, 0x30, + 0x2c, 0x2a, 0xde, 0xf4, 0x1e, 0x7b, 0x20, 0x0e, 0xfe, 0x32, 0xe2, 0xe2, 0x32, 0x4c, 0x79, 0xf9, + 0xe8, 0x35, 0x68, 0x0e, 0x09, 0x9c, 0xd4, 0x99, 0xde, 0x1c, 0xd2, 0xbe, 0x37, 0x87, 0x4a, 0x3f, + 0xa8, 0x26, 0xd8, 0xd2, 0x36, 0xd4, 0x14, 0x11, 0x61, 0xf5, 0x74, 0x20, 0xac, 0x32, 0xec, 0x43, + 0x46, 0xa8, 0xd0, 0xff, 0x81, 0xc0, 0xa2, 0x4e, 0x1d, 0xf8, 0xc2, 0x20, 0x3c, 0x05, 0x9e, 0x78, + 0xfc, 0xf8, 0x98, 0xbf, 0xcf, 0x8d, 0xe2, 0x7e, 0x37, 0x12, 0x56, 0x60, 0x29, 0x50, 0x36, 0x5d, + 0xfa, 0x87, 0x1c, 0x51, 0x6c, 0x1b, 0x9b, 0x7d, 0x64, 0xc5, 0x18, 0x75, 0x69, 0x3a, 0xe8, 0x5d, + 0x9a, 0x90, 0x6c, 0x0f, 0x1e, 0x33, 0x24, 0x84, 0x6d, 0x62, 0x06, 0xbf, 0x2a, 0xf4, 0xb2, 0xbd, + 0x01, 0xa9, 0x9e, 0x0d, 0x97, 0x92, 0xa2, 0x6b, 0x8e, 0x8d, 0x81, 0x7d, 0x6b, 0x48, 0x24, 0x1c, + 0xc2, 0xe7, 0x1c, 0x4c, 0x79, 0xc8, 0x68, 0x19, 0x26, 0x5d, 0xf4, 0xc2, 0xe9, 0x52, 0x5c, 0x82, + 0x75, 0x68, 0xa6, 0x66, 0xca, 0x6d, 0xfa, 0x02, 0x4f, 0x7e, 0x58, 0x8d, 0x65, 0xcf, 0xc0, 0xa4, + 0x88, 0x4d, 0x88, 0xf6, 0xff, 0xe8, 0x16, 0x24, 0x7b, 0xaa, 0x62, 0xda, 0xc1, 0x3a, 0xeb, 0x8f, + 0x42, 0x7b, 0xa9, 0xfc, 0xbe, 0xaa, 0x98, 0xa2, 0xcd, 0x25, 0xdc, 0x84, 0xa4, 0xf5, 0x8b, 0x6d, + 0xf2, 0x27, 0x21, 0xb5, 0xf9, 0xa4, 0x5e, 0xaa, 0x65, 0x39, 0x04, 0x90, 0x2e, 0x93, 0x96, 0x38, + 0x2e, 0x2c, 0xbb, 0x5b, 0x0f, 0x02, 0x51, 0x3e, 0x22, 0x67, 0x18, 0x06, 0x9f, 0x14, 0x02, 0xe1, + 0x93, 0x15, 0xe6, 0x36, 0x1b, 0x02, 0x9c, 0xfc, 0x8b, 0x83, 0x85, 0x40, 0x3e, 0x74, 0xd7, 0x0b, + 0x99, 0x5c, 0x89, 0x94, 0xe9, 0x05, 0x4b, 0x7e, 0xc6, 0x11, 0xb0, 0xe4, 0x1e, 0x03, 0x96, 0x5c, + 0x1f, 0x3a, 0xdf, 0x0b, 0x93, 0x1c, 0x84, 0xa0, 0x24, 0xb5, 0x7a, 0x61, 0xbb, 0x24, 0xed, 0x57, + 0xc9, 0x5f, 0x17, 0x25, 0x99, 0x87, 0xec, 0x76, 0xc9, 0xc1, 0x1d, 0xa4, 0x5a, 0xbd, 0x50, 0xaf, + 0x65, 0xe3, 0x83, 0x08, 0x45, 0xc2, 0xc5, 0x1f, 0xe6, 0x01, 0x51, 0xb3, 0x7a, 0x3f, 0x9d, 0xfd, + 0x94, 0x83, 0x39, 0x86, 0x4c, 0xad, 0xec, 0x79, 0x5d, 0xe3, 0x98, 0xd7, 0xb5, 0x3b, 0x30, 0x6f, + 0xb5, 0x50, 0xc4, 0xf1, 0x0d, 0xa9, 0x8b, 0x75, 0x1b, 0x23, 0xa5, 0xee, 0x74, 0xbe, 0x23, 0x9f, + 0x50, 0x0c, 0x73, 0x0f, 0xeb, 0x96, 0xe0, 0x17, 0x80, 0xff, 0x09, 0x3f, 0x8e, 0x93, 0x8b, 0x7a, + 0xec, 0x42, 0x7f, 0x68, 0xd0, 0x0e, 0x76, 0x02, 0x89, 0x31, 0x3a, 0x81, 0x90, 0x90, 0x4f, 0x8e, + 0x55, 0x1d, 0x8e, 0x7d, 0xc9, 0x09, 0x05, 0x52, 0x14, 0x9c, 0xa1, 0x48, 0x5f, 0xff, 0x37, 0x07, + 0x99, 0x72, 0x13, 0xab, 0xa6, 0xe5, 0xf4, 0x55, 0x98, 0x61, 0xbe, 0x68, 0x46, 0xcb, 0x21, 0x1f, + 0x3a, 0xdb, 0x16, 0xe7, 0x57, 0x22, 0x3f, 0x83, 0x16, 0x62, 0xe8, 0xc8, 0xf3, 0x35, 0x36, 0x03, + 0x2b, 0xbf, 0x32, 0x30, 0x33, 0x20, 0xfe, 0xf9, 0x6b, 0x43, 0xb8, 0xdc, 0x75, 0xde, 0x82, 0x94, + 0xfd, 0xed, 0x2a, 0x9a, 0x77, 0xbf, 0x9f, 0xf5, 0x7c, 0xda, 0xca, 0x2f, 0xf8, 0xa8, 0xce, 0xbc, + 0xf5, 0xbf, 0x67, 0x00, 0xfa, 0xbd, 0x0e, 0x7a, 0x08, 0xd3, 0xde, 0xcf, 0xe7, 0xd0, 0x52, 0xc4, + 0xc7, 0x9b, 0xfc, 0x72, 0xf0, 0xa0, 0xab, 0xd3, 0x43, 0x98, 0xf6, 0x7e, 0xac, 0xd1, 0x17, 0x16, + 0xf0, 0xc1, 0x48, 0x5f, 0x58, 0xe0, 0xf7, 0x1d, 0x31, 0xd4, 0x86, 0x0b, 0x21, 0xcf, 0xf5, 0xe8, + 0xfa, 0x68, 0x1f, 0x35, 0xf0, 0xaf, 0x8e, 0xf8, 0xee, 0x2f, 0xc4, 0x90, 0x0e, 0x17, 0x43, 0x5f, + 0xa9, 0xd1, 0xda, 0xa8, 0x6f, 0xe8, 0xfc, 0x8d, 0x11, 0x38, 0xdd, 0x35, 0x7b, 0xc0, 0x87, 0x3f, + 0x8d, 0xa1, 0x1b, 0x23, 0xbf, 0xd9, 0xf2, 0x37, 0x47, 0x7f, 0x69, 0x13, 0x62, 0x68, 0x07, 0xa6, + 0x3c, 0xaf, 0x2d, 0x88, 0x0f, 0x7c, 0x82, 0x21, 0x82, 0x97, 0x22, 0x9e, 0x67, 0x88, 0x24, 0xcf, + 0xdb, 0x42, 0x5f, 0xd2, 0xe0, 0x23, 0x49, 0x5f, 0x52, 0xc0, 0x63, 0x84, 0xdf, 0xfc, 0xbe, 0xcb, + 0x2f, 0xc8, 0xfc, 0xc1, 0xb7, 0x67, 0x90, 0xf9, 0x43, 0x6e, 0x52, 0x21, 0x86, 0xde, 0x87, 0x59, + 0x16, 0x26, 0x45, 0x2b, 0x91, 0x70, 0x2f, 0x7f, 0x29, 0x6c, 0xd8, 0x2b, 0x92, 0x45, 0xe5, 0xfa, + 0x22, 0x03, 0xd1, 0xc1, 0xbe, 0xc8, 0x10, 0x30, 0x2f, 0x66, 0xe5, 0x27, 0x06, 0x6b, 0xea, 0xe7, + 0xa7, 0x20, 0x88, 0xac, 0x9f, 0x9f, 0x02, 0x01, 0x2a, 0x21, 0x86, 0x14, 0x58, 0x0c, 0x86, 0x3a, + 0xd0, 0xb5, 0x91, 0x90, 0x1c, 0xfe, 0xfa, 0x30, 0x36, 0x37, 0xd5, 0xfc, 0x2d, 0x05, 0x49, 0xfb, + 0x16, 0xac, 0xc3, 0x39, 0x5f, 0xab, 0x89, 0x2e, 0x45, 0x37, 0xe0, 0xfc, 0xe5, 0xd0, 0x71, 0x77, + 0x27, 0x4f, 0xe1, 0xfc, 0x40, 0xf3, 0x88, 0x56, 0xbd, 0xf3, 0x82, 0x1a, 0x58, 0xfe, 0x4a, 0x04, + 0x87, 0x5f, 0x36, 0x9b, 0x76, 0x56, 0x87, 0x75, 0x37, 0xac, 0xec, 0xb0, 0x54, 0xf3, 0x11, 0x29, + 0x3a, 0xfc, 0x49, 0x46, 0x60, 0xf5, 0x0a, 0x4c, 0x2f, 0x57, 0x23, 0x79, 0xdc, 0x15, 0x3e, 0x74, + 0xab, 0x1d, 0x4f, 0x75, 0x8d, 0x18, 0xe5, 0x02, 0x9b, 0x00, 0x5e, 0x88, 0x62, 0x71, 0xc5, 0x3f, + 0x86, 0xac, 0xff, 0x0a, 0x46, 0xcc, 0x79, 0x05, 0xb9, 0xcd, 0x6a, 0x38, 0x83, 0xdf, 0x32, 0xfe, + 0xf8, 0xf7, 0x6b, 0x15, 0x14, 0xf9, 0x57, 0x23, 0x79, 0xbc, 0x19, 0xcb, 0x53, 0xf0, 0xf5, 0x33, + 0xd6, 0x60, 0x71, 0xd8, 0xcf, 0x58, 0x01, 0x15, 0xa2, 0x10, 0xbb, 0xf7, 0x0e, 0x40, 0xc3, 0x50, + 0x24, 0xd2, 0x11, 0xa3, 0x95, 0x81, 0xc7, 0x89, 0x07, 0x0a, 0x6e, 0x37, 0x77, 0xbb, 0xa6, 0xa2, + 0xa9, 0x46, 0xee, 0x17, 0x19, 0xbb, 0x1d, 0x9f, 0x6c, 0x18, 0x0a, 0x69, 0x4c, 0x37, 0x53, 0x4f, + 0x13, 0x0d, 0x43, 0x39, 0x4c, 0xdb, 0xfc, 0x6f, 0xfc, 0x37, 0x00, 0x00, 0xff, 0xff, 0x16, 0x7c, + 0xfe, 0x46, 0x7f, 0x36, 0x00, 0x00, +} + // Reference imports to suppress errors if they are not otherwise used. var _ context.Context var _ grpc.ClientConn @@ -4698,6 +4842,20 @@ type IdentityServer interface { Probe(context.Context, *ProbeRequest) (*ProbeResponse, error) } +// UnimplementedIdentityServer can be embedded to have forward compatible implementations. +type UnimplementedIdentityServer struct { +} + +func (*UnimplementedIdentityServer) GetPluginInfo(ctx context.Context, req *GetPluginInfoRequest) (*GetPluginInfoResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetPluginInfo not implemented") +} +func (*UnimplementedIdentityServer) GetPluginCapabilities(ctx context.Context, req *GetPluginCapabilitiesRequest) (*GetPluginCapabilitiesResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetPluginCapabilities not implemented") +} +func (*UnimplementedIdentityServer) Probe(ctx context.Context, req *ProbeRequest) (*ProbeResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method Probe not implemented") +} + func RegisterIdentityServer(s *grpc.Server, srv IdentityServer) { s.RegisterService(&_Identity_serviceDesc, srv) } @@ -4927,6 +5085,47 @@ type ControllerServer interface { ControllerExpandVolume(context.Context, *ControllerExpandVolumeRequest) (*ControllerExpandVolumeResponse, error) } +// UnimplementedControllerServer can be embedded to have forward compatible implementations. +type UnimplementedControllerServer struct { +} + +func (*UnimplementedControllerServer) CreateVolume(ctx context.Context, req *CreateVolumeRequest) (*CreateVolumeResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method CreateVolume not implemented") +} +func (*UnimplementedControllerServer) DeleteVolume(ctx context.Context, req *DeleteVolumeRequest) (*DeleteVolumeResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method DeleteVolume not implemented") +} +func (*UnimplementedControllerServer) ControllerPublishVolume(ctx context.Context, req *ControllerPublishVolumeRequest) (*ControllerPublishVolumeResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ControllerPublishVolume not implemented") +} +func (*UnimplementedControllerServer) ControllerUnpublishVolume(ctx context.Context, req *ControllerUnpublishVolumeRequest) (*ControllerUnpublishVolumeResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ControllerUnpublishVolume not implemented") +} +func (*UnimplementedControllerServer) ValidateVolumeCapabilities(ctx context.Context, req *ValidateVolumeCapabilitiesRequest) (*ValidateVolumeCapabilitiesResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ValidateVolumeCapabilities not implemented") +} +func (*UnimplementedControllerServer) ListVolumes(ctx context.Context, req *ListVolumesRequest) (*ListVolumesResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ListVolumes not implemented") +} +func (*UnimplementedControllerServer) GetCapacity(ctx context.Context, req *GetCapacityRequest) (*GetCapacityResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetCapacity not implemented") +} +func (*UnimplementedControllerServer) ControllerGetCapabilities(ctx context.Context, req *ControllerGetCapabilitiesRequest) (*ControllerGetCapabilitiesResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ControllerGetCapabilities not implemented") +} +func (*UnimplementedControllerServer) CreateSnapshot(ctx context.Context, req *CreateSnapshotRequest) (*CreateSnapshotResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method CreateSnapshot not implemented") +} +func (*UnimplementedControllerServer) DeleteSnapshot(ctx context.Context, req *DeleteSnapshotRequest) (*DeleteSnapshotResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method DeleteSnapshot not implemented") +} +func (*UnimplementedControllerServer) ListSnapshots(ctx context.Context, req *ListSnapshotsRequest) (*ListSnapshotsResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ListSnapshots not implemented") +} +func (*UnimplementedControllerServer) ControllerExpandVolume(ctx context.Context, req *ControllerExpandVolumeRequest) (*ControllerExpandVolumeResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ControllerExpandVolume not implemented") +} + func RegisterControllerServer(s *grpc.Server, srv ControllerServer) { s.RegisterService(&_Controller_serviceDesc, srv) } @@ -5310,6 +5509,35 @@ type NodeServer interface { NodeGetInfo(context.Context, *NodeGetInfoRequest) (*NodeGetInfoResponse, error) } +// UnimplementedNodeServer can be embedded to have forward compatible implementations. +type UnimplementedNodeServer struct { +} + +func (*UnimplementedNodeServer) NodeStageVolume(ctx context.Context, req *NodeStageVolumeRequest) (*NodeStageVolumeResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method NodeStageVolume not implemented") +} +func (*UnimplementedNodeServer) NodeUnstageVolume(ctx context.Context, req *NodeUnstageVolumeRequest) (*NodeUnstageVolumeResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method NodeUnstageVolume not implemented") +} +func (*UnimplementedNodeServer) NodePublishVolume(ctx context.Context, req *NodePublishVolumeRequest) (*NodePublishVolumeResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method NodePublishVolume not implemented") +} +func (*UnimplementedNodeServer) NodeUnpublishVolume(ctx context.Context, req *NodeUnpublishVolumeRequest) (*NodeUnpublishVolumeResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method NodeUnpublishVolume not implemented") +} +func (*UnimplementedNodeServer) NodeGetVolumeStats(ctx context.Context, req *NodeGetVolumeStatsRequest) (*NodeGetVolumeStatsResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method NodeGetVolumeStats not implemented") +} +func (*UnimplementedNodeServer) NodeExpandVolume(ctx context.Context, req *NodeExpandVolumeRequest) (*NodeExpandVolumeResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method NodeExpandVolume not implemented") +} +func (*UnimplementedNodeServer) NodeGetCapabilities(ctx context.Context, req *NodeGetCapabilitiesRequest) (*NodeGetCapabilitiesResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method NodeGetCapabilities not implemented") +} +func (*UnimplementedNodeServer) NodeGetInfo(ctx context.Context, req *NodeGetInfoRequest) (*NodeGetInfoResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method NodeGetInfo not implemented") +} + func RegisterNodeServer(s *grpc.Server, srv NodeServer) { s.RegisterService(&_Node_serviceDesc, srv) } @@ -5498,216 +5726,3 @@ var _Node_serviceDesc = grpc.ServiceDesc{ Streams: []grpc.StreamDesc{}, Metadata: "github.com/container-storage-interface/spec/csi.proto", } - -func init() { - proto.RegisterFile("github.com/container-storage-interface/spec/csi.proto", fileDescriptor_csi_2c5455657a82ae49) -} - -var fileDescriptor_csi_2c5455657a82ae49 = []byte{ - // 3276 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe4, 0x5a, 0x4d, 0x70, 0xdb, 0xc6, - 0x15, 0x26, 0xf8, 0x23, 0x51, 0x4f, 0x3f, 0xa6, 0x57, 0x3f, 0xa6, 0x21, 0xc9, 0x96, 0xe1, 0xd8, - 0x51, 0x1c, 0x9b, 0x6a, 0x94, 0x38, 0xd3, 0xd8, 0x4e, 0x1b, 0x8a, 0xa2, 0x25, 0xc6, 0x34, 0xa9, - 0x80, 0x94, 0x1c, 0xbb, 0xcd, 0x20, 0x10, 0xb9, 0xa2, 0x31, 0x21, 0x01, 0x06, 0x00, 0x55, 0xa9, - 0x97, 0xce, 0xb4, 0xa7, 0x4c, 0xcf, 0x9d, 0xb6, 0xa7, 0xce, 0xa4, 0xbd, 0xb4, 0xd3, 0x4c, 0x4f, - 0x9d, 0x1e, 0x3b, 0xd3, 0x63, 0x0f, 0xbd, 0xb6, 0x93, 0x4b, 0xae, 0x9d, 0x4c, 0x3b, 0x93, 0xe9, - 0xb1, 0xa7, 0x0e, 0xb0, 0x0b, 0x10, 0x0b, 0x02, 0x20, 0x69, 0xd9, 0xe3, 0x43, 0x4f, 0x24, 0xde, - 0xbe, 0x7d, 0xfb, 0xf6, 0xe1, 0xbd, 0xb7, 0xef, 0x7d, 0x0b, 0xb8, 0xdd, 0x52, 0xcc, 0xa7, 0xbd, - 0xc3, 0x5c, 0x43, 0xeb, 0x6c, 0x34, 0x34, 0xd5, 0x94, 0x15, 0x15, 0xeb, 0xb7, 0x0c, 0x53, 0xd3, - 0xe5, 0x16, 0xbe, 0xa5, 0xa8, 0x26, 0xd6, 0x8f, 0xe4, 0x06, 0xde, 0x30, 0xba, 0xb8, 0xb1, 0xd1, - 0x30, 0x94, 0x5c, 0x57, 0xd7, 0x4c, 0x0d, 0x4d, 0x58, 0x7f, 0x8f, 0xdf, 0xe0, 0xd7, 0x5a, 0x9a, - 0xd6, 0x6a, 0xe3, 0x0d, 0x9b, 0x7a, 0xd8, 0x3b, 0xda, 0x68, 0x62, 0xa3, 0xa1, 0x2b, 0x5d, 0x53, - 0xd3, 0x09, 0x27, 0x7f, 0xd9, 0xcf, 0x61, 0x2a, 0x1d, 0x6c, 0x98, 0x72, 0xa7, 0x4b, 0x19, 0x2e, - 0xf9, 0x19, 0x7e, 0xa0, 0xcb, 0xdd, 0x2e, 0xd6, 0x0d, 0x32, 0x2e, 0x2c, 0xc1, 0xc2, 0x0e, 0x36, - 0xf7, 0xda, 0xbd, 0x96, 0xa2, 0x96, 0xd4, 0x23, 0x4d, 0xc4, 0x9f, 0xf6, 0xb0, 0x61, 0x0a, 0xff, - 0xe0, 0x60, 0xd1, 0x37, 0x60, 0x74, 0x35, 0xd5, 0xc0, 0x08, 0x41, 0x52, 0x95, 0x3b, 0x38, 0xcb, - 0xad, 0x71, 0xeb, 0x53, 0xa2, 0xfd, 0x1f, 0x5d, 0x83, 0xb9, 0x63, 0xac, 0x36, 0x35, 0x5d, 0x3a, - 0xc6, 0xba, 0xa1, 0x68, 0x6a, 0x36, 0x6e, 0x8f, 0xce, 0x12, 0xea, 0x01, 0x21, 0xa2, 0x1d, 0x48, - 0x77, 0x64, 0x55, 0x39, 0xc2, 0x86, 0x99, 0x4d, 0xac, 0x25, 0xd6, 0xa7, 0x37, 0x5f, 0xcf, 0x91, - 0xad, 0xe6, 0x02, 0xd7, 0xca, 0x3d, 0xa4, 0xdc, 0x45, 0xd5, 0xd4, 0x4f, 0x45, 0x77, 0x32, 0x7f, - 0x17, 0x66, 0x99, 0x21, 0x94, 0x81, 0xc4, 0x27, 0xf8, 0x94, 0xea, 0x64, 0xfd, 0x45, 0x0b, 0x90, - 0x3a, 0x96, 0xdb, 0x3d, 0x4c, 0x35, 0x21, 0x0f, 0x77, 0xe2, 0xdf, 0xe6, 0x84, 0x4b, 0xb0, 0xe2, - 0xae, 0x56, 0x90, 0xbb, 0xf2, 0xa1, 0xd2, 0x56, 0x4c, 0x05, 0x1b, 0xce, 0xd6, 0x3f, 0x82, 0xd5, - 0x90, 0x71, 0x6a, 0x81, 0x7b, 0x30, 0xd3, 0xf0, 0xd0, 0xb3, 0x9c, 0xbd, 0x95, 0xac, 0xb3, 0x15, - 0xdf, 0xcc, 0x53, 0x91, 0xe1, 0x16, 0xfe, 0x96, 0x80, 0x8c, 0x9f, 0x05, 0xdd, 0x83, 0x49, 0x03, - 0xeb, 0xc7, 0x4a, 0x83, 0xd8, 0x75, 0x7a, 0x73, 0x2d, 0x4c, 0x5a, 0xae, 0x46, 0xf8, 0x76, 0x63, - 0xa2, 0x33, 0x05, 0xed, 0x43, 0xe6, 0x58, 0x6b, 0xf7, 0x3a, 0x58, 0xc2, 0x27, 0x5d, 0x59, 0x75, - 0x5f, 0xc0, 0xf4, 0xe6, 0x7a, 0xa8, 0x98, 0x03, 0x7b, 0x42, 0xd1, 0xe1, 0xdf, 0x8d, 0x89, 0xe7, - 0x8e, 0x59, 0x12, 0xff, 0x73, 0x0e, 0x26, 0xe9, 0x6a, 0xe8, 0x1d, 0x48, 0x9a, 0xa7, 0x5d, 0xa2, - 0xdd, 0xdc, 0xe6, 0xb5, 0x61, 0xda, 0xe5, 0xea, 0xa7, 0x5d, 0x2c, 0xda, 0x53, 0x84, 0x0f, 0x20, - 0x69, 0x3d, 0xa1, 0x69, 0x98, 0xdc, 0xaf, 0x3c, 0xa8, 0x54, 0x1f, 0x55, 0x32, 0x31, 0xb4, 0x04, - 0xa8, 0x50, 0xad, 0xd4, 0xc5, 0x6a, 0xb9, 0x5c, 0x14, 0xa5, 0x5a, 0x51, 0x3c, 0x28, 0x15, 0x8a, - 0x19, 0x0e, 0xbd, 0x02, 0x6b, 0x07, 0xd5, 0xf2, 0xfe, 0xc3, 0xa2, 0x94, 0x2f, 0x14, 0x8a, 0xb5, - 0x5a, 0x69, 0xab, 0x54, 0x2e, 0xd5, 0x1f, 0x4b, 0x85, 0x6a, 0xa5, 0x56, 0x17, 0xf3, 0xa5, 0x4a, - 0xbd, 0x96, 0x89, 0xf3, 0x3f, 0xe6, 0xe0, 0x9c, 0x6f, 0x03, 0x28, 0xcf, 0x68, 0x78, 0x6b, 0xd4, - 0x8d, 0x7b, 0x35, 0xbd, 0x19, 0xa4, 0x29, 0xc0, 0x44, 0xb5, 0x52, 0x2e, 0x55, 0x2c, 0xed, 0xa6, - 0x61, 0xb2, 0x7a, 0xff, 0xbe, 0xfd, 0x10, 0xdf, 0x9a, 0x20, 0x0b, 0x0a, 0x73, 0x30, 0xb3, 0xa7, - 0x6b, 0x87, 0xd8, 0xf1, 0x9f, 0x3c, 0xcc, 0xd2, 0x67, 0xea, 0x2f, 0xdf, 0x82, 0x94, 0x8e, 0xe5, - 0xe6, 0x29, 0x7d, 0xb5, 0x7c, 0x8e, 0xc4, 0x64, 0xce, 0x89, 0xc9, 0xdc, 0x96, 0xa6, 0xb5, 0x0f, - 0x2c, 0xff, 0x14, 0x09, 0xa3, 0xf0, 0x4d, 0x12, 0xe6, 0x0b, 0x3a, 0x96, 0x4d, 0x4c, 0xb4, 0xa5, - 0xa2, 0x03, 0x63, 0xef, 0x1e, 0xcc, 0x59, 0xfe, 0xd5, 0x50, 0xcc, 0x53, 0x49, 0x97, 0xd5, 0x16, - 0xa6, 0xaf, 0x7e, 0xd1, 0xb1, 0x40, 0x81, 0x8e, 0x8a, 0xd6, 0xa0, 0x38, 0xdb, 0xf0, 0x3e, 0xa2, - 0x12, 0xcc, 0x53, 0xd7, 0x61, 0x5c, 0x3a, 0xc1, 0xba, 0x34, 0xd1, 0xc2, 0xe3, 0xd2, 0xe8, 0x98, - 0xa5, 0x28, 0xd8, 0x40, 0x0f, 0x00, 0xba, 0xb2, 0x2e, 0x77, 0xb0, 0x89, 0x75, 0x23, 0x9b, 0x64, - 0xe3, 0x3b, 0x60, 0x37, 0xb9, 0x3d, 0x97, 0x9b, 0xc4, 0xb7, 0x67, 0x3a, 0xda, 0xb1, 0x02, 0xa2, - 0xa1, 0x63, 0xd3, 0xc8, 0xa6, 0x6c, 0x49, 0xeb, 0x51, 0x92, 0x6a, 0x84, 0xd5, 0x16, 0xb3, 0x95, - 0xf8, 0xc5, 0x16, 0x27, 0x3a, 0xb3, 0x51, 0x15, 0x16, 0x9d, 0x0d, 0x6a, 0xaa, 0x89, 0x55, 0x53, - 0x32, 0xb4, 0x9e, 0xde, 0xc0, 0xd9, 0x09, 0xdb, 0x4a, 0xcb, 0xbe, 0x2d, 0x12, 0x9e, 0x9a, 0xcd, - 0x22, 0x52, 0xd3, 0x30, 0x44, 0xf4, 0x04, 0x78, 0xb9, 0xd1, 0xc0, 0x86, 0xa1, 0x10, 0x5b, 0x48, - 0x3a, 0xfe, 0xb4, 0xa7, 0xe8, 0xb8, 0x83, 0x55, 0xd3, 0xc8, 0x4e, 0xb2, 0x52, 0xeb, 0x5a, 0x57, - 0x6b, 0x6b, 0xad, 0x53, 0xb1, 0xcf, 0x23, 0x5e, 0x64, 0xa6, 0x7b, 0x46, 0x0c, 0xfe, 0x5d, 0x38, - 0xe7, 0x33, 0xca, 0x38, 0x99, 0x8d, 0xbf, 0x03, 0x33, 0x5e, 0x4b, 0x8c, 0x95, 0x15, 0x7f, 0x1a, - 0x87, 0xf9, 0x00, 0x1b, 0xa0, 0x5d, 0x48, 0x1b, 0xaa, 0xdc, 0x35, 0x9e, 0x6a, 0x26, 0xf5, 0xdf, - 0x1b, 0x11, 0x26, 0xcb, 0xd5, 0x28, 0x2f, 0x79, 0xdc, 0x8d, 0x89, 0xee, 0x6c, 0xb4, 0x05, 0x13, - 0xc4, 0x9e, 0xfe, 0xdc, 0x14, 0x24, 0x87, 0xd0, 0x5c, 0x29, 0x74, 0x26, 0xff, 0x06, 0xcc, 0xb1, - 0x2b, 0xa0, 0xcb, 0x30, 0xed, 0xac, 0x20, 0x29, 0x4d, 0xba, 0x57, 0x70, 0x48, 0xa5, 0x26, 0xff, - 0x3a, 0xcc, 0x78, 0x85, 0xa1, 0x65, 0x98, 0xa2, 0x0e, 0xe1, 0xb2, 0xa7, 0x09, 0xa1, 0xd4, 0x74, - 0x63, 0xfa, 0x3b, 0xb0, 0xc0, 0xfa, 0x19, 0x0d, 0xe5, 0xeb, 0xee, 0x1e, 0x88, 0x2d, 0xe6, 0xd8, - 0x3d, 0x38, 0x7a, 0x0a, 0xbf, 0x4b, 0x42, 0xc6, 0x1f, 0x34, 0xe8, 0x1e, 0xa4, 0x0e, 0xdb, 0x5a, - 0xe3, 0x13, 0x3a, 0xf7, 0x95, 0xb0, 0xe8, 0xca, 0x6d, 0x59, 0x5c, 0x84, 0xba, 0x1b, 0x13, 0xc9, - 0x24, 0x6b, 0x76, 0x47, 0xeb, 0xa9, 0x26, 0xb5, 0x5e, 0xf8, 0xec, 0x87, 0x16, 0x57, 0x7f, 0xb6, - 0x3d, 0x09, 0x6d, 0xc3, 0x34, 0x71, 0x3b, 0xa9, 0xa3, 0x35, 0x71, 0x36, 0x61, 0xcb, 0xb8, 0x1a, - 0x2a, 0x23, 0x6f, 0xf3, 0x3e, 0xd4, 0x9a, 0x58, 0x04, 0xd9, 0xfd, 0xcf, 0xcf, 0xc2, 0xb4, 0x47, - 0x37, 0x7e, 0x07, 0xa6, 0x3d, 0x8b, 0xa1, 0x0b, 0x30, 0x79, 0x64, 0x48, 0x6e, 0x12, 0x9e, 0x12, - 0x27, 0x8e, 0x0c, 0x3b, 0x9f, 0x5e, 0x86, 0x69, 0x5b, 0x0b, 0xe9, 0xa8, 0x2d, 0xb7, 0x8c, 0x6c, - 0x7c, 0x2d, 0x61, 0xbd, 0x23, 0x9b, 0x74, 0xdf, 0xa2, 0xf0, 0xff, 0xe2, 0x00, 0xfa, 0x4b, 0xa2, - 0x7b, 0x90, 0xb4, 0xb5, 0x24, 0xa9, 0x7c, 0x7d, 0x04, 0x2d, 0x73, 0xb6, 0xaa, 0xf6, 0x2c, 0xe1, - 0x57, 0x1c, 0x24, 0x6d, 0x31, 0xfe, 0x03, 0xa7, 0x56, 0xaa, 0xec, 0x94, 0x8b, 0x52, 0xa5, 0xba, - 0x5d, 0x94, 0x1e, 0x89, 0xa5, 0x7a, 0x51, 0xcc, 0x70, 0x68, 0x19, 0x2e, 0x78, 0xe9, 0x62, 0x31, - 0xbf, 0x5d, 0x14, 0xa5, 0x6a, 0xa5, 0xfc, 0x38, 0x13, 0x47, 0x3c, 0x2c, 0x3d, 0xdc, 0x2f, 0xd7, - 0x4b, 0x83, 0x63, 0x09, 0xb4, 0x02, 0x59, 0xcf, 0x18, 0x95, 0x41, 0xc5, 0x26, 0x2d, 0xb1, 0x9e, - 0x51, 0xf2, 0x97, 0x0e, 0xa6, 0xb6, 0x66, 0xdd, 0x97, 0x61, 0x3b, 0xdb, 0x23, 0x98, 0x65, 0x72, - 0xb4, 0x55, 0x4e, 0xd1, 0xa4, 0xd2, 0x94, 0x0e, 0x4f, 0x4d, 0xbb, 0xc4, 0xe0, 0xd6, 0x13, 0xe2, - 0xac, 0x43, 0xdd, 0xb2, 0x88, 0x96, 0x59, 0xdb, 0x4a, 0x47, 0x31, 0x29, 0x4f, 0xdc, 0xe6, 0x01, - 0x9b, 0x64, 0x33, 0x08, 0x5f, 0xc5, 0x61, 0x82, 0xbe, 0x9b, 0x6b, 0x9e, 0x53, 0x82, 0x11, 0xe9, - 0x50, 0x89, 0x48, 0x26, 0x38, 0xe2, 0x6c, 0x70, 0xa0, 0x5d, 0x98, 0xf3, 0xa6, 0xd2, 0x13, 0xa7, - 0x88, 0xbb, 0xc2, 0xbe, 0x20, 0x6f, 0x3c, 0x9f, 0xd0, 0xd2, 0x6d, 0xf6, 0xd8, 0x4b, 0x43, 0x5b, - 0x30, 0xe7, 0xcb, 0xc6, 0xc9, 0xe1, 0xd9, 0x78, 0xb6, 0xc1, 0x24, 0xa6, 0x3c, 0xcc, 0x3b, 0x89, - 0xb4, 0x8d, 0x25, 0x93, 0x26, 0x5a, 0x7a, 0x5a, 0x64, 0x06, 0x12, 0x30, 0xea, 0x33, 0x3b, 0x34, - 0xfe, 0x3d, 0x40, 0x83, 0xba, 0x8e, 0x95, 0x35, 0x7b, 0x30, 0x1f, 0x90, 0xe2, 0x51, 0x0e, 0xa6, - 0xec, 0x57, 0x65, 0x28, 0x26, 0xa6, 0xe5, 0xe1, 0xa0, 0x46, 0x7d, 0x16, 0x8b, 0xbf, 0xab, 0xe3, - 0x23, 0xac, 0xeb, 0xb8, 0x69, 0x87, 0x47, 0x20, 0xbf, 0xcb, 0x22, 0xfc, 0x84, 0x83, 0xb4, 0x43, - 0x47, 0x77, 0x20, 0x6d, 0xe0, 0x16, 0x39, 0x7e, 0xc8, 0x5a, 0x97, 0xfc, 0x73, 0x73, 0x35, 0xca, - 0x40, 0x0b, 0x69, 0x87, 0xdf, 0x2a, 0xa4, 0x99, 0xa1, 0xb1, 0x36, 0xff, 0x27, 0x0e, 0xe6, 0xb7, - 0x71, 0x1b, 0xfb, 0xab, 0x94, 0xa8, 0x0c, 0xeb, 0x3d, 0xd8, 0xe3, 0xec, 0xc1, 0x1e, 0x20, 0x2a, - 0xe2, 0x60, 0x3f, 0xd3, 0x61, 0xb7, 0x04, 0x0b, 0xec, 0x6a, 0x24, 0xbd, 0x0b, 0xff, 0x4e, 0xc0, - 0x25, 0xcb, 0x17, 0x74, 0xad, 0xdd, 0xc6, 0xfa, 0x5e, 0xef, 0xb0, 0xad, 0x18, 0x4f, 0xc7, 0xd8, - 0xdc, 0x05, 0x98, 0x54, 0xb5, 0xa6, 0x27, 0x78, 0x26, 0xac, 0xc7, 0x52, 0x13, 0x15, 0xe1, 0xbc, - 0xbf, 0xcc, 0x3a, 0xa5, 0x49, 0x38, 0xbc, 0xc8, 0xca, 0x1c, 0xfb, 0x4f, 0x10, 0x1e, 0xd2, 0x56, - 0x81, 0xa8, 0xa9, 0xed, 0x53, 0x3b, 0x62, 0xd2, 0xa2, 0xfb, 0x8c, 0x44, 0x7f, 0xc5, 0xf4, 0xa6, - 0x5b, 0x31, 0x45, 0xee, 0x28, 0xaa, 0x78, 0xfa, 0x78, 0x20, 0xe2, 0x27, 0x6c, 0xd1, 0xef, 0x8c, - 0x28, 0x7a, 0x68, 0x26, 0x38, 0xcb, 0x5b, 0x7c, 0x0e, 0xe1, 0xfb, 0x57, 0x0e, 0x2e, 0x87, 0x6e, - 0x81, 0x1e, 0xf9, 0x4d, 0x38, 0xd7, 0x25, 0x03, 0xae, 0x11, 0x48, 0x94, 0xdd, 0x1d, 0x6a, 0x04, - 0xda, 0xc5, 0x52, 0x2a, 0x63, 0x86, 0xb9, 0x2e, 0x43, 0xe4, 0xf3, 0x30, 0x1f, 0xc0, 0x36, 0xd6, - 0x66, 0xbe, 0xe6, 0x60, 0xad, 0xaf, 0xca, 0xbe, 0xda, 0x7d, 0x7e, 0xee, 0x5b, 0xef, 0xfb, 0x16, - 0x49, 0xf9, 0xb7, 0x07, 0xf7, 0x1e, 0xbc, 0xe0, 0x8b, 0x8a, 0xe0, 0xab, 0x70, 0x25, 0x62, 0x69, - 0x1a, 0xce, 0x5f, 0x25, 0xe1, 0xca, 0x81, 0xdc, 0x56, 0x9a, 0x6e, 0x21, 0x17, 0xd0, 0xef, 0x47, - 0x9b, 0xa4, 0x31, 0x10, 0x01, 0x24, 0x6b, 0xdd, 0x73, 0xa3, 0x76, 0x98, 0xfc, 0x11, 0x8e, 0xc3, - 0xe7, 0xd8, 0x84, 0x3d, 0x0e, 0x68, 0xc2, 0xde, 0x19, 0x5d, 0xd7, 0xa8, 0x96, 0x6c, 0xdf, 0x9f, - 0x60, 0xde, 0x1e, 0x5d, 0x6e, 0x84, 0x17, 0x9c, 0x39, 0x8a, 0x5f, 0x66, 0xd7, 0xf4, 0x97, 0x24, - 0x08, 0x51, 0xbb, 0xa7, 0x39, 0x44, 0x84, 0xa9, 0x86, 0xa6, 0x1e, 0x29, 0x7a, 0x07, 0x37, 0x69, - 0xf5, 0xff, 0xd6, 0x28, 0xc6, 0xa3, 0x09, 0xa4, 0xe0, 0xcc, 0x15, 0xfb, 0x62, 0x50, 0x16, 0x26, - 0x3b, 0xd8, 0x30, 0xe4, 0x96, 0xa3, 0x96, 0xf3, 0xc8, 0x7f, 0x91, 0x80, 0x29, 0x77, 0x0a, 0x52, - 0x07, 0x3c, 0x98, 0xa4, 0xaf, 0x9d, 0x67, 0x51, 0xe0, 0xd9, 0x9d, 0x39, 0xfe, 0x0c, 0xce, 0xdc, - 0x64, 0x9c, 0x99, 0x84, 0xc3, 0xf6, 0x33, 0xa9, 0x1d, 0xe1, 0xd7, 0x2f, 0xdd, 0x01, 0x85, 0xef, - 0x03, 0x2a, 0x2b, 0x06, 0xed, 0xa2, 0xdc, 0xb4, 0x64, 0x35, 0x4d, 0xf2, 0x89, 0x84, 0x55, 0x53, - 0x57, 0x68, 0xb9, 0x9e, 0x12, 0xa1, 0x23, 0x9f, 0x14, 0x09, 0xc5, 0x2a, 0xe9, 0x0d, 0x53, 0xd6, - 0x4d, 0x45, 0x6d, 0x49, 0xa6, 0xf6, 0x09, 0x76, 0x41, 0x57, 0x87, 0x5a, 0xb7, 0x88, 0xc2, 0xe7, - 0x1c, 0xcc, 0x33, 0xe2, 0xa9, 0x4f, 0xde, 0x85, 0xc9, 0xbe, 0x6c, 0xa6, 0x8c, 0x0f, 0xe0, 0xce, - 0x11, 0xb3, 0x39, 0x33, 0xd0, 0x2a, 0x80, 0x8a, 0x4f, 0x4c, 0x66, 0xdd, 0x29, 0x8b, 0x62, 0xaf, - 0xc9, 0x6f, 0x40, 0x8a, 0x98, 0x61, 0xd4, 0x7e, 0xf9, 0x8b, 0x38, 0xa0, 0x1d, 0x6c, 0xba, 0x6d, - 0x10, 0xb5, 0x41, 0x88, 0x2f, 0x71, 0xcf, 0xe0, 0x4b, 0xef, 0x33, 0xbe, 0x44, 0xbc, 0xf1, 0x86, - 0x07, 0x7d, 0xf6, 0x2d, 0x1d, 0x99, 0x09, 0x43, 0x5a, 0x0f, 0x52, 0xcf, 0x8d, 0xd6, 0x7a, 0x9c, - 0xd1, 0x65, 0xb6, 0x61, 0x9e, 0xd1, 0x99, 0xbe, 0xd3, 0x5b, 0x80, 0xe4, 0x63, 0x59, 0x69, 0xcb, - 0x96, 0x5e, 0x4e, 0x67, 0x47, 0x3b, 0xbd, 0xf3, 0xee, 0x88, 0x33, 0x4d, 0x10, 0xbc, 0x05, 0x03, - 0x95, 0xe7, 0x47, 0xc3, 0xdb, 0xde, 0x83, 0x76, 0x80, 0x87, 0xae, 0xbb, 0x13, 0x88, 0x88, 0x5f, - 0x1d, 0x2c, 0x12, 0x28, 0x3c, 0x1c, 0x0a, 0x8e, 0x7f, 0x96, 0x80, 0xe5, 0x08, 0x6e, 0x74, 0x17, - 0x12, 0x7a, 0xb7, 0x41, 0x9d, 0xe9, 0xd5, 0x11, 0xe4, 0xe7, 0xc4, 0xbd, 0xc2, 0x6e, 0x4c, 0xb4, - 0x66, 0xf1, 0xbf, 0x89, 0x43, 0x42, 0xdc, 0x2b, 0xa0, 0xf7, 0x18, 0xa4, 0xf8, 0xe6, 0x88, 0x52, - 0xbc, 0x40, 0xf1, 0x97, 0x5c, 0x10, 0x52, 0x9c, 0x85, 0x85, 0x82, 0x58, 0xcc, 0xd7, 0x8b, 0xd2, - 0x76, 0xb1, 0x5c, 0xac, 0x17, 0x25, 0x82, 0x64, 0x67, 0x38, 0xb4, 0x02, 0xd9, 0xbd, 0xfd, 0xad, - 0x72, 0xa9, 0xb6, 0x2b, 0xed, 0x57, 0x9c, 0x7f, 0x74, 0x34, 0x8e, 0x32, 0x30, 0x53, 0x2e, 0xd5, - 0xea, 0x94, 0x50, 0xcb, 0x24, 0x2c, 0xca, 0x4e, 0xb1, 0x2e, 0x15, 0xf2, 0x7b, 0xf9, 0x42, 0xa9, - 0xfe, 0x38, 0x93, 0x44, 0x3c, 0x2c, 0xb1, 0xb2, 0x6b, 0x95, 0xfc, 0x5e, 0x6d, 0xb7, 0x5a, 0xcf, - 0xa4, 0x10, 0x82, 0x39, 0x7b, 0xbe, 0x43, 0xaa, 0x65, 0x26, 0x2c, 0x09, 0x85, 0x72, 0xb5, 0xe2, - 0xea, 0x30, 0x89, 0x16, 0x20, 0xe3, 0xac, 0x2c, 0x16, 0xf3, 0xdb, 0x36, 0x8a, 0x91, 0x46, 0xe7, - 0x61, 0xb6, 0xf8, 0xe1, 0x5e, 0xbe, 0xb2, 0xed, 0x30, 0x4e, 0xb9, 0x18, 0xd8, 0xd7, 0x71, 0x58, - 0x24, 0x20, 0x98, 0x03, 0xb9, 0x39, 0x61, 0xb9, 0x0e, 0x19, 0xd2, 0xb6, 0x4b, 0xfe, 0xc2, 0x69, - 0x8e, 0xd0, 0x0f, 0x9c, 0xf2, 0xc9, 0x01, 0xac, 0xe3, 0x1e, 0xc0, 0xba, 0xe4, 0x2f, 0x26, 0x6f, - 0xb0, 0xd0, 0xae, 0x6f, 0xb5, 0xa8, 0xfe, 0xe4, 0x61, 0x40, 0xb5, 0x73, 0x2b, 0x5a, 0x5a, 0xd4, - 0x49, 0x70, 0x96, 0x66, 0xe4, 0x8c, 0x01, 0x7d, 0x1f, 0x96, 0xfc, 0xfa, 0xd2, 0xd8, 0xba, 0x39, - 0x00, 0xc0, 0xba, 0x19, 0xc6, 0xe5, 0x75, 0x39, 0x84, 0xbf, 0x73, 0x90, 0x76, 0xc8, 0x56, 0x96, - 0x36, 0x94, 0x1f, 0x62, 0x06, 0xf0, 0x99, 0xb2, 0x28, 0x2e, 0x7e, 0xe4, 0x85, 0x4e, 0xe3, 0x7e, - 0xe8, 0x34, 0xf0, 0x3d, 0x27, 0x02, 0xdf, 0xf3, 0x77, 0x61, 0xb6, 0x61, 0xa9, 0xaf, 0x68, 0xaa, - 0x64, 0x2a, 0x1d, 0x07, 0xcf, 0x19, 0xbc, 0xea, 0xa8, 0x3b, 0xf7, 0x93, 0xe2, 0x8c, 0x33, 0xc1, - 0x22, 0xa1, 0x35, 0x98, 0xb1, 0xaf, 0x3e, 0x24, 0x53, 0x93, 0x7a, 0x06, 0xce, 0xa6, 0xec, 0xee, - 0x16, 0x6c, 0x5a, 0x5d, 0xdb, 0x37, 0xb0, 0xf0, 0x67, 0x0e, 0x16, 0x49, 0xd3, 0xee, 0x77, 0xc7, - 0x61, 0x10, 0xb0, 0xd7, 0xe3, 0x7c, 0x89, 0x3f, 0x50, 0xe0, 0x8b, 0xea, 0x59, 0xb2, 0xb0, 0xe4, - 0x5f, 0x8f, 0x36, 0x2a, 0xbf, 0xe5, 0x60, 0xc1, 0x3a, 0x75, 0x9d, 0x81, 0xe7, 0x5d, 0x04, 0x8c, - 0xf1, 0x26, 0x7d, 0xc6, 0x4c, 0xfa, 0x8d, 0x29, 0xfc, 0x9e, 0x83, 0x45, 0x9f, 0xae, 0xd4, 0x53, - 0xdf, 0xf5, 0x57, 0x14, 0x57, 0xbd, 0x15, 0xc5, 0x00, 0xff, 0x98, 0x35, 0xc5, 0x6d, 0xa7, 0xa6, - 0x18, 0x2f, 0x20, 0x3e, 0x8b, 0xc3, 0x6a, 0x3f, 0xb3, 0xdb, 0xd7, 0x7e, 0xcd, 0x31, 0x5a, 0xe2, - 0xb3, 0xdd, 0xae, 0x7d, 0xe0, 0x4f, 0x75, 0x9b, 0x83, 0x87, 0x4d, 0x80, 0x4a, 0x2f, 0xca, 0x01, - 0x7f, 0xe4, 0x45, 0xb7, 0xd8, 0x75, 0xe9, 0x2b, 0x1c, 0x11, 0x26, 0x7e, 0x1b, 0x2e, 0xd8, 0x40, - 0x81, 0x7b, 0xdd, 0xec, 0x5c, 0x82, 0x91, 0x2c, 0x92, 0x16, 0x17, 0xad, 0x61, 0xf7, 0x8e, 0x95, - 0x42, 0xa3, 0x4d, 0xe1, 0x9b, 0x24, 0x2c, 0x55, 0xb4, 0x26, 0xae, 0x99, 0x72, 0x6b, 0x1c, 0xd0, - 0xf0, 0x7b, 0x83, 0x18, 0x4c, 0x9c, 0xb5, 0x67, 0xb0, 0xd4, 0x51, 0xa0, 0x17, 0x94, 0x83, 0x79, - 0xc3, 0x94, 0x5b, 0x76, 0x04, 0xc9, 0x7a, 0x0b, 0x9b, 0x52, 0x57, 0x36, 0x9f, 0xd2, 0xf0, 0x38, - 0x4f, 0x87, 0xea, 0xf6, 0xc8, 0x9e, 0x6c, 0x3e, 0x0d, 0xc6, 0xf2, 0x92, 0x63, 0x63, 0x79, 0xef, - 0xfb, 0xdb, 0xe9, 0xd7, 0x87, 0xec, 0x25, 0xe2, 0x1c, 0xfc, 0x30, 0x04, 0xa7, 0x7b, 0x63, 0x88, - 0xc8, 0xe1, 0xf8, 0xdc, 0xd9, 0x71, 0xa9, 0x97, 0x0c, 0xf1, 0x5d, 0x84, 0x0b, 0x03, 0x9b, 0xa7, - 0x59, 0xb7, 0x05, 0x59, 0x6b, 0x68, 0x5f, 0x35, 0xc6, 0x74, 0xc7, 0x10, 0x8f, 0x89, 0x87, 0x78, - 0x8c, 0xb0, 0x0c, 0x17, 0x03, 0x16, 0xa2, 0x5a, 0xfc, 0x31, 0x45, 0xd4, 0x18, 0x1f, 0x6d, 0xfe, - 0x28, 0x2c, 0x2a, 0xde, 0xf2, 0xbe, 0xf6, 0x40, 0x60, 0xf6, 0x45, 0xc4, 0xc5, 0x65, 0x98, 0xf6, - 0xf2, 0xd1, 0x93, 0xc3, 0x1c, 0x12, 0x38, 0xa9, 0x33, 0x81, 0xe0, 0x13, 0x3e, 0x10, 0xbc, 0xdc, - 0x0f, 0xaa, 0x49, 0xb6, 0x1a, 0x0c, 0x35, 0x45, 0x44, 0x58, 0x3d, 0x19, 0x08, 0xab, 0x34, 0x8b, - 0xac, 0x87, 0x0a, 0xfd, 0x3f, 0x08, 0x2c, 0xea, 0xd4, 0x81, 0x90, 0xb7, 0xf0, 0x04, 0x78, 0xe2, - 0xf1, 0xe3, 0x83, 0xd0, 0x3e, 0x37, 0x8a, 0xfb, 0xdd, 0x48, 0x58, 0x85, 0xe5, 0x40, 0xd9, 0x74, - 0xe9, 0xc7, 0x44, 0xaf, 0x1d, 0x4c, 0x31, 0x8c, 0x9a, 0x29, 0x9b, 0xc6, 0xa8, 0x2b, 0xd3, 0x41, - 0xef, 0xca, 0x84, 0x64, 0xaf, 0xbc, 0x43, 0x76, 0xe5, 0x17, 0x4d, 0xcf, 0xce, 0xd7, 0x20, 0xd5, - 0xb3, 0xe1, 0x38, 0x52, 0xfc, 0xcc, 0xb3, 0x2e, 0xbd, 0x6f, 0x0d, 0x89, 0x84, 0x43, 0xf8, 0x03, - 0x07, 0xd3, 0x1e, 0x32, 0x5a, 0x81, 0x29, 0xb7, 0x3b, 0x77, 0xea, 0x74, 0x97, 0x60, 0xbd, 0x03, - 0x53, 0x33, 0xe5, 0x36, 0xbd, 0xe1, 0x25, 0x0f, 0x56, 0x6b, 0xd5, 0x33, 0x30, 0x29, 0xe3, 0x12, - 0xa2, 0xfd, 0x1f, 0xdd, 0x84, 0x64, 0x4f, 0x55, 0x4c, 0x3b, 0xf6, 0xe6, 0xfc, 0x41, 0x65, 0x2f, - 0x95, 0xdb, 0x57, 0x15, 0x53, 0xb4, 0xb9, 0x84, 0x1b, 0x90, 0xb4, 0x9e, 0xd8, 0x26, 0x76, 0x0a, - 0x52, 0x5b, 0x8f, 0xeb, 0xc5, 0x5a, 0x86, 0x43, 0x00, 0x13, 0xa5, 0x4a, 0x75, 0xbb, 0x58, 0xcb, - 0xc4, 0x85, 0x15, 0x77, 0xeb, 0x41, 0x20, 0xc1, 0xc7, 0xe4, 0x95, 0x84, 0xc1, 0x03, 0xf9, 0x40, - 0x78, 0x60, 0x95, 0x39, 0x9c, 0x86, 0x00, 0x03, 0xff, 0xe1, 0x60, 0x31, 0x90, 0x0f, 0xdd, 0xf6, - 0x42, 0x02, 0x57, 0x22, 0x65, 0x7a, 0xc1, 0x80, 0x5f, 0x72, 0x04, 0x0c, 0xb8, 0xc3, 0x80, 0x01, - 0xd7, 0x87, 0xce, 0xf7, 0xc2, 0x00, 0x07, 0x21, 0x28, 0x40, 0xad, 0x9e, 0xdf, 0x29, 0x4a, 0xfb, - 0x15, 0xf2, 0xeb, 0xa2, 0x00, 0x0b, 0x90, 0xb1, 0xba, 0x7a, 0xfa, 0x7d, 0x5b, 0xad, 0x9e, 0xaf, - 0xd7, 0x32, 0xf1, 0xc1, 0x0e, 0x3c, 0xe1, 0x76, 0xe0, 0x0b, 0x80, 0xa8, 0x59, 0xbd, 0x9f, 0x66, - 0x7e, 0xce, 0xc1, 0x3c, 0x43, 0xa6, 0x56, 0xf6, 0xdc, 0xde, 0x70, 0xcc, 0xed, 0xcd, 0x06, 0x2c, - 0x58, 0x4d, 0x04, 0x71, 0x64, 0x43, 0xea, 0x62, 0x5d, 0xb2, 0x46, 0xa8, 0x3b, 0x9d, 0xef, 0xc8, - 0x27, 0x14, 0xed, 0xdb, 0xc3, 0xba, 0x25, 0xf8, 0x39, 0xe0, 0x5b, 0xc2, 0xcf, 0x38, 0x72, 0xee, - 0x8e, 0x5d, 0x70, 0x0f, 0x0b, 0xc2, 0x80, 0x8a, 0x3c, 0x31, 0x7a, 0x45, 0x2e, 0xe4, 0xc9, 0x61, - 0x7b, 0x86, 0xe2, 0x77, 0xf3, 0xbf, 0x1c, 0xa4, 0x4b, 0x4d, 0xac, 0x9a, 0x96, 0xf7, 0x55, 0x60, - 0x96, 0xf9, 0x74, 0x15, 0xad, 0x84, 0x7c, 0xd1, 0x6a, 0x6f, 0x9d, 0x5f, 0x8d, 0xfc, 0xde, 0x55, - 0x88, 0xa1, 0x23, 0xcf, 0x67, 0xb7, 0x0c, 0x7e, 0xf9, 0xca, 0xc0, 0xcc, 0x80, 0x40, 0xe4, 0xaf, - 0x0d, 0xe1, 0x72, 0xd7, 0x79, 0x1b, 0x52, 0xf6, 0x47, 0x8a, 0x68, 0xc1, 0xfd, 0x50, 0xd2, 0xf3, - 0x0d, 0x23, 0xbf, 0xe8, 0xa3, 0x3a, 0xf3, 0x36, 0xff, 0x99, 0x06, 0xe8, 0xf7, 0x10, 0xe8, 0x01, - 0xcc, 0x78, 0xbf, 0x93, 0x42, 0xcb, 0x11, 0x5f, 0xe9, 0xf1, 0x2b, 0xc1, 0x83, 0xae, 0x4e, 0x0f, - 0x60, 0xc6, 0x7b, 0x2b, 0xdf, 0x17, 0x16, 0xf0, 0x65, 0x40, 0x5f, 0x58, 0xe0, 0x45, 0x7e, 0x0c, - 0xb5, 0xe1, 0x42, 0xc8, 0xbd, 0x2c, 0xba, 0x3e, 0xda, 0xed, 0x35, 0xff, 0xea, 0x88, 0x17, 0xbc, - 0x42, 0x0c, 0xe9, 0x70, 0x31, 0xf4, 0x3a, 0x12, 0xad, 0x8f, 0x7a, 0x59, 0xca, 0xbf, 0x36, 0x02, - 0xa7, 0xbb, 0x66, 0x0f, 0xf8, 0xf0, 0x3b, 0x10, 0xf4, 0xda, 0xc8, 0x97, 0x73, 0xfc, 0x8d, 0xd1, - 0xaf, 0x54, 0x84, 0x18, 0xda, 0x85, 0x69, 0xcf, 0x05, 0x01, 0xe2, 0x03, 0x6f, 0x0d, 0x88, 0xe0, - 0xe5, 0x88, 0x1b, 0x05, 0x22, 0xc9, 0x03, 0x62, 0xf7, 0x25, 0x0d, 0xa2, 0xf1, 0x7d, 0x49, 0x01, - 0xa8, 0xb7, 0xdf, 0xfc, 0xbe, 0x53, 0x28, 0xc8, 0xfc, 0xc1, 0xc7, 0x58, 0x90, 0xf9, 0x43, 0x8e, - 0x34, 0x21, 0x86, 0x3e, 0x80, 0x39, 0x16, 0xb1, 0x43, 0xab, 0x91, 0xc8, 0x23, 0x7f, 0x29, 0x6c, - 0xd8, 0x2b, 0x92, 0x05, 0x88, 0xfa, 0x22, 0x03, 0x81, 0xaa, 0xbe, 0xc8, 0x10, 0x5c, 0x29, 0x66, - 0xe5, 0x27, 0x06, 0x7c, 0xe9, 0xe7, 0xa7, 0x20, 0xbc, 0xa9, 0x9f, 0x9f, 0x02, 0x11, 0x1b, 0x21, - 0x86, 0x14, 0x58, 0x0a, 0x86, 0x10, 0xd0, 0xb5, 0x91, 0xa0, 0x0d, 0xfe, 0xfa, 0x30, 0x36, 0x37, - 0xd5, 0x7c, 0x99, 0x82, 0xa4, 0x7d, 0x1c, 0xd5, 0xe1, 0x9c, 0xaf, 0x85, 0x43, 0x97, 0xa2, 0x1b, - 0x5b, 0xfe, 0x72, 0xe8, 0xb8, 0xbb, 0x93, 0x27, 0x70, 0x7e, 0xa0, 0x29, 0x43, 0x6b, 0xde, 0x79, - 0x41, 0x8d, 0x21, 0x7f, 0x25, 0x82, 0xc3, 0x2f, 0x9b, 0x4d, 0x3b, 0x6b, 0xc3, 0xba, 0x06, 0x56, - 0x76, 0x58, 0xaa, 0xf9, 0x98, 0x9c, 0xfe, 0xfe, 0x24, 0x23, 0xb0, 0x7a, 0x05, 0xa6, 0x97, 0xab, - 0x91, 0x3c, 0xee, 0x0a, 0x1f, 0xb9, 0x65, 0x87, 0xa7, 0xcc, 0x45, 0x8c, 0x72, 0x81, 0xd5, 0x35, - 0x2f, 0x44, 0xb1, 0xb8, 0xe2, 0x1f, 0x41, 0xc6, 0x7f, 0x04, 0x23, 0xe6, 0x7d, 0x05, 0xb9, 0xcd, - 0x5a, 0x38, 0x83, 0xdf, 0x32, 0xfe, 0xf8, 0xf7, 0x6b, 0x15, 0x14, 0xf9, 0x57, 0x23, 0x79, 0xbc, - 0x19, 0xcb, 0x53, 0x79, 0xf5, 0x33, 0xd6, 0x60, 0x95, 0xd6, 0xcf, 0x58, 0x01, 0xa5, 0x9a, 0x10, - 0xbb, 0xf3, 0x2e, 0x40, 0xc3, 0x50, 0x24, 0xd2, 0x69, 0xa2, 0xd5, 0x01, 0x9c, 0xfc, 0xbe, 0x82, - 0xdb, 0xcd, 0x6a, 0xd7, 0x54, 0x34, 0xd5, 0xc8, 0xfe, 0x3a, 0x6d, 0xb7, 0xb9, 0x53, 0x0d, 0x43, - 0x21, 0x0d, 0xdf, 0x56, 0xea, 0x49, 0xa2, 0x61, 0x28, 0x87, 0x13, 0x36, 0xff, 0x9b, 0xff, 0x0b, - 0x00, 0x00, 0xff, 0xff, 0x95, 0x8e, 0xb8, 0x49, 0x68, 0x34, 0x00, 0x00, -} diff --git a/vendor/modules.txt b/vendor/modules.txt index 10bcf4dc8..fb2ef37f4 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -6,7 +6,7 @@ github.com/PuerkitoBio/urlesc github.com/beorn7/perks/quantile # github.com/blang/semver v3.5.0+incompatible github.com/blang/semver -# github.com/container-storage-interface/spec v1.1.0 +# github.com/container-storage-interface/spec v1.2.0 github.com/container-storage-interface/spec/lib/go/csi # github.com/davecgh/go-spew v1.1.1 github.com/davecgh/go-spew/spew From 84f78b120e891b76b3e368cb0d20502baec026ea Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Mon, 10 Feb 2020 13:08:33 +0100 Subject: [PATCH 19/56] prow.sh: generic driver installation This relies on a slightly different deployment script: a "deploy.sh" must exist which knows that it has to dump a test driver configurion into the file pointed to with CSI_PROW_TEST_DRIVER, if that env variable is set. That way, we no longer need to know what capabilities the installed driver has. --- prow.sh | 104 +++++++++++++++++++++++--------------------------------- 1 file changed, 43 insertions(+), 61 deletions(-) diff --git a/prow.sh b/prow.sh index 118c5bd19..bb80741e2 100755 --- a/prow.sh +++ b/prow.sh @@ -157,7 +157,9 @@ csi_prow_kubernetes_version_suffix="$(echo "${CSI_PROW_KUBERNETES_VERSION}" | tr # the caller. configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csiprow.XXXXXXXXXX")" "work directory" -# The hostpath deployment script is searched for in several places. +# By default, this script tests sidecars with the CSI hostpath driver, +# using the install_csi_driver function. That function depends on +# a deployment script that it searches for in several places: # # - The "deploy" directory in the current repository: this is useful # for the situation that a component becomes incompatible with the @@ -165,11 +167,11 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip # own example until the shared one can be updated; it's also how # csi-driver-host-path itself provides the example. # -# - CSI_PROW_HOSTPATH_VERSION of the CSI_PROW_HOSTPATH_REPO is checked +# - CSI_PROW_DRIVER_VERSION of the CSI_PROW_DRIVER_REPO is checked # out: this allows other repos to reference a version of the example # that is known to be compatible. # -# - The csi-driver-host-path/deploy directory has multiple sub-directories, +# - The /deploy directory can have multiple sub-directories, # each with different deployments (stable set of images for Kubernetes 1.13, # stable set of images for Kubernetes 1.14, canary for latest Kubernetes, etc.). # This is necessary because there may be incompatible changes in the @@ -186,16 +188,26 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip # "none" disables the deployment of the hostpath driver. # # When no deploy script is found (nothing in `deploy` directory, -# CSI_PROW_HOSTPATH_REPO=none), nothing gets deployed. -configvar CSI_PROW_HOSTPATH_VERSION "v1.3.0-rc3" "hostpath driver" -configvar CSI_PROW_HOSTPATH_REPO https://github.com/kubernetes-csi/csi-driver-host-path "hostpath repo" +# CSI_PROW_DRIVER_REPO=none), nothing gets deployed. +# +# If the deployment script is called with CSI_PROW_TEST_DRIVER= as +# environment variable, then it must write a suitable test driver configuration +# into that file in addition to installing the driver. +configvar CSI_PROW_DRIVER_VERSION "v1.3.0-rc4" "CSI driver version" +configvar CSI_PROW_DRIVER_REPO https://github.com/kubernetes-csi/csi-driver-host-path "CSI driver repo" configvar CSI_PROW_DEPLOYMENT "" "deployment" -configvar CSI_PROW_HOSTPATH_DRIVER_NAME "hostpath.csi.k8s.io" "the hostpath driver name" -# If CSI_PROW_HOSTPATH_CANARY is set (typically to "canary", but also -# "1.0-canary"), then all image versions are replaced with that -# version tag. -configvar CSI_PROW_HOSTPATH_CANARY "" "hostpath image" +# The install_csi_driver function may work also for other CSI drivers, +# as long as they follow the conventions of the CSI hostpath driver. +# If they don't, then a different install function can be provided in +# a .prow.sh file and this config variable can be overridden. +configvar CSI_PROW_DRIVER_INSTALL "install_csi_driver" "name of the shell function which installs the CSI driver" + +# If CSI_PROW_DRIVER_CANARY is set (typically to "canary", but also +# version tag. Usually empty. CSI_PROW_HOSTPATH_CANARY is +# accepted as alternative name because some test-infra jobs +# still use that name. +configvar CSI_PROW_DRIVER_CANARY "${CSI_PROW_HOSTPATH_CANARY}" "driver image override for canary images" # The E2E testing can come from an arbitrary repo. The expectation is that # the repo supports "go test ./test/e2e -args --storage.testdriver" (https://github.com/kubernetes/kubernetes/pull/72836) @@ -613,7 +625,7 @@ find_deployment () { # Fixed deployment name? Use it if it exists, otherwise fail. if [ "${CSI_PROW_DEPLOYMENT}" ]; then - file="$dir/${CSI_PROW_DEPLOYMENT}/deploy-hostpath.sh" + file="$dir/${CSI_PROW_DEPLOYMENT}/deploy.sh" if ! [ -e "$file" ]; then return 1 fi @@ -623,9 +635,9 @@ find_deployment () { # Ignore: See if you can use ${variable//search/replace} instead. # shellcheck disable=SC2001 - file="$dir/kubernetes-$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/\([0-9]*\)\.\([0-9]*\).*/\1.\2/')/deploy-hostpath.sh" + file="$dir/kubernetes-$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/\([0-9]*\)\.\([0-9]*\).*/\1.\2/')/deploy.sh" if ! [ -e "$file" ]; then - file="$dir/kubernetes-latest/deploy-hostpath.sh" + file="$dir/kubernetes-latest/deploy.sh" if ! [ -e "$file" ]; then return 1 fi @@ -633,12 +645,11 @@ find_deployment () { echo "$file" } -# This installs the hostpath driver example. CSI_PROW_HOSTPATH_CANARY overrides all -# image versions with that canary version. The parameters of install_hostpath can be -# used to override registry and/or tag of individual images (CSI_PROVISIONER_REGISTRY=localhost:9000 -# CSI_PROVISIONER_TAG=latest). -install_hostpath () { - local images deploy_hostpath +# This installs the CSI driver. It's called with a list of env variables +# that override the default images. CSI_PROW_DRIVER_CANARY overrides all +# image versions with that canary version. +install_csi_driver () { + local images deploy_driver images="$*" if [ "${CSI_PROW_DEPLOYMENT}" = "none" ]; then @@ -654,31 +665,31 @@ install_hostpath () { done fi - if deploy_hostpath="$(find_deployment "$(pwd)/deploy")"; then + if deploy_driver="$(find_deployment "$(pwd)/deploy")"; then : - elif [ "${CSI_PROW_HOSTPATH_REPO}" = "none" ]; then + elif [ "${CSI_PROW_DRIVER_REPO}" = "none" ]; then return 1 else - git_checkout "${CSI_PROW_HOSTPATH_REPO}" "${CSI_PROW_WORK}/hostpath" "${CSI_PROW_HOSTPATH_VERSION}" --depth=1 || die "checking out hostpath repo failed" - if deploy_hostpath="$(find_deployment "${CSI_PROW_WORK}/hostpath/deploy")"; then + git_checkout "${CSI_PROW_DRIVER_REPO}" "${CSI_PROW_WORK}/csi-driver" "${CSI_PROW_DRIVER_VERSION}" --depth=1 || die "checking out CSI driver repo failed" + if deploy_driver="$(find_deployment "${CSI_PROW_WORK}/csi-driver/deploy")"; then : else - die "deploy-hostpath.sh not found in ${CSI_PROW_HOSTPATH_REPO} ${CSI_PROW_HOSTPATH_VERSION}. To disable E2E testing, set CSI_PROW_HOSTPATH_REPO=none" + die "deploy.sh not found in ${CSI_PROW_DRIVER_REPO} ${CSI_PROW_DRIVER_VERSION}. To disable E2E testing, set CSI_PROW_DRIVER_REPO=none" fi fi - if [ "${CSI_PROW_HOSTPATH_CANARY}" != "stable" ]; then - images="$images IMAGE_TAG=${CSI_PROW_HOSTPATH_CANARY}" + if [ "${CSI_PROW_DRIVER_CANARY}" != "stable" ]; then + images="$images IMAGE_TAG=${CSI_PROW_DRIVER_CANARY}" fi # Ignore: Double quote to prevent globbing and word splitting. # It's intentional here for $images. # shellcheck disable=SC2086 - if ! run env $images "${deploy_hostpath}"; then + if ! run env "CSI_PROW_TEST_DRIVER=${CSI_PROW_WORK}/test-driver.yaml" $images "${deploy_driver}"; then # Collect information about failed deployment before failing. collect_cluster_info (start_loggers >/dev/null; wait) info "For container output see job artifacts." - die "deploying the hostpath driver with ${deploy_hostpath} failed" + die "deploying the CSI driver with ${deploy_driver} failed" fi } @@ -804,33 +815,6 @@ install_sanity () ( run_with_go "${CSI_PROW_GO_VERSION_SANITY}" go test -c -o "${CSI_PROW_WORK}/csi-sanity" "${CSI_PROW_SANITY_IMPORT_PATH}/cmd/csi-sanity" || die "building csi-sanity failed" ) -# The default implementation of this function generates a external -# driver test configuration for the hostpath driver. -# -# The content depends on both what the E2E suite expects and what the -# installed hostpath driver supports. Generating it here seems prone -# to breakage, but it is uncertain where a better place might be. -generate_test_driver () { - cat <"${CSI_PROW_WORK}/test-driver.yaml" || die "generating test-driver.yaml failed" - # Rename, merge and filter JUnit files. Necessary in case that we run the E2E suite again # and to avoid the large number of "skipped" tests that we get from using # the full Kubernetes E2E testsuite while only running a few tests. @@ -1063,7 +1045,7 @@ main () { cmds="$(grep '^\s*CMDS\s*=' Makefile | sed -e 's/\s*CMDS\s*=//')" # Get the image that was just built (if any) from the # top-level Makefile CMDS variable and set the - # deploy-hostpath.sh env variables for it. We also need to + # deploy.sh env variables for it. We also need to # side-load those images into the cluster. for i in $cmds; do e=$(echo "$i" | tr '[:lower:]' '[:upper:]' | tr - _) @@ -1101,7 +1083,7 @@ main () { fi # Installing the driver might be disabled. - if install_hostpath "$images"; then + if ${CSI_PROW_DRIVER_INSTALL} "$images"; then collect_cluster_info if sanity_enabled; then @@ -1158,7 +1140,7 @@ main () { fi # Installing the driver might be disabled. - if install_hostpath "$images"; then + if ${CSI_PROW_DRIVER_INSTALL} "$images"; then collect_cluster_info if tests_enabled "parallel-alpha"; then From 5f74333a466f0767cec812945c970c943a59cf7f Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Mon, 10 Feb 2020 11:06:31 +0100 Subject: [PATCH 20/56] prow.sh: also configure feature gates for kubelet That this hasn't been done before is an oversight. Apparently it hasn't been a problem because there never have been feature gates that mattered? --- prow.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/prow.sh b/prow.sh index 118c5bd19..6778e8b8e 100755 --- a/prow.sh +++ b/prow.sh @@ -568,6 +568,13 @@ kubeadmConfigPatches: nodeRegistration: kubeletExtraArgs: "feature-gates": "$gates" +- | + apiVersion: kubelet.config.k8s.io/v1beta1 + kind: KubeletConfiguration + metadata: + name: config + featureGates: +$(list_gates "$gates") - | apiVersion: kubeproxy.config.k8s.io/v1alpha1 kind: KubeProxyConfiguration From 13dee9bc89a34abc4ca4c2bcf988412c9e70f6bd Mon Sep 17 00:00:00 2001 From: Grant Griffiths Date: Fri, 7 Feb 2020 14:05:38 -0800 Subject: [PATCH 21/56] Add snapshot controller delete test Signed-off-by: Grant Griffiths --- pkg/common-controller/framework_test.go | 17 ++-- pkg/common-controller/snapshot_create_test.go | 28 +++---- pkg/common-controller/snapshot_delete_test.go | 84 ++++++++++++------- .../snapshot_finalizer_test.go | 14 ++-- pkg/common-controller/snapshot_update_test.go | 56 ++++++------- pkg/common-controller/snapshotclass_test.go | 16 ++-- 6 files changed, 120 insertions(+), 95 deletions(-) diff --git a/pkg/common-controller/framework_test.go b/pkg/common-controller/framework_test.go index e815aa13a..1a40219e0 100644 --- a/pkg/common-controller/framework_test.go +++ b/pkg/common-controller/framework_test.go @@ -863,14 +863,15 @@ func newContentWithUnmatchDriverArray(contentName, boundToSnapshotUID, boundToSn func newSnapshot( snapshotName, snapshotUID, pvcName, targetContentName, snapshotClassName, boundContentName string, readyToUse *bool, creationTime *metav1.Time, restoreSize *resource.Quantity, - err *crdv1.VolumeSnapshotError, nilStatus bool, withFinalizer bool) *crdv1.VolumeSnapshot { + err *crdv1.VolumeSnapshotError, nilStatus bool, withFinalizer bool, deletionTimestamp *metav1.Time) *crdv1.VolumeSnapshot { snapshot := crdv1.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ - Name: snapshotName, - Namespace: testNamespace, - UID: types.UID(snapshotUID), - ResourceVersion: "1", - SelfLink: "/apis/snapshot.storage.k8s.io/v1beta1/namespaces/" + testNamespace + "/volumesnapshots/" + snapshotName, + Name: snapshotName, + Namespace: testNamespace, + UID: types.UID(snapshotUID), + ResourceVersion: "1", + SelfLink: "/apis/snapshot.storage.k8s.io/v1beta1/namespaces/" + testNamespace + "/volumesnapshots/" + snapshotName, + DeletionTimestamp: deletionTimestamp, }, Spec: crdv1.VolumeSnapshotSpec{ VolumeSnapshotClassName: nil, @@ -912,9 +913,9 @@ func newSnapshot( func newSnapshotArray( snapshotName, snapshotUID, pvcName, targetContentName, snapshotClassName, boundContentName string, readyToUse *bool, creationTime *metav1.Time, restoreSize *resource.Quantity, - err *crdv1.VolumeSnapshotError, nilStatus bool, withFinalizer bool) []*crdv1.VolumeSnapshot { + err *crdv1.VolumeSnapshotError, nilStatus bool, withFinalizer bool, deletionTimestamp *metav1.Time) []*crdv1.VolumeSnapshot { return []*crdv1.VolumeSnapshot{ - newSnapshot(snapshotName, snapshotUID, pvcName, targetContentName, snapshotClassName, boundContentName, readyToUse, creationTime, restoreSize, err, nilStatus, withFinalizer), + newSnapshot(snapshotName, snapshotUID, pvcName, targetContentName, snapshotClassName, boundContentName, readyToUse, creationTime, restoreSize, err, nilStatus, withFinalizer, deletionTimestamp), } } diff --git a/pkg/common-controller/snapshot_create_test.go b/pkg/common-controller/snapshot_create_test.go index 2a952200c..ab16f6447 100644 --- a/pkg/common-controller/snapshot_create_test.go +++ b/pkg/common-controller/snapshot_create_test.go @@ -71,8 +71,8 @@ func TestCreateSnapshotSync(t *testing.T) { name: "6-1 - successful create snapshot with snapshot class gold", initialContents: nocontents, expectedContents: newContentArrayNoStatus("snapcontent-snapuid6-1", "snapuid6-1", "snap6-1", "sid6-1", classGold, "", "pv-handle6-1", deletionPolicy, nil, nil, false, false), - initialSnapshots: newSnapshotArray("snap6-1", "snapuid6-1", "claim6-1", "", classGold, "", &False, nil, nil, nil, false, true), - expectedSnapshots: newSnapshotArray("snap6-1", "snapuid6-1", "claim6-1", "", classGold, "snapcontent-snapuid6-1", &False, nil, nil, nil, false, true), + initialSnapshots: newSnapshotArray("snap6-1", "snapuid6-1", "claim6-1", "", classGold, "", &False, nil, nil, nil, false, true, nil), + expectedSnapshots: newSnapshotArray("snap6-1", "snapuid6-1", "claim6-1", "", classGold, "snapcontent-snapuid6-1", &False, nil, nil, nil, false, true, nil), initialClaims: newClaimArray("claim6-1", "pvc-uid6-1", "1Gi", "volume6-1", v1.ClaimBound, &classEmpty), initialVolumes: newVolumeArray("volume6-1", "pv-uid6-1", "pv-handle6-1", "1Gi", "pvc-uid6-1", "claim6-1", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), errors: noerrors, @@ -82,8 +82,8 @@ func TestCreateSnapshotSync(t *testing.T) { name: "6-2 - successful create snapshot with snapshot class silver", initialContents: nocontents, expectedContents: newContentArrayNoStatus("snapcontent-snapuid6-2", "snapuid6-2", "snap6-2", "sid6-2", classSilver, "", "pv-handle6-2", deletionPolicy, nil, nil, false, false), - initialSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "", &False, nil, nil, nil, false, true), - expectedSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "snapcontent-snapuid6-2", &False, nil, nil, nil, false, true), + initialSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "", &False, nil, nil, nil, false, true, nil), + expectedSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "snapcontent-snapuid6-2", &False, nil, nil, nil, false, true, nil), initialClaims: newClaimArray("claim6-2", "pvc-uid6-2", "1Gi", "volume6-2", v1.ClaimBound, &classEmpty), initialVolumes: newVolumeArray("volume6-2", "pv-uid6-2", "pv-handle6-2", "1Gi", "pvc-uid6-2", "claim6-2", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), errors: noerrors, @@ -93,8 +93,8 @@ func TestCreateSnapshotSync(t *testing.T) { name: "7-1 - fail to create snapshot with non-existing snapshot class", initialContents: nocontents, expectedContents: nocontents, - initialSnapshots: newSnapshotArray("snap7-1", "snapuid7-1", "claim7-1", "", classNonExisting, "", &False, nil, nil, nil, false, true), - expectedSnapshots: newSnapshotArray("snap7-1", "snapuid7-1", "claim7-1", "", classNonExisting, "", &False, nil, nil, newVolumeError("Failed to create snapshot content with error failed to get input parameters to create snapshot snap7-1: \"failed to retrieve snapshot class non-existing from the informer: \\\"volumesnapshotclass.snapshot.storage.k8s.io \\\\\\\"non-existing\\\\\\\" not found\\\"\""), false, true), + initialSnapshots: newSnapshotArray("snap7-1", "snapuid7-1", "claim7-1", "", classNonExisting, "", &False, nil, nil, nil, false, true, nil), + expectedSnapshots: newSnapshotArray("snap7-1", "snapuid7-1", "claim7-1", "", classNonExisting, "", &False, nil, nil, newVolumeError("Failed to create snapshot content with error failed to get input parameters to create snapshot snap7-1: \"failed to retrieve snapshot class non-existing from the informer: \\\"volumesnapshotclass.snapshot.storage.k8s.io \\\\\\\"non-existing\\\\\\\" not found\\\"\""), false, true, nil), initialClaims: newClaimArray("claim7-1", "pvc-uid7-1", "1Gi", "volume7-1", v1.ClaimBound, &classEmpty), initialVolumes: newVolumeArray("volume7-1", "pv-uid7-1", "pv-handle7-1", "1Gi", "pvc-uid7-1", "claim7-1", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), expectedEvents: []string{"Warning SnapshotContentCreationFailed"}, @@ -107,8 +107,8 @@ func TestCreateSnapshotSync(t *testing.T) { name: "7-2 - fail to update snapshot reports warning event", initialContents: newContentArrayWithReadyToUse("snapcontent-snapuid7-2", "snapuid7-2", "snap7-2", "sid7-2", classGold, "sid7-2", "pv-handle7-2", deletionPolicy, nil, nil, &True, false), expectedContents: newContentArrayWithReadyToUse("snapcontent-snapuid7-2", "snapuid7-2", "snap7-2", "sid7-2", classGold, "sid7-2", "pv-handle7-2", deletionPolicy, nil, nil, &True, false), - initialSnapshots: newSnapshotArray("snap7-2", "snapuid7-2", "claim7-2", "", classGold, "snapcontent-snapuid7-2", &False, nil, nil, nil, false, true), - expectedSnapshots: newSnapshotArray("snap7-2", "snapuid7-2", "claim7-2", "", classGold, "snapcontent-snapuid7-2", &False, nil, nil, newVolumeError("Snapshot status update failed, snapshot controller failed to update default/snap7-2 on API server: mock update error"), false, true), + initialSnapshots: newSnapshotArray("snap7-2", "snapuid7-2", "claim7-2", "", classGold, "snapcontent-snapuid7-2", &False, nil, nil, nil, false, true, nil), + expectedSnapshots: newSnapshotArray("snap7-2", "snapuid7-2", "claim7-2", "", classGold, "snapcontent-snapuid7-2", &False, nil, nil, newVolumeError("Snapshot status update failed, snapshot controller failed to update default/snap7-2 on API server: mock update error"), false, true, nil), initialClaims: newClaimArray("claim7-2", "pvc-uid7-2", "1Gi", "volume7-2", v1.ClaimBound, &classGold), initialVolumes: newVolumeArray("volume7-2", "pv-uid7-2", "pv-handle7-2", "1Gi", "pvc-uid7-2", "claim7-2", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classGold), expectedEvents: []string{"Warning SnapshotStatusUpdateFailed"}, @@ -138,8 +138,8 @@ func TestCreateSnapshotSync(t *testing.T) { name: "7-4 - fail create snapshot with no-existing claim", initialContents: nocontents, expectedContents: nocontents, - initialSnapshots: newSnapshotArray("snap7-4", "snapuid7-4", "claim7-4", "", classGold, "", &False, nil, nil, nil, false, true), - expectedSnapshots: newSnapshotArray("snap7-4", "snapuid7-4", "claim7-4", "", classGold, "", &False, nil, nil, newVolumeError("Failed to create snapshot content with error snapshot controller failed to update snap7-4 on API server: cannot get claim from snapshot"), false, true), + initialSnapshots: newSnapshotArray("snap7-4", "snapuid7-4", "claim7-4", "", classGold, "", &False, nil, nil, nil, false, true, nil), + expectedSnapshots: newSnapshotArray("snap7-4", "snapuid7-4", "claim7-4", "", classGold, "", &False, nil, nil, newVolumeError("Failed to create snapshot content with error snapshot controller failed to update snap7-4 on API server: cannot get claim from snapshot"), false, true, nil), initialVolumes: newVolumeArray("volume7-4", "pv-uid7-4", "pv-handle7-4", "1Gi", "pvc-uid7-4", "claim7-4", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), expectedEvents: []string{"Warning SnapshotContentCreationFailed"}, errors: noerrors, @@ -150,8 +150,8 @@ func TestCreateSnapshotSync(t *testing.T) { name: "7-5 - fail create snapshot with no-existing volume", initialContents: nocontents, expectedContents: nocontents, - initialSnapshots: newSnapshotArray("snap7-5", "snapuid7-5", "claim7-5", "", classGold, "", &False, nil, nil, nil, false, true), - expectedSnapshots: newSnapshotArray("snap7-5", "snapuid7-5", "claim7-5", "", classGold, "", &False, nil, nil, newVolumeError("Failed to create snapshot content with error failed to get input parameters to create snapshot snap7-5: \"failed to retrieve PV volume7-5 from the API server: \\\"cannot find volume volume7-5\\\"\""), false, true), + initialSnapshots: newSnapshotArray("snap7-5", "snapuid7-5", "claim7-5", "", classGold, "", &False, nil, nil, nil, false, true, nil), + expectedSnapshots: newSnapshotArray("snap7-5", "snapuid7-5", "claim7-5", "", classGold, "", &False, nil, nil, newVolumeError("Failed to create snapshot content with error failed to get input parameters to create snapshot snap7-5: \"failed to retrieve PV volume7-5 from the API server: \\\"cannot find volume volume7-5\\\"\""), false, true, nil), initialClaims: newClaimArray("claim7-5", "pvc-uid7-5", "1Gi", "volume7-5", v1.ClaimBound, &classEmpty), expectedEvents: []string{"Warning SnapshotContentCreationFailed"}, errors: noerrors, @@ -163,8 +163,8 @@ func TestCreateSnapshotSync(t *testing.T) { name: "7-6 - fail create snapshot with claim that is not yet bound", initialContents: nocontents, expectedContents: nocontents, - initialSnapshots: newSnapshotArray("snap7-6", "snapuid7-6", "claim7-6", "", classGold, "", &False, nil, nil, nil, false, true), - expectedSnapshots: newSnapshotArray("snap7-6", "snapuid7-6", "claim7-6", "", classGold, "", &False, nil, nil, newVolumeError("Failed to create snapshot content with error failed to get input parameters to create snapshot snap7-6: \"the PVC claim7-6 is not yet bound to a PV, will not attempt to take a snapshot\""), false, true), + initialSnapshots: newSnapshotArray("snap7-6", "snapuid7-6", "claim7-6", "", classGold, "", &False, nil, nil, nil, false, true, nil), + expectedSnapshots: newSnapshotArray("snap7-6", "snapuid7-6", "claim7-6", "", classGold, "", &False, nil, nil, newVolumeError("Failed to create snapshot content with error failed to get input parameters to create snapshot snap7-6: \"the PVC claim7-6 is not yet bound to a PV, will not attempt to take a snapshot\""), false, true, nil), initialClaims: newClaimArray("claim7-6", "pvc-uid7-6", "1Gi", "", v1.ClaimPending, &classEmpty), expectedEvents: []string{"Warning SnapshotContentCreationFailed"}, errors: noerrors, diff --git a/pkg/common-controller/snapshot_delete_test.go b/pkg/common-controller/snapshot_delete_test.go index be80e01cd..0fd3c8f87 100644 --- a/pkg/common-controller/snapshot_delete_test.go +++ b/pkg/common-controller/snapshot_delete_test.go @@ -24,6 +24,7 @@ import ( "github.com/kubernetes-csi/external-snapshotter/v2/pkg/utils" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" ) var class1Parameters = map[string]string{ @@ -49,6 +50,11 @@ var class5Parameters = map[string]string{ //utils.SnapshotterSecretNamespaceKey: "default", } +var timeNowMetav1 = metav1.Now() + +var content31 = "content3-1" +var claim31 = "claim3-1" + var snapshotClasses = []*crdv1.VolumeSnapshotClass{ { TypeMeta: metav1.TypeMeta{ @@ -169,30 +175,7 @@ func TestDeleteSync(t *testing.T) { initialSecrets: []*v1.Secret{secret()}, //expectedDeleteCalls: []deleteCall{{"sid1-3", map[string]string{"foo": "bar"}, nil}}, test: testSyncContent, - }, /* - { - name: "1-4 - fail delete with snapshot class that has invalid secret parameter", - initialContents: newContentArray("content1-4", "sid1-4", "snapuid1-4", "snap1-4", &deletePolicy, nil, nil, true), - expectedContents: newContentArray("content1-4", "sid1-4", "snapuid1-4", "snap1-4", &deletePolicy, nil, nil, true), - initialSnapshots: nosnapshots, - expectedSnapshots: nosnapshots, - expectedEvents: noevents, - errors: noerrors, - test: testSyncContent, - }, - { - name: "1-5 - csi driver delete snapshot returns error", - initialContents: newContentArray("content1-5", "sid1-5", "snap1-5", "sid1-5", validSecretClass, "", "", deletionPolicy, nil, nil, true), - expectedContents: newContentArray("content1-5", "sid1-5", "snap1-5", "sid1-5", validSecretClass, "", "", deletionPolicy, nil, nil, true), - initialSnapshots: nosnapshots, - expectedSnapshots: nosnapshots, - initialSecrets: []*v1.Secret{secret()}, - expectedDeleteCalls: []deleteCall{{"sid1-5", map[string]string{"foo": "bar"}, errors.New("mock csi driver delete error")}}, - expectedEvents: []string{"Warning SnapshotDeleteError"}, - errors: noerrors, - test: testSyncContent, - },*/ - /*{ + }, /*{ name: "1-6 - api server delete content returns error", initialContents: newContentArray("content1-6", "sid1-6", "snap1-6", "sid1-6", validSecretClass, "", "", deletionPolicy, nil, nil, true), expectedContents: newContentArray("content1-6", "sid1-6", "snap1-6", "sid1-6", validSecretClass, "", "", deletionPolicy, nil, nil, true), @@ -214,8 +197,8 @@ func TestDeleteSync(t *testing.T) { name: "1-7 - prebound content is deleted while the snapshot exists", initialContents: newContentArray("content1-7", "sid1-7", "snap1-7", "sid1-7", emptySecretClass, "", "", deletionPolicy, nil, nil, true), expectedContents: newContentArray("content1-7", "sid1-7", "snap1-7", "sid1-7", emptySecretClass, "", "", deletionPolicy, nil, nil, true), - initialSnapshots: newSnapshotArray("snap1-7", "snapuid1-7-x", "claim1-7", "", validSecretClass, "", &False, nil, nil, nil, false, true), - expectedSnapshots: newSnapshotArray("snap1-7", "snapuid1-7-x", "claim1-7", "", validSecretClass, "", &False, nil, nil, nil, false, true), + initialSnapshots: newSnapshotArray("snap1-7", "snapuid1-7-x", "claim1-7", "", validSecretClass, "", &False, nil, nil, nil, false, true, nil), + expectedSnapshots: newSnapshotArray("snap1-7", "snapuid1-7-x", "claim1-7", "", validSecretClass, "", &False, nil, nil, nil, false, true, nil), initialSecrets: []*v1.Secret{secret()}, //expectedDeleteCalls: []deleteCall{{"sid1-7", map[string]string{"foo": "bar"}, nil}}, expectedEvents: noevents, @@ -255,8 +238,8 @@ func TestDeleteSync(t *testing.T) { name: "1-10 - will not delete content with retain policy set which is bound to a snapshot incorrectly", initialContents: newContentArray("content1-10", "snapuid1-10-x", "snap1-10", "sid1-10", validSecretClass, "", "", retainPolicy, nil, nil, true), expectedContents: newContentArray("content1-10", "snapuid1-10-x", "snap1-10", "sid1-10", validSecretClass, "", "", retainPolicy, nil, nil, true), - initialSnapshots: newSnapshotArray("snap1-10", "snapuid1-10", "claim1-10", "", validSecretClass, "content1-10", &False, nil, nil, nil, false, true), - expectedSnapshots: newSnapshotArray("snap1-10", "snapuid1-10", "claim1-10", "", validSecretClass, "content1-10", &False, nil, nil, nil, false, true), + initialSnapshots: newSnapshotArray("snap1-10", "snapuid1-10", "claim1-10", "", validSecretClass, "content1-10", &False, nil, nil, nil, false, true, nil), + expectedSnapshots: newSnapshotArray("snap1-10", "snapuid1-10", "claim1-10", "", validSecretClass, "content1-10", &False, nil, nil, nil, false, true, nil), expectedEvents: noevents, initialSecrets: []*v1.Secret{secret()}, errors: noerrors, @@ -266,8 +249,8 @@ func TestDeleteSync(t *testing.T) { name: "1-11 - content will not be deleted if it is bound to a snapshot correctly, snapsht uid is not specified", initialContents: newContentArray("content1-11", "", "snap1-11", "sid1-11", validSecretClass, "", "", deletePolicy, nil, nil, true), expectedContents: newContentArray("content1-11", "", "snap1-11", "sid1-11", validSecretClass, "", "", deletePolicy, nil, nil, true), - initialSnapshots: newSnapshotArray("snap1-11", "snapuid1-11", "claim1-11", "", validSecretClass, "content1-11", &False, nil, nil, nil, false, true), - expectedSnapshots: newSnapshotArray("snap1-11", "snapuid1-11", "claim1-11", "", validSecretClass, "content1-11", &False, nil, nil, nil, false, true), + initialSnapshots: newSnapshotArray("snap1-11", "snapuid1-11", "claim1-11", "", validSecretClass, "content1-11", &False, nil, nil, nil, false, true, nil), + expectedSnapshots: newSnapshotArray("snap1-11", "snapuid1-11", "claim1-11", "", validSecretClass, "content1-11", &False, nil, nil, nil, false, true, nil), expectedEvents: noevents, initialSecrets: []*v1.Secret{secret()}, errors: noerrors, @@ -316,6 +299,47 @@ func TestDeleteSync(t *testing.T) { //expectedDeleteCalls: []deleteCall{{"sid1-16", nil, nil}}, test: testSyncContent, }, + { + name: "3-1 - content will be deleted if snapshot deletion timestamp is set", + initialContents: newContentArray("content3-1", "", "snap3-1", "sid3-1", validSecretClass, "", "", deletePolicy, nil, nil, true), + expectedContents: nocontents, + initialSnapshots: newSnapshotArray("snap3-1", "snapuid3-1", "claim3-1", "", validSecretClass, "content3-1", &False, nil, nil, nil, false, true, &timeNowMetav1), + expectedSnapshots: []*crdv1.VolumeSnapshot{ + &crdv1.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "snap3-1", + Namespace: testNamespace, + UID: types.UID("snapuid3-1"), + ResourceVersion: "1", + Finalizers: []string{ + "snapshot.storage.kubernetes.io/volumesnapshotcontent-bound-protection", + "snapshot.storage.kubernetes.io/volumesnapshot-bound-protection", + }, + SelfLink: "/apis/snapshot.storage.k8s.io/v1beta1/namespaces/" + testNamespace + "/volumesnapshots/" + "snap3-1", + DeletionTimestamp: &timeNowMetav1, + }, + Spec: crdv1.VolumeSnapshotSpec{ + VolumeSnapshotClassName: &validSecretClass, + Source: crdv1.VolumeSnapshotSource{ + PersistentVolumeClaimName: &claim31, + }, + }, + + Status: &crdv1.VolumeSnapshotStatus{ + CreationTime: nil, + ReadyToUse: &False, + Error: nil, + RestoreSize: nil, + BoundVolumeSnapshotContentName: &content31, + }, + }, + }, + initialClaims: newClaimArray("claim3-1", "pvc-uid3-1", "1Gi", "volume3-1", v1.ClaimBound, &classEmpty), + expectedEvents: noevents, + initialSecrets: []*v1.Secret{secret()}, + errors: noerrors, + test: testSyncSnapshot, + }, } runSyncTests(t, tests, snapshotClasses) } diff --git a/pkg/common-controller/snapshot_finalizer_test.go b/pkg/common-controller/snapshot_finalizer_test.go index 6265c26cb..ecaf02461 100644 --- a/pkg/common-controller/snapshot_finalizer_test.go +++ b/pkg/common-controller/snapshot_finalizer_test.go @@ -29,49 +29,49 @@ func TestSnapshotFinalizer(t *testing.T) { tests := []controllerTest{ { name: "1-1 - successful add PVC finalizer", - initialSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "", &False, nil, nil, nil, false, true), + initialSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "", &False, nil, nil, nil, false, true, nil), initialClaims: newClaimArray("claim6-2", "pvc-uid6-2", "1Gi", "volume6-2", v1.ClaimBound, &classEmpty), test: testAddPVCFinalizer, expectSuccess: true, }, { name: "1-2 - won't add PVC finalizer; already added", - initialSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "", &False, nil, nil, nil, false, true), + initialSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "", &False, nil, nil, nil, false, true, nil), initialClaims: newClaimArrayFinalizer("claim6-2", "pvc-uid6-2", "1Gi", "volume6-2", v1.ClaimBound, &classEmpty), test: testAddPVCFinalizer, expectSuccess: false, }, { name: "1-3 - successful remove PVC finalizer", - initialSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "", &False, nil, nil, nil, false, true), + initialSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "", &False, nil, nil, nil, false, true, nil), initialClaims: newClaimArrayFinalizer("claim6-2", "pvc-uid6-2", "1Gi", "volume6-2", v1.ClaimBound, &classEmpty), test: testRemovePVCFinalizer, expectSuccess: true, }, { name: "1-4 - won't remove PVC finalizer; already removed", - initialSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "", &False, nil, nil, nil, false, true), + initialSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "", &False, nil, nil, nil, false, true, nil), initialClaims: newClaimArray("claim6-2", "pvc-uid6-2", "1Gi", "volume6-2", v1.ClaimBound, &classEmpty), test: testRemovePVCFinalizer, expectSuccess: false, }, { name: "1-5 - won't remove PVC finalizer; PVC in-use", - initialSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "", &False, nil, nil, nil, false, true), + initialSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "", &False, nil, nil, nil, false, true, nil), initialClaims: newClaimArray("claim6-2", "pvc-uid6-2", "1Gi", "volume6-2", v1.ClaimBound, &classEmpty), test: testRemovePVCFinalizer, expectSuccess: false, }, { name: "2-1 - successful add Snapshot finalizer", - initialSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "", &False, nil, nil, nil, false, false), + initialSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "", &False, nil, nil, nil, false, false, nil), initialClaims: newClaimArray("claim6-2", "pvc-uid6-2", "1Gi", "volume6-2", v1.ClaimBound, &classEmpty), test: testAddSnapshotFinalizer, expectSuccess: true, }, { name: "2-1 - successful remove Snapshot finalizer", - initialSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "", &False, nil, nil, nil, false, true), + initialSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "", &False, nil, nil, nil, false, true, nil), initialClaims: newClaimArray("claim6-2", "pvc-uid6-2", "1Gi", "volume6-2", v1.ClaimBound, &classEmpty), test: testRemoveSnapshotFinalizer, expectSuccess: true, diff --git a/pkg/common-controller/snapshot_update_test.go b/pkg/common-controller/snapshot_update_test.go index 78d0dceea..cbb1b5c16 100644 --- a/pkg/common-controller/snapshot_update_test.go +++ b/pkg/common-controller/snapshot_update_test.go @@ -48,8 +48,8 @@ func TestSync(t *testing.T) { name: "2-1 - snapshot is bound to a non-existing content", initialContents: nocontents, expectedContents: nocontents, - initialSnapshots: newSnapshotArray("snap2-1", "snapuid2-1", "claim2-1", "", validSecretClass, "content2-1", &True, nil, nil, nil, false, true), - expectedSnapshots: newSnapshotArray("snap2-1", "snapuid2-1", "claim2-1", "", validSecretClass, "content2-1", &False, nil, nil, newVolumeError("VolumeSnapshotContent is missing"), false, true), + initialSnapshots: newSnapshotArray("snap2-1", "snapuid2-1", "claim2-1", "", validSecretClass, "content2-1", &True, nil, nil, nil, false, true, nil), + expectedSnapshots: newSnapshotArray("snap2-1", "snapuid2-1", "claim2-1", "", validSecretClass, "content2-1", &False, nil, nil, newVolumeError("VolumeSnapshotContent is missing"), false, true, nil), expectedEvents: []string{"Warning SnapshotContentMissing"}, errors: noerrors, test: testSyncSnapshot, @@ -68,8 +68,8 @@ func TestSync(t *testing.T) { name: "2-3 - success bind snapshot and content but not ready, no status changed", initialContents: newContentArray("content2-3", "snapuid2-3", "snap2-3", "sid2-3", validSecretClass, "", "", deletionPolicy, nil, nil, false), expectedContents: newContentArrayWithReadyToUse("content2-3", "snapuid2-3", "snap2-3", "sid2-3", validSecretClass, "", "", deletionPolicy, &timeNowStamp, nil, &True, false), - initialSnapshots: newSnapshotArray("snap2-3", "snapuid2-3", "claim2-3", "", validSecretClass, "content2-3", &False, metaTimeNow, nil, nil, false, true), - expectedSnapshots: newSnapshotArray("snap2-3", "snapuid2-3", "claim2-3", "", validSecretClass, "content2-3", &True, metaTimeNow, nil, nil, false, true), + initialSnapshots: newSnapshotArray("snap2-3", "snapuid2-3", "claim2-3", "", validSecretClass, "content2-3", &False, metaTimeNow, nil, nil, false, true, nil), + expectedSnapshots: newSnapshotArray("snap2-3", "snapuid2-3", "claim2-3", "", validSecretClass, "content2-3", &True, metaTimeNow, nil, nil, false, true, nil), initialClaims: newClaimArray("claim2-3", "pvc-uid2-3", "1Gi", "volume2-3", v1.ClaimBound, &classEmpty), initialVolumes: newVolumeArray("volume2-3", "pv-uid2-3", "pv-handle2-3", "1Gi", "pvc-uid2-3", "claim2-3", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), initialSecrets: []*v1.Secret{secret()}, @@ -95,8 +95,8 @@ func TestSync(t *testing.T) { name: "2-4 - noop", initialContents: newContentArray("content2-4", "snapuid2-4", "snap2-4", "sid2-4", validSecretClass, "", "", deletionPolicy, nil, nil, false), expectedContents: newContentArray("content2-4", "snapuid2-4", "snap2-4", "sid2-4", validSecretClass, "", "", deletionPolicy, nil, nil, false), - initialSnapshots: newSnapshotArray("snap2-4", "snapuid2-4", "claim2-4", "", validSecretClass, "content2-4", &True, metaTimeNow, nil, nil, false, true), - expectedSnapshots: newSnapshotArray("snap2-4", "snapuid2-4", "claim2-4", "", validSecretClass, "content2-4", &True, metaTimeNow, nil, nil, false, true), + initialSnapshots: newSnapshotArray("snap2-4", "snapuid2-4", "claim2-4", "", validSecretClass, "content2-4", &True, metaTimeNow, nil, nil, false, true, nil), + expectedSnapshots: newSnapshotArray("snap2-4", "snapuid2-4", "claim2-4", "", validSecretClass, "content2-4", &True, metaTimeNow, nil, nil, false, true, nil), errors: noerrors, test: testSyncSnapshot, }, @@ -104,8 +104,8 @@ func TestSync(t *testing.T) { name: "2-5 - snapshot and content bound, status ready false -> true", initialContents: newContentArrayWithReadyToUse("content2-5", "snapuid2-5", "snap2-5", "sid2-5", validSecretClass, "", "", deletionPolicy, &timeNowStamp, nil, &False, false), expectedContents: newContentArrayWithReadyToUse("content2-5", "snapuid2-5", "snap2-5", "sid2-5", validSecretClass, "", "", deletionPolicy, &timeNowStamp, nil, &False, false), - initialSnapshots: newSnapshotArray("snap2-5", "snapuid2-5", "claim2-5", "", validSecretClass, "content2-5", &False, metaTimeNow, nil, nil, false, true), - expectedSnapshots: newSnapshotArray("snap2-5", "snapuid2-5", "claim2-5", "", validSecretClass, "content2-5", &False, metaTimeNow, nil, nil, false, true), + initialSnapshots: newSnapshotArray("snap2-5", "snapuid2-5", "claim2-5", "", validSecretClass, "content2-5", &False, metaTimeNow, nil, nil, false, true, nil), + expectedSnapshots: newSnapshotArray("snap2-5", "snapuid2-5", "claim2-5", "", validSecretClass, "content2-5", &False, metaTimeNow, nil, nil, false, true, nil), initialClaims: newClaimArray("claim2-5", "pvc-uid2-5", "1Gi", "volume2-5", v1.ClaimBound, &classEmpty), initialVolumes: newVolumeArray("volume2-5", "pv-uid2-5", "pv-handle2-5", "1Gi", "pvc-uid2-5", "claim2-5", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), initialSecrets: []*v1.Secret{secret()}, @@ -130,8 +130,8 @@ func TestSync(t *testing.T) { name: "2-6 - snapshot bound to prebound content correctly, status ready false -> true, ref.UID '' -> 'snapuid2-6'", initialContents: newContentArrayWithReadyToUse("content2-6", "snapuid2-6", "snap2-6", "sid2-6", validSecretClass, "", "", deletionPolicy, &timeNowStamp, nil, &False, false), expectedContents: newContentArrayWithReadyToUse("content2-6", "snapuid2-6", "snap2-6", "sid2-6", validSecretClass, "", "", deletionPolicy, &timeNowStamp, nil, &False, false), - initialSnapshots: newSnapshotArray("snap2-6", "snapuid2-6", "", "content2-6", validSecretClass, "content2-6", &False, metaTimeNow, nil, nil, false, true), - expectedSnapshots: newSnapshotArray("snap2-6", "snapuid2-6", "", "content2-6", validSecretClass, "content2-6", &False, metaTimeNow, nil, nil, false, true), + initialSnapshots: newSnapshotArray("snap2-6", "snapuid2-6", "", "content2-6", validSecretClass, "content2-6", &False, metaTimeNow, nil, nil, false, true, nil), + expectedSnapshots: newSnapshotArray("snap2-6", "snapuid2-6", "", "content2-6", validSecretClass, "content2-6", &False, metaTimeNow, nil, nil, false, true, nil), errors: noerrors, test: testSyncSnapshot, }, @@ -193,8 +193,8 @@ func TestSync(t *testing.T) { name: "2-9 - fail on status update as there is not pvc provided", initialContents: newContentArray("content2-9", "snapuid2-9", "snap2-9", "sid2-9", validSecretClass, "", "", deletionPolicy, nil, nil, false), expectedContents: newContentArray("content2-9", "snapuid2-9", "snap2-9", "sid2-9", validSecretClass, "", "", deletionPolicy, nil, nil, false), - initialSnapshots: newSnapshotArray("snap2-9", "snapuid2-9", "claim2-9", "", validSecretClass, "", &False, nil, nil, nil, false, true), - expectedSnapshots: newSnapshotArray("snap2-9", "snapuid2-9", "claim2-9", "", validSecretClass, "content2-9", &True, nil, nil, nil, false, true), + initialSnapshots: newSnapshotArray("snap2-9", "snapuid2-9", "claim2-9", "", validSecretClass, "", &False, nil, nil, nil, false, true, nil), + expectedSnapshots: newSnapshotArray("snap2-9", "snapuid2-9", "claim2-9", "", validSecretClass, "content2-9", &True, nil, nil, nil, false, true, nil), //expectedSnapshots: newSnapshotArray("snap2-9", "snapuid2-9", "claim2-9", "", validSecretClass, "content2-9", &False, nil, nil, newVolumeError("Failed to check and update snapshot: failed to get input parameters to create snapshot snap2-9: \"failed to retrieve PVC claim2-9 from the lister: \\\"persistentvolumeclaim \\\\\\\"claim2-9\\\\\\\" not found\\\"\"")), errors: noerrors, test: testSyncSnapshot, @@ -203,8 +203,8 @@ func TestSync(t *testing.T) { name: "2-10 - do not bind when snapshot and content not match", initialContents: newContentArray("content2-10", "snapuid2-10-x", "snap2-10", "sid2-10", validSecretClass, "", "", deletionPolicy, nil, nil, false), expectedContents: newContentArray("content2-10", "snapuid2-10-x", "snap2-10", "sid2-10", validSecretClass, "", "", deletionPolicy, nil, nil, false), - initialSnapshots: newSnapshotArray("snap2-10", "snapuid2-10", "claim2-10", "", validSecretClass, "", &False, nil, nil, newVolumeError("mock driver error"), false, true), - expectedSnapshots: newSnapshotArray("snap2-10", "snapuid2-10", "claim2-10", "", validSecretClass, "", &False, nil, nil, newVolumeError("mock driver error"), false, true), + initialSnapshots: newSnapshotArray("snap2-10", "snapuid2-10", "claim2-10", "", validSecretClass, "", &False, nil, nil, newVolumeError("mock driver error"), false, true, nil), + expectedSnapshots: newSnapshotArray("snap2-10", "snapuid2-10", "claim2-10", "", validSecretClass, "", &False, nil, nil, newVolumeError("mock driver error"), false, true, nil), errors: noerrors, test: testSyncSnapshot, }, @@ -212,8 +212,8 @@ func TestSync(t *testing.T) { name: "3-1 - ready snapshot lost reference to VolumeSnapshotContent", initialContents: nocontents, expectedContents: nocontents, - initialSnapshots: newSnapshotArray("snap3-1", "snapuid3-1", "claim3-1", "", validSecretClass, "content3-1", &True, metaTimeNow, nil, nil, false, true), - expectedSnapshots: newSnapshotArray("snap3-1", "snapuid3-1", "claim3-1", "", validSecretClass, "content3-1", &False, metaTimeNow, nil, newVolumeError("VolumeSnapshotContent is missing"), false, true), + initialSnapshots: newSnapshotArray("snap3-1", "snapuid3-1", "claim3-1", "", validSecretClass, "content3-1", &True, metaTimeNow, nil, nil, false, true, nil), + expectedSnapshots: newSnapshotArray("snap3-1", "snapuid3-1", "claim3-1", "", validSecretClass, "content3-1", &False, metaTimeNow, nil, newVolumeError("VolumeSnapshotContent is missing"), false, true, nil), errors: noerrors, expectedEvents: []string{"Warning SnapshotContentMissing"}, test: testSyncSnapshot, @@ -222,8 +222,8 @@ func TestSync(t *testing.T) { name: "3-2 - ready snapshot bound to none-exist content", initialContents: nocontents, expectedContents: nocontents, - initialSnapshots: newSnapshotArray("snap3-2", "snapuid3-2", "claim3-2", "", validSecretClass, "content3-2", &True, metaTimeNow, nil, nil, false, true), - expectedSnapshots: newSnapshotArray("snap3-2", "snapuid3-2", "claim3-2", "", validSecretClass, "content3-2", &False, metaTimeNow, nil, newVolumeError("VolumeSnapshotContent is missing"), false, true), + initialSnapshots: newSnapshotArray("snap3-2", "snapuid3-2", "claim3-2", "", validSecretClass, "content3-2", &True, metaTimeNow, nil, nil, false, true, nil), + expectedSnapshots: newSnapshotArray("snap3-2", "snapuid3-2", "claim3-2", "", validSecretClass, "content3-2", &False, metaTimeNow, nil, newVolumeError("VolumeSnapshotContent is missing"), false, true, nil), errors: noerrors, expectedEvents: []string{"Warning SnapshotContentMissing"}, test: testSyncSnapshot, @@ -232,8 +232,8 @@ func TestSync(t *testing.T) { name: "3-3 - ready snapshot(everything is well, do nothing)", initialContents: newContentArray("content3-3", "snapuid3-3", "snap3-3", "sid3-3", validSecretClass, "", "", deletionPolicy, nil, nil, false), expectedContents: newContentArray("content3-3", "snapuid3-3", "snap3-3", "sid3-3", validSecretClass, "", "", deletionPolicy, nil, nil, false), - initialSnapshots: newSnapshotArray("snap3-3", "snapuid3-3", "claim3-3", "", validSecretClass, "content3-3", &True, metaTimeNow, nil, nil, false, true), - expectedSnapshots: newSnapshotArray("snap3-3", "snapuid3-3", "claim3-3", "", validSecretClass, "content3-3", &True, metaTimeNow, nil, nil, false, true), + initialSnapshots: newSnapshotArray("snap3-3", "snapuid3-3", "claim3-3", "", validSecretClass, "content3-3", &True, metaTimeNow, nil, nil, false, true, nil), + expectedSnapshots: newSnapshotArray("snap3-3", "snapuid3-3", "claim3-3", "", validSecretClass, "content3-3", &True, metaTimeNow, nil, nil, false, true, nil), errors: noerrors, test: testSyncSnapshot, }, @@ -241,8 +241,8 @@ func TestSync(t *testing.T) { name: "3-4 - ready snapshot misbound to VolumeSnapshotContent", initialContents: newContentArray("content3-4", "snapuid3-4-x", "snap3-4", "sid3-4", validSecretClass, "", "", deletionPolicy, nil, nil, false), expectedContents: newContentArray("content3-4", "snapuid3-4-x", "snap3-4", "sid3-4", validSecretClass, "", "", deletionPolicy, nil, nil, false), - initialSnapshots: newSnapshotArray("snap3-4", "snapuid3-4", "claim3-4", "", validSecretClass, "content3-4", &True, metaTimeNow, nil, nil, false, true), - expectedSnapshots: newSnapshotArray("snap3-4", "snapuid3-4", "claim3-4", "", validSecretClass, "content3-4", &False, metaTimeNow, nil, newVolumeError("VolumeSnapshotContent is not bound to the VolumeSnapshot correctly"), false, true), + initialSnapshots: newSnapshotArray("snap3-4", "snapuid3-4", "claim3-4", "", validSecretClass, "content3-4", &True, metaTimeNow, nil, nil, false, true, nil), + expectedSnapshots: newSnapshotArray("snap3-4", "snapuid3-4", "claim3-4", "", validSecretClass, "content3-4", &False, metaTimeNow, nil, newVolumeError("VolumeSnapshotContent is not bound to the VolumeSnapshot correctly"), false, true, nil), errors: noerrors, test: testSyncSnapshot, }, @@ -250,8 +250,8 @@ func TestSync(t *testing.T) { name: "4-1 - content bound to snapshot, snapshot status missing and rebuilt", initialContents: newContentArrayWithReadyToUse("content2-5", "snapuid2-5", "snap2-5", "sid2-5", validSecretClass, "", "", deletionPolicy, nil, &size, &True, false), expectedContents: newContentArrayWithReadyToUse("content2-5", "snapuid2-5", "snap2-5", "sid2-5", validSecretClass, "", "", deletionPolicy, nil, &size, &True, false), - initialSnapshots: newSnapshotArray("snap2-5", "snapuid2-5", "claim2-5", "", validSecretClass, "", &False, nil, nil, nil, true, true), - expectedSnapshots: newSnapshotArray("snap2-5", "snapuid2-5", "claim2-5", "", validSecretClass, "content2-5", &True, nil, getSize(1), nil, false, true), + initialSnapshots: newSnapshotArray("snap2-5", "snapuid2-5", "claim2-5", "", validSecretClass, "", &False, nil, nil, nil, true, true, nil), + expectedSnapshots: newSnapshotArray("snap2-5", "snapuid2-5", "claim2-5", "", validSecretClass, "content2-5", &True, nil, getSize(1), nil, false, true, nil), initialClaims: newClaimArray("claim2-5", "pvc-uid2-5", "1Gi", "volume2-5", v1.ClaimBound, &classEmpty), initialVolumes: newVolumeArray("volume2-5", "pv-uid2-5", "pv-handle2-5", "1Gi", "pvc-uid2-5", "claim2-5", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), initialSecrets: []*v1.Secret{secret()}, @@ -262,8 +262,8 @@ func TestSync(t *testing.T) { name: "4-2 - snapshot and content bound, ReadyToUse in snapshot status missing and rebuilt", initialContents: newContentArrayWithReadyToUse("content2-5", "snapuid2-5", "snap2-5", "sid2-5", validSecretClass, "", "", deletionPolicy, nil, nil, &True, false), expectedContents: newContentArrayWithReadyToUse("content2-5", "snapuid2-5", "snap2-5", "sid2-5", validSecretClass, "", "", deletionPolicy, nil, nil, &True, false), - initialSnapshots: newSnapshotArray("snap2-5", "snapuid2-5", "claim2-5", "", validSecretClass, "content2-5", &False, nil, nil, nil, false, true), - expectedSnapshots: newSnapshotArray("snap2-5", "snapuid2-5", "claim2-5", "", validSecretClass, "content2-5", &True, nil, nil, nil, false, true), + initialSnapshots: newSnapshotArray("snap2-5", "snapuid2-5", "claim2-5", "", validSecretClass, "content2-5", &False, nil, nil, nil, false, true, nil), + expectedSnapshots: newSnapshotArray("snap2-5", "snapuid2-5", "claim2-5", "", validSecretClass, "content2-5", &True, nil, nil, nil, false, true, nil), initialClaims: newClaimArray("claim2-5", "pvc-uid2-5", "1Gi", "volume2-5", v1.ClaimBound, &classEmpty), initialVolumes: newVolumeArray("volume2-5", "pv-uid2-5", "pv-handle2-5", "1Gi", "pvc-uid2-5", "claim2-5", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), initialSecrets: []*v1.Secret{secret()}, @@ -274,8 +274,8 @@ func TestSync(t *testing.T) { name: "4-3 - content bound to snapshot, fields in snapshot status missing and rebuilt", initialContents: newContentArrayWithReadyToUse("content2-5", "snapuid2-5", "snap2-5", "sid2-5", validSecretClass, "", "", deletionPolicy, nil, &size, &True, false), expectedContents: newContentArrayWithReadyToUse("content2-5", "snapuid2-5", "snap2-5", "sid2-5", validSecretClass, "", "", deletionPolicy, nil, &size, &True, false), - initialSnapshots: newSnapshotArray("snap2-5", "snapuid2-5", "claim2-5", "", validSecretClass, "", &False, nil, nil, nil, false, true), - expectedSnapshots: newSnapshotArray("snap2-5", "snapuid2-5", "claim2-5", "", validSecretClass, "content2-5", &True, nil, getSize(1), nil, false, true), + initialSnapshots: newSnapshotArray("snap2-5", "snapuid2-5", "claim2-5", "", validSecretClass, "", &False, nil, nil, nil, false, true, nil), + expectedSnapshots: newSnapshotArray("snap2-5", "snapuid2-5", "claim2-5", "", validSecretClass, "content2-5", &True, nil, getSize(1), nil, false, true, nil), initialClaims: newClaimArray("claim2-5", "pvc-uid2-5", "1Gi", "volume2-5", v1.ClaimBound, &classEmpty), initialVolumes: newVolumeArray("volume2-5", "pv-uid2-5", "pv-handle2-5", "1Gi", "pvc-uid2-5", "claim2-5", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), initialSecrets: []*v1.Secret{secret()}, diff --git a/pkg/common-controller/snapshotclass_test.go b/pkg/common-controller/snapshotclass_test.go index 65a64da9d..72ea3b5ae 100644 --- a/pkg/common-controller/snapshotclass_test.go +++ b/pkg/common-controller/snapshotclass_test.go @@ -36,8 +36,8 @@ func TestUpdateSnapshotClass(t *testing.T) { // defualt snapshot class name should be set name: "1-1 - default snapshot class name should be set", initialContents: nocontents, - initialSnapshots: newSnapshotArray("snap1-1", "snapuid1-1", "claim1-1", "content1-1", "", "content1-1", &True, nil, nil, nil, false, true), - expectedSnapshots: newSnapshotArray("snap1-1", "snapuid1-1", "claim1-1", "content1-1", defaultClass, "content1-1", &True, nil, nil, nil, false, true), + initialSnapshots: newSnapshotArray("snap1-1", "snapuid1-1", "claim1-1", "content1-1", "", "content1-1", &True, nil, nil, nil, false, true, nil), + expectedSnapshots: newSnapshotArray("snap1-1", "snapuid1-1", "claim1-1", "content1-1", defaultClass, "content1-1", &True, nil, nil, nil, false, true, nil), initialClaims: newClaimArray("claim1-1", "pvc-uid1-1", "1Gi", "volume1-1", v1.ClaimBound, &sameDriver), initialVolumes: newVolumeArray("volume1-1", "pv-uid1-1", "pv-handle1-1", "1Gi", "pvc-uid1-1", "claim1-1", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), initialStorageClasses: []*storage.StorageClass{sameDriverStorageClass}, @@ -49,8 +49,8 @@ func TestUpdateSnapshotClass(t *testing.T) { // snapshot class name already set name: "1-2 - snapshot class name already set", initialContents: nocontents, - initialSnapshots: newSnapshotArray("snap1-1", "snapuid1-1", "claim1-1", "content1-1", defaultClass, "content1-1", &True, nil, nil, nil, false, true), - expectedSnapshots: newSnapshotArray("snap1-1", "snapuid1-1", "claim1-1", "content1-1", defaultClass, "content1-1", &True, nil, nil, nil, false, true), + initialSnapshots: newSnapshotArray("snap1-1", "snapuid1-1", "claim1-1", "content1-1", defaultClass, "content1-1", &True, nil, nil, nil, false, true, nil), + expectedSnapshots: newSnapshotArray("snap1-1", "snapuid1-1", "claim1-1", "content1-1", defaultClass, "content1-1", &True, nil, nil, nil, false, true, nil), initialClaims: newClaimArray("claim1-1", "pvc-uid1-1", "1Gi", "volume1-1", v1.ClaimBound, &sameDriver), initialVolumes: newVolumeArray("volume1-1", "pv-uid1-1", "pv-handle1-1", "1Gi", "pvc-uid1-1", "claim1-1", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), initialStorageClasses: []*storage.StorageClass{sameDriverStorageClass}, @@ -62,8 +62,8 @@ func TestUpdateSnapshotClass(t *testing.T) { // default snapshot class not found name: "1-3 - snapshot class name not found", initialContents: nocontents, - initialSnapshots: newSnapshotArray("snap1-1", "snapuid1-1", "claim1-1", "content1-1", "missing-class", "content1-1", &True, nil, nil, nil, false, true), - expectedSnapshots: newSnapshotArray("snap1-1", "snapuid1-1", "claim1-1", "content1-1", "missing-class", "content1-1", &False, nil, nil, newVolumeError("Failed to get snapshot class with error failed to retrieve snapshot class missing-class from the informer: \"volumesnapshotclass.snapshot.storage.k8s.io \\\"missing-class\\\" not found\""), false, true), + initialSnapshots: newSnapshotArray("snap1-1", "snapuid1-1", "claim1-1", "content1-1", "missing-class", "content1-1", &True, nil, nil, nil, false, true, nil), + expectedSnapshots: newSnapshotArray("snap1-1", "snapuid1-1", "claim1-1", "content1-1", "missing-class", "content1-1", &False, nil, nil, newVolumeError("Failed to get snapshot class with error failed to retrieve snapshot class missing-class from the informer: \"volumesnapshotclass.snapshot.storage.k8s.io \\\"missing-class\\\" not found\""), false, true, nil), initialClaims: newClaimArray("claim1-1", "pvc-uid1-1", "1Gi", "volume1-1", v1.ClaimBound, &sameDriver), initialVolumes: newVolumeArray("volume1-1", "pv-uid1-1", "pv-handle1-1", "1Gi", "pvc-uid1-1", "claim1-1", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), initialStorageClasses: []*storage.StorageClass{sameDriverStorageClass}, @@ -75,8 +75,8 @@ func TestUpdateSnapshotClass(t *testing.T) { // failed to get snapshot class from name name: "1-4 - snapshot update with default class name failed because storageclass not found", initialContents: nocontents, - initialSnapshots: newSnapshotArray("snap1-1", "snapuid1-1", "claim1-1", "content1-1", "", "content1-1", &True, nil, nil, nil, false, true), - expectedSnapshots: newSnapshotArray("snap1-1", "snapuid1-1", "claim1-1", "content1-1", "", "content1-1", &False, nil, nil, newVolumeError("Failed to set default snapshot class with error mock update error"), false, true), + initialSnapshots: newSnapshotArray("snap1-1", "snapuid1-1", "claim1-1", "content1-1", "", "content1-1", &True, nil, nil, nil, false, true, nil), + expectedSnapshots: newSnapshotArray("snap1-1", "snapuid1-1", "claim1-1", "content1-1", "", "content1-1", &False, nil, nil, newVolumeError("Failed to set default snapshot class with error mock update error"), false, true, nil), initialClaims: newClaimArray("claim1-1", "pvc-uid1-1", "1Gi", "volume1-1", v1.ClaimBound, &sameDriver), initialVolumes: newVolumeArray("volume1-1", "pv-uid1-1", "pv-handle1-1", "1Gi", "pvc-uid1-1", "claim1-1", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), initialStorageClasses: []*storage.StorageClass{sameDriverStorageClass}, From 7811e59803fa9888acc4abc616b3ce58b1bfb120 Mon Sep 17 00:00:00 2001 From: Grant Griffiths Date: Wed, 12 Feb 2020 15:44:54 -0800 Subject: [PATCH 22/56] Refactor delete test to use withSnapshotFinalizers function Signed-off-by: Grant Griffiths --- pkg/common-controller/framework_test.go | 22 ++++++------ pkg/common-controller/snapshot_delete_test.go | 34 ++----------------- 2 files changed, 15 insertions(+), 41 deletions(-) diff --git a/pkg/common-controller/framework_test.go b/pkg/common-controller/framework_test.go index 1a40219e0..14313f367 100644 --- a/pkg/common-controller/framework_test.go +++ b/pkg/common-controller/framework_test.go @@ -164,11 +164,13 @@ type reactorError struct { error error } -func withSnapshotFinalizer(snapshot *crdv1.VolumeSnapshot) *crdv1.VolumeSnapshot { - snapshot.ObjectMeta.Finalizers = append(snapshot.ObjectMeta.Finalizers, utils.VolumeSnapshotContentFinalizer) - snapshot.ObjectMeta.Finalizers = append(snapshot.ObjectMeta.Finalizers, utils.VolumeSnapshotAsSourceFinalizer) - snapshot.ObjectMeta.Finalizers = append(snapshot.ObjectMeta.Finalizers, utils.VolumeSnapshotBoundFinalizer) - return snapshot +func withSnapshotFinalizers(snapshots []*crdv1.VolumeSnapshot, finalizers ...string) []*crdv1.VolumeSnapshot { + for i := range snapshots { + for _, f := range finalizers { + snapshots[i].ObjectMeta.Finalizers = append(snapshots[i].ObjectMeta.Finalizers, f) + } + } + return snapshots } func withContentFinalizer(content *crdv1.VolumeSnapshotContent) *crdv1.VolumeSnapshotContent { @@ -863,7 +865,7 @@ func newContentWithUnmatchDriverArray(contentName, boundToSnapshotUID, boundToSn func newSnapshot( snapshotName, snapshotUID, pvcName, targetContentName, snapshotClassName, boundContentName string, readyToUse *bool, creationTime *metav1.Time, restoreSize *resource.Quantity, - err *crdv1.VolumeSnapshotError, nilStatus bool, withFinalizer bool, deletionTimestamp *metav1.Time) *crdv1.VolumeSnapshot { + err *crdv1.VolumeSnapshotError, nilStatus bool, withAllFinalizers bool, deletionTimestamp *metav1.Time) *crdv1.VolumeSnapshot { snapshot := crdv1.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: snapshotName, @@ -904,8 +906,8 @@ func newSnapshot( VolumeSnapshotContentName: &targetContentName, } } - if withFinalizer { - return withSnapshotFinalizer(&snapshot) + if withAllFinalizers { + return withSnapshotFinalizers([]*crdv1.VolumeSnapshot{&snapshot}, utils.VolumeSnapshotContentFinalizer, utils.VolumeSnapshotAsSourceFinalizer, utils.VolumeSnapshotBoundFinalizer)[0] } return &snapshot } @@ -913,9 +915,9 @@ func newSnapshot( func newSnapshotArray( snapshotName, snapshotUID, pvcName, targetContentName, snapshotClassName, boundContentName string, readyToUse *bool, creationTime *metav1.Time, restoreSize *resource.Quantity, - err *crdv1.VolumeSnapshotError, nilStatus bool, withFinalizer bool, deletionTimestamp *metav1.Time) []*crdv1.VolumeSnapshot { + err *crdv1.VolumeSnapshotError, nilStatus bool, withAllFinalizers bool, deletionTimestamp *metav1.Time) []*crdv1.VolumeSnapshot { return []*crdv1.VolumeSnapshot{ - newSnapshot(snapshotName, snapshotUID, pvcName, targetContentName, snapshotClassName, boundContentName, readyToUse, creationTime, restoreSize, err, nilStatus, withFinalizer, deletionTimestamp), + newSnapshot(snapshotName, snapshotUID, pvcName, targetContentName, snapshotClassName, boundContentName, readyToUse, creationTime, restoreSize, err, nilStatus, withAllFinalizers, deletionTimestamp), } } diff --git a/pkg/common-controller/snapshot_delete_test.go b/pkg/common-controller/snapshot_delete_test.go index 0fd3c8f87..250c34a2e 100644 --- a/pkg/common-controller/snapshot_delete_test.go +++ b/pkg/common-controller/snapshot_delete_test.go @@ -24,7 +24,6 @@ import ( "github.com/kubernetes-csi/external-snapshotter/v2/pkg/utils" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" ) var class1Parameters = map[string]string{ @@ -304,36 +303,9 @@ func TestDeleteSync(t *testing.T) { initialContents: newContentArray("content3-1", "", "snap3-1", "sid3-1", validSecretClass, "", "", deletePolicy, nil, nil, true), expectedContents: nocontents, initialSnapshots: newSnapshotArray("snap3-1", "snapuid3-1", "claim3-1", "", validSecretClass, "content3-1", &False, nil, nil, nil, false, true, &timeNowMetav1), - expectedSnapshots: []*crdv1.VolumeSnapshot{ - &crdv1.VolumeSnapshot{ - ObjectMeta: metav1.ObjectMeta{ - Name: "snap3-1", - Namespace: testNamespace, - UID: types.UID("snapuid3-1"), - ResourceVersion: "1", - Finalizers: []string{ - "snapshot.storage.kubernetes.io/volumesnapshotcontent-bound-protection", - "snapshot.storage.kubernetes.io/volumesnapshot-bound-protection", - }, - SelfLink: "/apis/snapshot.storage.k8s.io/v1beta1/namespaces/" + testNamespace + "/volumesnapshots/" + "snap3-1", - DeletionTimestamp: &timeNowMetav1, - }, - Spec: crdv1.VolumeSnapshotSpec{ - VolumeSnapshotClassName: &validSecretClass, - Source: crdv1.VolumeSnapshotSource{ - PersistentVolumeClaimName: &claim31, - }, - }, - - Status: &crdv1.VolumeSnapshotStatus{ - CreationTime: nil, - ReadyToUse: &False, - Error: nil, - RestoreSize: nil, - BoundVolumeSnapshotContentName: &content31, - }, - }, - }, + expectedSnapshots: withSnapshotFinalizers(newSnapshotArray("snap3-1", "snapuid3-1", "claim3-1", "", validSecretClass, "content3-1", &False, nil, nil, nil, false, false, &timeNowMetav1), + utils.VolumeSnapshotContentFinalizer, utils.VolumeSnapshotBoundFinalizer, + ), initialClaims: newClaimArray("claim3-1", "pvc-uid3-1", "1Gi", "volume3-1", v1.ClaimBound, &classEmpty), expectedEvents: noevents, initialSecrets: []*v1.Secret{secret()}, From fdb32183fea99d452a2e110ef9ca6bfd6f3e9fa3 Mon Sep 17 00:00:00 2001 From: Jan Wozniak Date: Thu, 13 Feb 2020 11:21:45 +0100 Subject: [PATCH 23/56] Change 'make test-vet' to call 'go vet' --- build.make | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.make b/build.make index a9b9d25dc..3bf3391c2 100644 --- a/build.make +++ b/build.make @@ -113,7 +113,7 @@ test-go: test: test-vet test-vet: @ echo; echo "### $@:" - go test $(GOFLAGS_VENDOR) `go list $(GOFLAGS_VENDOR) ./... | grep -v vendor $(TEST_VET_FILTER_CMD)` + go vet $(GOFLAGS_VENDOR) `go list $(GOFLAGS_VENDOR) ./... | grep -v vendor $(TEST_VET_FILTER_CMD)` .PHONY: test-fmt test: test-fmt From 7c5a89c8fcd634c4413837bc1042bd4bf27a93e6 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Fri, 14 Feb 2020 09:38:16 +0100 Subject: [PATCH 24/56] prow.sh: use 1.3.0 hostpath driver for testing The final 1.3.0 release of the hostpath driver really uses the 1.3.0 driver image in its deployment, in contrast to the previous -rc candidates which still used 1.2.0. --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index 0331ac3f8..d54dea518 100755 --- a/prow.sh +++ b/prow.sh @@ -193,7 +193,7 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip # If the deployment script is called with CSI_PROW_TEST_DRIVER= as # environment variable, then it must write a suitable test driver configuration # into that file in addition to installing the driver. -configvar CSI_PROW_DRIVER_VERSION "v1.3.0-rc4" "CSI driver version" +configvar CSI_PROW_DRIVER_VERSION "v1.3.0" "CSI driver version" configvar CSI_PROW_DRIVER_REPO https://github.com/kubernetes-csi/csi-driver-host-path "CSI driver repo" configvar CSI_PROW_DEPLOYMENT "" "deployment" From 05efba2a3073cb7549d454aca866f63cb0995ca7 Mon Sep 17 00:00:00 2001 From: Grant Griffiths Date: Wed, 12 Feb 2020 18:47:43 -0800 Subject: [PATCH 25/56] Add more snapshot and content sync tests Signed-off-by: Grant Griffiths --- pkg/common-controller/framework_test.go | 23 +++++++- pkg/common-controller/snapshot_delete_test.go | 38 +++++++------ pkg/common-controller/snapshot_update_test.go | 56 ++++++++++++++++++- 3 files changed, 96 insertions(+), 21 deletions(-) diff --git a/pkg/common-controller/framework_test.go b/pkg/common-controller/framework_test.go index 14313f367..9acc3eea6 100644 --- a/pkg/common-controller/framework_test.go +++ b/pkg/common-controller/framework_test.go @@ -175,7 +175,6 @@ func withSnapshotFinalizers(snapshots []*crdv1.VolumeSnapshot, finalizers ...str func withContentFinalizer(content *crdv1.VolumeSnapshotContent) *crdv1.VolumeSnapshotContent { content.ObjectMeta.Finalizers = append(content.ObjectMeta.Finalizers, utils.VolumeSnapshotContentFinalizer) - metav1.SetMetaDataAnnotation(&content.ObjectMeta, utils.AnnVolumeSnapshotBeingDeleted, "yes") return content } @@ -826,6 +825,18 @@ func newContent(contentName, boundToSnapshotUID, boundToSnapshotName, snapshotHa return &content } +func withContentAnnotations(contents []*crdv1.VolumeSnapshotContent, annotations map[string]string) []*crdv1.VolumeSnapshotContent { + for i := range contents { + if contents[i].ObjectMeta.Annotations == nil { + contents[i].ObjectMeta.Annotations = make(map[string]string) + } + for k, v := range annotations { + contents[i].ObjectMeta.Annotations[k] = v + } + } + return contents +} + func newContentArray(contentName, boundToSnapshotUID, boundToSnapshotName, snapshotHandle, snapshotClassName, desiredSnapshotHandle, volumeHandle string, deletionPolicy crdv1.DeletionPolicy, size, creationTime *int64, withFinalizer bool) []*crdv1.VolumeSnapshotContent { @@ -907,7 +918,7 @@ func newSnapshot( } } if withAllFinalizers { - return withSnapshotFinalizers([]*crdv1.VolumeSnapshot{&snapshot}, utils.VolumeSnapshotContentFinalizer, utils.VolumeSnapshotAsSourceFinalizer, utils.VolumeSnapshotBoundFinalizer)[0] + return withSnapshotFinalizers([]*crdv1.VolumeSnapshot{&snapshot}, utils.VolumeSnapshotAsSourceFinalizer, utils.VolumeSnapshotBoundFinalizer)[0] } return &snapshot } @@ -1073,6 +1084,14 @@ func testSyncContent(ctrl *csiSnapshotCommonController, reactor *snapshotReactor return ctrl.syncContent(test.initialContents[0]) } +func testSyncContentError(ctrl *csiSnapshotCommonController, reactor *snapshotReactor, test controllerTest) error { + err := ctrl.syncContent(test.initialContents[0]) + if err != nil { + return nil + } + return fmt.Errorf("syncContent succeeded when failure was expected") +} + func testAddPVCFinalizer(ctrl *csiSnapshotCommonController, reactor *snapshotReactor, test controllerTest) error { return ctrl.ensurePVCFinalizer(test.initialSnapshots[0]) } diff --git a/pkg/common-controller/snapshot_delete_test.go b/pkg/common-controller/snapshot_delete_test.go index 250c34a2e..782dff151 100644 --- a/pkg/common-controller/snapshot_delete_test.go +++ b/pkg/common-controller/snapshot_delete_test.go @@ -17,7 +17,7 @@ limitations under the License. package common_controller import ( - //"errors" + "errors" "testing" crdv1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" @@ -174,22 +174,7 @@ func TestDeleteSync(t *testing.T) { initialSecrets: []*v1.Secret{secret()}, //expectedDeleteCalls: []deleteCall{{"sid1-3", map[string]string{"foo": "bar"}, nil}}, test: testSyncContent, - }, /*{ - name: "1-6 - api server delete content returns error", - initialContents: newContentArray("content1-6", "sid1-6", "snap1-6", "sid1-6", validSecretClass, "", "", deletionPolicy, nil, nil, true), - expectedContents: newContentArray("content1-6", "sid1-6", "snap1-6", "sid1-6", validSecretClass, "", "", deletionPolicy, nil, nil, true), - initialSnapshots: nosnapshots, - expectedSnapshots: nosnapshots, - initialSecrets: []*v1.Secret{secret()}, - //expectedDeleteCalls: []deleteCall{{"sid1-6", map[string]string{"foo": "bar"}, nil}}, - expectedEvents: []string{"Warning SnapshotContentObjectDeleteError"}, - errors: []reactorError{ - // Inject error to the first client.VolumesnapshotV1beta1().VolumeSnapshotContents().Delete call. - // All other calls will succeed. - {"delete", "volumesnapshotcontents", errors.New("mock delete error")}, - }, - test: testSyncContent, - },*/ + }, { // delete success - snapshot that the content was pointing to was deleted, and another // with the same name created. @@ -304,7 +289,7 @@ func TestDeleteSync(t *testing.T) { expectedContents: nocontents, initialSnapshots: newSnapshotArray("snap3-1", "snapuid3-1", "claim3-1", "", validSecretClass, "content3-1", &False, nil, nil, nil, false, true, &timeNowMetav1), expectedSnapshots: withSnapshotFinalizers(newSnapshotArray("snap3-1", "snapuid3-1", "claim3-1", "", validSecretClass, "content3-1", &False, nil, nil, nil, false, false, &timeNowMetav1), - utils.VolumeSnapshotContentFinalizer, utils.VolumeSnapshotBoundFinalizer, + utils.VolumeSnapshotBoundFinalizer, ), initialClaims: newClaimArray("claim3-1", "pvc-uid3-1", "1Gi", "volume3-1", v1.ClaimBound, &classEmpty), expectedEvents: noevents, @@ -312,6 +297,23 @@ func TestDeleteSync(t *testing.T) { errors: noerrors, test: testSyncSnapshot, }, + { + name: "3-2 - content will not be deleted if deletion API call fails", + initialContents: newContentArray("content3-2", "", "snap3-2", "sid3-2", validSecretClass, "", "", deletePolicy, nil, nil, true), + expectedContents: newContentArray("content3-2", "", "snap3-2", "sid3-2", validSecretClass, "", "", deletePolicy, nil, nil, true), + initialSnapshots: newSnapshotArray("snap3-2", "snapuid3-2", "claim3-2", "", validSecretClass, "content3-2", &False, nil, nil, nil, false, true, &timeNowMetav1), + expectedSnapshots: newSnapshotArray("snap3-2", "snapuid3-2", "claim3-2", "", validSecretClass, "content3-2", &False, nil, nil, nil, false, true, &timeNowMetav1), + initialClaims: newClaimArray("claim3-2", "pvc-uid3-2", "1Gi", "volume3-2", v1.ClaimBound, &classEmpty), + expectedEvents: []string{"Warning SnapshotContentObjectDeleteError"}, + initialSecrets: []*v1.Secret{secret()}, + errors: []reactorError{ + // Inject error to the first client.VolumesnapshotV1beta1().VolumeSnapshotContents().Delete call. + // All other calls will succeed. + {"delete", "volumesnapshotcontents", errors.New("mock delete error")}, + }, + expectSuccess: false, + test: testSyncSnapshotError, + }, } runSyncTests(t, tests, snapshotClasses) } diff --git a/pkg/common-controller/snapshot_update_test.go b/pkg/common-controller/snapshot_update_test.go index cbb1b5c16..9c261ce9a 100644 --- a/pkg/common-controller/snapshot_update_test.go +++ b/pkg/common-controller/snapshot_update_test.go @@ -17,10 +17,11 @@ limitations under the License. package common_controller import ( - //"errors" + "errors" "testing" "time" + "github.com/kubernetes-csi/external-snapshotter/v2/pkg/utils" v1 "k8s.io/api/core/v1" storagev1beta1 "k8s.io/api/storage/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -282,6 +283,59 @@ func TestSync(t *testing.T) { errors: noerrors, test: testSyncSnapshot, }, + { + name: "5-1 - content missing finalizer is updated to have finalizer", + initialContents: newContentArray("content5-1", "snapuid5-1", "snap5-1", "sid5-1", validSecretClass, "", "", deletionPolicy, nil, nil, false), + expectedContents: newContentArray("content5-1", "snapuid5-1", "snap5-1", "sid5-1", validSecretClass, "", "", deletionPolicy, nil, nil, true), + initialClaims: newClaimArray("claim5-1", "pvc-uid5-1", "1Gi", "volume5-1", v1.ClaimBound, &classEmpty), + initialVolumes: newVolumeArray("volume5-1", "pv-uid5-1", "pv-handle5-1", "1Gi", "pvc-uid5-1", "claim5-1", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), + initialSecrets: []*v1.Secret{secret()}, + errors: noerrors, + test: testSyncContent, + }, + { + name: "5-2 - content missing finalizer update attempt fails because of failed API call", + initialContents: newContentArray("content5-2", "snapuid5-2", "snap5-2", "sid5-2", validSecretClass, "", "", deletionPolicy, nil, nil, false), + expectedContents: newContentArray("content5-2", "snapuid5-2", "snap5-2", "sid5-2", validSecretClass, "", "", deletionPolicy, nil, nil, false), + initialClaims: newClaimArray("claim5-2", "pvc-uid5-2", "1Gi", "volume5-2", v1.ClaimBound, &classEmpty), + initialVolumes: newVolumeArray("volume5-2", "pv-uid5-2", "pv-handle5-2", "1Gi", "pvc-uid5-2", "claim5-2", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), + initialSecrets: []*v1.Secret{secret()}, + errors: []reactorError{ + // Inject error to the forth client.VolumesnapshotV1beta1().VolumeSnapshots().Update call. + {"update", "volumesnapshotcontents", errors.New("mock update error")}, + }, + expectSuccess: false, + test: testSyncContentError, + }, + { + name: "5-3 - snapshot deletion candidate marked for deletion", + initialSnapshots: newSnapshotArray("snap5-3", "snapuid5-3", "claim5-3", "", validSecretClass, "content5-3", &False, nil, nil, nil, false, true, &timeNowMetav1), + expectedSnapshots: newSnapshotArray("snap5-3", "snapuid5-3", "claim5-3", "", validSecretClass, "content5-3", &False, nil, nil, nil, false, true, &timeNowMetav1), + initialContents: newContentArray("content5-3", "snapuid5-3", "snap5-3", "sid5-3", validSecretClass, "", "", deletionPolicy, nil, nil, true), + expectedContents: withContentAnnotations(newContentArray("content5-3", "snapuid5-3", "snap5-3", "sid5-3", validSecretClass, "", "", deletionPolicy, nil, nil, true), map[string]string{utils.AnnVolumeSnapshotBeingDeleted: "yes"}), + initialClaims: newClaimArray("claim5-3", "pvc-uid5-3", "1Gi", "volume5-3", v1.ClaimBound, &classEmpty), + initialVolumes: newVolumeArray("volume5-3", "pv-uid5-3", "pv-handle5-3", "1Gi", "pvc-uid5-3", "claim5-3", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), + initialSecrets: []*v1.Secret{secret()}, + expectSuccess: true, + test: testSyncContent, + }, + { + name: "5-4 - snapshot deletion candidate fail to mark for deletion due to failed API call", + initialSnapshots: newSnapshotArray("snap5-4", "snapuid5-4", "claim5-4", "", validSecretClass, "content5-4", &False, nil, nil, nil, false, true, &timeNowMetav1), + expectedSnapshots: newSnapshotArray("snap5-4", "snapuid5-4", "claim5-4", "", validSecretClass, "content5-4", &False, nil, nil, nil, false, true, &timeNowMetav1), + initialContents: newContentArray("content5-4", "snapuid5-4", "snap5-4", "sid5-4", validSecretClass, "", "", deletionPolicy, nil, nil, true), + // result of the test framework - annotation is still set in memory, but update call fails. + expectedContents: withContentAnnotations(newContentArray("content5-4", "snapuid5-4", "snap5-4", "sid5-4", validSecretClass, "", "", deletionPolicy, nil, nil, true), map[string]string{utils.AnnVolumeSnapshotBeingDeleted: "yes"}), + initialClaims: newClaimArray("claim5-4", "pvc-uid5-4", "1Gi", "volume5-4", v1.ClaimBound, &classEmpty), + initialVolumes: newVolumeArray("volume5-4", "pv-uid5-4", "pv-handle5-4", "1Gi", "pvc-uid5-4", "claim5-4", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), + initialSecrets: []*v1.Secret{secret()}, + errors: []reactorError{ + // Inject error to the forth client.VolumesnapshotV1beta1().VolumeSnapshots().Update call. + {"update", "volumesnapshotcontents", errors.New("mock update error")}, + }, + expectSuccess: false, + test: testSyncContentError, + }, } runSyncTests(t, tests, snapshotClasses) From 401732912f806b5c43de67363fa4acdd320f26d7 Mon Sep 17 00:00:00 2001 From: bells17 Date: Wed, 19 Feb 2020 20:19:37 +0900 Subject: [PATCH 26/56] Improve error messages --- pkg/sidecar-controller/snapshot_controller.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/sidecar-controller/snapshot_controller.go b/pkg/sidecar-controller/snapshot_controller.go index 136109efd..99ed16f24 100644 --- a/pkg/sidecar-controller/snapshot_controller.go +++ b/pkg/sidecar-controller/snapshot_controller.go @@ -260,8 +260,8 @@ func (ctrl *csiSnapshotSideCarController) checkandUpdateContentStatusOperation(c if content.Spec.VolumeSnapshotClassName != nil { class, err := ctrl.getSnapshotClass(*content.Spec.VolumeSnapshotClassName) if err != nil { - klog.Errorf("failed to getSnapshotClass %s", err) - return nil, fmt.Errorf("cannot get snapshot class for snapshot content %s", err) + klog.Errorf("Failed to getSnapshotClass %s", err) + return nil, fmt.Errorf("cannot get snapshot class %s for snapshot content %s", *content.Spec.VolumeSnapshotClassName, err) } snapshotterListSecretRef, err := utils.GetSecretReference(utils.SnapshotterListSecretParams, class.Parameters, content.GetObjectMeta().GetName(), nil) @@ -273,7 +273,7 @@ func (ctrl *csiSnapshotSideCarController) checkandUpdateContentStatusOperation(c snapshotterListCredentials, err = utils.GetCredentials(ctrl.client, snapshotterListSecretRef) if err != nil { // Continue with deletion, as the secret may have already been deleted. - klog.Errorf("Failed to credentials for snapshot %s: %s", content.Name, err.Error()) + klog.Errorf("Failed to get credentials for snapshot %s: %s", content.Name, err.Error()) return nil, fmt.Errorf("cannot get credentials for snapshot content %#v", content.Name) } } From 2045ef16bada145ed5463e0d5bdd11a4a7fdc024 Mon Sep 17 00:00:00 2001 From: bells17 Date: Wed, 19 Feb 2020 20:22:26 +0900 Subject: [PATCH 27/56] Fix error message s/Delete/List/ --- pkg/sidecar-controller/framework_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/sidecar-controller/framework_test.go b/pkg/sidecar-controller/framework_test.go index 5d853907e..d59f0f4a6 100644 --- a/pkg/sidecar-controller/framework_test.go +++ b/pkg/sidecar-controller/framework_test.go @@ -928,7 +928,7 @@ func (f *fakeSnapshotter) GetSnapshotStatus(ctx context.Context, snapshotID stri if !reflect.DeepEqual(call.secrets, snapshotterListCredentials) { f.t.Errorf("Wrong CSI List Snapshot call: snapshotID=%s, expected secrets %+v, got %+v", snapshotID, call.secrets, snapshotterListCredentials) - err = fmt.Errorf("unexpected Delete Snapshot call") + err = fmt.Errorf("unexpected List Snapshot call") } if err != nil { From 0a1579d0f2a9b01b08b3f502ead4d270fc2dc195 Mon Sep 17 00:00:00 2001 From: bells17 Date: Thu, 20 Feb 2020 01:00:22 +0900 Subject: [PATCH 28/56] Add support for list secret parameters to RemovePrefixedParameters --- pkg/sidecar-controller/snapshot_delete_test.go | 8 ++++++++ pkg/utils/util.go | 2 ++ 2 files changed, 10 insertions(+) diff --git a/pkg/sidecar-controller/snapshot_delete_test.go b/pkg/sidecar-controller/snapshot_delete_test.go index 85283f5e9..012408928 100644 --- a/pkg/sidecar-controller/snapshot_delete_test.go +++ b/pkg/sidecar-controller/snapshot_delete_test.go @@ -61,6 +61,13 @@ var class5Parameters = map[string]string{ utils.AnnDeletionSecretRefNamespace: "default", } +var class6Parameters = map[string]string{ + "csi.storage.k8s.io/snapshotter-secret-name": "secret", + "csi.storage.k8s.io/snapshotter-secret-namespace": "default", + "csi.storage.k8s.io/snapshotter-list-secret-name": "secret", + "csi.storage.k8s.io/snapshotter-list-secret-namespace": "default", +} + var snapshotClasses = []*crdv1.VolumeSnapshotClass{ { TypeMeta: metav1.TypeMeta{ @@ -126,6 +133,7 @@ var snapshotClasses = []*crdv1.VolumeSnapshotClass{ Annotations: map[string]string{utils.IsDefaultSnapshotClassAnnotation: "true"}, }, Driver: mockDriverName, + Parameters: class6Parameters, DeletionPolicy: crdv1.VolumeSnapshotContentDelete, }, } diff --git a/pkg/utils/util.go b/pkg/utils/util.go index c2a155abe..b97a73834 100644 --- a/pkg/utils/util.go +++ b/pkg/utils/util.go @@ -368,6 +368,8 @@ func RemovePrefixedParameters(param map[string]string) (map[string]string, error switch k { case prefixedSnapshotterSecretNameKey: case prefixedSnapshotterSecretNamespaceKey: + case prefixedSnapshotterListSecretNameKey: + case prefixedSnapshotterListSecretNamespaceKey: default: return map[string]string{}, fmt.Errorf("found unknown parameter key \"%s\" with reserved namespace %s", k, csiParameterPrefix) } From dd390a3e388bdc368677296ac3504cfa604eedd5 Mon Sep 17 00:00:00 2001 From: bells17 Date: Thu, 20 Feb 2020 01:09:40 +0900 Subject: [PATCH 29/56] Improve error messages --- pkg/sidecar-controller/snapshot_controller.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/sidecar-controller/snapshot_controller.go b/pkg/sidecar-controller/snapshot_controller.go index 99ed16f24..085f16f9b 100644 --- a/pkg/sidecar-controller/snapshot_controller.go +++ b/pkg/sidecar-controller/snapshot_controller.go @@ -260,21 +260,21 @@ func (ctrl *csiSnapshotSideCarController) checkandUpdateContentStatusOperation(c if content.Spec.VolumeSnapshotClassName != nil { class, err := ctrl.getSnapshotClass(*content.Spec.VolumeSnapshotClassName) if err != nil { - klog.Errorf("Failed to getSnapshotClass %s", err) - return nil, fmt.Errorf("cannot get snapshot class %s for snapshot content %s", *content.Spec.VolumeSnapshotClassName, err) + klog.Errorf("Failed to get snapshot class %s for snapshot content %s", *content.Spec.VolumeSnapshotClassName, err) + return nil, fmt.Errorf("failed to get snapshot class %s for snapshot content %s", *content.Spec.VolumeSnapshotClassName, err) } snapshotterListSecretRef, err := utils.GetSecretReference(utils.SnapshotterListSecretParams, class.Parameters, content.GetObjectMeta().GetName(), nil) if err != nil { - klog.Errorf("Failed to get secret reference for snapshot %s: %s", content.Name, err.Error()) - return nil, fmt.Errorf("cannot get secret reference for snapshot content %#v", content.Name) + klog.Errorf("Failed to get secret reference for snapshot %s: %s", content.Name, err) + return nil, fmt.Errorf("failed to get secret reference for snapshot %s: %s", content.Name, err) } snapshotterListCredentials, err = utils.GetCredentials(ctrl.client, snapshotterListSecretRef) if err != nil { // Continue with deletion, as the secret may have already been deleted. - klog.Errorf("Failed to get credentials for snapshot %s: %s", content.Name, err.Error()) - return nil, fmt.Errorf("cannot get credentials for snapshot content %#v", content.Name) + klog.Errorf("Failed to get credentials for snapshot %s: %s", content.Name, err) + return nil, fmt.Errorf("failed to get credentials for snapshot %s: %s", content.Name, err) } } From 8e9680affd6ae254f7a74c58649ba642c63427f4 Mon Sep 17 00:00:00 2001 From: bells17 Date: Thu, 20 Feb 2020 23:13:12 +0900 Subject: [PATCH 30/56] Modify to upper case variable name from lower case for unit tests --- .../snapshot_delete_test.go | 8 +++---- pkg/utils/util.go | 24 +++++++++---------- pkg/utils/util_test.go | 16 ++++++------- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/pkg/sidecar-controller/snapshot_delete_test.go b/pkg/sidecar-controller/snapshot_delete_test.go index 012408928..5c4310371 100644 --- a/pkg/sidecar-controller/snapshot_delete_test.go +++ b/pkg/sidecar-controller/snapshot_delete_test.go @@ -62,10 +62,10 @@ var class5Parameters = map[string]string{ } var class6Parameters = map[string]string{ - "csi.storage.k8s.io/snapshotter-secret-name": "secret", - "csi.storage.k8s.io/snapshotter-secret-namespace": "default", - "csi.storage.k8s.io/snapshotter-list-secret-name": "secret", - "csi.storage.k8s.io/snapshotter-list-secret-namespace": "default", + utils.PrefixedSnapshotterSecretNameKey: "secret", + utils.PrefixedSnapshotterSecretNamespaceKey: "default", + utils.PrefixedSnapshotterListSecretNameKey: "secret", + utils.PrefixedSnapshotterListSecretNamespaceKey: "default", } var snapshotClasses = []*crdv1.VolumeSnapshotClass{ diff --git a/pkg/utils/util.go b/pkg/utils/util.go index b97a73834..4c1356629 100644 --- a/pkg/utils/util.go +++ b/pkg/utils/util.go @@ -53,11 +53,11 @@ const ( // fields in subsequent CSI calls or Kubernetes API objects. csiParameterPrefix = "csi.storage.k8s.io/" - prefixedSnapshotterSecretNameKey = csiParameterPrefix + "snapshotter-secret-name" - prefixedSnapshotterSecretNamespaceKey = csiParameterPrefix + "snapshotter-secret-namespace" + PrefixedSnapshotterSecretNameKey = csiParameterPrefix + "snapshotter-secret-name" + PrefixedSnapshotterSecretNamespaceKey = csiParameterPrefix + "snapshotter-secret-namespace" - prefixedSnapshotterListSecretNameKey = csiParameterPrefix + "snapshotter-list-secret-name" - prefixedSnapshotterListSecretNamespaceKey = csiParameterPrefix + "snapshotter-list-secret-namespace" + PrefixedSnapshotterListSecretNameKey = csiParameterPrefix + "snapshotter-list-secret-name" + PrefixedSnapshotterListSecretNamespaceKey = csiParameterPrefix + "snapshotter-list-secret-namespace" // Name of finalizer on VolumeSnapshotContents that are bound by VolumeSnapshots VolumeSnapshotContentFinalizer = "snapshot.storage.kubernetes.io/volumesnapshotcontent-bound-protection" @@ -86,14 +86,14 @@ const ( var SnapshotterSecretParams = secretParamsMap{ name: "Snapshotter", - secretNameKey: prefixedSnapshotterSecretNameKey, - secretNamespaceKey: prefixedSnapshotterSecretNamespaceKey, + secretNameKey: PrefixedSnapshotterSecretNameKey, + secretNamespaceKey: PrefixedSnapshotterSecretNamespaceKey, } var SnapshotterListSecretParams = secretParamsMap{ name: "SnapshotterList", - secretNameKey: prefixedSnapshotterListSecretNameKey, - secretNamespaceKey: prefixedSnapshotterListSecretNamespaceKey, + secretNameKey: PrefixedSnapshotterListSecretNameKey, + secretNamespaceKey: PrefixedSnapshotterListSecretNamespaceKey, } func SnapshotKey(vs *crdv1.VolumeSnapshot) string { @@ -366,10 +366,10 @@ func RemovePrefixedParameters(param map[string]string) (map[string]string, error if strings.HasPrefix(k, csiParameterPrefix) { // Check if its well known switch k { - case prefixedSnapshotterSecretNameKey: - case prefixedSnapshotterSecretNamespaceKey: - case prefixedSnapshotterListSecretNameKey: - case prefixedSnapshotterListSecretNamespaceKey: + case PrefixedSnapshotterSecretNameKey: + case PrefixedSnapshotterSecretNamespaceKey: + case PrefixedSnapshotterListSecretNameKey: + case PrefixedSnapshotterListSecretNamespaceKey: default: return map[string]string{}, fmt.Errorf("found unknown parameter key \"%s\" with reserved namespace %s", k, csiParameterPrefix) } diff --git a/pkg/utils/util_test.go b/pkg/utils/util_test.go index d8e853b59..d3a371919 100644 --- a/pkg/utils/util_test.go +++ b/pkg/utils/util_test.go @@ -41,18 +41,18 @@ func TestGetSecretReference(t *testing.T) { }, "namespace, no name": { secretParams: SnapshotterSecretParams, - params: map[string]string{prefixedSnapshotterSecretNamespaceKey: "foo"}, + params: map[string]string{PrefixedSnapshotterSecretNamespaceKey: "foo"}, expectErr: true, }, "simple - valid": { secretParams: SnapshotterSecretParams, - params: map[string]string{prefixedSnapshotterSecretNameKey: "name", prefixedSnapshotterSecretNamespaceKey: "ns"}, + params: map[string]string{PrefixedSnapshotterSecretNameKey: "name", PrefixedSnapshotterSecretNamespaceKey: "ns"}, snapshot: &crdv1.VolumeSnapshot{}, expectRef: &v1.SecretReference{Name: "name", Namespace: "ns"}, }, "simple - invalid name": { secretParams: SnapshotterSecretParams, - params: map[string]string{prefixedSnapshotterSecretNameKey: "bad name", prefixedSnapshotterSecretNamespaceKey: "ns"}, + params: map[string]string{PrefixedSnapshotterSecretNameKey: "bad name", PrefixedSnapshotterSecretNamespaceKey: "ns"}, snapshot: &crdv1.VolumeSnapshot{}, expectRef: nil, expectErr: true, @@ -60,8 +60,8 @@ func TestGetSecretReference(t *testing.T) { "template - invalid": { secretParams: SnapshotterSecretParams, params: map[string]string{ - prefixedSnapshotterSecretNameKey: "static-${volumesnapshotcontent.name}-${volumesnapshot.namespace}-${volumesnapshot.name}-${volumesnapshot.annotations['akey']}", - prefixedSnapshotterSecretNamespaceKey: "static-${volumesnapshotcontent.name}-${volumesnapshot.namespace}", + PrefixedSnapshotterSecretNameKey: "static-${volumesnapshotcontent.name}-${volumesnapshot.namespace}-${volumesnapshot.name}-${volumesnapshot.annotations['akey']}", + PrefixedSnapshotterSecretNamespaceKey: "static-${volumesnapshotcontent.name}-${volumesnapshot.namespace}", }, snapContentName: "snapcontentname", snapshot: &crdv1.VolumeSnapshot{ @@ -111,14 +111,14 @@ func TestRemovePrefixedCSIParams(t *testing.T) { }, { name: "one prefixed", - params: map[string]string{prefixedSnapshotterSecretNameKey: "bar", "bim": "baz"}, + params: map[string]string{PrefixedSnapshotterSecretNameKey: "bar", "bim": "baz"}, expectedParams: map[string]string{"bim": "baz"}, }, { name: "all known prefixed", params: map[string]string{ - prefixedSnapshotterSecretNameKey: "csiBar", - prefixedSnapshotterSecretNamespaceKey: "csiBar", + PrefixedSnapshotterSecretNameKey: "csiBar", + PrefixedSnapshotterSecretNamespaceKey: "csiBar", }, expectedParams: map[string]string{}, }, From 7cbcf6aa41e7511c8d022414e259f7f7df9fe5f5 Mon Sep 17 00:00:00 2001 From: bells17 Date: Fri, 21 Feb 2020 00:42:15 +0900 Subject: [PATCH 31/56] Fix log messages and error messages --- pkg/sidecar-controller/snapshot_controller.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/sidecar-controller/snapshot_controller.go b/pkg/sidecar-controller/snapshot_controller.go index 085f16f9b..bf02e70ba 100644 --- a/pkg/sidecar-controller/snapshot_controller.go +++ b/pkg/sidecar-controller/snapshot_controller.go @@ -260,21 +260,21 @@ func (ctrl *csiSnapshotSideCarController) checkandUpdateContentStatusOperation(c if content.Spec.VolumeSnapshotClassName != nil { class, err := ctrl.getSnapshotClass(*content.Spec.VolumeSnapshotClassName) if err != nil { - klog.Errorf("Failed to get snapshot class %s for snapshot content %s", *content.Spec.VolumeSnapshotClassName, err) - return nil, fmt.Errorf("failed to get snapshot class %s for snapshot content %s", *content.Spec.VolumeSnapshotClassName, err) + klog.Errorf("Failed to get snapshot class %s for snapshot content %s: %v", *content.Spec.VolumeSnapshotClassName, content.Name, err) + return nil, fmt.Errorf("failed to get snapshot class %s for snapshot content %s: %v", *content.Spec.VolumeSnapshotClassName, content.Name, err) } snapshotterListSecretRef, err := utils.GetSecretReference(utils.SnapshotterListSecretParams, class.Parameters, content.GetObjectMeta().GetName(), nil) if err != nil { - klog.Errorf("Failed to get secret reference for snapshot %s: %s", content.Name, err) - return nil, fmt.Errorf("failed to get secret reference for snapshot %s: %s", content.Name, err) + klog.Errorf("Failed to get secret reference for snapshot content %s: %v", content.Name, err) + return nil, fmt.Errorf("failed to get secret reference for snapshot content %s: %v", content.Name, err) } snapshotterListCredentials, err = utils.GetCredentials(ctrl.client, snapshotterListSecretRef) if err != nil { // Continue with deletion, as the secret may have already been deleted. - klog.Errorf("Failed to get credentials for snapshot %s: %s", content.Name, err) - return nil, fmt.Errorf("failed to get credentials for snapshot %s: %s", content.Name, err) + klog.Errorf("Failed to get credentials for snapshot content %s: %v", content.Name, err) + return nil, fmt.Errorf("failed to get credentials for snapshot content %s: %v", content.Name, err) } } From 3c2baf4e9657ed49978ff7f5ab3255bc85ff9ba7 Mon Sep 17 00:00:00 2001 From: bells17 Date: Fri, 21 Feb 2020 00:44:44 +0900 Subject: [PATCH 32/56] Add comments --- pkg/utils/util.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/utils/util.go b/pkg/utils/util.go index 4c1356629..0bd64435b 100644 --- a/pkg/utils/util.go +++ b/pkg/utils/util.go @@ -53,11 +53,11 @@ const ( // fields in subsequent CSI calls or Kubernetes API objects. csiParameterPrefix = "csi.storage.k8s.io/" - PrefixedSnapshotterSecretNameKey = csiParameterPrefix + "snapshotter-secret-name" - PrefixedSnapshotterSecretNamespaceKey = csiParameterPrefix + "snapshotter-secret-namespace" + PrefixedSnapshotterSecretNameKey = csiParameterPrefix + "snapshotter-secret-name" // Prefixed name key for DeleteSnapshot secret + PrefixedSnapshotterSecretNamespaceKey = csiParameterPrefix + "snapshotter-secret-namespace" // Prefixed namespace key for DeleteSnapshot secret - PrefixedSnapshotterListSecretNameKey = csiParameterPrefix + "snapshotter-list-secret-name" - PrefixedSnapshotterListSecretNamespaceKey = csiParameterPrefix + "snapshotter-list-secret-namespace" + PrefixedSnapshotterListSecretNameKey = csiParameterPrefix + "snapshotter-list-secret-name" // Prefixed name key for ListSnapshots secret + PrefixedSnapshotterListSecretNamespaceKey = csiParameterPrefix + "snapshotter-list-secret-namespace" // Prefixed namespace key for ListSnapshots secret // Name of finalizer on VolumeSnapshotContents that are bound by VolumeSnapshots VolumeSnapshotContentFinalizer = "snapshot.storage.kubernetes.io/volumesnapshotcontent-bound-protection" From c3beea55ba5a0f6c493ab77a3ab6546b5a6c0cea Mon Sep 17 00:00:00 2001 From: Christian Huffman Date: Tue, 25 Feb 2020 09:34:03 -0500 Subject: [PATCH 33/56] Update CRDs to include additional get information --- ....storage.k8s.io_volumesnapshotclasses.yaml | 23 ++++++-- ...storage.k8s.io_volumesnapshotcontents.yaml | 46 +++++++++++++--- ...apshot.storage.k8s.io_volumesnapshots.yaml | 52 ++++++++++++++++--- hack/update-crd.sh | 2 +- pkg/apis/volumesnapshot/v1beta1/types.go | 18 +++++++ 5 files changed, 122 insertions(+), 19 deletions(-) diff --git a/config/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml b/config/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml index c0908cccd..4aa980cc7 100644 --- a/config/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml +++ b/config/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml @@ -4,19 +4,32 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: (devel) - api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/139" + controller-gen.kubebuilder.io/version: v0.2.5 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/260" creationTimestamp: null name: volumesnapshotclasses.snapshot.storage.k8s.io spec: + additionalPrinterColumns: + - JSONPath: .driver + name: Driver + type: string + - JSONPath: .deletionPolicy + description: Determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass + should be deleted when its bound VolumeSnapshot is deleted. + name: DeletionPolicy + type: string + - JSONPath: .metadata.creationTimestamp + name: Age + type: date group: snapshot.storage.k8s.io names: kind: VolumeSnapshotClass listKind: VolumeSnapshotClassList plural: volumesnapshotclasses singular: volumesnapshotclass - scope: Cluster preserveUnknownFields: false + scope: Cluster + subresources: {} validation: openAPIV3Schema: description: VolumeSnapshotClass specifies parameters that a underlying storage @@ -27,7 +40,7 @@ spec: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string deletionPolicy: description: deletionPolicy determines whether a VolumeSnapshotContent created @@ -47,7 +60,7 @@ spec: kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string parameters: additionalProperties: diff --git a/config/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml b/config/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml index 5614f9457..34c51ad62 100644 --- a/config/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml +++ b/config/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml @@ -4,21 +4,53 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: (devel) - api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/139" + controller-gen.kubebuilder.io/version: v0.2.5 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/260" creationTimestamp: null name: volumesnapshotcontents.snapshot.storage.k8s.io spec: + additionalPrinterColumns: + - JSONPath: .status.readyToUse + description: Indicates if a snapshot is ready to be used to restore a volume. + name: ReadyToUse + type: boolean + - JSONPath: .status.restoreSize + description: Represents the complete size of the snapshot in bytes + name: RestoreSize + type: integer + - JSONPath: .spec.deletionPolicy + description: Determines whether this VolumeSnapshotContent and its physical snapshot + on the underlying storage system should be deleted when its bound VolumeSnapshot + is deleted. + name: DeletionPolicy + type: string + - JSONPath: .spec.driver + description: Name of the CSI driver used to create the physical snapshot on the + underlying storage system. + name: Driver + type: string + - JSONPath: .spec.volumeSnapshotClassName + description: Name of the VolumeSnapshotClass to which this snapshot belongs. + name: VolumeSnapshotClass + type: string + - JSONPath: .spec.volumeSnapshotRef.name + description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent + object is bound. + name: VolumeSnapshot + type: string + - JSONPath: .metadata.creationTimestamp + name: Age + type: date group: snapshot.storage.k8s.io names: kind: VolumeSnapshotContent listKind: VolumeSnapshotContentList plural: volumesnapshotcontents singular: volumesnapshotcontent + preserveUnknownFields: false scope: Cluster subresources: status: {} - preserveUnknownFields: false validation: openAPIV3Schema: description: VolumeSnapshotContent represents the actual "on-disk" snapshot @@ -27,12 +59,12 @@ spec: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string spec: description: spec defines properties of a VolumeSnapshotContent created @@ -104,7 +136,7 @@ spec: in the future.' type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' @@ -114,7 +146,7 @@ spec: type: string resourceVersion: description: 'Specific resourceVersion to which this reference is - made, if any. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency' + made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' type: string uid: description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' diff --git a/config/crd/snapshot.storage.k8s.io_volumesnapshots.yaml b/config/crd/snapshot.storage.k8s.io_volumesnapshots.yaml index 5b3d760a1..483706f16 100644 --- a/config/crd/snapshot.storage.k8s.io_volumesnapshots.yaml +++ b/config/crd/snapshot.storage.k8s.io_volumesnapshots.yaml @@ -4,21 +4,57 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: (devel) - api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/139" + controller-gen.kubebuilder.io/version: v0.2.5 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/260" creationTimestamp: null name: volumesnapshots.snapshot.storage.k8s.io spec: + additionalPrinterColumns: + - JSONPath: .status.readyToUse + description: Indicates if a snapshot is ready to be used to restore a volume. + name: ReadyToUse + type: boolean + - JSONPath: .spec.source.persistentVolumeClaimName + description: Name of the source PVC from where a dynamically taken snapshot will + be created. + name: SourcePVC + type: string + - JSONPath: .spec.source.volumeSnapshotContentName + description: Name of the VolumeSnapshotContent which represents a pre-provisioned + snapshot. + name: SourceSnapshotContent + type: string + - JSONPath: .status.restoreSize + description: Represents the complete size of the snapshot. + name: RestoreSize + type: string + - JSONPath: .spec.volumeSnapshotClassName + description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot. + name: SnapshotClass + type: string + - JSONPath: .status.boundVolumeSnapshotContentName + description: The name of the VolumeSnapshotContent to which this VolumeSnapshot + is bound. + name: SnapshotContent + type: string + - JSONPath: .status.creationTime + description: Timestamp when the point-in-time snapshot is taken by the underlying + storage system. + name: CreationTime + type: date + - JSONPath: .metadata.creationTimestamp + name: Age + type: date group: snapshot.storage.k8s.io names: kind: VolumeSnapshot listKind: VolumeSnapshotList plural: volumesnapshots singular: volumesnapshot + preserveUnknownFields: false scope: Namespaced subresources: status: {} - preserveUnknownFields: false validation: openAPIV3Schema: description: VolumeSnapshot is a user's request for either creating a point-in-time @@ -27,12 +63,12 @@ spec: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string spec: description: 'spec defines the desired characteristics of a snapshot requested @@ -121,6 +157,9 @@ spec: of a snapshot is unknown. type: boolean restoreSize: + anyOf: + - type: integer + - type: string description: restoreSize represents the complete size of the snapshot in bytes. In dynamic snapshot creation case, this field will be filled in with the "size_bytes" value returned from CSI "CreateSnapshotRequest" @@ -130,7 +169,8 @@ spec: this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object required: - spec diff --git a/hack/update-crd.sh b/hack/update-crd.sh index b4ffd9c1f..8118707b8 100755 --- a/hack/update-crd.sh +++ b/hack/update-crd.sh @@ -26,7 +26,7 @@ then TMP_DIR=$(mktemp -d); cd $TMP_DIR; go mod init tmp; - go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.2.4; + go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.2.5; rm -rf $TMP_DIR; CONTROLLER_GEN=$(which controller-gen) fi diff --git a/pkg/apis/volumesnapshot/v1beta1/types.go b/pkg/apis/volumesnapshot/v1beta1/types.go index 3c1dc1e27..1299d3655 100644 --- a/pkg/apis/volumesnapshot/v1beta1/types.go +++ b/pkg/apis/volumesnapshot/v1beta1/types.go @@ -31,6 +31,14 @@ import ( // +kubebuilder:object:root=true // +kubebuilder:resource:scope=Namespaced // +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="ReadyToUse",type=boolean,JSONPath=`.status.readyToUse`,description="Indicates if a snapshot is ready to be used to restore a volume." +// +kubebuilder:printcolumn:name="SourcePVC",type=string,JSONPath=`.spec.source.persistentVolumeClaimName`,description="Name of the source PVC from where a dynamically taken snapshot will be created." +// +kubebuilder:printcolumn:name="SourceSnapshotContent",type=string,JSONPath=`.spec.source.volumeSnapshotContentName`,description="Name of the VolumeSnapshotContent which represents a pre-provisioned snapshot." +// +kubebuilder:printcolumn:name="RestoreSize",type=string,JSONPath=`.status.restoreSize`,description="Represents the complete size of the snapshot." +// +kubebuilder:printcolumn:name="SnapshotClass",type=string,JSONPath=`.spec.volumeSnapshotClassName`,description="The name of the VolumeSnapshotClass requested by the VolumeSnapshot." +// +kubebuilder:printcolumn:name="SnapshotContent",type=string,JSONPath=`.status.boundVolumeSnapshotContentName`,description="The name of the VolumeSnapshotContent to which this VolumeSnapshot is bound." +// +kubebuilder:printcolumn:name="CreationTime",type=date,JSONPath=`.status.creationTime`,description="Timestamp when the point-in-time snapshot is taken by the underlying storage system." +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` type VolumeSnapshot struct { metav1.TypeMeta `json:",inline"` // Standard object's metadata. @@ -163,6 +171,9 @@ type VolumeSnapshotStatus struct { // VolumeSnapshotClasses are non-namespaced // +kubebuilder:object:root=true // +kubebuilder:resource:scope=Cluster +// +kubebuilder:printcolumn:name="Driver",type=string,JSONPath=`.driver` +// +kubebuilder:printcolumn:name="DeletionPolicy",type=string,JSONPath=`.deletionPolicy`,description="Determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted." +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` type VolumeSnapshotClass struct { metav1.TypeMeta `json:",inline"` // Standard object's metadata. @@ -212,6 +223,13 @@ type VolumeSnapshotClassList struct { // +kubebuilder:object:root=true // +kubebuilder:resource:scope=Cluster // +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="ReadyToUse",type=boolean,JSONPath=`.status.readyToUse`,description="Indicates if a snapshot is ready to be used to restore a volume." +// +kubebuilder:printcolumn:name="RestoreSize",type=integer,JSONPath=`.status.restoreSize`,description="Represents the complete size of the snapshot in bytes" +// +kubebuilder:printcolumn:name="DeletionPolicy",type=string,JSONPath=`.spec.deletionPolicy`,description="Determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted." +// +kubebuilder:printcolumn:name="Driver",type=string,JSONPath=`.spec.driver`,description="Name of the CSI driver used to create the physical snapshot on the underlying storage system." +// +kubebuilder:printcolumn:name="VolumeSnapshotClass",type=string,JSONPath=`.spec.volumeSnapshotClassName`,description="Name of the VolumeSnapshotClass to which this snapshot belongs." +// +kubebuilder:printcolumn:name="VolumeSnapshot",type=string,JSONPath=`.spec.volumeSnapshotRef.name`,description="Name of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound." +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` type VolumeSnapshotContent struct { metav1.TypeMeta `json:",inline"` // Standard object's metadata. From 8f4904b94542e739feb23c6af4d7ccd0c7ee47fd Mon Sep 17 00:00:00 2001 From: windayski <599059134@qq.com> Date: Sat, 29 Feb 2020 00:05:28 +0800 Subject: [PATCH 34/56] Update CONTRIBUTING.md fix broken link --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 15fe07e44..433be2c0b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,7 +10,7 @@ We have full documentation on how to get started contributing here: - [Contributor License Agreement](https://git.k8s.io/community/CLA.md) Kubernetes projects require that you sign a Contributor License Agreement (CLA) before we can accept your pull requests - [Kubernetes Contributor Guide](http://git.k8s.io/community/contributors/guide) - Main contributor documentation, or you can just jump directly to the [contributing section](http://git.k8s.io/community/contributors/guide#contributing) -- [Contributor Cheat Sheet](https://git.k8s.io/community/contributors/guide/contributor-cheatsheet.md) - Common resources for existing developers +- [Contributor Cheat Sheet](https://github.com/kubernetes/community/blob/master/contributors/guide/contributor-cheatsheet/README.md) - Common resources for existing developers ## Mentorship From b154cc77e7fed93f2af90e3275d19a6dacba9fb2 Mon Sep 17 00:00:00 2001 From: xing-yang Date: Fri, 7 Feb 2020 15:03:59 +0000 Subject: [PATCH 35/56] Add VolumeSnapshotBeingDeleted annotation to content --- pkg/common-controller/snapshot_controller.go | 4 ++++ pkg/common-controller/snapshot_update_test.go | 15 +++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/pkg/common-controller/snapshot_controller.go b/pkg/common-controller/snapshot_controller.go index 433027fef..c66e2b099 100644 --- a/pkg/common-controller/snapshot_controller.go +++ b/pkg/common-controller/snapshot_controller.go @@ -249,6 +249,7 @@ func (ctrl *csiSnapshotCommonController) checkandRemoveSnapshotFinalizersAndChec } klog.V(5).Infof("checkandRemoveSnapshotFinalizersAndCheckandDeleteContent[%s]: set DeletionTimeStamp on content.", utils.SnapshotKey(snapshot)) + // If content exists, set DeletionTimeStamp on the content; // content won't be deleted immediately due to the finalizer if content != nil && deleteContent && !inUse { @@ -261,6 +262,9 @@ func (ctrl *csiSnapshotCommonController) checkandRemoveSnapshotFinalizersAndChec } if !inUse { + klog.V(5).Infof("checkandRemoveSnapshotFinalizersAndCheckandDeleteContent: Set VolumeSnapshotBeingDeleted annotation on the content [%s]", content.Name) + ctrl.setAnnVolumeSnapshotBeingDeleted(content) + klog.V(5).Infof("checkandRemoveSnapshotFinalizersAndCheckandDeleteContent: Remove Finalizer for VolumeSnapshot[%s]", utils.SnapshotKey(snapshot)) doesContentExist := false if content != nil { diff --git a/pkg/common-controller/snapshot_update_test.go b/pkg/common-controller/snapshot_update_test.go index 9c261ce9a..5ba641bf5 100644 --- a/pkg/common-controller/snapshot_update_test.go +++ b/pkg/common-controller/snapshot_update_test.go @@ -21,6 +21,7 @@ import ( "testing" "time" + crdv1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" "github.com/kubernetes-csi/external-snapshotter/v2/pkg/utils" v1 "k8s.io/api/core/v1" storagev1beta1 "k8s.io/api/storage/v1beta1" @@ -336,6 +337,20 @@ func TestSync(t *testing.T) { expectSuccess: false, test: testSyncContentError, }, + { + name: "5-5 - snapshot deletion candidate marked for deletion by syncSnapshot", + initialSnapshots: newSnapshotArray("snap5-5", "snapuid5-5", "claim5-5", "", validSecretClass, "content5-5", &False, nil, nil, nil, false, true, &timeNowMetav1), + expectedSnapshots: withSnapshotFinalizers(newSnapshotArray("snap5-5", "snapuid5-5", "claim5-5", "", validSecretClass, "content5-5", &False, nil, nil, nil, false, false, &timeNowMetav1), + utils.VolumeSnapshotBoundFinalizer, + ), + initialContents: newContentArray("content5-5", "snapuid5-5", "snap5-5", "sid5-5", validSecretClass, "", "", crdv1.VolumeSnapshotContentRetain, nil, nil, true), + expectedContents: withContentAnnotations(newContentArray("content5-5", "snapuid5-5", "snap5-5", "sid5-5", validSecretClass, "", "", crdv1.VolumeSnapshotContentRetain, nil, nil, true), map[string]string{utils.AnnVolumeSnapshotBeingDeleted: "yes"}), + initialClaims: newClaimArray("claim5-5", "pvc-uid5-5", "1Gi", "volume5-5", v1.ClaimBound, &classEmpty), + initialVolumes: newVolumeArray("volume5-5", "pv-uid5-5", "pv-handle5-5", "1Gi", "pvc-uid5-5", "claim5-5", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), + initialSecrets: []*v1.Secret{secret()}, + expectSuccess: true, + test: testSyncSnapshot, + }, } runSyncTests(t, tests, snapshotClasses) From 9830b6ec23a86add44886110cecb7d0ba1128efc Mon Sep 17 00:00:00 2001 From: Grant Griffiths Date: Wed, 4 Mar 2020 10:47:27 -0800 Subject: [PATCH 36/56] Fix error check for checkandAddSnapshotFinalizers and add tests Signed-off-by: Grant Griffiths --- pkg/common-controller/framework_test.go | 17 +- pkg/common-controller/snapshot_controller.go | 30 +-- pkg/common-controller/snapshot_create_test.go | 179 +++++++++--------- pkg/common-controller/snapshot_delete_test.go | 176 ++++------------- pkg/common-controller/snapshot_update_test.go | 156 +++++++-------- 5 files changed, 218 insertions(+), 340 deletions(-) diff --git a/pkg/common-controller/framework_test.go b/pkg/common-controller/framework_test.go index 9acc3eea6..0eadcd9a2 100644 --- a/pkg/common-controller/framework_test.go +++ b/pkg/common-controller/framework_test.go @@ -173,11 +173,6 @@ func withSnapshotFinalizers(snapshots []*crdv1.VolumeSnapshot, finalizers ...str return snapshots } -func withContentFinalizer(content *crdv1.VolumeSnapshotContent) *crdv1.VolumeSnapshotContent { - content.ObjectMeta.Finalizers = append(content.ObjectMeta.Finalizers, utils.VolumeSnapshotContentFinalizer) - return content -} - func withPVCFinalizer(pvc *v1.PersistentVolumeClaim) *v1.PersistentVolumeClaim { pvc.ObjectMeta.Finalizers = append(pvc.ObjectMeta.Finalizers, utils.PVCFinalizer) return pvc @@ -837,6 +832,18 @@ func withContentAnnotations(contents []*crdv1.VolumeSnapshotContent, annotations return contents } +func withContentSpecSnapshotClassName(contents []*crdv1.VolumeSnapshotContent, volumeSnapshotClassName *string) []*crdv1.VolumeSnapshotContent { + for i := range contents { + contents[i].Spec.VolumeSnapshotClassName = volumeSnapshotClassName + } + return contents +} + +func withContentFinalizer(content *crdv1.VolumeSnapshotContent) *crdv1.VolumeSnapshotContent { + content.ObjectMeta.Finalizers = append(content.ObjectMeta.Finalizers, utils.VolumeSnapshotContentFinalizer) + return content +} + func newContentArray(contentName, boundToSnapshotUID, boundToSnapshotName, snapshotHandle, snapshotClassName, desiredSnapshotHandle, volumeHandle string, deletionPolicy crdv1.DeletionPolicy, size, creationTime *int64, withFinalizer bool) []*crdv1.VolumeSnapshotContent { diff --git a/pkg/common-controller/snapshot_controller.go b/pkg/common-controller/snapshot_controller.go index c66e2b099..0a4c7fa54 100644 --- a/pkg/common-controller/snapshot_controller.go +++ b/pkg/common-controller/snapshot_controller.go @@ -173,7 +173,11 @@ func (ctrl *csiSnapshotCommonController) syncSnapshot(snapshot *crdv1.VolumeSnap } } else { klog.V(5).Infof("syncSnapshot[%s]: check if we should add finalizers on snapshot", utils.SnapshotKey(snapshot)) - ctrl.checkandAddSnapshotFinalizers(snapshot) + if err := ctrl.checkandAddSnapshotFinalizers(snapshot); err != nil { + klog.Errorf("error check and add Snapshot finalizers for snapshot [%s]: %v", snapshot.Name, errFinalizer) + ctrl.eventRecorder.Event(snapshot, v1.EventTypeWarning, "SnapshotCheckandUpdateFailed", fmt.Sprintf("Failed to check and update snapshot: %s", err.Error())) + return err + } // Need to build or update snapshot.Status in following cases: // 1) snapshot.Status is nil // 2) snapshot.Status.ReadyToUse is false @@ -301,7 +305,7 @@ func (ctrl *csiSnapshotCommonController) checkandAddSnapshotFinalizers(snapshot if addSourceFinalizer || addBoundFinalizer { // Snapshot is not being deleted -> it should have the finalizer. klog.V(5).Infof("checkandAddSnapshotFinalizers: Add Finalizer for VolumeSnapshot[%s]", utils.SnapshotKey(snapshot)) - ctrl.addSnapshotFinalizer(snapshot, addSourceFinalizer, addBoundFinalizer) + return ctrl.addSnapshotFinalizer(snapshot, addSourceFinalizer, addBoundFinalizer) } return nil } @@ -665,26 +669,6 @@ func (ctrl *csiSnapshotCommonController) updateSnapshotErrorStatusWithEvent(snap return nil } -// isSnapshotConentBeingUsed checks if snapshot content is bound to snapshot. -func (ctrl *csiSnapshotCommonController) isSnapshotContentBeingUsed(content *crdv1.VolumeSnapshotContent) bool { - if content.Spec.VolumeSnapshotRef.Name != "" && content.Spec.VolumeSnapshotRef.Namespace != "" { - snapshotObj, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshots(content.Spec.VolumeSnapshotRef.Namespace).Get(content.Spec.VolumeSnapshotRef.Name, metav1.GetOptions{}) - if err != nil { - klog.Infof("isSnapshotContentBeingUsed: Cannot get snapshot %s from api server: [%v]. VolumeSnapshot object may be deleted already.", content.Spec.VolumeSnapshotRef.Name, err) - return false - } - - // Check if the snapshot content is bound to the snapshot - if utils.IsSnapshotBound(snapshotObj, content) { - klog.Infof("isSnapshotContentBeingUsed: VolumeSnapshot %s is bound to volumeSnapshotContent [%s]", snapshotObj.Name, content.Name) - return true - } - } - - klog.V(5).Infof("isSnapshotContentBeingUsed: Snapshot content %s is not being used", content.Name) - return false -} - // addContentFinalizer adds a Finalizer for VolumeSnapshotContent. func (ctrl *csiSnapshotCommonController) addContentFinalizer(content *crdv1.VolumeSnapshotContent) error { contentClone := content.DeepCopy() @@ -858,7 +842,7 @@ func (ctrl *csiSnapshotCommonController) checkandBindSnapshotContent(snapshot *c } newContent, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().Update(contentClone) if err != nil { - klog.V(4).Infof("updating VolumeSnapshotContent[%s] error status failed %v", newContent.Name, err) + klog.V(4).Infof("updating VolumeSnapshotContent[%s] error status failed %v", contentClone.Name, err) return nil, err } diff --git a/pkg/common-controller/snapshot_create_test.go b/pkg/common-controller/snapshot_create_test.go index ab16f6447..f0a3b0a80 100644 --- a/pkg/common-controller/snapshot_create_test.go +++ b/pkg/common-controller/snapshot_create_test.go @@ -79,13 +79,18 @@ func TestCreateSnapshotSync(t *testing.T) { test: testSyncSnapshot, }, { - name: "6-2 - successful create snapshot with snapshot class silver", - initialContents: nocontents, - expectedContents: newContentArrayNoStatus("snapcontent-snapuid6-2", "snapuid6-2", "snap6-2", "sid6-2", classSilver, "", "pv-handle6-2", deletionPolicy, nil, nil, false, false), - initialSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "", &False, nil, nil, nil, false, true, nil), - expectedSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "snapcontent-snapuid6-2", &False, nil, nil, nil, false, true, nil), + name: "6-2 - successful create snapshot with validSecretClass and initial secret", + initialContents: nocontents, + expectedContents: withContentAnnotations(newContentArrayNoStatus("snapcontent-snapuid6-2", "snapuid6-2", "snap6-2", "sid6-2", validSecretClass, "", "pv-handle6-2", deletionPolicy, nil, nil, false, false), + map[string]string{ + "snapshot.storage.kubernetes.io/deletion-secret-name": "secret", + "snapshot.storage.kubernetes.io/deletion-secret-namespace": "default", + }), + initialSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", validSecretClass, "", &False, nil, nil, nil, false, true, nil), + expectedSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", validSecretClass, "snapcontent-snapuid6-2", &False, nil, nil, nil, false, true, nil), initialClaims: newClaimArray("claim6-2", "pvc-uid6-2", "1Gi", "volume6-2", v1.ClaimBound, &classEmpty), initialVolumes: newVolumeArray("volume6-2", "pv-uid6-2", "pv-handle6-2", "1Gi", "pvc-uid6-2", "claim6-2", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), + initialSecrets: []*v1.Secret{secret()}, // no initial secret created errors: noerrors, test: testSyncSnapshot, }, @@ -120,12 +125,12 @@ func TestCreateSnapshotSync(t *testing.T) { {"update", "volumesnapshots", errors.New("mock update error")}, }, test: testSyncSnapshot, }, - /*{ + { name: "7-3 - fail to create snapshot without snapshot class ", initialContents: nocontents, expectedContents: nocontents, - initialSnapshots: newSnapshotArray("snap7-3", "snapuid7-3", "claim7-3", "", "", "", &False, nil, nil, nil, false, true), - expectedSnapshots: newSnapshotArray("snap7-3", "snapuid7-3", "claim7-3", "", "", "", &False, nil, nil, newVolumeError("Failed to create snapshot content with error failed to get input parameters to create snapshot snap7-3: \"failed to retrieve snapshot class from the informer: \\\"volumesnapshotclass.snapshot.storage.k8s.io \\\\\\\"\\\\\\\" not found\\\"\""), false, true), + initialSnapshots: newSnapshotArray("snap7-3", "snapuid7-3", "claim7-3", "", "", "", &False, nil, nil, nil, false, true, nil), + expectedSnapshots: newSnapshotArray("snap7-3", "snapuid7-3", "claim7-3", "", "", "", &False, nil, nil, newVolumeError("Failed to create snapshot content with error failed to get input parameters to create snapshot snap7-3: \"failed to take snapshot snap7-3 without a snapshot class\""), false, true, nil), initialClaims: newClaimArray("claim7-3", "pvc-uid7-3", "1Gi", "volume7-3", v1.ClaimBound, &classEmpty), initialVolumes: newVolumeArray("volume7-3", "pv-uid7-3", "pv-handle7-3", "1Gi", "pvc-uid7-3", "claim7-3", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), initialStorageClasses: []*storage.StorageClass{diffDriverStorageClass}, @@ -133,7 +138,7 @@ func TestCreateSnapshotSync(t *testing.T) { errors: noerrors, expectSuccess: false, test: testSyncSnapshot, - },*/ + }, { name: "7-4 - fail create snapshot with no-existing claim", initialContents: nocontents, @@ -171,83 +176,87 @@ func TestCreateSnapshotSync(t *testing.T) { expectSuccess: false, test: testSyncSnapshot, }, - /*{ - name: "7-8 - fail create snapshot due to cannot update snapshot status", - initialContents: nocontents, - expectedContents: nocontents, - initialSnapshots: newSnapshotArray("snap7-8", "snapuid7-8", "claim7-8", "", classGold, "", &False, nil, nil, nil), - expectedSnapshots: newSnapshotArray("snap7-8", "snapuid7-8", "claim7-8", "", classGold, "", &False, nil, nil, newVolumeError("Failed to create snapshot: snapshot controller failed to update default/snap7-8 on API server: mock update error")), - initialClaims: newClaimArray("claim7-8", "pvc-uid7-8", "1Gi", "volume7-8", v1.ClaimBound, &classEmpty), - initialVolumes: newVolumeArray("volume7-8", "pv-uid7-8", "pv-handle7-8", "1Gi", "pvc-uid7-8", "claim7-8", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), - expectedCreateCalls: []createCall{ - { - snapshotName: "snapshot-snapuid7-8", - volume: newVolume("volume7-8", "pv-uid7-8", "pv-handle7-8", "1Gi", "pvc-uid7-8", "claim7-8", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), - parameters: map[string]string{"param1": "value1"}, - // information to return - driverName: mockDriverName, - size: defaultSize, - snapshotId: "sid7-8", - creationTime: timeNow, - readyToUse: True, - }, - }, - /*errors: []reactorError{ - // Inject error to the forth client.VolumesnapshotV1beta1().VolumeSnapshots().Update call. - // All other calls will succeed. - {"update", "volumesnapshots", errors.New("mock update error")}, - {"update", "volumesnapshots", errors.New("mock update error")}, - {"update", "volumesnapshots", errors.New("mock update error")}, - }, - expectedEvents: []string{"Warning SnapshotContentCreationFailed"}, - expectSuccess: false, - test: testSyncSnapshot, - }, - /*{ - // TODO(xiangqian): this test case needs to be revisited the scenario - // of VolumeSnapshotContent saving failure. Since there will be no content object - // in API server, it could potentially cause leaking issue - name: "7-9 - fail create snapshot due to cannot save snapshot content", - initialContents: nocontents, - expectedContents: nocontents, - initialSnapshots: newSnapshotArray("snap7-9", "snapuid7-9", "claim7-9", "", classGold, "", &False, nil, nil, nil), - expectedSnapshots: newSnapshotArray("snap7-9", "snapuid7-9", "claim7-9", "", classGold, "snapcontent-snapuid7-9", &True, metaTimeNowUnix, getSize(defaultSize), nil), - initialClaims: newClaimArray("claim7-9", "pvc-uid7-9", "1Gi", "volume7-9", v1.ClaimBound, &classEmpty), - initialVolumes: newVolumeArray("volume7-9", "pv-uid7-9", "pv-handle7-9", "1Gi", "pvc-uid7-9", "claim7-9", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), - expectedCreateCalls: []createCall{ - { - snapshotName: "snapshot-snapuid7-9", - volume: newVolume("volume7-9", "pv-uid7-9", "pv-handle7-9", "1Gi", "pvc-uid7-9", "claim7-9", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), - parameters: map[string]string{"param1": "value1"}, - // information to return - driverName: mockDriverName, - size: defaultSize, - snapshotId: "sid7-9", - creationTime: timeNow, - readyToUse: True, - }, - }, - errors: []reactorError{ - {"create", "volumesnapshotcontents", errors.New("mock create error")}, - {"create", "volumesnapshotcontents", errors.New("mock create error")}, - {"create", "volumesnapshotcontents", errors.New("mock create error")}, - }, - expectedEvents: []string{"Warning CreateSnapshotContentFailed"}, - test: testSyncSnapshot, - }, - { - name: "7-10 - fail create snapshot with secret not found", - initialContents: nocontents, - expectedContents: nocontents, - initialSnapshots: newSnapshotArray("snap7-10", "snapuid7-10", "claim7-10", "", validSecretClass, "", &False, nil, nil, nil, false, true), - expectedSnapshots: newSnapshotArray("snap7-10", "snapuid7-10", "claim7-10", "", validSecretClass, "", &False, nil, nil, newVolumeError("Failed to create snapshot: error getting secret secret in namespace default: cannot find secret secret"), false, true), - initialClaims: newClaimArray("claim7-10", "pvc-uid7-10", "1Gi", "volume7-10", v1.ClaimBound, &classEmpty), - initialVolumes: newVolumeArray("volume7-10", "pv-uid7-10", "pv-handle7-10", "1Gi", "pvc-uid7-10", "claim7-10", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), - initialSecrets: []*v1.Secret{}, // no initial secret created - errors: noerrors, - test: testSyncSnapshot, - },*/ + { + name: "7-7 - remove pvc finalizer failed", + initialContents: newContentArray("snapcontent-snapuid7-7", "snapuid7-7", "snap7-7", "sid7-7", classGold, "", "pv-handle7-7", deletionPolicy, nil, nil, false), + expectedContents: newContentArray("snapcontent-snapuid7-7", "snapuid7-7", "snap7-7", "sid7-7", classGold, "", "pv-handle7-7", deletionPolicy, nil, nil, false), + initialSnapshots: newSnapshotArray("snap7-7", "snapuid7-7", "claim7-7", "", classGold, "snapcontent-snapuid7-7", &True, nil, nil, nil, false, true, nil), + expectedSnapshots: newSnapshotArray("snap7-7", "snapuid7-7", "claim7-7", "", classGold, "snapcontent-snapuid7-7", &True, nil, nil, nil, false, true, nil), + initialClaims: newClaimArrayFinalizer("claim7-7", "pvc-uid7-7", "1Gi", "volume7-7", v1.ClaimBound, &classEmpty), + initialVolumes: newVolumeArray("volume7-7", "pv-uid7-7", "pv-handle7-7", "1Gi", "pvc-uid7-7", "claim7-7", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), + errors: []reactorError{ + {"update", "persistentvolumeclaims", errors.New("mock update error")}, + {"update", "persistentvolumeclaims", errors.New("mock update error")}, + {"update", "persistentvolumeclaims", errors.New("mock update error")}, + }, + expectSuccess: false, + test: testSyncSnapshot, + }, + { + name: "7-8 - fail create snapshot due to cannot update snapshot status", + initialContents: nocontents, + expectedContents: newContentArrayNoStatus("snapcontent-snapuid7-8", "snapuid7-8", "snap7-8", "sid7-8", classGold, "", "pv-handle7-8", deletionPolicy, nil, nil, false, false), + initialSnapshots: newSnapshotArray("snap7-8", "snapuid7-8", "claim7-8", "", classGold, "", &False, nil, nil, nil, false, true, nil), + expectedSnapshots: newSnapshotArray("snap7-8", "snapuid7-8", "claim7-8", "", classGold, "", &False, nil, nil, newVolumeError("Snapshot status update failed, snapshot controller failed to update default/snap7-8 on API server: mock update error"), false, true, nil), + initialClaims: newClaimArray("claim7-8", "pvc-uid7-8", "1Gi", "volume7-8", v1.ClaimBound, &classEmpty), + initialVolumes: newVolumeArray("volume7-8", "pv-uid7-8", "pv-handle7-8", "1Gi", "pvc-uid7-8", "claim7-8", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), + errors: []reactorError{ + {"update", "volumesnapshots", errors.New("mock update error")}, + {"update", "volumesnapshots", errors.New("mock update error")}, + {"update", "volumesnapshots", errors.New("mock update error")}, + }, + expectedEvents: []string{"Warning SnapshotStatusUpdateFailed"}, + expectSuccess: false, + test: testSyncSnapshot, + }, + { + name: "7-9 - fail create snapshot due to cannot update snapshot status, and failure cannot be recorded either due to additional status update failure.", + initialContents: nocontents, + expectedContents: newContentArrayNoStatus("snapcontent-snapuid7-9", "snapuid7-9", "snap7-9", "sid7-9", classGold, "", "pv-handle7-9", deletionPolicy, nil, nil, false, false), + initialSnapshots: newSnapshotArray("snap7-9", "snapuid7-9", "claim7-9", "", classGold, "", &False, nil, nil, nil, false, true, nil), + expectedSnapshots: newSnapshotArray("snap7-9", "snapuid7-9", "claim7-9", "", classGold, "", &False, nil, nil, nil, false, true, nil), + initialClaims: newClaimArray("claim7-9", "pvc-uid7-9", "1Gi", "volume7-9", v1.ClaimBound, &classEmpty), + initialVolumes: newVolumeArray("volume7-9", "pv-uid7-9", "pv-handle7-9", "1Gi", "pvc-uid7-9", "claim7-9", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), + errors: []reactorError{ + {"update", "volumesnapshots", errors.New("mock update error")}, + {"update", "volumesnapshots", errors.New("mock update error")}, + {"update", "volumesnapshots", errors.New("mock update error")}, + {"update", "volumesnapshots", errors.New("mock update error")}, + }, + expectSuccess: false, + test: testSyncSnapshot, + }, + { + name: "7-10 - fail create snapshot with invalid secret", + initialContents: nocontents, + expectedContents: nocontents, + initialSnapshots: newSnapshotArray("snap7-10", "snapuid7-10", "claim7-10", "", invalidSecretClass, "", &False, nil, nil, nil, false, true, nil), + expectedSnapshots: newSnapshotArray("snap7-10", "snapuid7-10", "claim7-10", "snap7-10", invalidSecretClass, "", &False, nil, nil, newVolumeError("Failed to create snapshot content with error failed to get input parameters to create snapshot snap7-10: \"failed to get name and namespace template from params: either name and namespace for Snapshotter secrets specified, Both must be specified\""), false, true, nil), + initialClaims: newClaimArray("claim7-10", "pvc-uid7-10", "1Gi", "volume7-10", v1.ClaimBound, &classEmpty), + initialVolumes: newVolumeArray("volume7-10", "pv-uid7-10", "pv-handle7-10", "1Gi", "pvc-uid7-10", "claim7-10", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), + initialSecrets: []*v1.Secret{}, // no initial secret created + test: testSyncSnapshot, + }, + { + // TODO(xiangqian): this test case needs to be revisited the scenario + // of VolumeSnapshotContent saving failure. Since there will be no content object + // in API server, it could potentially cause leaking issue + name: "7-11 - fail create snapshot due to cannot save snapshot content", + initialContents: nocontents, + expectedContents: nocontents, + initialSnapshots: newSnapshotArray("snap7-11", "snapuid7-11", "claim7-11", "", classGold, "", &False, nil, nil, nil, false, true, nil), + expectedSnapshots: newSnapshotArray("snap7-11", "snapuid7-11", "claim7-11", "", classGold, "", &False, nil, nil, newVolumeError("Failed to create snapshot content with error snapshot controller failed to update default/snap7-11 on API server: mock create error"), false, true, nil), + initialClaims: newClaimArray("claim7-11", "pvc-uid7-11", "1Gi", "volume7-11", v1.ClaimBound, &classEmpty), + initialVolumes: newVolumeArray("volume7-11", "pv-uid7-11", "pv-handle7-11", "1Gi", "pvc-uid7-11", "claim7-11", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), + errors: []reactorError{ + {"create", "volumesnapshotcontents", errors.New("mock create error")}, + {"create", "volumesnapshotcontents", errors.New("mock create error")}, + {"create", "volumesnapshotcontents", errors.New("mock create error")}, + }, + expectedEvents: []string{"Warning CreateSnapshotContentFailed"}, + test: testSyncSnapshot, + }, } runSyncTests(t, tests, snapshotClasses) } diff --git a/pkg/common-controller/snapshot_delete_test.go b/pkg/common-controller/snapshot_delete_test.go index 782dff151..17ce7ffef 100644 --- a/pkg/common-controller/snapshot_delete_test.go +++ b/pkg/common-controller/snapshot_delete_test.go @@ -35,8 +35,8 @@ var class2Parameters = map[string]string{ } var class3Parameters = map[string]string{ - "param3": "value3", - //utils.SnapshotterSecretNameKey: "name", + "param3": "value3", + utils.PrefixedSnapshotterSecretNameKey: "name", } var class4Parameters = map[string]string{ @@ -45,8 +45,8 @@ var class4Parameters = map[string]string{ } var class5Parameters = map[string]string{ - //utils.SnapshotterSecretNameKey: "secret", - //utils.SnapshotterSecretNamespaceKey: "default", + utils.PrefixedSnapshotterSecretNameKey: "secret", + utils.PrefixedSnapshotterSecretNamespaceKey: "default", } var timeNowMetav1 = metav1.Now() @@ -130,159 +130,44 @@ var snapshotClasses = []*crdv1.VolumeSnapshotClass{ func TestDeleteSync(t *testing.T) { tests := []controllerTest{ { - name: "1-1 - content with empty snapshot class is deleted if it is bound to a non-exist snapshot and also has a snapshot uid specified", - initialContents: newContentArray("content1-1", "snapuid1-1", "snap1-1", "sid1-1", classGold, "", "", deletionPolicy, nil, nil, true), - expectedContents: newContentArray("content1-1", "snapuid1-1", "snap1-1", "sid1-1", classGold, "", "", deletionPolicy, nil, nil, true), - initialSnapshots: nosnapshots, - expectedSnapshots: nosnapshots, - expectedEvents: noevents, - errors: noerrors, - //expectedDeleteCalls: []deleteCall{{"sid1-1", nil, nil}}, - test: testSyncContent, - }, - { - name: "2-1 - content with empty snapshot class will not be deleted if it is bound to a non-exist snapshot but it does not have a snapshot uid specified", - initialContents: newContentArray("content2-1", "", "snap2-1", "sid2-1", "", "", "", deletionPolicy, nil, nil, true), - expectedContents: newContentArray("content2-1", "", "snap2-1", "sid2-1", "", "", "", deletionPolicy, nil, nil, true), - initialSnapshots: nosnapshots, - expectedSnapshots: nosnapshots, - expectedEvents: noevents, - errors: noerrors, - //expectedDeleteCalls: []deleteCall{{"sid2-1", nil, nil}}, - test: testSyncContent, - }, - { - name: "1-2 - successful delete with snapshot class that has empty secret parameter", - initialContents: newContentArray("content1-2", "sid1-2", "snap1-2", "sid1-2", emptySecretClass, "", "", deletionPolicy, nil, nil, true), - expectedContents: newContentArray("content1-2", "sid1-2", "snap1-2", "sid1-2", emptySecretClass, "", "", deletionPolicy, nil, nil, true), - initialSnapshots: nosnapshots, - expectedSnapshots: nosnapshots, - initialSecrets: []*v1.Secret{emptySecret()}, - expectedEvents: noevents, - errors: noerrors, - //expectedDeleteCalls: []deleteCall{{"sid1-2", map[string]string{}, nil}}, - test: testSyncContent, - }, - { - name: "1-3 - successful delete with snapshot class that has valid secret parameter", - initialContents: newContentArray("content1-3", "sid1-3", "snap1-3", "sid1-3", validSecretClass, "", "", deletionPolicy, nil, nil, true), - expectedContents: newContentArray("content1-3", "sid1-3", "snap1-3", "sid1-3", validSecretClass, "", "", deletionPolicy, nil, nil, true), - initialSnapshots: nosnapshots, - expectedSnapshots: nosnapshots, + name: "1-1 - noop: content will not be deleted if it is bound to a snapshot correctly, snapshot uid is not specified", + initialContents: newContentArray("content1-1", "", "snap1-1", "sid1-1", validSecretClass, "", "", deletePolicy, nil, nil, true), + expectedContents: newContentArray("content1-1", "", "snap1-1", "sid1-1", validSecretClass, "", "", deletePolicy, nil, nil, true), + initialSnapshots: newSnapshotArray("snap1-1", "snapuid1-1", "claim1-1", "", validSecretClass, "content1-1", &False, nil, nil, nil, false, true, &timeNowMetav1), + expectedSnapshots: newSnapshotArray("snap1-1", "snapuid1-1", "claim1-1", "", validSecretClass, "content1-1", &False, nil, nil, nil, false, true, &timeNowMetav1), expectedEvents: noevents, - errors: noerrors, - initialSecrets: []*v1.Secret{secret()}, - //expectedDeleteCalls: []deleteCall{{"sid1-3", map[string]string{"foo": "bar"}, nil}}, - test: testSyncContent, - }, - { - // delete success - snapshot that the content was pointing to was deleted, and another - // with the same name created. - name: "1-7 - prebound content is deleted while the snapshot exists", - initialContents: newContentArray("content1-7", "sid1-7", "snap1-7", "sid1-7", emptySecretClass, "", "", deletionPolicy, nil, nil, true), - expectedContents: newContentArray("content1-7", "sid1-7", "snap1-7", "sid1-7", emptySecretClass, "", "", deletionPolicy, nil, nil, true), - initialSnapshots: newSnapshotArray("snap1-7", "snapuid1-7-x", "claim1-7", "", validSecretClass, "", &False, nil, nil, nil, false, true, nil), - expectedSnapshots: newSnapshotArray("snap1-7", "snapuid1-7-x", "claim1-7", "", validSecretClass, "", &False, nil, nil, nil, false, true, nil), initialSecrets: []*v1.Secret{secret()}, - //expectedDeleteCalls: []deleteCall{{"sid1-7", map[string]string{"foo": "bar"}, nil}}, - expectedEvents: noevents, - errors: noerrors, - test: testSyncContent, + errors: noerrors, + test: testSyncContent, }, { - // delete success(?) - content is deleted before doDelete() starts - name: "1-8 - content is deleted before deleting", - initialContents: newContentArray("content1-8", "sid1-8", "snap1-8", "sid1-8", validSecretClass, "", "", deletionPolicy, nil, nil, true), + // delete success - content is deleted before doDelete() starts + name: "1-2 - content is deleted before deleting", + initialContents: newContentArray("content1-2", "sid1-2", "snap1-2", "sid1-2", validSecretClass, "", "", deletionPolicy, nil, nil, true), expectedContents: nocontents, initialSnapshots: nosnapshots, expectedSnapshots: nosnapshots, initialSecrets: []*v1.Secret{secret()}, - //expectedDeleteCalls: []deleteCall{{"sid1-8", map[string]string{"foo": "bar"}, nil}}, - expectedEvents: noevents, - errors: noerrors, + expectedEvents: noevents, + errors: noerrors, test: wrapTestWithInjectedOperation(testSyncContent, func(ctrl *csiSnapshotCommonController, reactor *snapshotReactor) { // Delete the volume before delete operation starts reactor.lock.Lock() - delete(reactor.contents, "content1-8") + delete(reactor.contents, "content1-2") reactor.lock.Unlock() }), }, - /*{ - name: "1-9 - content will not be deleted if it is bound to a snapshot correctly, snapshot uid is specified", - initialContents: newContentArray("content1-9", "snapuid1-9", "snap1-9", "sid1-9", validSecretClass, "", "", deletionPolicy, nil, nil, true), - expectedContents: newContentArray("content1-9", "snapuid1-9", "snap1-9", "sid1-9", validSecretClass, "", "", deletionPolicy, nil, nil, true), - initialSnapshots: newSnapshotArray("snap1-9", "snapuid1-9", "claim1-9", "", validSecretClass, "content1-9", &False, nil, nil, nil), - expectedSnapshots: newSnapshotArray("snap1-9", "snapuid1-9", "claim1-9", "", validSecretClass, "content1-9", &True, nil, nil, nil), - expectedEvents: noevents, - initialSecrets: []*v1.Secret{secret()}, - errors: noerrors, - test: testSyncContent, - },*/ { - name: "1-10 - will not delete content with retain policy set which is bound to a snapshot incorrectly", - initialContents: newContentArray("content1-10", "snapuid1-10-x", "snap1-10", "sid1-10", validSecretClass, "", "", retainPolicy, nil, nil, true), - expectedContents: newContentArray("content1-10", "snapuid1-10-x", "snap1-10", "sid1-10", validSecretClass, "", "", retainPolicy, nil, nil, true), - initialSnapshots: newSnapshotArray("snap1-10", "snapuid1-10", "claim1-10", "", validSecretClass, "content1-10", &False, nil, nil, nil, false, true, nil), - expectedSnapshots: newSnapshotArray("snap1-10", "snapuid1-10", "claim1-10", "", validSecretClass, "content1-10", &False, nil, nil, nil, false, true, nil), + name: "1-3 - will not delete content with retain policy set which is bound to a snapshot incorrectly", + initialContents: newContentArray("content1-3", "snapuid1-3-x", "snap1-3", "sid1-3", validSecretClass, "", "", retainPolicy, nil, nil, true), + expectedContents: newContentArray("content1-3", "snapuid1-3-x", "snap1-3", "sid1-3", validSecretClass, "", "", retainPolicy, nil, nil, true), + initialSnapshots: newSnapshotArray("snap1-3", "snapuid1-3", "claim1-3", "", validSecretClass, "content1-3", &False, nil, nil, nil, false, true, &timeNowMetav1), + expectedSnapshots: newSnapshotArray("snap1-3", "snapuid1-3", "claim1-3", "", validSecretClass, "content1-3", &False, nil, nil, nil, false, true, &timeNowMetav1), expectedEvents: noevents, initialSecrets: []*v1.Secret{secret()}, errors: noerrors, test: testSyncContent, }, - { - name: "1-11 - content will not be deleted if it is bound to a snapshot correctly, snapsht uid is not specified", - initialContents: newContentArray("content1-11", "", "snap1-11", "sid1-11", validSecretClass, "", "", deletePolicy, nil, nil, true), - expectedContents: newContentArray("content1-11", "", "snap1-11", "sid1-11", validSecretClass, "", "", deletePolicy, nil, nil, true), - initialSnapshots: newSnapshotArray("snap1-11", "snapuid1-11", "claim1-11", "", validSecretClass, "content1-11", &False, nil, nil, nil, false, true, nil), - expectedSnapshots: newSnapshotArray("snap1-11", "snapuid1-11", "claim1-11", "", validSecretClass, "content1-11", &False, nil, nil, nil, false, true, nil), - expectedEvents: noevents, - initialSecrets: []*v1.Secret{secret()}, - errors: noerrors, - test: testSyncContent, - }, - { - name: "1-12 - content with retain policy will not be deleted if it is bound to a non-exist snapshot and also has a snapshot uid specified", - initialContents: newContentArray("content1-12", "sid1-12", "snap1-11", "sid1-11", validSecretClass, "", "", retainPolicy, nil, nil, true), - expectedContents: newContentArray("content1-12", "sid1-12", "snap1-11", "sid1-11", validSecretClass, "", "", retainPolicy, nil, nil, true), - initialSnapshots: nosnapshots, - expectedSnapshots: nosnapshots, - expectedEvents: noevents, - errors: noerrors, - test: testSyncContent, - }, - { - name: "1-13 - content with empty snapshot class is not deleted when Deletion policy is not set even if it is bound to a non-exist snapshot and also has a snapshot uid specified", - initialContents: newContentArray("content1-13", "sid1-13", "snap1-13", "sid1-13", validSecretClass, "", "", retainPolicy, nil, nil, true), - expectedContents: newContentArray("content1-13", "sid1-13", "snap1-13", "sid1-13", validSecretClass, "", "", retainPolicy, nil, nil, true), - initialSnapshots: nosnapshots, - expectedSnapshots: nosnapshots, - expectedEvents: noevents, - errors: noerrors, - test: testSyncContent, - }, - /*{ - name: "1-14 - content will not be deleted if it is bound to a snapshot correctly, snapshot uid is specified", - initialContents: newContentArray("content1-14", "snapuid1-14", "snap1-14", "sid1-14", validSecretClass, "", "", retainPolicy, nil, nil, true), - expectedContents: newContentArray("content1-14", "snapuid1-14", "snap1-14", "sid1-14", validSecretClass, "", "", retainPolicy, nil, nil, true), - initialSnapshots: newSnapshotArray("snap1-14", "snapuid1-14", "claim1-14", "", validSecretClass, "content1-14", &False, nil, nil, nil), - expectedSnapshots: newSnapshotArray("snap1-14", "snapuid1-14", "claim1-14", "", validSecretClass, "content1-14", &True, nil, nil, nil), - expectedEvents: noevents, - initialSecrets: []*v1.Secret{secret()}, - errors: noerrors, - test: testSyncContent, - },*/ - { - name: "1-16 - continue delete with snapshot class that has nonexistent secret", - initialContents: newContentArray("content1-16", "sid1-16", "snap1-16", "sid1-16", emptySecretClass, "", "", deletePolicy, nil, nil, true), - expectedContents: newContentArray("content1-16", "sid1-16", "snap1-16", "sid1-16", emptySecretClass, "", "", deletePolicy, nil, nil, true), - initialSnapshots: nosnapshots, - expectedSnapshots: nosnapshots, - expectedEvents: noevents, - errors: noerrors, - initialSecrets: []*v1.Secret{}, // secret does not exist - //expectedDeleteCalls: []deleteCall{{"sid1-16", nil, nil}}, - test: testSyncContent, - }, { name: "3-1 - content will be deleted if snapshot deletion timestamp is set", initialContents: newContentArray("content3-1", "", "snap3-1", "sid3-1", validSecretClass, "", "", deletePolicy, nil, nil, true), @@ -314,6 +199,23 @@ func TestDeleteSync(t *testing.T) { expectSuccess: false, test: testSyncSnapshotError, }, + { + name: "3-3 - content will not be deleted if retainPolicy is set", + initialContents: newContentArray("content3-3", "", "snap3-3", "sid3-3", validSecretClass, "", "", retainPolicy, nil, nil, true), + expectedContents: withContentAnnotations(newContentArray("content3-3", "", "snap3-3", "sid3-3", validSecretClass, "", "", retainPolicy, nil, nil, true), + map[string]string{ + "snapshot.storage.kubernetes.io/volumesnapshot-being-deleted": "yes", + }), + initialSnapshots: newSnapshotArray("snap3-3", "snapuid3-3", "claim3-3", "", validSecretClass, "content3-3", &False, nil, nil, nil, false, true, &timeNowMetav1), + expectedSnapshots: withSnapshotFinalizers(newSnapshotArray("snap3-3", "snapuid3-3", "claim3-3", "", validSecretClass, "content3-3", &False, nil, nil, nil, false, false, &timeNowMetav1), + utils.VolumeSnapshotBoundFinalizer, + ), + initialClaims: newClaimArray("claim3-3", "pvc-uid3-3", "1Gi", "volume3-3", v1.ClaimBound, &classEmpty), + expectedEvents: noevents, + initialSecrets: []*v1.Secret{secret()}, + errors: noerrors, + test: testSyncSnapshot, + }, } runSyncTests(t, tests, snapshotClasses) } diff --git a/pkg/common-controller/snapshot_update_test.go b/pkg/common-controller/snapshot_update_test.go index 5ba641bf5..06cdb7e1c 100644 --- a/pkg/common-controller/snapshot_update_test.go +++ b/pkg/common-controller/snapshot_update_test.go @@ -37,6 +37,8 @@ var volumeErr = &storagev1beta1.VolumeError{ Message: "Failed to upload the snapshot", } +var emptyString = "" + // Test single call to syncSnapshot and syncContent methods. // 1. Fill in the controller with initial data // 2. Call the tested function (syncSnapshot/syncContent) via @@ -56,16 +58,17 @@ func TestSync(t *testing.T) { errors: noerrors, test: testSyncSnapshot, }, - /*{ + { name: "2-2 - snapshot points to a content but content does not point to snapshot(VolumeSnapshotRef does not match)", initialContents: newContentArray("content2-2", "snapuid2-2-x", "snap2-2", "sid2-2", validSecretClass, "", "", deletionPolicy, nil, nil, false), expectedContents: newContentArray("content2-2", "snapuid2-2-x", "snap2-2", "sid2-2", validSecretClass, "", "", deletionPolicy, nil, nil, false), - initialSnapshots: newSnapshotArray("snap2-2", "snapuid2-2", "claim2-2", "", validSecretClass, "content2-2", &False, nil, nil, nil), - expectedSnapshots: newSnapshotArray("snap2-2", "snapuid2-2", "claim2-2", "", validSecretClass, "content2-2", &False, nil, nil, newVolumeError("Snapshot is bound to a VolumeSnapshotContent which is bound to other Snapshot")), - expectedEvents: []string{"Warning InvalidSnapshotBinding"}, + initialSnapshots: newSnapshotArray("snap2-2", "snapuid2-2", "", "content2-2", validSecretClass, "content2-2", &False, nil, nil, nil, false, true, nil), + expectedSnapshots: newSnapshotArray("snap2-2", "snapuid2-2", "", "content2-2", validSecretClass, "content2-2", &False, nil, nil, newVolumeError("Snapshot failed to bind VolumeSnapshotContent, Could not bind snapshot snap2-2 and content content2-2, the VolumeSnapshotRef does not match"), false, true, nil), + initialClaims: newClaimArray("claim2-2", "pvc-uid2-2", "1Gi", "volume2-2", v1.ClaimBound, &classEmpty), + expectedEvents: []string{"Warning SnapshotBindFailed"}, errors: noerrors, test: testSyncSnapshotError, - },*/ + }, { name: "2-3 - success bind snapshot and content but not ready, no status changed", initialContents: newContentArray("content2-3", "snapuid2-3", "snap2-3", "sid2-3", validSecretClass, "", "", deletionPolicy, nil, nil, false), @@ -75,22 +78,8 @@ func TestSync(t *testing.T) { initialClaims: newClaimArray("claim2-3", "pvc-uid2-3", "1Gi", "volume2-3", v1.ClaimBound, &classEmpty), initialVolumes: newVolumeArray("volume2-3", "pv-uid2-3", "pv-handle2-3", "1Gi", "pvc-uid2-3", "claim2-3", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), initialSecrets: []*v1.Secret{secret()}, - /*expectedCreateCalls: []createCall{ - { - snapshotName: "snapshot-snapuid2-3", - volume: newVolume("volume2-3", "pv-uid2-3", "pv-handle2-3", "1Gi", "pvc-uid2-3", "claim2-3", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), - parameters: class5Parameters, - secrets: map[string]string{"foo": "bar"}, - // information to return - driverName: mockDriverName, - size: defaultSize, - snapshotId: "sid2-3", - creationTime: timeNow, - readyToUse: False, - }, - },*/ - errors: noerrors, - test: testSyncSnapshot, + errors: noerrors, + test: testSyncSnapshot, }, { // nothing changed @@ -111,22 +100,8 @@ func TestSync(t *testing.T) { initialClaims: newClaimArray("claim2-5", "pvc-uid2-5", "1Gi", "volume2-5", v1.ClaimBound, &classEmpty), initialVolumes: newVolumeArray("volume2-5", "pv-uid2-5", "pv-handle2-5", "1Gi", "pvc-uid2-5", "claim2-5", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), initialSecrets: []*v1.Secret{secret()}, - /*expectedCreateCalls: []createCall{ - { - snapshotName: "snapshot-snapuid2-5", - volume: newVolume("volume2-5", "pv-uid2-5", "pv-handle2-5", "1Gi", "pvc-uid2-5", "claim2-5", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), - parameters: class5Parameters, - secrets: map[string]string{"foo": "bar"}, - // information to return - driverName: mockDriverName, - size: defaultSize, - snapshotId: "sid2-5", - creationTime: timeNow, - readyToUse: True, - }, - },*/ - errors: noerrors, - test: testSyncSnapshot, + errors: noerrors, + test: testSyncSnapshot, }, { name: "2-6 - snapshot bound to prebound content correctly, status ready false -> true, ref.UID '' -> 'snapuid2-6'", @@ -137,69 +112,47 @@ func TestSync(t *testing.T) { errors: noerrors, test: testSyncSnapshot, }, - /*{ - name: "2-7 - snapshot and content bound, csi driver get status error", - initialContents: newContentArrayWithReadyToUse("content2-7", "snapuid2-7", "snap2-7", "sid2-7", validSecretClass, "", "", deletionPolicy, &timeNowStamp, nil, &False, false), - expectedContents: newContentArrayWithReadyToUse("content2-7", "snapuid2-7", "snap2-7", "sid2-7", validSecretClass, "", "", deletionPolicy, &timeNowStamp, nil, &False, false), - initialSnapshots: newSnapshotArray("snap2-7", "snapuid2-7", "claim2-7", "", validSecretClass, "content2-7", &False, metaTimeNow, nil, nil), - expectedSnapshots: newSnapshotArray("snap2-7", "snapuid2-7", "claim2-7", "", validSecretClass, "content2-7", &False, metaTimeNow, nil, newVolumeError("Failed to check and update snapshot: mock create snapshot error")), + { + name: "2-8 - snapshot and content bound, apiserver update status error", + initialContents: newContentArrayWithReadyToUse("content2-8", "snapuid2-8", "snap2-8", "sid2-8", validSecretClass, "", "", deletionPolicy, &timeNowStamp, nil, &False, false), + expectedContents: newContentArrayWithReadyToUse("content2-8", "snapuid2-8", "snap2-8", "sid2-8", validSecretClass, "", "", deletionPolicy, &timeNowStamp, nil, &False, false), + initialSnapshots: newSnapshotArray("snap2-8", "snapuid2-8", "claim2-8", "", validSecretClass, "content2-8", &False, metaTimeNow, nil, nil, false, false, nil), + expectedSnapshots: newSnapshotArray("snap2-8", "snapuid2-8", "claim2-8", "", validSecretClass, "content2-8", &False, metaTimeNow, nil, nil, false, false, nil), expectedEvents: []string{"Warning SnapshotCheckandUpdateFailed"}, - initialClaims: newClaimArray("claim2-7", "pvc-uid2-7", "1Gi", "volume2-7", v1.ClaimBound, &classEmpty), - initialVolumes: newVolumeArray("volume2-7", "pv-uid2-7", "pv-handle2-7", "1Gi", "pvc-uid2-7", "claim2-7", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), + initialClaims: newClaimArray("claim2-8", "pvc-uid2-8", "1Gi", "volume2-8", v1.ClaimBound, &classEmpty), + initialVolumes: newVolumeArray("volume2-8", "pv-uid2-8", "pv-handle2-8", "1Gi", "pvc-uid2-8", "claim2-8", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), initialSecrets: []*v1.Secret{secret()}, - expectedCreateCalls: []createCall{ - { - snapshotName: "snapshot-snapuid2-7", - volume: newVolume("volume2-7", "pv-uid2-7", "pv-handle2-7", "1Gi", "pvc-uid2-7", "claim2-7", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), - parameters: class5Parameters, - secrets: map[string]string{"foo": "bar"}, - // information to return - err: errors.New("mock create snapshot error"), - }, - }, - errors: noerrors, - test: testSyncSnapshot, - },*/ - /*{ - name: "2-8 - snapshot and content bound, apiserver update status error", - initialContents: newContentArrayWithReadyToUse("content2-8", "snapuid2-8", "snap2-8", "sid2-8", validSecretClass, "", "", deletionPolicy, &timeNowStamp, nil, &False, false), - expectedContents: newContentArrayWithReadyToUse("content2-8", "snapuid2-8", "snap2-8", "sid2-8", validSecretClass, "", "", deletionPolicy, &timeNowStamp, nil, &False, false), - initialSnapshots: newSnapshotArray("snap2-8", "snapuid2-8", "claim2-8", "", validSecretClass, "content2-8", &False, metaTimeNow, nil, nil), - expectedSnapshots: newSnapshotArray("snap2-8", "snapuid2-8", "claim2-8", "", validSecretClass, "content2-8", &False, metaTimeNow, nil, newVolumeError("Failed to check and update snapshot: snapshot controller failed to update default/snap2-8 on API server: mock update error")), - expectedEvents: []string{"Warning SnapshotCheckandUpdateFailed"}, - initialClaims: newClaimArray("claim2-8", "pvc-uid2-8", "1Gi", "volume2-8", v1.ClaimBound, &classEmpty), - initialVolumes: newVolumeArray("volume2-8", "pv-uid2-8", "pv-handle2-8", "1Gi", "pvc-uid2-8", "claim2-8", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), - initialSecrets: []*v1.Secret{secret()}, - /*expectedCreateCalls: []createCall{ - { - snapshotName: "snapshot-snapuid2-8", - volume: newVolume("volume2-8", "pv-uid2-8", "pv-handle2-8", "1Gi", "pvc-uid2-8", "claim2-8", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), - parameters: class5Parameters, - secrets: map[string]string{"foo": "bar"}, - // information to return - driverName: mockDriverName, - size: defaultSize, - snapshotId: "sid2-8", - creationTime: timeNow, - readyToUse: true, - }, - },*/ /* errors: []reactorError{ // Inject error to the first client.VolumesnapshotV1beta1().VolumeSnapshots().Update call. // All other calls will succeed. {"update", "volumesnapshots", errors.New("mock update error")}, }, - test: testSyncSnapshot, - },*/ + test: testSyncSnapshotError, + }, { name: "2-9 - fail on status update as there is not pvc provided", - initialContents: newContentArray("content2-9", "snapuid2-9", "snap2-9", "sid2-9", validSecretClass, "", "", deletionPolicy, nil, nil, false), - expectedContents: newContentArray("content2-9", "snapuid2-9", "snap2-9", "sid2-9", validSecretClass, "", "", deletionPolicy, nil, nil, false), + initialContents: nocontents, + expectedContents: nocontents, initialSnapshots: newSnapshotArray("snap2-9", "snapuid2-9", "claim2-9", "", validSecretClass, "", &False, nil, nil, nil, false, true, nil), - expectedSnapshots: newSnapshotArray("snap2-9", "snapuid2-9", "claim2-9", "", validSecretClass, "content2-9", &True, nil, nil, nil, false, true, nil), - //expectedSnapshots: newSnapshotArray("snap2-9", "snapuid2-9", "claim2-9", "", validSecretClass, "content2-9", &False, nil, nil, newVolumeError("Failed to check and update snapshot: failed to get input parameters to create snapshot snap2-9: \"failed to retrieve PVC claim2-9 from the lister: \\\"persistentvolumeclaim \\\\\\\"claim2-9\\\\\\\" not found\\\"\"")), - errors: noerrors, - test: testSyncSnapshot, + expectedSnapshots: newSnapshotArray("snap2-9", "snapuid2-9", "claim2-9", "", validSecretClass, "", &False, nil, nil, newVolumeError("Failed to create snapshot content with error snapshot controller failed to update snap2-9 on API server: cannot get claim from snapshot"), false, true, nil), + errors: []reactorError{ + {"get", "persistentvolumeclaims", errors.New("mock update error")}, + {"get", "persistentvolumeclaims", errors.New("mock update error")}, + {"get", "persistentvolumeclaims", errors.New("mock update error")}, + }, test: testSyncSnapshot, + }, + { + name: "7-1 - fail to create snapshot with non-existing snapshot class", + initialContents: nocontents, + expectedContents: nocontents, + initialSnapshots: newSnapshotArray("snap7-1", "snapuid7-1", "claim7-1", "", classNonExisting, "", &False, nil, nil, nil, false, true, nil), + expectedSnapshots: newSnapshotArray("snap7-1", "snapuid7-1", "claim7-1", "", classNonExisting, "", &False, nil, nil, newVolumeError("Failed to create snapshot content with error failed to get input parameters to create snapshot snap7-1: \"failed to retrieve snapshot class non-existing from the informer: \\\"volumesnapshotclass.snapshot.storage.k8s.io \\\\\\\"non-existing\\\\\\\" not found\\\"\""), false, true, nil), + initialClaims: newClaimArray("claim7-1", "pvc-uid7-1", "1Gi", "volume7-1", v1.ClaimBound, &classEmpty), + initialVolumes: newVolumeArray("volume7-1", "pv-uid7-1", "pv-handle7-1", "1Gi", "pvc-uid7-1", "claim7-1", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), + expectedEvents: []string{"Warning SnapshotContentCreationFailed"}, + errors: noerrors, + expectSuccess: false, + test: testSyncSnapshot, }, { name: "2-10 - do not bind when snapshot and content not match", @@ -210,6 +163,29 @@ func TestSync(t *testing.T) { errors: noerrors, test: testSyncSnapshot, }, + { + name: "2-11 - successful bind snapshot content with volume snapshot classname", + initialContents: withContentSpecSnapshotClassName(newContentArray("content2-11", "snapuid2-11", "snap2-11", "sid2-11", validSecretClass, "", "", deletionPolicy, nil, nil, false), nil), + expectedContents: newContentArray("content2-11", "snapuid2-11", "snap2-11", "sid2-11", validSecretClass, "", "", deletionPolicy, nil, nil, false), + initialSnapshots: newSnapshotArray("snap2-11", "snapuid2-11", "", "content2-11", validSecretClass, "content2-11", &False, nil, nil, nil, false, true, nil), + expectedSnapshots: newSnapshotArray("snap2-11", "snapuid2-11", "", "content2-11", validSecretClass, "content2-11", &True, nil, nil, nil, false, true, nil), + initialClaims: newClaimArray("claim2-11", "pvc-uid2-11", "1Gi", "volume2-11", v1.ClaimBound, &classEmpty), + errors: noerrors, + test: testSyncSnapshot, + }, + { + name: "2-12 - fail bind snapshot content with volume snapshot classname due to API call failed", + initialContents: withContentSpecSnapshotClassName(newContentArray("content2-12", "snapuid2-12", "snap2-12", "sid2-12", validSecretClass, "", "", deletionPolicy, nil, nil, false), nil), + expectedContents: withContentSpecSnapshotClassName(newContentArray("content2-12", "snapuid2-12", "snap2-12", "sid2-12", validSecretClass, "", "", deletionPolicy, nil, nil, false), nil), + initialSnapshots: newSnapshotArray("snap2-12", "snapuid2-12", "", "content2-12", validSecretClass, "content2-12", &False, nil, nil, nil, false, true, nil), + expectedSnapshots: newSnapshotArray("snap2-12", "snapuid2-12", "", "content2-12", validSecretClass, "content2-12", &False, nil, nil, newVolumeError("Snapshot failed to bind VolumeSnapshotContent, mock update error"), false, true, nil), + initialClaims: newClaimArray("claim2-12", "pvc-uid2-12", "1Gi", "volume2-12", v1.ClaimBound, &classEmpty), + errors: []reactorError{ + // Inject error to the forth client.VolumesnapshotV1beta1().VolumeSnapshots().Update call. + {"update", "volumesnapshotcontents", errors.New("mock update error")}, + }, + test: testSyncSnapshot, + }, { name: "3-1 - ready snapshot lost reference to VolumeSnapshotContent", initialContents: nocontents, From 14eaac289b1a700107c11b583b2ee6d2eb611fde Mon Sep 17 00:00:00 2001 From: Alex Szakaly Date: Wed, 4 Mar 2020 22:25:25 +0100 Subject: [PATCH 37/56] doc: add workaround for hack/update-generated-code.sh code-generator incorrectly resolve package path: wants to write into v2/ directory instead of resolving v2 as package path Provide steps to successfully execute code-generator Signed-off-by: Alex Szakaly --- hack/README.md | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/hack/README.md b/hack/README.md index abbf27064..f2189685a 100644 --- a/hack/README.md +++ b/hack/README.md @@ -10,7 +10,35 @@ This is the script to update clientset/informers/listers and API deepcopy code u Make sure to run this script after making changes to /pkg/apis/volumesnapshot/v1beta1/types.go. -To run this script, simply run: ./hack/update-generated-code.sh from the project root directory. +To run this script, you have to patch it: +```patch +diff --git a/hack/update-generated-code.sh b/hack/update-generated-code.sh +--- a/hack/update-generated-code.sh ++++ b/hack/update-generated-code.sh +@@ -27,7 +27,7 @@ CODEGEN_PKG=${CODEGEN_PKG:-$(cd ${SCRIPT_ROOT}; ls -d -1 ./vendor/k8s.io/code-ge + # k8s.io/kubernetes. The output-base is needed for the generators to output into the vendor dir + # instead of the $GOPATH directly. For normal projects this can be dropped. + bash ${CODEGEN_PKG}/generate-groups.sh "deepcopy,client,informer,lister" \ +- github.com/kubernetes-csi/external-snapshotter/pkg/client github.com/kubernetes-csi/external-snapshotter/pkg/apis \ ++ github.com/kubernetes-csi/external-snapshotter/v2/pkg/client github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis \ + volumesnapshot:v1beta1 \ + --go-header-file ${SCRIPT_ROOT}/hack/boilerplate.go.txt +``` + +Once you are done with patching, continue: +```bash +rm -fr v2/ +ln -sfvn $(pwd) v2 +rm -fr pkg/client +``` + +Run: ./hack/update-generated-code.sh from the project root directory. + +Do not forget to revert previously applied workaround: +```bash +git checkout -- v2/ +git checkout -- hack/update-generated-code.sh +``` ## update-crd.sh From fe1c355fa96626a188adb5980e9b763da1e9dcff Mon Sep 17 00:00:00 2001 From: Grant Griffiths Date: Wed, 4 Mar 2020 20:25:31 -0800 Subject: [PATCH 38/56] Add more sidecar tests and cleanup sidecar code Signed-off-by: Grant Griffiths --- pkg/sidecar-controller/content_create_test.go | 174 ++++++++++++++++-- pkg/sidecar-controller/framework_test.go | 56 ++++-- pkg/sidecar-controller/snapshot_controller.go | 17 +- .../snapshot_delete_test.go | 10 +- 4 files changed, 210 insertions(+), 47 deletions(-) diff --git a/pkg/sidecar-controller/content_create_test.go b/pkg/sidecar-controller/content_create_test.go index 7358c693a..5185b5ed2 100644 --- a/pkg/sidecar-controller/content_create_test.go +++ b/pkg/sidecar-controller/content_create_test.go @@ -17,32 +17,168 @@ limitations under the License. package sidecar_controller import ( + "errors" "testing" "time" + + crdv1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" + "github.com/kubernetes-csi/external-snapshotter/v2/pkg/utils" + v1 "k8s.io/api/core/v1" ) func TestSyncContent(t *testing.T) { - var tests []controllerTest - - tests = append(tests, controllerTest{ - name: "Basic content update ready to use", - initialContents: newContentArrayWithReadyToUse("content1-1", "snapuid1-1", "snap1-1", "sid1-1", defaultClass, "", "volume-handle-1-1", retainPolicy, nil, &defaultSize, &False, true), - expectedContents: newContentArrayWithReadyToUse("content1-1", "snapuid1-1", "snap1-1", "sid1-1", defaultClass, "", "volume-handle-1-1", retainPolicy, nil, &defaultSize, &True, true), - expectedEvents: noevents, - expectedCreateCalls: []createCall{ - { - volumeHandle: "volume-handle-1-1", - snapshotName: "snapshot-snapuid1-1", - driverName: mockDriverName, - snapshotId: "snapuid1-1", - creationTime: timeNow, - readyToUse: true, + + tests := []controllerTest{ + { + name: "1-1: Basic content update ready to use", + initialContents: newContentArrayWithReadyToUse("content1-1", "snapuid1-1", "snap1-1", "sid1-1", defaultClass, "", "volume-handle-1-1", retainPolicy, nil, &defaultSize, &False, true), + expectedContents: newContentArrayWithReadyToUse("content1-1", "snapuid1-1", "snap1-1", "sid1-1", defaultClass, "", "volume-handle-1-1", retainPolicy, nil, &defaultSize, &True, true), + expectedEvents: noevents, + expectedCreateCalls: []createCall{ + { + volumeHandle: "volume-handle-1-1", + snapshotName: "snapshot-snapuid1-1", + driverName: mockDriverName, + snapshotId: "snapuid1-1", + creationTime: timeNow, + readyToUse: true, + }, + }, + expectedListCalls: []listCall{{"sid1-1", map[string]string{}, true, time.Now(), 1, nil}}, + errors: noerrors, + test: testSyncContent, + }, + { + name: "1-2: Basic sync content create snapshot", + initialContents: withContentStatus(newContentArray("content1-2", "snapuid1-2", "snap1-2", "sid1-2", defaultClass, "", "volume-handle-1-2", retainPolicy, nil, &defaultSize, true), + nil), + expectedContents: withContentStatus(newContentArray("content1-2", "snapuid1-2", "snap1-2", "sid1-2", defaultClass, "", "volume-handle-1-2", retainPolicy, nil, &defaultSize, true), + &crdv1.VolumeSnapshotContentStatus{SnapshotHandle: toStringPointer("snapuid1-2"), RestoreSize: &defaultSize, ReadyToUse: &True}), + expectedEvents: noevents, + expectedCreateCalls: []createCall{ + { + volumeHandle: "volume-handle-1-2", + snapshotName: "snapshot-snapuid1-2", + driverName: mockDriverName, + snapshotId: "snapuid1-2", + creationTime: timeNow, + readyToUse: true, + size: defaultSize, + }, + }, + expectedListCalls: []listCall{{"sid1-2", map[string]string{}, true, time.Now(), 1, nil}}, + errors: noerrors, + test: testSyncContent, + }, + { + name: "1-3: Basic sync content create snapshot with non-existent secret", + initialContents: withContentAnnotations(withContentStatus(newContentArray("content1-3", "snapuid1-3", "snap1-3", "sid1-3", invalidSecretClass, "", "volume-handle-1-3", retainPolicy, nil, &defaultSize, true), + nil), map[string]string{ + utils.AnnDeletionSecretRefName: "", + utils.AnnDeletionSecretRefNamespace: "", + }), + expectedContents: withContentAnnotations(withContentStatus(newContentArray("content1-3", "snapuid1-3", "snap1-3", "sid1-3", invalidSecretClass, "", "volume-handle-1-3", retainPolicy, nil, &defaultSize, true), + &crdv1.VolumeSnapshotContentStatus{ + SnapshotHandle: nil, + RestoreSize: nil, + ReadyToUse: &False, + Error: newSnapshotError("Failed to create snapshot: failed to get input parameters to create snapshot for content content1-3: \"cannot retrieve secrets for snapshot content \\\"content1-3\\\", err: secret name or namespace not specified\""), + }), map[string]string{ + utils.AnnDeletionSecretRefName: "", + utils.AnnDeletionSecretRefNamespace: "", + }), initialSecrets: []*v1.Secret{}, // no initial secret created + expectedEvents: []string{"Warning SnapshotCreationFailed"}, + errors: noerrors, + test: testSyncContent, + }, + { + name: "1-4: Basic sync content create snapshot with valid secret", + initialContents: withContentAnnotations(withContentStatus(newContentArray("content1-4", "snapuid1-4", "snap1-4", "sid1-4", validSecretClass, "", "volume-handle-1-4", retainPolicy, nil, &defaultSize, true), + nil), map[string]string{ + utils.AnnDeletionSecretRefName: "secret", + utils.AnnDeletionSecretRefNamespace: "default", + }), + expectedContents: withContentAnnotations(withContentStatus(newContentArray("content1-4", "snapuid1-4", "snap1-4", "sid1-4", validSecretClass, "", "volume-handle-1-4", retainPolicy, nil, &defaultSize, true), + &crdv1.VolumeSnapshotContentStatus{ + SnapshotHandle: toStringPointer("snapuid1-4"), + RestoreSize: &defaultSize, + ReadyToUse: &True, + Error: nil, + }), map[string]string{ + utils.AnnDeletionSecretRefName: "secret", + utils.AnnDeletionSecretRefNamespace: "default", + }), + expectedCreateCalls: []createCall{ + { + volumeHandle: "volume-handle-1-4", + snapshotName: "snapshot-snapuid1-4", + parameters: class5Parameters, + secrets: map[string]string{ + "foo": "bar", + }, + driverName: mockDriverName, + snapshotId: "snapuid1-4", + creationTime: timeNow, + readyToUse: true, + size: defaultSize, + }, + }, + initialSecrets: []*v1.Secret{secret()}, + expectedEvents: noevents, + errors: noerrors, + test: testSyncContent, + }, + { + name: "1-5: Basic sync content create snapshot with failed secret call", + initialContents: withContentAnnotations(withContentStatus(newContentArray("content1-5", "snapuid1-5", "snap1-5", "sid1-5", invalidSecretClass, "", "volume-handle-1-5", retainPolicy, nil, &defaultSize, true), + nil), map[string]string{ + utils.AnnDeletionSecretRefName: "secret", + utils.AnnDeletionSecretRefNamespace: "default", + }), + expectedContents: withContentAnnotations(withContentStatus(newContentArray("content1-5", "snapuid1-5", "snap1-5", "sid1-5", invalidSecretClass, "", "volume-handle-1-5", retainPolicy, nil, &defaultSize, true), + &crdv1.VolumeSnapshotContentStatus{ + SnapshotHandle: nil, + RestoreSize: nil, + ReadyToUse: &False, + Error: newSnapshotError("Failed to create snapshot: failed to get input parameters to create snapshot for content content1-5: \"cannot get credentials for snapshot content \\\"content1-5\\\"\""), + }), map[string]string{ + utils.AnnDeletionSecretRefName: "secret", + utils.AnnDeletionSecretRefNamespace: "default", + }), initialSecrets: []*v1.Secret{}, // no initial secret created + expectedEvents: []string{"Warning SnapshotCreationFailed"}, + errors: []reactorError{ + // Inject error to the first client.VolumesnapshotV1beta1().VolumeSnapshots().Update call. + // All other calls will succeed. + {"get", "secrets", errors.New("mock secrets error")}, + }, + test: testSyncContent, + }, + { + name: "1-6: Basic content update ready to use bad snapshot class", + initialContents: newContentArrayWithReadyToUse("content1-6", "snapuid1-6", "snap1-6", "sid1-6", "bad-class", "", "volume-handle-1-6", retainPolicy, nil, &defaultSize, &False, true), + expectedContents: withContentStatus(newContentArray("content1-6", "snapuid1-6", "snap1-6", "sid1-6", "bad-class", "", "volume-handle-1-6", retainPolicy, nil, &defaultSize, true), + &crdv1.VolumeSnapshotContentStatus{ + SnapshotHandle: toStringPointer("sid1-6"), + RestoreSize: &defaultSize, + ReadyToUse: &False, + Error: newSnapshotError("Failed to check and update snapshot content: failed to get input parameters to create snapshot content1-6: \"failed to retrieve snapshot class bad-class from the informer: \\\"volumesnapshotclass.snapshot.storage.k8s.io \\\\\\\"bad-class\\\\\\\" not found\\\"\""), + }), + expectedEvents: []string{"Warning SnapshotContentCheckandUpdateFailed"}, + expectedCreateCalls: []createCall{ + { + volumeHandle: "volume-handle-1-6", + snapshotName: "snapshot-snapuid1-6", + driverName: mockDriverName, + snapshotId: "snapuid1-6", + creationTime: timeNow, + readyToUse: true, + }, }, + expectedListCalls: []listCall{{"sid1-6", map[string]string{}, true, time.Now(), 1, nil}}, + errors: noerrors, + test: testSyncContent, }, - expectedListCalls: []listCall{{"sid1-1", map[string]string{}, true, time.Now(), 1, nil}}, - errors: noerrors, - test: testSyncContent, - }) + } runSyncContentTests(t, tests, snapshotClasses) } diff --git a/pkg/sidecar-controller/framework_test.go b/pkg/sidecar-controller/framework_test.go index d59f0f4a6..264b989ac 100644 --- a/pkg/sidecar-controller/framework_test.go +++ b/pkg/sidecar-controller/framework_test.go @@ -123,6 +123,7 @@ var noerrors = []reactorError{} // the list. type snapshotReactor struct { secrets map[string]*v1.Secret + snapshotClasses map[string]*crdv1.VolumeSnapshotClass contents map[string]*crdv1.VolumeSnapshotContent changedObjects []interface{} changedSinceLastSync int @@ -283,7 +284,12 @@ func (r *snapshotReactor) checkContents(expectedContents []*crdv1.VolumeSnapshot v := v.DeepCopy() v.ResourceVersion = "" v.Spec.VolumeSnapshotRef.ResourceVersion = "" - v.Status.CreationTime = nil + if v.Status != nil { + v.Status.CreationTime = nil + } + if v.Status.Error != nil { + v.Status.Error.Time = &metav1.Time{} + } expectedMap[v.Name] = v } for _, v := range r.contents { @@ -292,7 +298,13 @@ func (r *snapshotReactor) checkContents(expectedContents []*crdv1.VolumeSnapshot v := v.DeepCopy() v.ResourceVersion = "" v.Spec.VolumeSnapshotRef.ResourceVersion = "" - v.Status.CreationTime = nil + if v.Status != nil { + v.Status.CreationTime = nil + if v.Status.Error != nil { + v.Status.Error.Time = &metav1.Time{} + } + } + gotMap[v.Name] = v } if !reflect.DeepEqual(expectedMap, gotMap) { @@ -487,6 +499,7 @@ func (r *snapshotReactor) modifyContentEvent(content *crdv1.VolumeSnapshotConten func newSnapshotReactor(kubeClient *kubefake.Clientset, client *fake.Clientset, ctrl *csiSnapshotSideCarController, fakeVolumeWatch, fakeClaimWatch *watch.FakeWatcher, errors []reactorError) *snapshotReactor { reactor := &snapshotReactor{ secrets: make(map[string]*v1.Secret), + snapshotClasses: make(map[string]*crdv1.VolumeSnapshotClass), contents: make(map[string]*crdv1.VolumeSnapshotContent), ctrl: ctrl, fakeContentWatch: fakeVolumeWatch, @@ -497,6 +510,7 @@ func newSnapshotReactor(kubeClient *kubefake.Clientset, client *fake.Clientset, client.AddReactor("update", "volumesnapshotcontents", reactor.React) client.AddReactor("get", "volumesnapshotcontents", reactor.React) client.AddReactor("delete", "volumesnapshotcontents", reactor.React) + kubeClient.AddReactor("get", "secrets", reactor.React) return reactor } @@ -615,16 +629,6 @@ func newContentArrayWithReadyToUse(contentName, boundToSnapshotUID, boundToSnaps } } -func newContentWithUnmatchDriverArray(contentName, boundToSnapshotUID, boundToSnapshotName, snapshotHandle, snapshotClassName, desiredSnapshotHandle, volumeHandle string, - deletionPolicy crdv1.DeletionPolicy, size, creationTime *int64, - withFinalizer bool) []*crdv1.VolumeSnapshotContent { - content := newContent(contentName, boundToSnapshotUID, boundToSnapshotName, snapshotHandle, snapshotClassName, desiredSnapshotHandle, volumeHandle, deletionPolicy, size, creationTime, withFinalizer, nil) - content.Spec.Driver = "fake" - return []*crdv1.VolumeSnapshotContent{ - content, - } -} - func newContentArrayWithDeletionTimestamp(contentName, boundToSnapshotUID, boundToSnapshotName, snapshotHandle, snapshotClassName, desiredSnapshotHandle, volumeHandle string, deletionPolicy crdv1.DeletionPolicy, size, creationTime *int64, withFinalizer bool, deletionTime *metav1.Time) []*crdv1.VolumeSnapshotContent { @@ -633,6 +637,22 @@ func newContentArrayWithDeletionTimestamp(contentName, boundToSnapshotUID, bound } } +func withContentStatus(content []*crdv1.VolumeSnapshotContent, status *crdv1.VolumeSnapshotContentStatus) []*crdv1.VolumeSnapshotContent { + for i := range content { + content[i].Status = status + } + + return content +} + +func withContentAnnotations(content []*crdv1.VolumeSnapshotContent, annotations map[string]string) []*crdv1.VolumeSnapshotContent { + for i := range content { + content[i].ObjectMeta.Annotations = annotations + } + + return content +} + func testSyncContent(ctrl *csiSnapshotSideCarController, reactor *snapshotReactor, test controllerTest) error { return ctrl.syncContent(test.initialContents[0]) } @@ -742,6 +762,7 @@ func runSyncContentTests(t *testing.T, tests []controllerTest, snapshotClasses [ indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{}) for _, class := range snapshotClasses { indexer.Add(class) + reactor.snapshotClasses[class.Name] = class } ctrl.classLister = storagelisters.NewVolumeSnapshotClassLister(indexer) @@ -875,7 +896,7 @@ func (f *fakeSnapshotter) CreateSnapshot(ctx context.Context, snapshotName strin err = fmt.Errorf("unexpected create snapshot call") } - if !reflect.DeepEqual(call.secrets, snapshotterCredentials) { + if !reflect.DeepEqual(call.secrets, snapshotterCredentials) && !(len(call.secrets) == 0 && len(snapshotterCredentials) == 0) { f.t.Errorf("Wrong CSI Create Snapshot call: snapshotName=%s, volumeHandle=%s, expected secrets %+v, got %+v", snapshotName, volumeHandle, call.secrets, snapshotterCredentials) err = fmt.Errorf("unexpected create snapshot call") } @@ -937,3 +958,12 @@ func (f *fakeSnapshotter) GetSnapshotStatus(ctx context.Context, snapshotID stri return call.readyToUse, call.createTime, call.size, call.err } + +func newSnapshotError(message string) *crdv1.VolumeSnapshotError { + return &crdv1.VolumeSnapshotError{ + Time: &metav1.Time{}, + Message: &message, + } +} + +func toStringPointer(str string) *string { return &str } diff --git a/pkg/sidecar-controller/snapshot_controller.go b/pkg/sidecar-controller/snapshot_controller.go index bf02e70ba..8c2963b68 100644 --- a/pkg/sidecar-controller/snapshot_controller.go +++ b/pkg/sidecar-controller/snapshot_controller.go @@ -55,7 +55,6 @@ const controllerUpdateFailMsg = "snapshot controller failed to update" func (ctrl *csiSnapshotSideCarController) syncContent(content *crdv1.VolumeSnapshotContent) error { klog.V(5).Infof("synchronizing VolumeSnapshotContent[%s]", content.Name) - var err error if ctrl.shouldDelete(content) { klog.V(4).Infof("VolumeSnapshotContent[%s]: the policy is %s", content.Name, content.Spec.DeletionPolicy) if content.Spec.DeletionPolicy == crdv1.VolumeSnapshotContentDelete && @@ -75,10 +74,7 @@ func (ctrl *csiSnapshotSideCarController) syncContent(content *crdv1.VolumeSnaps } else { if content.Spec.Source.VolumeHandle != nil && content.Status == nil { klog.V(5).Infof("syncContent: Call CreateSnapshot for content %s", content.Name) - if err = ctrl.createSnapshot(content); err != nil { - ctrl.updateContentErrorStatusWithEvent(content, v1.EventTypeWarning, "SnapshotCreationFailed", fmt.Sprintf("Failed to create snapshot with error %v", err)) - return err - } + ctrl.createSnapshot(content) } else { // Skip checkandUpdateContentStatus() if ReadyToUse is // already true. We don't want to keep calling CreateSnapshot @@ -87,10 +83,7 @@ func (ctrl *csiSnapshotSideCarController) syncContent(content *crdv1.VolumeSnaps if content.Status != nil && content.Status.ReadyToUse != nil && *content.Status.ReadyToUse == true { return nil } - if err = ctrl.checkandUpdateContentStatus(content); err != nil { - ctrl.updateContentErrorStatusWithEvent(content, v1.EventTypeWarning, "SnapshotContentStatusUpdateFailed", fmt.Sprintf("Failed to update snapshot content status with error %v", err)) - return err - } + ctrl.checkandUpdateContentStatus(content) } } return nil @@ -129,7 +122,7 @@ func (ctrl *csiSnapshotSideCarController) storeContentUpdate(content interface{} } // createSnapshot starts new asynchronous operation to create snapshot -func (ctrl *csiSnapshotSideCarController) createSnapshot(content *crdv1.VolumeSnapshotContent) error { +func (ctrl *csiSnapshotSideCarController) createSnapshot(content *crdv1.VolumeSnapshotContent) { klog.V(5).Infof("createSnapshot for content [%s]: started", content.Name) opName := fmt.Sprintf("create-%s", content.Name) ctrl.scheduleOperation(opName, func() error { @@ -147,10 +140,9 @@ func (ctrl *csiSnapshotSideCarController) createSnapshot(content *crdv1.VolumeSn } return nil }) - return nil } -func (ctrl *csiSnapshotSideCarController) checkandUpdateContentStatus(content *crdv1.VolumeSnapshotContent) error { +func (ctrl *csiSnapshotSideCarController) checkandUpdateContentStatus(content *crdv1.VolumeSnapshotContent) { klog.V(5).Infof("checkandUpdateContentStatus[%s] started", content.Name) opName := fmt.Sprintf("check-%s", content.Name) ctrl.scheduleOperation(opName, func() error { @@ -168,7 +160,6 @@ func (ctrl *csiSnapshotSideCarController) checkandUpdateContentStatus(content *c return nil }) - return nil } // updateContentStatusWithEvent saves new content.Status to API server and emits diff --git a/pkg/sidecar-controller/snapshot_delete_test.go b/pkg/sidecar-controller/snapshot_delete_test.go index 5c4310371..ae25a8bb8 100644 --- a/pkg/sidecar-controller/snapshot_delete_test.go +++ b/pkg/sidecar-controller/snapshot_delete_test.go @@ -68,6 +68,11 @@ var class6Parameters = map[string]string{ utils.PrefixedSnapshotterListSecretNamespaceKey: "default", } +var class7Annotations = map[string]string{ + utils.AnnDeletionSecretRefName: "secret-x", + utils.AnnDeletionSecretRefNamespace: "default-x", +} + var snapshotClasses = []*crdv1.VolumeSnapshotClass{ { TypeMeta: metav1.TypeMeta{ @@ -107,10 +112,11 @@ var snapshotClasses = []*crdv1.VolumeSnapshotClass{ Kind: "VolumeSnapshotClass", }, ObjectMeta: metav1.ObjectMeta{ - Name: invalidSecretClass, + Name: invalidSecretClass, + Annotations: class7Annotations, }, Driver: mockDriverName, - Parameters: class3Parameters, + Parameters: class2Parameters, DeletionPolicy: crdv1.VolumeSnapshotContentDelete, }, { From 5d722431cd20847102d73d5a83f2194023045fe4 Mon Sep 17 00:00:00 2001 From: xing-yang Date: Tue, 7 Jan 2020 20:18:20 +0000 Subject: [PATCH 39/56] Update images --- deploy/kubernetes/csi-snapshotter/setup-csi-snapshotter.yaml | 5 ++--- .../snapshot-controller/setup-snapshot-controller.yaml | 3 +-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/deploy/kubernetes/csi-snapshotter/setup-csi-snapshotter.yaml b/deploy/kubernetes/csi-snapshotter/setup-csi-snapshotter.yaml index 271ef9b05..5b2e2403b 100644 --- a/deploy/kubernetes/csi-snapshotter/setup-csi-snapshotter.yaml +++ b/deploy/kubernetes/csi-snapshotter/setup-csi-snapshotter.yaml @@ -72,7 +72,7 @@ spec: serviceAccount: csi-snapshotter containers: - name: csi-provisioner - image: quay.io/k8scsi/csi-provisioner:v1.5.0-rc1 + image: quay.io/k8scsi/csi-provisioner:v1.5.0 args: - "--v=5" - "--csi-address=$(ADDRESS)" @@ -84,8 +84,7 @@ spec: - name: socket-dir mountPath: /csi - name: csi-snapshotter - # NOTE: replace with official image when released: quay.io/k8scsi/csi-snapshotter:v2.0.0 - image: quay.io/k8scsi/csi-snapshotter:v2.0.0-rc4 + image: quay.io/k8scsi/csi-snapshotter:v2.0.1 args: - "--v=5" - "--csi-address=$(ADDRESS)" diff --git a/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml b/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml index f3adafa6b..6d000f68c 100644 --- a/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml +++ b/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml @@ -19,8 +19,7 @@ spec: serviceAccount: snapshot-controller containers: - name: snapshot-controller - # NOTE: replace with official image when released: quay.io/k8scsi/snapshot-controller:v2.0.0 - image: quay.io/k8scsi/snapshot-controller:v2.0.0-rc4 + image: quay.io/k8scsi/snapshot-controller:v2.0.1 args: - "--v=5" - "--leader-election=false" From 806ccc0b29fb734e05c53da49829a27b7f68be2a Mon Sep 17 00:00:00 2001 From: Christian Huffman Date: Wed, 18 Mar 2020 11:12:23 -0400 Subject: [PATCH 40/56] Include a nil check for the snapshotContent --- pkg/common-controller/snapshot_controller.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/common-controller/snapshot_controller.go b/pkg/common-controller/snapshot_controller.go index 0a4c7fa54..cc3b00665 100644 --- a/pkg/common-controller/snapshot_controller.go +++ b/pkg/common-controller/snapshot_controller.go @@ -23,7 +23,7 @@ import ( crdv1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" "github.com/kubernetes-csi/external-snapshotter/v2/pkg/utils" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" @@ -266,7 +266,7 @@ func (ctrl *csiSnapshotCommonController) checkandRemoveSnapshotFinalizersAndChec } if !inUse { - klog.V(5).Infof("checkandRemoveSnapshotFinalizersAndCheckandDeleteContent: Set VolumeSnapshotBeingDeleted annotation on the content [%s]", content.Name) + klog.V(5).Infof("checkandRemoveSnapshotFinalizersAndCheckandDeleteContent: Set VolumeSnapshotBeingDeleted annotation on the content") ctrl.setAnnVolumeSnapshotBeingDeleted(content) klog.V(5).Infof("checkandRemoveSnapshotFinalizersAndCheckandDeleteContent: Remove Finalizer for VolumeSnapshot[%s]", utils.SnapshotKey(snapshot)) @@ -1281,6 +1281,9 @@ func (ctrl *csiSnapshotCommonController) getSnapshotFromStore(snapshotName strin func (ctrl *csiSnapshotCommonController) setAnnVolumeSnapshotBeingDeleted(content *crdv1.VolumeSnapshotContent) error { // Set AnnVolumeSnapshotBeingDeleted if it is not set yet if !metav1.HasAnnotation(content.ObjectMeta, utils.AnnVolumeSnapshotBeingDeleted) { + if content == nil { + return nil + } klog.V(5).Infof("setAnnVolumeSnapshotBeingDeleted: set annotation [%s] on content [%s].", utils.AnnVolumeSnapshotBeingDeleted, content.Name) metav1.SetMetaDataAnnotation(&content.ObjectMeta, utils.AnnVolumeSnapshotBeingDeleted, "yes") From 91a55c0e1341582d3a9d69f654c3b1420668637b Mon Sep 17 00:00:00 2001 From: Christian Huffman Date: Fri, 13 Mar 2020 11:36:04 -0400 Subject: [PATCH 41/56] Allows VolumeSnapshot to be deleted if the class isn't found --- pkg/common-controller/snapshot_controller.go | 4 ++-- .../snapshot_controller_base.go | 17 ++++++++++++----- pkg/common-controller/snapshot_create_test.go | 2 +- pkg/common-controller/snapshot_update_test.go | 2 +- pkg/common-controller/snapshotclass_test.go | 2 +- 5 files changed, 17 insertions(+), 10 deletions(-) diff --git a/pkg/common-controller/snapshot_controller.go b/pkg/common-controller/snapshot_controller.go index 0a4c7fa54..0d136ede0 100644 --- a/pkg/common-controller/snapshot_controller.go +++ b/pkg/common-controller/snapshot_controller.go @@ -23,7 +23,7 @@ import ( crdv1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" "github.com/kubernetes-csi/external-snapshotter/v2/pkg/utils" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" @@ -1076,7 +1076,7 @@ func (ctrl *csiSnapshotCommonController) getSnapshotClass(className string) (*cr class, err := ctrl.classLister.Get(className) if err != nil { klog.Errorf("failed to retrieve snapshot class %s from the informer: %q", className, err) - return nil, fmt.Errorf("failed to retrieve snapshot class %s from the informer: %q", className, err) + return nil, err } return class, nil diff --git a/pkg/common-controller/snapshot_controller_base.go b/pkg/common-controller/snapshot_controller_base.go index 97659d73b..01d6c596a 100644 --- a/pkg/common-controller/snapshot_controller_base.go +++ b/pkg/common-controller/snapshot_controller_base.go @@ -26,7 +26,7 @@ import ( storagelisters "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/listers/volumesnapshot/v1beta1" "github.com/kubernetes-csi/external-snapshotter/v2/pkg/utils" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/wait" @@ -215,8 +215,11 @@ func (ctrl *csiSnapshotCommonController) snapshotWorker() { // The volume snapshot still exists in informer cache, the event must have // been add/update/sync newSnapshot, err := ctrl.checkAndUpdateSnapshotClass(snapshot) - if err == nil { - klog.V(5).Infof("passed checkAndUpdateSnapshotClass for snapshot %q", key) + if err == nil || (newSnapshot.ObjectMeta.DeletionTimestamp != nil && errors.IsNotFound(err)) { + // If the VolumeSnapshotClass is not found, we still need to process an update + // so that syncSnapshot can delete the snapshot, should it still exist in the + // cluster after it's been removed from the informer cache + klog.V(5).Infof("updating snapshot %q; snapshotClass may have already been removed", key) ctrl.updateSnapshot(newSnapshot) } return false @@ -243,7 +246,10 @@ func (ctrl *csiSnapshotCommonController) snapshotWorker() { return false } newSnapshot, err := ctrl.checkAndUpdateSnapshotClass(snapshot) - if err == nil { + if err == nil || errors.IsNotFound(err) { + // We should still handle deletion events even if the VolumeSnapshotClass + // is not found in the cluster + klog.V(5).Infof("deleting snapshot %q; snapshotClass may have already been removed", key) ctrl.deleteSnapshot(newSnapshot) } return false @@ -329,7 +335,8 @@ func (ctrl *csiSnapshotCommonController) checkAndUpdateSnapshotClass(snapshot *c if err != nil { klog.Errorf("checkAndUpdateSnapshotClass failed to getSnapshotClass %v", err) ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "GetSnapshotClassFailed", fmt.Sprintf("Failed to get snapshot class with error %v", err)) - return nil, err + // we need to return the original snapshot even if the class isn't found, as it may need to be deleted + return newSnapshot, err } } else { klog.V(5).Infof("checkAndUpdateSnapshotClass [%s]: SetDefaultSnapshotClass", snapshot.Name) diff --git a/pkg/common-controller/snapshot_create_test.go b/pkg/common-controller/snapshot_create_test.go index f0a3b0a80..6573db982 100644 --- a/pkg/common-controller/snapshot_create_test.go +++ b/pkg/common-controller/snapshot_create_test.go @@ -99,7 +99,7 @@ func TestCreateSnapshotSync(t *testing.T) { initialContents: nocontents, expectedContents: nocontents, initialSnapshots: newSnapshotArray("snap7-1", "snapuid7-1", "claim7-1", "", classNonExisting, "", &False, nil, nil, nil, false, true, nil), - expectedSnapshots: newSnapshotArray("snap7-1", "snapuid7-1", "claim7-1", "", classNonExisting, "", &False, nil, nil, newVolumeError("Failed to create snapshot content with error failed to get input parameters to create snapshot snap7-1: \"failed to retrieve snapshot class non-existing from the informer: \\\"volumesnapshotclass.snapshot.storage.k8s.io \\\\\\\"non-existing\\\\\\\" not found\\\"\""), false, true, nil), + expectedSnapshots: newSnapshotArray("snap7-1", "snapuid7-1", "claim7-1", "", classNonExisting, "", &False, nil, nil, newVolumeError("Failed to create snapshot content with error failed to get input parameters to create snapshot snap7-1: \"volumesnapshotclass.snapshot.storage.k8s.io \\\"non-existing\\\" not found\""), false, true, nil), initialClaims: newClaimArray("claim7-1", "pvc-uid7-1", "1Gi", "volume7-1", v1.ClaimBound, &classEmpty), initialVolumes: newVolumeArray("volume7-1", "pv-uid7-1", "pv-handle7-1", "1Gi", "pvc-uid7-1", "claim7-1", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), expectedEvents: []string{"Warning SnapshotContentCreationFailed"}, diff --git a/pkg/common-controller/snapshot_update_test.go b/pkg/common-controller/snapshot_update_test.go index 06cdb7e1c..7f71ef063 100644 --- a/pkg/common-controller/snapshot_update_test.go +++ b/pkg/common-controller/snapshot_update_test.go @@ -146,7 +146,7 @@ func TestSync(t *testing.T) { initialContents: nocontents, expectedContents: nocontents, initialSnapshots: newSnapshotArray("snap7-1", "snapuid7-1", "claim7-1", "", classNonExisting, "", &False, nil, nil, nil, false, true, nil), - expectedSnapshots: newSnapshotArray("snap7-1", "snapuid7-1", "claim7-1", "", classNonExisting, "", &False, nil, nil, newVolumeError("Failed to create snapshot content with error failed to get input parameters to create snapshot snap7-1: \"failed to retrieve snapshot class non-existing from the informer: \\\"volumesnapshotclass.snapshot.storage.k8s.io \\\\\\\"non-existing\\\\\\\" not found\\\"\""), false, true, nil), + expectedSnapshots: newSnapshotArray("snap7-1", "snapuid7-1", "claim7-1", "", classNonExisting, "", &False, nil, nil, newVolumeError("Failed to create snapshot content with error failed to get input parameters to create snapshot snap7-1: \"volumesnapshotclass.snapshot.storage.k8s.io \\\"non-existing\\\" not found\""), false, true, nil), initialClaims: newClaimArray("claim7-1", "pvc-uid7-1", "1Gi", "volume7-1", v1.ClaimBound, &classEmpty), initialVolumes: newVolumeArray("volume7-1", "pv-uid7-1", "pv-handle7-1", "1Gi", "pvc-uid7-1", "claim7-1", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), expectedEvents: []string{"Warning SnapshotContentCreationFailed"}, diff --git a/pkg/common-controller/snapshotclass_test.go b/pkg/common-controller/snapshotclass_test.go index 72ea3b5ae..83fdfb4fe 100644 --- a/pkg/common-controller/snapshotclass_test.go +++ b/pkg/common-controller/snapshotclass_test.go @@ -63,7 +63,7 @@ func TestUpdateSnapshotClass(t *testing.T) { name: "1-3 - snapshot class name not found", initialContents: nocontents, initialSnapshots: newSnapshotArray("snap1-1", "snapuid1-1", "claim1-1", "content1-1", "missing-class", "content1-1", &True, nil, nil, nil, false, true, nil), - expectedSnapshots: newSnapshotArray("snap1-1", "snapuid1-1", "claim1-1", "content1-1", "missing-class", "content1-1", &False, nil, nil, newVolumeError("Failed to get snapshot class with error failed to retrieve snapshot class missing-class from the informer: \"volumesnapshotclass.snapshot.storage.k8s.io \\\"missing-class\\\" not found\""), false, true, nil), + expectedSnapshots: newSnapshotArray("snap1-1", "snapuid1-1", "claim1-1", "content1-1", "missing-class", "content1-1", &False, nil, nil, newVolumeError("Failed to get snapshot class with error volumesnapshotclass.snapshot.storage.k8s.io \"missing-class\" not found"), false, true, nil), initialClaims: newClaimArray("claim1-1", "pvc-uid1-1", "1Gi", "volume1-1", v1.ClaimBound, &sameDriver), initialVolumes: newVolumeArray("volume1-1", "pv-uid1-1", "pv-handle1-1", "1Gi", "pvc-uid1-1", "claim1-1", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), initialStorageClasses: []*storage.StorageClass{sameDriverStorageClass}, From 25b65ea8a838249f1b321cc0d6cd409a9e00524c Mon Sep 17 00:00:00 2001 From: Christian Huffman Date: Thu, 19 Mar 2020 16:21:46 -0400 Subject: [PATCH 42/56] Moving nil check to correct location --- pkg/common-controller/snapshot_controller.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/common-controller/snapshot_controller.go b/pkg/common-controller/snapshot_controller.go index a5ae32132..467f3240e 100644 --- a/pkg/common-controller/snapshot_controller.go +++ b/pkg/common-controller/snapshot_controller.go @@ -1279,11 +1279,11 @@ func (ctrl *csiSnapshotCommonController) getSnapshotFromStore(snapshotName strin } func (ctrl *csiSnapshotCommonController) setAnnVolumeSnapshotBeingDeleted(content *crdv1.VolumeSnapshotContent) error { + if content == nil { + return nil + } // Set AnnVolumeSnapshotBeingDeleted if it is not set yet if !metav1.HasAnnotation(content.ObjectMeta, utils.AnnVolumeSnapshotBeingDeleted) { - if content == nil { - return nil - } klog.V(5).Infof("setAnnVolumeSnapshotBeingDeleted: set annotation [%s] on content [%s].", utils.AnnVolumeSnapshotBeingDeleted, content.Name) metav1.SetMetaDataAnnotation(&content.ObjectMeta, utils.AnnVolumeSnapshotBeingDeleted, "yes") From 12f4e1c114790f2875b5277159813c385b6effc3 Mon Sep 17 00:00:00 2001 From: Christian Huffman Date: Mon, 30 Mar 2020 16:13:11 -0400 Subject: [PATCH 43/56] Makes the number of worker threads configurable. --- cmd/csi-snapshotter/main.go | 6 ++---- cmd/snapshot-controller/main.go | 8 ++------ 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/cmd/csi-snapshotter/main.go b/cmd/csi-snapshotter/main.go index 82f584361..74b6ffc94 100644 --- a/cmd/csi-snapshotter/main.go +++ b/cmd/csi-snapshotter/main.go @@ -48,9 +48,6 @@ import ( ) const ( - // Number of worker threads - threads = 10 - // Default timeout of short CSI calls like GetPluginInfo defaultCSITimeout = time.Minute ) @@ -63,6 +60,7 @@ var ( snapshotNamePrefix = flag.String("snapshot-name-prefix", "snapshot", "Prefix to apply to the name of a created snapshot") snapshotNameUUIDLength = flag.Int("snapshot-name-uuid-length", -1, "Length in characters for the generated uuid of a created snapshot. Defaults behavior is to NOT truncate.") showVersion = flag.Bool("version", false, "Show version.") + threads = flag.Int("worker-threads", 10, "Number of worker threads.") csiTimeout = flag.Duration("timeout", defaultCSITimeout, "The timeout for any RPCs to the CSI driver. Default is 1 minute.") leaderElection = flag.Bool("leader-election", false, "Enables leader election.") @@ -182,7 +180,7 @@ func main() { stopCh := make(chan struct{}) factory.Start(stopCh) coreFactory.Start(stopCh) - go ctrl.Run(threads, stopCh) + go ctrl.Run(*threads, stopCh) // ...until SIGINT c := make(chan os.Signal, 1) diff --git a/cmd/snapshot-controller/main.go b/cmd/snapshot-controller/main.go index 0831ae659..a8a87de3b 100644 --- a/cmd/snapshot-controller/main.go +++ b/cmd/snapshot-controller/main.go @@ -39,16 +39,12 @@ import ( coreinformers "k8s.io/client-go/informers" ) -const ( - // Number of worker threads - threads = 10 -) - // Command line flags var ( kubeconfig = flag.String("kubeconfig", "", "Absolute path to the kubeconfig file. Required only when running out of cluster.") resyncPeriod = flag.Duration("resync-period", 60*time.Second, "Resync interval of the controller.") showVersion = flag.Bool("version", false, "Show version.") + threads = flag.Int("worker-threads", 10, "Number of worker threads.") leaderElection = flag.Bool("leader-election", false, "Enables leader election.") leaderElectionNamespace = flag.String("leader-election-namespace", "", "The namespace where the leader election resource exists. Defaults to the pod namespace if not set.") @@ -111,7 +107,7 @@ func main() { stopCh := make(chan struct{}) factory.Start(stopCh) coreFactory.Start(stopCh) - go ctrl.Run(threads, stopCh) + go ctrl.Run(*threads, stopCh) // ...until SIGINT c := make(chan os.Signal, 1) From 8ae4d0c27882d98fc81c9181dc67e600cbf24ee8 Mon Sep 17 00:00:00 2001 From: xing-yang Date: Wed, 1 Apr 2020 04:23:53 +0000 Subject: [PATCH 44/56] Check if PVC is being used by a snapshot as source --- pkg/common-controller/snapshot_controller.go | 20 ++++++++++++++++++- pkg/common-controller/snapshot_delete_test.go | 12 +++++------ pkg/common-controller/snapshot_update_test.go | 4 ++-- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/pkg/common-controller/snapshot_controller.go b/pkg/common-controller/snapshot_controller.go index 467f3240e..ebd472d7d 100644 --- a/pkg/common-controller/snapshot_controller.go +++ b/pkg/common-controller/snapshot_controller.go @@ -166,7 +166,10 @@ func (ctrl *csiSnapshotCommonController) syncSnapshot(snapshot *crdv1.VolumeSnap ctrl.eventRecorder.Event(snapshot, v1.EventTypeWarning, "ErrorPVCFinalizer", "Error check and remove PVC Finalizer for VolumeSnapshot") } - if snapshot.ObjectMeta.DeletionTimestamp != nil { + // Proceed with snapshot deletion only if snapshot is not in the middled of being + // created from a PVC. This is to ensure that the PVC finalizer can be removed even + // if a delete snapshot request is received before create snapshot has completed. + if snapshot.ObjectMeta.DeletionTimestamp != nil && !ctrl.isPVCInUseByCurrentSnapshot(snapshot) { err := ctrl.processSnapshotWithDeletionTimestamp(snapshot) if err != nil { return err @@ -190,6 +193,21 @@ func (ctrl *csiSnapshotCommonController) syncSnapshot(snapshot *crdv1.VolumeSnap return nil } +// Check if PVC is being used by the current snapshot as source +func (ctrl *csiSnapshotCommonController) isPVCInUseByCurrentSnapshot(snapshot *crdv1.VolumeSnapshot) bool { + // Get snapshot source which is a PVC + pvc, err := ctrl.getClaimFromVolumeSnapshot(snapshot) + if err != nil { + klog.Infof("cannot get claim from snapshot [%s]: [%v] Claim may be deleted already.", snapshot.Name, err) + return false + } + if snapshot.Spec.Source.PersistentVolumeClaimName != nil && pvc.Name == *snapshot.Spec.Source.PersistentVolumeClaimName && !utils.IsSnapshotReady(snapshot) { + klog.V(2).Infof("PVC %s/%s is being used by snapshot %s/%s as source", pvc.Namespace, pvc.Name, snapshot.Namespace, snapshot.Name) + return true + } + return false +} + // checkContentAndBoundStatus is a helper function that checks the following: // - It checks if content exists and returns the content object if it exists and nil otherwise. // - It checks the deletionPolicy and returns true if policy is Delete and false otherwise. diff --git a/pkg/common-controller/snapshot_delete_test.go b/pkg/common-controller/snapshot_delete_test.go index 17ce7ffef..35427874e 100644 --- a/pkg/common-controller/snapshot_delete_test.go +++ b/pkg/common-controller/snapshot_delete_test.go @@ -172,8 +172,8 @@ func TestDeleteSync(t *testing.T) { name: "3-1 - content will be deleted if snapshot deletion timestamp is set", initialContents: newContentArray("content3-1", "", "snap3-1", "sid3-1", validSecretClass, "", "", deletePolicy, nil, nil, true), expectedContents: nocontents, - initialSnapshots: newSnapshotArray("snap3-1", "snapuid3-1", "claim3-1", "", validSecretClass, "content3-1", &False, nil, nil, nil, false, true, &timeNowMetav1), - expectedSnapshots: withSnapshotFinalizers(newSnapshotArray("snap3-1", "snapuid3-1", "claim3-1", "", validSecretClass, "content3-1", &False, nil, nil, nil, false, false, &timeNowMetav1), + initialSnapshots: newSnapshotArray("snap3-1", "snapuid3-1", "claim3-1", "", validSecretClass, "content3-1", &True, nil, nil, nil, false, true, &timeNowMetav1), + expectedSnapshots: withSnapshotFinalizers(newSnapshotArray("snap3-1", "snapuid3-1", "claim3-1", "", validSecretClass, "content3-1", &True, nil, nil, nil, false, false, &timeNowMetav1), utils.VolumeSnapshotBoundFinalizer, ), initialClaims: newClaimArray("claim3-1", "pvc-uid3-1", "1Gi", "volume3-1", v1.ClaimBound, &classEmpty), @@ -186,8 +186,8 @@ func TestDeleteSync(t *testing.T) { name: "3-2 - content will not be deleted if deletion API call fails", initialContents: newContentArray("content3-2", "", "snap3-2", "sid3-2", validSecretClass, "", "", deletePolicy, nil, nil, true), expectedContents: newContentArray("content3-2", "", "snap3-2", "sid3-2", validSecretClass, "", "", deletePolicy, nil, nil, true), - initialSnapshots: newSnapshotArray("snap3-2", "snapuid3-2", "claim3-2", "", validSecretClass, "content3-2", &False, nil, nil, nil, false, true, &timeNowMetav1), - expectedSnapshots: newSnapshotArray("snap3-2", "snapuid3-2", "claim3-2", "", validSecretClass, "content3-2", &False, nil, nil, nil, false, true, &timeNowMetav1), + initialSnapshots: newSnapshotArray("snap3-2", "snapuid3-2", "claim3-2", "", validSecretClass, "content3-2", &True, nil, nil, nil, false, true, &timeNowMetav1), + expectedSnapshots: newSnapshotArray("snap3-2", "snapuid3-2", "claim3-2", "", validSecretClass, "content3-2", &True, nil, nil, nil, false, true, &timeNowMetav1), initialClaims: newClaimArray("claim3-2", "pvc-uid3-2", "1Gi", "volume3-2", v1.ClaimBound, &classEmpty), expectedEvents: []string{"Warning SnapshotContentObjectDeleteError"}, initialSecrets: []*v1.Secret{secret()}, @@ -206,8 +206,8 @@ func TestDeleteSync(t *testing.T) { map[string]string{ "snapshot.storage.kubernetes.io/volumesnapshot-being-deleted": "yes", }), - initialSnapshots: newSnapshotArray("snap3-3", "snapuid3-3", "claim3-3", "", validSecretClass, "content3-3", &False, nil, nil, nil, false, true, &timeNowMetav1), - expectedSnapshots: withSnapshotFinalizers(newSnapshotArray("snap3-3", "snapuid3-3", "claim3-3", "", validSecretClass, "content3-3", &False, nil, nil, nil, false, false, &timeNowMetav1), + initialSnapshots: newSnapshotArray("snap3-3", "snapuid3-3", "claim3-3", "", validSecretClass, "content3-3", &True, nil, nil, nil, false, true, &timeNowMetav1), + expectedSnapshots: withSnapshotFinalizers(newSnapshotArray("snap3-3", "snapuid3-3", "claim3-3", "", validSecretClass, "content3-3", &True, nil, nil, nil, false, false, &timeNowMetav1), utils.VolumeSnapshotBoundFinalizer, ), initialClaims: newClaimArray("claim3-3", "pvc-uid3-3", "1Gi", "volume3-3", v1.ClaimBound, &classEmpty), diff --git a/pkg/common-controller/snapshot_update_test.go b/pkg/common-controller/snapshot_update_test.go index 7f71ef063..f0f9550c0 100644 --- a/pkg/common-controller/snapshot_update_test.go +++ b/pkg/common-controller/snapshot_update_test.go @@ -315,8 +315,8 @@ func TestSync(t *testing.T) { }, { name: "5-5 - snapshot deletion candidate marked for deletion by syncSnapshot", - initialSnapshots: newSnapshotArray("snap5-5", "snapuid5-5", "claim5-5", "", validSecretClass, "content5-5", &False, nil, nil, nil, false, true, &timeNowMetav1), - expectedSnapshots: withSnapshotFinalizers(newSnapshotArray("snap5-5", "snapuid5-5", "claim5-5", "", validSecretClass, "content5-5", &False, nil, nil, nil, false, false, &timeNowMetav1), + initialSnapshots: newSnapshotArray("snap5-5", "snapuid5-5", "claim5-5", "", validSecretClass, "content5-5", &True, nil, nil, nil, false, true, &timeNowMetav1), + expectedSnapshots: withSnapshotFinalizers(newSnapshotArray("snap5-5", "snapuid5-5", "claim5-5", "", validSecretClass, "content5-5", &True, nil, nil, nil, false, false, &timeNowMetav1), utils.VolumeSnapshotBoundFinalizer, ), initialContents: newContentArray("content5-5", "snapuid5-5", "snap5-5", "sid5-5", validSecretClass, "", "", crdv1.VolumeSnapshotContentRetain, nil, nil, true), From c82c8e0c7eae9d8f40803d4dfc500e713f664e6d Mon Sep 17 00:00:00 2001 From: xing-yang Date: Thu, 2 Apr 2020 02:01:52 +0000 Subject: [PATCH 45/56] Add a check to see if PVC has a finalizer --- pkg/common-controller/snapshot_controller.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/pkg/common-controller/snapshot_controller.go b/pkg/common-controller/snapshot_controller.go index ebd472d7d..eaaf1235e 100644 --- a/pkg/common-controller/snapshot_controller.go +++ b/pkg/common-controller/snapshot_controller.go @@ -167,9 +167,10 @@ func (ctrl *csiSnapshotCommonController) syncSnapshot(snapshot *crdv1.VolumeSnap } // Proceed with snapshot deletion only if snapshot is not in the middled of being - // created from a PVC. This is to ensure that the PVC finalizer can be removed even - // if a delete snapshot request is received before create snapshot has completed. - if snapshot.ObjectMeta.DeletionTimestamp != nil && !ctrl.isPVCInUseByCurrentSnapshot(snapshot) { + // created from a PVC with a finalizer. This is to ensure that the PVC finalizer + // can be removed even if a delete snapshot request is received before create + // snapshot has completed. + if snapshot.ObjectMeta.DeletionTimestamp != nil && !ctrl.isPVCwithFinalizerInUseByCurrentSnapshot(snapshot) { err := ctrl.processSnapshotWithDeletionTimestamp(snapshot) if err != nil { return err @@ -193,15 +194,21 @@ func (ctrl *csiSnapshotCommonController) syncSnapshot(snapshot *crdv1.VolumeSnap return nil } -// Check if PVC is being used by the current snapshot as source -func (ctrl *csiSnapshotCommonController) isPVCInUseByCurrentSnapshot(snapshot *crdv1.VolumeSnapshot) bool { +// Check if PVC has a finalizer and if it is being used by the current snapshot as source. +func (ctrl *csiSnapshotCommonController) isPVCwithFinalizerInUseByCurrentSnapshot(snapshot *crdv1.VolumeSnapshot) bool { // Get snapshot source which is a PVC pvc, err := ctrl.getClaimFromVolumeSnapshot(snapshot) if err != nil { klog.Infof("cannot get claim from snapshot [%s]: [%v] Claim may be deleted already.", snapshot.Name, err) return false } - if snapshot.Spec.Source.PersistentVolumeClaimName != nil && pvc.Name == *snapshot.Spec.Source.PersistentVolumeClaimName && !utils.IsSnapshotReady(snapshot) { + + // Check if there is a Finalizer on PVC. If not, return false + if !slice.ContainsString(pvc.ObjectMeta.Finalizers, utils.PVCFinalizer, nil) { + return false + } + + if !utils.IsSnapshotReady(snapshot) { klog.V(2).Infof("PVC %s/%s is being used by snapshot %s/%s as source", pvc.Namespace, pvc.Name, snapshot.Namespace, snapshot.Name) return true } From 313507f866f6d9d573b6b963108176207d6346b8 Mon Sep 17 00:00:00 2001 From: xing-yang Date: Thu, 27 Feb 2020 21:02:26 +0000 Subject: [PATCH 46/56] Add AnnVolumeSnapshotBeingCreated --- pkg/sidecar-controller/snapshot_controller.go | 72 ++++++++++++++++++- pkg/utils/util.go | 16 +++++ 2 files changed, 87 insertions(+), 1 deletion(-) diff --git a/pkg/sidecar-controller/snapshot_controller.go b/pkg/sidecar-controller/snapshot_controller.go index 8c2963b68..c5ac96524 100644 --- a/pkg/sidecar-controller/snapshot_controller.go +++ b/pkg/sidecar-controller/snapshot_controller.go @@ -322,14 +322,45 @@ func (ctrl *csiSnapshotSideCarController) createSnapshotOperation(content *crdv1 return nil, fmt.Errorf("failed to get input parameters to create snapshot for content %s: %q", content.Name, err) } + // NOTE(xyang): handle create timeout + // Add annotation and set to Yes to indicate create snapshot request is + // sent to the storage system and wait for a response of success or failure. + // Annotation will be set to No only after storage system has responded + // with success or failure. If the request times out, retry will happen + // and annotation will stay as Yes to avoid leaking of snapshot + // resources on the storage system + err = ctrl.setAnnVolumeSnapshotBeingCreated(content, utils.AnnVolumeSnapshotBeingCreated_Yes) + if err != nil { + return nil, fmt.Errorf("failed to set VolumeSnapshotBeingCreated annotation to Yes on the content %s: %q", content.Name, err) + } + driverName, snapshotID, creationTime, size, readyToUse, err := ctrl.handler.CreateSnapshot(content, class.Parameters, snapshotterCredentials) if err != nil { + // NOTE(xyang): handle create timeout + // If it is not a timeout error, set annotation to No to indicate + // storage system has responded with an error + errStr := fmt.Sprintf("%q", err) + if !strings.Contains(errStr, "DeadlineExceeded") { + err = ctrl.setAnnVolumeSnapshotBeingCreated(content, utils.AnnVolumeSnapshotBeingCreated_No) + if err != nil { + return nil, fmt.Errorf("failed to set VolumeSnapshotBeingCreated annotation to No on the content %s: %q", content.Name, err) + } + } + return nil, fmt.Errorf("failed to take snapshot of the volume, %s: %q", *content.Spec.Source.VolumeHandle, err) } if driverName != class.Driver { return nil, fmt.Errorf("failed to take snapshot of the volume, %s: driver name %s returned from the driver is different from driver %s in snapshot class", *content.Spec.Source.VolumeHandle, driverName, class.Driver) } + // NOTE(xyang): handle create timeout + // Set annotation to No to indicate storage system has successfully + // cut the snapshot + err = ctrl.setAnnVolumeSnapshotBeingCreated(content, utils.AnnVolumeSnapshotBeingCreated_No) + if err != nil { + return nil, fmt.Errorf("failed to set VolumeSnapshotBeingCreated annotation to No on the content %s: %q", content.Name, err) + } + klog.V(5).Infof("Created snapshot: driver %s, snapshotId %s, creationTime %v, size %d, readyToUse %t", driverName, snapshotID, creationTime, size, readyToUse) timestamp := creationTime.UnixNano() @@ -564,9 +595,48 @@ func (ctrl *csiSnapshotSideCarController) shouldDelete(content *crdv1.VolumeSnap if content.Spec.Source.SnapshotHandle != nil && content.Spec.VolumeSnapshotRef.UID == "" { return true } - // 2) shouldDelete returns true if AnnVolumeSnapshotBeingDeleted annotation is set + + // NOTE(xyang): Handle create snapshot timeout + // 2) shouldDelete returns false if AnnVolumeSnapshotBeingCreated + // annotation is set and its value is Yes. This indicates a + // CreateSnapshot CSI RPC has not responded with success or failure. + // We need to keep waiting for a response from the CSI driver. + if metav1.HasAnnotation(content.ObjectMeta, utils.AnnVolumeSnapshotBeingCreated) && content.ObjectMeta.Annotations[utils.AnnVolumeSnapshotBeingCreated] == utils.AnnVolumeSnapshotBeingCreated_Yes { + return false + } + + // 3) shouldDelete returns true if AnnVolumeSnapshotBeingDeleted annotation is set if metav1.HasAnnotation(content.ObjectMeta, utils.AnnVolumeSnapshotBeingDeleted) { return true } return false } + +// setAnnVolumeSnapshotBeingCreated sets VolumeSnapshotBeingCreated annotation +// on VolumeSnapshotContent +// If beingCreated is true, it indicates snapshot is being created +// If beingCreated if false, it indicates CreateSnapshot CSI RPC returns +// success or failure +func (ctrl *csiSnapshotSideCarController) setAnnVolumeSnapshotBeingCreated(content *crdv1.VolumeSnapshotContent, annBeingCreated string) error { + // Set AnnVolumeSnapshotBeingCreated if it is not set yet or if it is + // set but has a different value + if !metav1.HasAnnotation(content.ObjectMeta, utils.AnnVolumeSnapshotBeingCreated) || content.ObjectMeta.Annotations[utils.AnnVolumeSnapshotBeingCreated] != annBeingCreated { + klog.V(5).Infof("setAnnVolumeSnapshotBeingCreated: set annotation [%s:%s] on content [%s].", utils.AnnVolumeSnapshotBeingCreated, annBeingCreated, content.Name) + metav1.SetMetaDataAnnotation(&content.ObjectMeta, utils.AnnVolumeSnapshotBeingCreated, annBeingCreated) + + updateContent, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().Update(content) + if err != nil { + return newControllerUpdateError(content.Name, err.Error()) + } + // update content if update is successful + content = updateContent + + _, err = ctrl.storeContentUpdate(content) + if err != nil { + klog.V(4).Infof("setAnnVolumeSnapshotBeingCreated for content [%s]: cannot update internal cache %v", content.Name, err) + return err + } + klog.V(5).Infof("setAnnVolumeSnapshotBeingCreated: volume snapshot content %+v", content) + } + return nil +} diff --git a/pkg/utils/util.go b/pkg/utils/util.go index 0bd64435b..09f9ee48e 100644 --- a/pkg/utils/util.go +++ b/pkg/utils/util.go @@ -78,6 +78,22 @@ const ( // backing the snapshot content. AnnVolumeSnapshotBeingDeleted = "snapshot.storage.kubernetes.io/volumesnapshot-being-deleted" + // AnnVolumeSnapshotBeingCreated annotation applies to VolumeSnapshotContents. + // If it is set and value is "yes", it indicates that the csi-snapshotter + // sidecar has sent the create snapshot request to the storage system and + // is waiting for a response of success or failure. + // This annotation will be set to "no" if the driver's CreateSnapshot + // CSI function returns success or failure. If the create snapshot + // request times out, retry will happen and the annotation value will + // stay as "yes". + // This only applies to dynamic provisioning of snapshots because + // the create snapshot CSI method will not be called for pre-provisioned + // snapshots. + AnnVolumeSnapshotBeingCreated = "snapshot.storage.kubernetes.io/volumesnapshot-being-created" + // VolumeSnapshotBeingCreated annotation values can be "yes" or "no" + AnnVolumeSnapshotBeingCreated_Yes = "yes" + AnnVolumeSnapshotBeingCreated_No = "no" + // Annotation for secret name and namespace will be added to the content // and used at snapshot content deletion time. AnnDeletionSecretRefName = "snapshot.storage.kubernetes.io/deletion-secret-name" From 4c38cc8c6e4228b347d421d4b0ba4b577e9ddaff Mon Sep 17 00:00:00 2001 From: xing-yang Date: Thu, 5 Mar 2020 21:18:37 +0000 Subject: [PATCH 47/56] Remove VolumeSnapshotBeingCreated annotation after driver response --- pkg/sidecar-controller/content_create_test.go | 3 +- pkg/sidecar-controller/snapshot_controller.go | 79 ++++++++++++------- pkg/utils/util.go | 10 +-- 3 files changed, 56 insertions(+), 36 deletions(-) diff --git a/pkg/sidecar-controller/content_create_test.go b/pkg/sidecar-controller/content_create_test.go index 5185b5ed2..475224b2e 100644 --- a/pkg/sidecar-controller/content_create_test.go +++ b/pkg/sidecar-controller/content_create_test.go @@ -52,8 +52,9 @@ func TestSyncContent(t *testing.T) { name: "1-2: Basic sync content create snapshot", initialContents: withContentStatus(newContentArray("content1-2", "snapuid1-2", "snap1-2", "sid1-2", defaultClass, "", "volume-handle-1-2", retainPolicy, nil, &defaultSize, true), nil), - expectedContents: withContentStatus(newContentArray("content1-2", "snapuid1-2", "snap1-2", "sid1-2", defaultClass, "", "volume-handle-1-2", retainPolicy, nil, &defaultSize, true), + expectedContents: withContentAnnotations(withContentStatus(newContentArray("content1-2", "snapuid1-2", "snap1-2", "sid1-2", defaultClass, "", "volume-handle-1-2", retainPolicy, nil, &defaultSize, true), &crdv1.VolumeSnapshotContentStatus{SnapshotHandle: toStringPointer("snapuid1-2"), RestoreSize: &defaultSize, ReadyToUse: &True}), + map[string]string{}), expectedEvents: noevents, expectedCreateCalls: []createCall{ { diff --git a/pkg/sidecar-controller/snapshot_controller.go b/pkg/sidecar-controller/snapshot_controller.go index c5ac96524..66c4ebff0 100644 --- a/pkg/sidecar-controller/snapshot_controller.go +++ b/pkg/sidecar-controller/snapshot_controller.go @@ -323,13 +323,13 @@ func (ctrl *csiSnapshotSideCarController) createSnapshotOperation(content *crdv1 } // NOTE(xyang): handle create timeout - // Add annotation and set to Yes to indicate create snapshot request is + // Add annotation to indicate create snapshot request is // sent to the storage system and wait for a response of success or failure. - // Annotation will be set to No only after storage system has responded + // Annotation will be removed only after storage system has responded // with success or failure. If the request times out, retry will happen - // and annotation will stay as Yes to avoid leaking of snapshot + // and annotation will remain to avoid leaking of snapshot // resources on the storage system - err = ctrl.setAnnVolumeSnapshotBeingCreated(content, utils.AnnVolumeSnapshotBeingCreated_Yes) + err = ctrl.setAnnVolumeSnapshotBeingCreated(content) if err != nil { return nil, fmt.Errorf("failed to set VolumeSnapshotBeingCreated annotation to Yes on the content %s: %q", content.Name, err) } @@ -337,11 +337,11 @@ func (ctrl *csiSnapshotSideCarController) createSnapshotOperation(content *crdv1 driverName, snapshotID, creationTime, size, readyToUse, err := ctrl.handler.CreateSnapshot(content, class.Parameters, snapshotterCredentials) if err != nil { // NOTE(xyang): handle create timeout - // If it is not a timeout error, set annotation to No to indicate + // If it is not a timeout error, remove annotation to indicate // storage system has responded with an error errStr := fmt.Sprintf("%q", err) if !strings.Contains(errStr, "DeadlineExceeded") { - err = ctrl.setAnnVolumeSnapshotBeingCreated(content, utils.AnnVolumeSnapshotBeingCreated_No) + err = ctrl.removeAnnVolumeSnapshotBeingCreated(content) if err != nil { return nil, fmt.Errorf("failed to set VolumeSnapshotBeingCreated annotation to No on the content %s: %q", content.Name, err) } @@ -353,14 +353,6 @@ func (ctrl *csiSnapshotSideCarController) createSnapshotOperation(content *crdv1 return nil, fmt.Errorf("failed to take snapshot of the volume, %s: driver name %s returned from the driver is different from driver %s in snapshot class", *content.Spec.Source.VolumeHandle, driverName, class.Driver) } - // NOTE(xyang): handle create timeout - // Set annotation to No to indicate storage system has successfully - // cut the snapshot - err = ctrl.setAnnVolumeSnapshotBeingCreated(content, utils.AnnVolumeSnapshotBeingCreated_No) - if err != nil { - return nil, fmt.Errorf("failed to set VolumeSnapshotBeingCreated annotation to No on the content %s: %q", content.Name, err) - } - klog.V(5).Infof("Created snapshot: driver %s, snapshotId %s, creationTime %v, size %d, readyToUse %t", driverName, snapshotID, creationTime, size, readyToUse) timestamp := creationTime.UnixNano() @@ -372,6 +364,14 @@ func (ctrl *csiSnapshotSideCarController) createSnapshotOperation(content *crdv1 content = newContent } + // NOTE(xyang): handle create timeout + // Remove annotation to indicate storage system has successfully + // cut the snapshot + err = ctrl.removeAnnVolumeSnapshotBeingCreated(content) + if err != nil { + return nil, fmt.Errorf("failed to set VolumeSnapshotBeingCreated annotation to No on the content %s: %q", content.Name, err) + } + // Update content in the cache store _, err = ctrl.storeContentUpdate(content) if err != nil { @@ -598,10 +598,10 @@ func (ctrl *csiSnapshotSideCarController) shouldDelete(content *crdv1.VolumeSnap // NOTE(xyang): Handle create snapshot timeout // 2) shouldDelete returns false if AnnVolumeSnapshotBeingCreated - // annotation is set and its value is Yes. This indicates a - // CreateSnapshot CSI RPC has not responded with success or failure. + // annotation is set. This indicates a CreateSnapshot CSI RPC has + // not responded with success or failure. // We need to keep waiting for a response from the CSI driver. - if metav1.HasAnnotation(content.ObjectMeta, utils.AnnVolumeSnapshotBeingCreated) && content.ObjectMeta.Annotations[utils.AnnVolumeSnapshotBeingCreated] == utils.AnnVolumeSnapshotBeingCreated_Yes { + if metav1.HasAnnotation(content.ObjectMeta, utils.AnnVolumeSnapshotBeingCreated) { return false } @@ -614,17 +614,15 @@ func (ctrl *csiSnapshotSideCarController) shouldDelete(content *crdv1.VolumeSnap // setAnnVolumeSnapshotBeingCreated sets VolumeSnapshotBeingCreated annotation // on VolumeSnapshotContent -// If beingCreated is true, it indicates snapshot is being created -// If beingCreated if false, it indicates CreateSnapshot CSI RPC returns -// success or failure -func (ctrl *csiSnapshotSideCarController) setAnnVolumeSnapshotBeingCreated(content *crdv1.VolumeSnapshotContent, annBeingCreated string) error { - // Set AnnVolumeSnapshotBeingCreated if it is not set yet or if it is - // set but has a different value - if !metav1.HasAnnotation(content.ObjectMeta, utils.AnnVolumeSnapshotBeingCreated) || content.ObjectMeta.Annotations[utils.AnnVolumeSnapshotBeingCreated] != annBeingCreated { - klog.V(5).Infof("setAnnVolumeSnapshotBeingCreated: set annotation [%s:%s] on content [%s].", utils.AnnVolumeSnapshotBeingCreated, annBeingCreated, content.Name) - metav1.SetMetaDataAnnotation(&content.ObjectMeta, utils.AnnVolumeSnapshotBeingCreated, annBeingCreated) - - updateContent, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().Update(content) +// If set, it indicates snapshot is being created +func (ctrl *csiSnapshotSideCarController) setAnnVolumeSnapshotBeingCreated(content *crdv1.VolumeSnapshotContent) error { + // Set AnnVolumeSnapshotBeingCreated if it is not set yet + if !metav1.HasAnnotation(content.ObjectMeta, utils.AnnVolumeSnapshotBeingCreated) { + klog.V(5).Infof("setAnnVolumeSnapshotBeingCreated: set annotation [%s:yes] on content [%s].", utils.AnnVolumeSnapshotBeingCreated, content.Name) + contentClone := content.DeepCopy() + metav1.SetMetaDataAnnotation(&contentClone.ObjectMeta, utils.AnnVolumeSnapshotBeingCreated, "yes") + + updateContent, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().Update(contentClone) if err != nil { return newControllerUpdateError(content.Name, err.Error()) } @@ -640,3 +638,28 @@ func (ctrl *csiSnapshotSideCarController) setAnnVolumeSnapshotBeingCreated(conte } return nil } + +// removeAnnVolumeSnapshotBeingCreated removes the VolumeSnapshotBeingCreated +// annotation from a content if there exists one. +func (ctrl csiSnapshotSideCarController) removeAnnVolumeSnapshotBeingCreated(content *crdv1.VolumeSnapshotContent) error { + if !metav1.HasAnnotation(content.ObjectMeta, utils.AnnVolumeSnapshotBeingCreated) { + // the annotation does not exit, return directly + return nil + } + contentClone := content.DeepCopy() + delete(contentClone.ObjectMeta.Annotations, utils.AnnVolumeSnapshotBeingCreated) + + updateContent, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().Update(contentClone) + if err != nil { + return newControllerUpdateError(content.Name, err.Error()) + } + // update content if update is successful + content = updateContent + + klog.V(5).Infof("Removed VolumeSnapshotBeingCreated annotation from volume snapshot content %s", content.Name) + _, err = ctrl.storeContentUpdate(content) + if err != nil { + klog.Errorf("failed to update content store %v", err) + } + return nil +} diff --git a/pkg/utils/util.go b/pkg/utils/util.go index 09f9ee48e..b0fc07496 100644 --- a/pkg/utils/util.go +++ b/pkg/utils/util.go @@ -79,20 +79,16 @@ const ( AnnVolumeSnapshotBeingDeleted = "snapshot.storage.kubernetes.io/volumesnapshot-being-deleted" // AnnVolumeSnapshotBeingCreated annotation applies to VolumeSnapshotContents. - // If it is set and value is "yes", it indicates that the csi-snapshotter + // If it is set, it indicates that the csi-snapshotter // sidecar has sent the create snapshot request to the storage system and // is waiting for a response of success or failure. - // This annotation will be set to "no" if the driver's CreateSnapshot + // This annotation will be removed if the driver's CreateSnapshot // CSI function returns success or failure. If the create snapshot - // request times out, retry will happen and the annotation value will - // stay as "yes". + // request times out, retry will happen and the annotation will remain. // This only applies to dynamic provisioning of snapshots because // the create snapshot CSI method will not be called for pre-provisioned // snapshots. AnnVolumeSnapshotBeingCreated = "snapshot.storage.kubernetes.io/volumesnapshot-being-created" - // VolumeSnapshotBeingCreated annotation values can be "yes" or "no" - AnnVolumeSnapshotBeingCreated_Yes = "yes" - AnnVolumeSnapshotBeingCreated_No = "no" // Annotation for secret name and namespace will be added to the content // and used at snapshot content deletion time. From aa2bc2fb9c2752ade87a8a078bfa4dac9deb82c5 Mon Sep 17 00:00:00 2001 From: xing-yang Date: Thu, 5 Mar 2020 21:55:57 +0000 Subject: [PATCH 48/56] Remove the unnecessary driver name check --- pkg/sidecar-controller/snapshot_controller.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/sidecar-controller/snapshot_controller.go b/pkg/sidecar-controller/snapshot_controller.go index 66c4ebff0..481c187e3 100644 --- a/pkg/sidecar-controller/snapshot_controller.go +++ b/pkg/sidecar-controller/snapshot_controller.go @@ -349,9 +349,6 @@ func (ctrl *csiSnapshotSideCarController) createSnapshotOperation(content *crdv1 return nil, fmt.Errorf("failed to take snapshot of the volume, %s: %q", *content.Spec.Source.VolumeHandle, err) } - if driverName != class.Driver { - return nil, fmt.Errorf("failed to take snapshot of the volume, %s: driver name %s returned from the driver is different from driver %s in snapshot class", *content.Spec.Source.VolumeHandle, driverName, class.Driver) - } klog.V(5).Infof("Created snapshot: driver %s, snapshotId %s, creationTime %v, size %d, readyToUse %t", driverName, snapshotID, creationTime, size, readyToUse) From 7930dd261d52f30092baa664d3ab399ef65c1d2a Mon Sep 17 00:00:00 2001 From: xing-yang Date: Fri, 6 Mar 2020 03:59:51 +0000 Subject: [PATCH 49/56] Check error code DeadlineExceeded --- pkg/sidecar-controller/snapshot_controller.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/pkg/sidecar-controller/snapshot_controller.go b/pkg/sidecar-controller/snapshot_controller.go index 481c187e3..6bf91cca5 100644 --- a/pkg/sidecar-controller/snapshot_controller.go +++ b/pkg/sidecar-controller/snapshot_controller.go @@ -23,6 +23,8 @@ import ( crdv1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" "github.com/kubernetes-csi/external-snapshotter/v2/pkg/utils" + codes "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog" @@ -331,7 +333,7 @@ func (ctrl *csiSnapshotSideCarController) createSnapshotOperation(content *crdv1 // resources on the storage system err = ctrl.setAnnVolumeSnapshotBeingCreated(content) if err != nil { - return nil, fmt.Errorf("failed to set VolumeSnapshotBeingCreated annotation to Yes on the content %s: %q", content.Name, err) + return nil, fmt.Errorf("failed to add VolumeSnapshotBeingCreated annotation on the content %s: %q", content.Name, err) } driverName, snapshotID, creationTime, size, readyToUse, err := ctrl.handler.CreateSnapshot(content, class.Parameters, snapshotterCredentials) @@ -339,11 +341,12 @@ func (ctrl *csiSnapshotSideCarController) createSnapshotOperation(content *crdv1 // NOTE(xyang): handle create timeout // If it is not a timeout error, remove annotation to indicate // storage system has responded with an error - errStr := fmt.Sprintf("%q", err) - if !strings.Contains(errStr, "DeadlineExceeded") { - err = ctrl.removeAnnVolumeSnapshotBeingCreated(content) - if err != nil { - return nil, fmt.Errorf("failed to set VolumeSnapshotBeingCreated annotation to No on the content %s: %q", content.Name, err) + if e, ok := status.FromError(err); ok { + if e.Code() == codes.DeadlineExceeded { + err = ctrl.removeAnnVolumeSnapshotBeingCreated(content) + if err != nil { + return nil, fmt.Errorf("failed to remove VolumeSnapshotBeingCreated annotation from the content %s: %q", content.Name, err) + } } } @@ -366,7 +369,7 @@ func (ctrl *csiSnapshotSideCarController) createSnapshotOperation(content *crdv1 // cut the snapshot err = ctrl.removeAnnVolumeSnapshotBeingCreated(content) if err != nil { - return nil, fmt.Errorf("failed to set VolumeSnapshotBeingCreated annotation to No on the content %s: %q", content.Name, err) + return nil, fmt.Errorf("failed to remove VolumeSnapshotBeingCreated annotation on the content %s: %q", content.Name, err) } // Update content in the cache store From 6ca7507bc3c70b75ef0ccf2a4f49c6b343fb2691 Mon Sep 17 00:00:00 2001 From: xing-yang Date: Sun, 8 Mar 2020 01:21:15 +0000 Subject: [PATCH 50/56] Address review comments --- pkg/sidecar-controller/snapshot_controller.go | 28 +++++++------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/pkg/sidecar-controller/snapshot_controller.go b/pkg/sidecar-controller/snapshot_controller.go index 6bf91cca5..b519c5658 100644 --- a/pkg/sidecar-controller/snapshot_controller.go +++ b/pkg/sidecar-controller/snapshot_controller.go @@ -303,11 +303,7 @@ func (ctrl *csiSnapshotSideCarController) checkandUpdateContentStatusOperation(c return updateContent, nil } -// The function goes through the whole snapshot creation process. -// 1. Trigger the snapshot through csi storage provider. -// 2. Update VolumeSnapshot status with creationtimestamp information -// 3. Create the VolumeSnapshotContent object with the snapshot id information. -// 4. Bind the VolumeSnapshot and VolumeSnapshotContent object +// The function goes through the snapshot creation process. func (ctrl *csiSnapshotSideCarController) createSnapshotOperation(content *crdv1.VolumeSnapshotContent) (*crdv1.VolumeSnapshotContent, error) { klog.Infof("createSnapshotOperation: Creating snapshot for content %s through the plugin ...", content.Name) @@ -325,12 +321,12 @@ func (ctrl *csiSnapshotSideCarController) createSnapshotOperation(content *crdv1 } // NOTE(xyang): handle create timeout - // Add annotation to indicate create snapshot request is - // sent to the storage system and wait for a response of success or failure. - // Annotation will be removed only after storage system has responded - // with success or failure. If the request times out, retry will happen - // and annotation will remain to avoid leaking of snapshot - // resources on the storage system + // Add an annotation to indicate the snapshot creation request has been + // sent to the storage system and the controller is waiting for a response. + // The annotation will be removed after the storage system has responded with + // success or permanent failure. If the request times out, annotation will + // remain on the content to avoid potential leaking of a snapshot resource on + // the storage system. err = ctrl.setAnnVolumeSnapshotBeingCreated(content) if err != nil { return nil, fmt.Errorf("failed to add VolumeSnapshotBeingCreated annotation on the content %s: %q", content.Name, err) @@ -341,8 +337,10 @@ func (ctrl *csiSnapshotSideCarController) createSnapshotOperation(content *crdv1 // NOTE(xyang): handle create timeout // If it is not a timeout error, remove annotation to indicate // storage system has responded with an error + klog.Infof("createSnapshotOperation: CreateSnapshot for content %s returned error: %v", content.Name, err) if e, ok := status.FromError(err); ok { - if e.Code() == codes.DeadlineExceeded { + klog.Infof("createSnapshotOperation: CreateSnapshot for content %s error status: %+v", content.Name, e) + if e.Code() != codes.DeadlineExceeded { err = ctrl.removeAnnVolumeSnapshotBeingCreated(content) if err != nil { return nil, fmt.Errorf("failed to remove VolumeSnapshotBeingCreated annotation from the content %s: %q", content.Name, err) @@ -372,12 +370,6 @@ func (ctrl *csiSnapshotSideCarController) createSnapshotOperation(content *crdv1 return nil, fmt.Errorf("failed to remove VolumeSnapshotBeingCreated annotation on the content %s: %q", content.Name, err) } - // Update content in the cache store - _, err = ctrl.storeContentUpdate(content) - if err != nil { - klog.Errorf("failed to update content store %v", err) - } - return content, nil } From 0189c77f5aa3e489250e2c8cc6f926d05bfe3f73 Mon Sep 17 00:00:00 2001 From: xing-yang Date: Mon, 16 Mar 2020 01:49:58 +0000 Subject: [PATCH 51/56] Check final error --- pkg/sidecar-controller/content_create_test.go | 11 +-- pkg/sidecar-controller/snapshot_controller.go | 79 +++++++++++-------- pkg/utils/util.go | 7 +- 3 files changed, 58 insertions(+), 39 deletions(-) diff --git a/pkg/sidecar-controller/content_create_test.go b/pkg/sidecar-controller/content_create_test.go index 475224b2e..babc144a8 100644 --- a/pkg/sidecar-controller/content_create_test.go +++ b/pkg/sidecar-controller/content_create_test.go @@ -30,10 +30,11 @@ func TestSyncContent(t *testing.T) { tests := []controllerTest{ { - name: "1-1: Basic content update ready to use", - initialContents: newContentArrayWithReadyToUse("content1-1", "snapuid1-1", "snap1-1", "sid1-1", defaultClass, "", "volume-handle-1-1", retainPolicy, nil, &defaultSize, &False, true), - expectedContents: newContentArrayWithReadyToUse("content1-1", "snapuid1-1", "snap1-1", "sid1-1", defaultClass, "", "volume-handle-1-1", retainPolicy, nil, &defaultSize, &True, true), - expectedEvents: noevents, + name: "1-1: Basic content update ready to use", + initialContents: newContentArrayWithReadyToUse("content1-1", "snapuid1-1", "snap1-1", "sid1-1", defaultClass, "", "volume-handle-1-1", retainPolicy, nil, &defaultSize, &False, true), + expectedContents: withContentAnnotations(newContentArrayWithReadyToUse("content1-1", "snapuid1-1", "snap1-1", "sid1-1", defaultClass, "", "volume-handle-1-1", retainPolicy, nil, &defaultSize, &True, true), + map[string]string{}), + expectedEvents: noevents, expectedCreateCalls: []createCall{ { volumeHandle: "volume-handle-1-1", @@ -162,7 +163,7 @@ func TestSyncContent(t *testing.T) { SnapshotHandle: toStringPointer("sid1-6"), RestoreSize: &defaultSize, ReadyToUse: &False, - Error: newSnapshotError("Failed to check and update snapshot content: failed to get input parameters to create snapshot content1-6: \"failed to retrieve snapshot class bad-class from the informer: \\\"volumesnapshotclass.snapshot.storage.k8s.io \\\\\\\"bad-class\\\\\\\" not found\\\"\""), + Error: newSnapshotError("Failed to check and update snapshot content: failed to get input parameters to create snapshot for content content1-6: \"failed to retrieve snapshot class bad-class from the informer: \\\"volumesnapshotclass.snapshot.storage.k8s.io \\\\\\\"bad-class\\\\\\\" not found\\\"\""), }), expectedEvents: []string{"Warning SnapshotContentCheckandUpdateFailed"}, expectedCreateCalls: []createCall{ diff --git a/pkg/sidecar-controller/snapshot_controller.go b/pkg/sidecar-controller/snapshot_controller.go index b519c5658..4742dbc1a 100644 --- a/pkg/sidecar-controller/snapshot_controller.go +++ b/pkg/sidecar-controller/snapshot_controller.go @@ -278,29 +278,21 @@ func (ctrl *csiSnapshotSideCarController) checkandUpdateContentStatusOperation(c } driverName = content.Spec.Driver snapshotID = *content.Spec.Source.SnapshotHandle - } else { - class, snapshotterCredentials, err := ctrl.getCSISnapshotInput(content) - if err != nil { - return nil, fmt.Errorf("failed to get input parameters to create snapshot %s: %q", content.Name, err) + + klog.V(5).Infof("checkandUpdateContentStatusOperation: driver %s, snapshotId %s, creationTime %v, size %d, readyToUse %t", driverName, snapshotID, creationTime, size, readyToUse) + + if creationTime.IsZero() { + creationTime = time.Now() } - driverName, snapshotID, creationTime, size, readyToUse, err = ctrl.handler.CreateSnapshot(content, class.Parameters, snapshotterCredentials) + updatedContent, err := ctrl.updateSnapshotContentStatus(content, snapshotID, readyToUse, creationTime.UnixNano(), size) if err != nil { - klog.Errorf("checkandUpdateContentStatusOperation: failed to call create snapshot to check whether the snapshot is ready to use %q", err) return nil, err } + return updatedContent, nil + } else { + return ctrl.createSnapshotOperation(content) } - klog.V(5).Infof("checkandUpdateContentStatusOperation: driver %s, snapshotId %s, creationTime %v, size %d, readyToUse %t", driverName, snapshotID, creationTime, size, readyToUse) - - if creationTime.IsZero() { - creationTime = time.Now() - } - - updateContent, err := ctrl.updateSnapshotContentStatus(content, snapshotID, readyToUse, creationTime.UnixNano(), size) - if err != nil { - return nil, err - } - return updateContent, nil } // The function goes through the snapshot creation process. @@ -335,16 +327,13 @@ func (ctrl *csiSnapshotSideCarController) createSnapshotOperation(content *crdv1 driverName, snapshotID, creationTime, size, readyToUse, err := ctrl.handler.CreateSnapshot(content, class.Parameters, snapshotterCredentials) if err != nil { // NOTE(xyang): handle create timeout - // If it is not a timeout error, remove annotation to indicate + // If it is a final error, remove annotation to indicate // storage system has responded with an error klog.Infof("createSnapshotOperation: CreateSnapshot for content %s returned error: %v", content.Name, err) - if e, ok := status.FromError(err); ok { - klog.Infof("createSnapshotOperation: CreateSnapshot for content %s error status: %+v", content.Name, e) - if e.Code() != codes.DeadlineExceeded { - err = ctrl.removeAnnVolumeSnapshotBeingCreated(content) - if err != nil { - return nil, fmt.Errorf("failed to remove VolumeSnapshotBeingCreated annotation from the content %s: %q", content.Name, err) - } + if isFinalError(err) { + err = ctrl.removeAnnVolumeSnapshotBeingCreated(content) + if err != nil { + return nil, fmt.Errorf("failed to remove VolumeSnapshotBeingCreated annotation from the content %s: %q", content.Name, err) } } @@ -353,8 +342,11 @@ func (ctrl *csiSnapshotSideCarController) createSnapshotOperation(content *crdv1 klog.V(5).Infof("Created snapshot: driver %s, snapshotId %s, creationTime %v, size %d, readyToUse %t", driverName, snapshotID, creationTime, size, readyToUse) - timestamp := creationTime.UnixNano() - newContent, err := ctrl.updateSnapshotContentStatus(content, snapshotID, readyToUse, timestamp, size) + if creationTime.IsZero() { + creationTime = time.Now() + } + + newContent, err := ctrl.updateSnapshotContentStatus(content, snapshotID, readyToUse, creationTime.UnixNano(), size) if err != nil { strerr := fmt.Sprintf("error updating volume snapshot content status for snapshot %s: %v.", content.Name, err) klog.Error(strerr) @@ -614,12 +606,12 @@ func (ctrl *csiSnapshotSideCarController) setAnnVolumeSnapshotBeingCreated(conte contentClone := content.DeepCopy() metav1.SetMetaDataAnnotation(&contentClone.ObjectMeta, utils.AnnVolumeSnapshotBeingCreated, "yes") - updateContent, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().Update(contentClone) + updatedContent, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().Update(contentClone) if err != nil { return newControllerUpdateError(content.Name, err.Error()) } // update content if update is successful - content = updateContent + content = updatedContent _, err = ctrl.storeContentUpdate(content) if err != nil { @@ -641,12 +633,12 @@ func (ctrl csiSnapshotSideCarController) removeAnnVolumeSnapshotBeingCreated(con contentClone := content.DeepCopy() delete(contentClone.ObjectMeta.Annotations, utils.AnnVolumeSnapshotBeingCreated) - updateContent, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().Update(contentClone) + updatedContent, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().Update(contentClone) if err != nil { return newControllerUpdateError(content.Name, err.Error()) } // update content if update is successful - content = updateContent + content = updatedContent klog.V(5).Infof("Removed VolumeSnapshotBeingCreated annotation from volume snapshot content %s", content.Name) _, err = ctrl.storeContentUpdate(content) @@ -655,3 +647,28 @@ func (ctrl csiSnapshotSideCarController) removeAnnVolumeSnapshotBeingCreated(con } return nil } + +// This function checks if the error is final +func isFinalError(err error) bool { + // Sources: + // https://github.com/grpc/grpc/blob/master/doc/statuscodes.md + // https://github.com/container-storage-interface/spec/blob/master/spec.md + st, ok := status.FromError(err) + if !ok { + // This is not gRPC error. The operation must have failed before gRPC + // method was called, otherwise we would get gRPC error. + // We don't know if any previous CreateSnapshot is in progress, be on the safe side. + return false + } + switch st.Code() { + case codes.Canceled, // gRPC: Client Application cancelled the request + codes.DeadlineExceeded, // gRPC: Timeout + codes.Unavailable, // gRPC: Server shutting down, TCP connection broken - previous CreateSnapshot() may be still in progress. + codes.ResourceExhausted, // gRPC: Server temporarily out of resources - previous CreateSnapshot() may be still in progress. + codes.Aborted: // CSI: Operation pending for Snapshot + return false + } + // All other errors mean that creating snapshot either did not + // even start or failed. It is for sure not in progress. + return true +} diff --git a/pkg/utils/util.go b/pkg/utils/util.go index b0fc07496..99fcdc9aa 100644 --- a/pkg/utils/util.go +++ b/pkg/utils/util.go @@ -82,9 +82,10 @@ const ( // If it is set, it indicates that the csi-snapshotter // sidecar has sent the create snapshot request to the storage system and // is waiting for a response of success or failure. - // This annotation will be removed if the driver's CreateSnapshot - // CSI function returns success or failure. If the create snapshot - // request times out, retry will happen and the annotation will remain. + // This annotation will be removed once the driver's CreateSnapshot + // CSI function returns success or a final error (determined by isFinalError()). + // If the create snapshot request fails with a non-final error such as timeout, + // retry will happen and the annotation will remain. // This only applies to dynamic provisioning of snapshots because // the create snapshot CSI method will not be called for pre-provisioned // snapshots. From f32313bdb8bd39bbb08e6626f2f04afca2100115 Mon Sep 17 00:00:00 2001 From: xing-yang Date: Wed, 1 Apr 2020 04:07:45 +0000 Subject: [PATCH 52/56] checkandUpdateContentStatusOperation should not call createSnapshotOperation --- pkg/sidecar-controller/snapshot_controller.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pkg/sidecar-controller/snapshot_controller.go b/pkg/sidecar-controller/snapshot_controller.go index 4742dbc1a..5db2e0bfa 100644 --- a/pkg/sidecar-controller/snapshot_controller.go +++ b/pkg/sidecar-controller/snapshot_controller.go @@ -291,7 +291,7 @@ func (ctrl *csiSnapshotSideCarController) checkandUpdateContentStatusOperation(c } return updatedContent, nil } else { - return ctrl.createSnapshotOperation(content) + return ctrl.createSnapshotWrapper(content) } } @@ -307,6 +307,13 @@ func (ctrl *csiSnapshotSideCarController) createSnapshotOperation(content *crdv1 return content, nil } + return ctrl.createSnapshotWrapper(content) +} + +// This is a wrapper function for the snapshot creation process. +func (ctrl *csiSnapshotSideCarController) createSnapshotWrapper(content *crdv1.VolumeSnapshotContent) (*crdv1.VolumeSnapshotContent, error) { + klog.Infof("createSnapshotWrapper: Creating snapshot for content %s through the plugin ...", content.Name) + class, snapshotterCredentials, err := ctrl.getCSISnapshotInput(content) if err != nil { return nil, fmt.Errorf("failed to get input parameters to create snapshot for content %s: %q", content.Name, err) @@ -329,7 +336,7 @@ func (ctrl *csiSnapshotSideCarController) createSnapshotOperation(content *crdv1 // NOTE(xyang): handle create timeout // If it is a final error, remove annotation to indicate // storage system has responded with an error - klog.Infof("createSnapshotOperation: CreateSnapshot for content %s returned error: %v", content.Name, err) + klog.Infof("createSnapshotWrapper: CreateSnapshot for content %s returned error: %v", content.Name, err) if isFinalError(err) { err = ctrl.removeAnnVolumeSnapshotBeingCreated(content) if err != nil { From 9f382acd4329d8cd2da5ba3a8e0769e4b4948eca Mon Sep 17 00:00:00 2001 From: xing-yang Date: Thu, 2 Apr 2020 03:22:01 +0000 Subject: [PATCH 53/56] Address review comments --- pkg/sidecar-controller/snapshot_controller.go | 79 ++++++++++--------- 1 file changed, 40 insertions(+), 39 deletions(-) diff --git a/pkg/sidecar-controller/snapshot_controller.go b/pkg/sidecar-controller/snapshot_controller.go index 5db2e0bfa..eedf60854 100644 --- a/pkg/sidecar-controller/snapshot_controller.go +++ b/pkg/sidecar-controller/snapshot_controller.go @@ -83,6 +83,11 @@ func (ctrl *csiSnapshotSideCarController) syncContent(content *crdv1.VolumeSnaps // or ListSnapshots CSI methods over and over again for // performance reasons. if content.Status != nil && content.Status.ReadyToUse != nil && *content.Status.ReadyToUse == true { + // Try to remove AnnVolumeSnapshotBeingCreated if it is not removed yet for some reason + err := ctrl.removeAnnVolumeSnapshotBeingCreated(content) + if err != nil { + return fmt.Errorf("failed to remove VolumeSnapshotBeingCreated annotation from the content %s: %q", content.Name, err) + } return nil } ctrl.checkandUpdateContentStatus(content) @@ -128,10 +133,17 @@ func (ctrl *csiSnapshotSideCarController) createSnapshot(content *crdv1.VolumeSn klog.V(5).Infof("createSnapshot for content [%s]: started", content.Name) opName := fmt.Sprintf("create-%s", content.Name) ctrl.scheduleOperation(opName, func() error { - contentObj, err := ctrl.createSnapshotOperation(content) + // content.Status will be created for the first time after a snapshot + // is created by the CSI driver. If content.Status is not nil, + // we should update content status without creating snapshot again. + if content.Status != nil && content.Status.Error != nil && content.Status.Error.Message != nil && !isControllerUpdateFailError(content.Status.Error) { + klog.V(4).Infof("error is already set in snapshot, do not retry to create: %s", *content.Status.Error.Message) + return nil + } + contentObj, err := ctrl.createSnapshotWrapper(content) if err != nil { ctrl.updateContentErrorStatusWithEvent(content, v1.EventTypeWarning, "SnapshotCreationFailed", fmt.Sprintf("Failed to create snapshot: %v", err)) - klog.Errorf("createSnapshot [%s]: error occurred in createSnapshotOperation: %v", opName, err) + klog.Errorf("createSnapshot [%s]: error occurred in createSnapshotWrapper: %v", opName, err) return err } @@ -295,21 +307,6 @@ func (ctrl *csiSnapshotSideCarController) checkandUpdateContentStatusOperation(c } } -// The function goes through the snapshot creation process. -func (ctrl *csiSnapshotSideCarController) createSnapshotOperation(content *crdv1.VolumeSnapshotContent) (*crdv1.VolumeSnapshotContent, error) { - klog.Infof("createSnapshotOperation: Creating snapshot for content %s through the plugin ...", content.Name) - - // content.Status will be created for the first time after a snapshot - // is created by the CSI driver. If content.Status is not nil, - // we should update content status without creating snapshot again. - if content.Status != nil && content.Status.Error != nil && content.Status.Error.Message != nil && !isControllerUpdateFailError(content.Status.Error) { - klog.V(4).Infof("error is already set in snapshot, do not retry to create: %s", *content.Status.Error.Message) - return content, nil - } - - return ctrl.createSnapshotWrapper(content) -} - // This is a wrapper function for the snapshot creation process. func (ctrl *csiSnapshotSideCarController) createSnapshotWrapper(content *crdv1.VolumeSnapshotContent) (*crdv1.VolumeSnapshotContent, error) { klog.Infof("createSnapshotWrapper: Creating snapshot for content %s through the plugin ...", content.Name) @@ -337,7 +334,7 @@ func (ctrl *csiSnapshotSideCarController) createSnapshotWrapper(content *crdv1.V // If it is a final error, remove annotation to indicate // storage system has responded with an error klog.Infof("createSnapshotWrapper: CreateSnapshot for content %s returned error: %v", content.Name, err) - if isFinalError(err) { + if isCSIFinalError(err) { err = ctrl.removeAnnVolumeSnapshotBeingCreated(content) if err != nil { return nil, fmt.Errorf("failed to remove VolumeSnapshotBeingCreated annotation from the content %s: %q", content.Name, err) @@ -355,8 +352,8 @@ func (ctrl *csiSnapshotSideCarController) createSnapshotWrapper(content *crdv1.V newContent, err := ctrl.updateSnapshotContentStatus(content, snapshotID, readyToUse, creationTime.UnixNano(), size) if err != nil { - strerr := fmt.Sprintf("error updating volume snapshot content status for snapshot %s: %v.", content.Name, err) - klog.Error(strerr) + klog.Errorf("error updating status for volume snapshot content %s: %v.", content.Name, err) + return nil, fmt.Errorf("error updating status for volume snapshot content %s: %v.", content.Name, err) } else { content = newContent } @@ -607,26 +604,30 @@ func (ctrl *csiSnapshotSideCarController) shouldDelete(content *crdv1.VolumeSnap // on VolumeSnapshotContent // If set, it indicates snapshot is being created func (ctrl *csiSnapshotSideCarController) setAnnVolumeSnapshotBeingCreated(content *crdv1.VolumeSnapshotContent) error { - // Set AnnVolumeSnapshotBeingCreated if it is not set yet - if !metav1.HasAnnotation(content.ObjectMeta, utils.AnnVolumeSnapshotBeingCreated) { - klog.V(5).Infof("setAnnVolumeSnapshotBeingCreated: set annotation [%s:yes] on content [%s].", utils.AnnVolumeSnapshotBeingCreated, content.Name) - contentClone := content.DeepCopy() - metav1.SetMetaDataAnnotation(&contentClone.ObjectMeta, utils.AnnVolumeSnapshotBeingCreated, "yes") + if metav1.HasAnnotation(content.ObjectMeta, utils.AnnVolumeSnapshotBeingCreated) { + // the annotation already exists, return directly + return nil + } - updatedContent, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().Update(contentClone) - if err != nil { - return newControllerUpdateError(content.Name, err.Error()) - } - // update content if update is successful - content = updatedContent + // Set AnnVolumeSnapshotBeingCreated + klog.V(5).Infof("setAnnVolumeSnapshotBeingCreated: set annotation [%s:yes] on content [%s].", utils.AnnVolumeSnapshotBeingCreated, content.Name) + contentClone := content.DeepCopy() + metav1.SetMetaDataAnnotation(&contentClone.ObjectMeta, utils.AnnVolumeSnapshotBeingCreated, "yes") - _, err = ctrl.storeContentUpdate(content) - if err != nil { - klog.V(4).Infof("setAnnVolumeSnapshotBeingCreated for content [%s]: cannot update internal cache %v", content.Name, err) - return err - } - klog.V(5).Infof("setAnnVolumeSnapshotBeingCreated: volume snapshot content %+v", content) + updatedContent, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().Update(contentClone) + if err != nil { + return newControllerUpdateError(content.Name, err.Error()) } + // update content if update is successful + content = updatedContent + + _, err = ctrl.storeContentUpdate(content) + if err != nil { + klog.V(4).Infof("setAnnVolumeSnapshotBeingCreated for content [%s]: cannot update internal cache %v", content.Name, err) + return err + } + klog.V(5).Infof("setAnnVolumeSnapshotBeingCreated: volume snapshot content %+v", content) + return nil } @@ -634,7 +635,7 @@ func (ctrl *csiSnapshotSideCarController) setAnnVolumeSnapshotBeingCreated(conte // annotation from a content if there exists one. func (ctrl csiSnapshotSideCarController) removeAnnVolumeSnapshotBeingCreated(content *crdv1.VolumeSnapshotContent) error { if !metav1.HasAnnotation(content.ObjectMeta, utils.AnnVolumeSnapshotBeingCreated) { - // the annotation does not exit, return directly + // the annotation does not exist, return directly return nil } contentClone := content.DeepCopy() @@ -656,7 +657,7 @@ func (ctrl csiSnapshotSideCarController) removeAnnVolumeSnapshotBeingCreated(con } // This function checks if the error is final -func isFinalError(err error) bool { +func isCSIFinalError(err error) bool { // Sources: // https://github.com/grpc/grpc/blob/master/doc/statuscodes.md // https://github.com/container-storage-interface/spec/blob/master/spec.md From 2295e6aea3b25d7421128ab3204113e68ab5e3d1 Mon Sep 17 00:00:00 2001 From: xing-yang Date: Fri, 3 Apr 2020 01:37:54 +0000 Subject: [PATCH 54/56] Add error status check in createSnapshotWrapper --- pkg/sidecar-controller/snapshot_controller.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/sidecar-controller/snapshot_controller.go b/pkg/sidecar-controller/snapshot_controller.go index eedf60854..04440b73c 100644 --- a/pkg/sidecar-controller/snapshot_controller.go +++ b/pkg/sidecar-controller/snapshot_controller.go @@ -133,13 +133,6 @@ func (ctrl *csiSnapshotSideCarController) createSnapshot(content *crdv1.VolumeSn klog.V(5).Infof("createSnapshot for content [%s]: started", content.Name) opName := fmt.Sprintf("create-%s", content.Name) ctrl.scheduleOperation(opName, func() error { - // content.Status will be created for the first time after a snapshot - // is created by the CSI driver. If content.Status is not nil, - // we should update content status without creating snapshot again. - if content.Status != nil && content.Status.Error != nil && content.Status.Error.Message != nil && !isControllerUpdateFailError(content.Status.Error) { - klog.V(4).Infof("error is already set in snapshot, do not retry to create: %s", *content.Status.Error.Message) - return nil - } contentObj, err := ctrl.createSnapshotWrapper(content) if err != nil { ctrl.updateContentErrorStatusWithEvent(content, v1.EventTypeWarning, "SnapshotCreationFailed", fmt.Sprintf("Failed to create snapshot: %v", err)) @@ -311,6 +304,14 @@ func (ctrl *csiSnapshotSideCarController) checkandUpdateContentStatusOperation(c func (ctrl *csiSnapshotSideCarController) createSnapshotWrapper(content *crdv1.VolumeSnapshotContent) (*crdv1.VolumeSnapshotContent, error) { klog.Infof("createSnapshotWrapper: Creating snapshot for content %s through the plugin ...", content.Name) + // content.Status will be created for the first time after a snapshot + // is created by the CSI driver. If content.Status is not nil, + // we should update content status without creating snapshot again. + if content.Status != nil && content.Status.Error != nil && content.Status.Error.Message != nil && !isControllerUpdateFailError(content.Status.Error) { + klog.V(4).Infof("error is already set in snapshot, do not retry to create: %s", *content.Status.Error.Message) + return content, nil + } + class, snapshotterCredentials, err := ctrl.getCSISnapshotInput(content) if err != nil { return nil, fmt.Errorf("failed to get input parameters to create snapshot for content %s: %q", content.Name, err) @@ -624,7 +625,6 @@ func (ctrl *csiSnapshotSideCarController) setAnnVolumeSnapshotBeingCreated(conte _, err = ctrl.storeContentUpdate(content) if err != nil { klog.V(4).Infof("setAnnVolumeSnapshotBeingCreated for content [%s]: cannot update internal cache %v", content.Name, err) - return err } klog.V(5).Infof("setAnnVolumeSnapshotBeingCreated: volume snapshot content %+v", content) From f688d7baef900d55b742f0c3b17811910522013d Mon Sep 17 00:00:00 2001 From: xing-yang Date: Fri, 3 Apr 2020 18:34:57 +0000 Subject: [PATCH 55/56] Remove error status check --- pkg/sidecar-controller/snapshot_controller.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pkg/sidecar-controller/snapshot_controller.go b/pkg/sidecar-controller/snapshot_controller.go index 04440b73c..68e48ac0f 100644 --- a/pkg/sidecar-controller/snapshot_controller.go +++ b/pkg/sidecar-controller/snapshot_controller.go @@ -304,14 +304,6 @@ func (ctrl *csiSnapshotSideCarController) checkandUpdateContentStatusOperation(c func (ctrl *csiSnapshotSideCarController) createSnapshotWrapper(content *crdv1.VolumeSnapshotContent) (*crdv1.VolumeSnapshotContent, error) { klog.Infof("createSnapshotWrapper: Creating snapshot for content %s through the plugin ...", content.Name) - // content.Status will be created for the first time after a snapshot - // is created by the CSI driver. If content.Status is not nil, - // we should update content status without creating snapshot again. - if content.Status != nil && content.Status.Error != nil && content.Status.Error.Message != nil && !isControllerUpdateFailError(content.Status.Error) { - klog.V(4).Infof("error is already set in snapshot, do not retry to create: %s", *content.Status.Error.Message) - return content, nil - } - class, snapshotterCredentials, err := ctrl.getCSISnapshotInput(content) if err != nil { return nil, fmt.Errorf("failed to get input parameters to create snapshot for content %s: %q", content.Name, err) From b4f4f1680559e12257a943dba506ed7fe3ec19b0 Mon Sep 17 00:00:00 2001 From: xing-yang Date: Mon, 30 Mar 2020 20:11:13 +0000 Subject: [PATCH 56/56] Changelog for 2.1 release --- CHANGELOG-2.1.md | 34 ++++++++++++++++++++++++++++++++++ README.md | 4 ++++ 2 files changed, 38 insertions(+) create mode 100644 CHANGELOG-2.1.md diff --git a/CHANGELOG-2.1.md b/CHANGELOG-2.1.md new file mode 100644 index 000000000..3acc129ad --- /dev/null +++ b/CHANGELOG-2.1.md @@ -0,0 +1,34 @@ +# Changelog since v2.0.0 + +## New Features + +### Snapshot Controller + +- The number of worker threads in the snapshot-controller is now configurable via the `worker-threads` flag. ([#282](https://github.com/kubernetes-csi/external-snapshotter/pull/282), [@huffmanca](https://github.com/huffmanca)) + +### CSI External-Snapshotter Sidecar + +- The number of worker threads in the csi-snapshotter is now configurable via the `worker-threads` flag. ([#282](https://github.com/kubernetes-csi/external-snapshotter/pull/282), [@huffmanca](https://github.com/huffmanca)) +- Adds support for ListSnapshots secrets ([#252](https://github.com/kubernetes-csi/external-snapshotter/pull/252), [@bells17](https://github.com/bells17)) + +## Breaking Changes + +- Update package path to v2. Vendoring with dep depends on https://github.com/golang/dep/pull/1963 or the workaround described in v2/README.md. ([#240](https://github.com/kubernetes-csi/external-snapshotter/pull/240), [@alex1989hu](https://github.com/alex1989hu)) + +## Bug Fixes + +### Snapshot Controller + +- Fixes a problem of not removing the PVC finalizer when it is no longer used by a snapshot as source. ([#283](https://github.com/kubernetes-csi/external-snapshotter/pull/283), [@xing-yang](https://github.com/xing-yang)) +- Fixes a problem deleting VolumeSnapshotContent with `Retain` policy for pre-provisioned snapshots. ([#249](https://github.com/kubernetes-csi/external-snapshotter/pull/249), [@xing-yang](https://github.com/xing-yang)) +- Allows the volume snapshot to be deleted if the volume snapshot class is not found. ([#275](https://github.com/kubernetes-csi/external-snapshotter/pull/275), [@huffmanca](https://github.com/huffmanca)) + +### CSI External-Snapshotter Sidecar + +- Fixes a create snapshot timeout issue. ([#261](https://github.com/kubernetes-csi/external-snapshotter/pull/261), [@xing-yang](https://github.com/xing-yang)) + +## Other Notable Changes + +### API Changes + +- Prints additional details when using kubectl get on VolumeSnapshot, VolumeSnapshotContent, and VolumeSnapshotClass API objects. ([#260](https://github.com/kubernetes-csi/external-snapshotter/pull/260), [@huffmanca](https://github.com/huffmanca)) diff --git a/README.md b/README.md index 325b08e8f..53b7610d3 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,8 @@ Install CSI Driver: * `--metrics-path`: The HTTP path where prometheus metrics will be exposed. Default is `/metrics`. +* `--worker-threads`: Number of worker threads. Default value is 10. + #### Other recognized arguments * `--kubeconfig `: Path to Kubernetes client configuration that the snapshot controller uses to connect to Kubernetes API server. When omitted, default token provided by Kubernetes will be used. This option is useful only when the snapshot controller does not run as a Kubernetes pod, e.g. for debugging. @@ -113,6 +115,8 @@ Install CSI Driver: * `snapshot-name-uuid-length`: Length in characters for the generated uuid of a created snapshot. Defaults behavior is to NOT truncate. +* `--worker-threads`: Number of worker threads for running create snapshot and delete snapshot operations. Default value is 10. + #### Other recognized arguments * `--kubeconfig `: Path to Kubernetes client configuration that the CSI external-snapshotter uses to connect to Kubernetes API server. When omitted, default token provided by Kubernetes will be used. This option is useful only when the external-snapshotter does not run as a Kubernetes pod, e.g. for debugging.