Skip to content

Commit

Permalink
prow.sh: generic driver installation
Browse files Browse the repository at this point in the history 
This relies on a slightly different deployment script: a "deploy.sh"
must exist which knows that it has to dump a test driver configurion
into the file pointed to with CSI_PROW_TEST_DRIVER, if that env
variable is set.

That way, we no longer need to know what capabilities the installed
driver has.
  • Loading branch information
@pohly
pohly committed Feb 10, 2020
1 parent 3c34b4f commit 84f78b1
Showing 1 changed file with 43 additions and 61 deletions.
104 changes: 43 additions & 61 deletions prow.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,19 +157,21 @@ csi_prow_kubernetes_version_suffix="$(echo "${CSI_PROW_KUBERNETES_VERSION}" | tr
# the caller.
configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csiprow.XXXXXXXXXX")" "work directory"

# The hostpath deployment script is searched for in several places.
# By default, this script tests sidecars with the CSI hostpath driver,
# using the install_csi_driver function. That function depends on
# a deployment script that it searches for in several places:
#
# - The "deploy" directory in the current repository: this is useful
# for the situation that a component becomes incompatible with the
# shared deployment, because then it can (temporarily!) provide its
# own example until the shared one can be updated; it's also how
# csi-driver-host-path itself provides the example.
#
# - CSI_PROW_HOSTPATH_VERSION of the CSI_PROW_HOSTPATH_REPO is checked
# - CSI_PROW_DRIVER_VERSION of the CSI_PROW_DRIVER_REPO is checked
# out: this allows other repos to reference a version of the example
# that is known to be compatible.
#
# - The csi-driver-host-path/deploy directory has multiple sub-directories,
# - The <driver repo>/deploy directory can have multiple sub-directories,
# each with different deployments (stable set of images for Kubernetes 1.13,
# stable set of images for Kubernetes 1.14, canary for latest Kubernetes, etc.).
# This is necessary because there may be incompatible changes in the
Expand All @@ -186,16 +188,26 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip
# "none" disables the deployment of the hostpath driver.
#
# When no deploy script is found (nothing in `deploy` directory,
# CSI_PROW_HOSTPATH_REPO=none), nothing gets deployed.
configvar CSI_PROW_HOSTPATH_VERSION "v1.3.0-rc3" "hostpath driver"
configvar CSI_PROW_HOSTPATH_REPO https://github.com/kubernetes-csi/csi-driver-host-path "hostpath repo"
# CSI_PROW_DRIVER_REPO=none), nothing gets deployed.
#
# If the deployment script is called with CSI_PROW_TEST_DRIVER=<file name> as
# environment variable, then it must write a suitable test driver configuration
# into that file in addition to installing the driver.
configvar CSI_PROW_DRIVER_VERSION "v1.3.0-rc4" "CSI driver version"
configvar CSI_PROW_DRIVER_REPO https://github.com/kubernetes-csi/csi-driver-host-path "CSI driver repo"
configvar CSI_PROW_DEPLOYMENT "" "deployment"
configvar CSI_PROW_HOSTPATH_DRIVER_NAME "hostpath.csi.k8s.io" "the hostpath driver name"

# If CSI_PROW_HOSTPATH_CANARY is set (typically to "canary", but also
# "1.0-canary"), then all image versions are replaced with that
# version tag.
configvar CSI_PROW_HOSTPATH_CANARY "" "hostpath image"
# The install_csi_driver function may work also for other CSI drivers,
# as long as they follow the conventions of the CSI hostpath driver.
# If they don't, then a different install function can be provided in
# a .prow.sh file and this config variable can be overridden.
configvar CSI_PROW_DRIVER_INSTALL "install_csi_driver" "name of the shell function which installs the CSI driver"

# If CSI_PROW_DRIVER_CANARY is set (typically to "canary", but also
# version tag. Usually empty. CSI_PROW_HOSTPATH_CANARY is
# accepted as alternative name because some test-infra jobs
# still use that name.
configvar CSI_PROW_DRIVER_CANARY "${CSI_PROW_HOSTPATH_CANARY}" "driver image override for canary images"

# The E2E testing can come from an arbitrary repo. The expectation is that
# the repo supports "go test ./test/e2e -args --storage.testdriver" (https://github.com/kubernetes/kubernetes/pull/72836)
Expand Down Expand Up @@ -613,7 +625,7 @@ find_deployment () {

# Fixed deployment name? Use it if it exists, otherwise fail.
if [ "${CSI_PROW_DEPLOYMENT}" ]; then
file="$dir/${CSI_PROW_DEPLOYMENT}/deploy-hostpath.sh"
file="$dir/${CSI_PROW_DEPLOYMENT}/deploy.sh"
if ! [ -e "$file" ]; then
return 1
fi
Expand All @@ -623,22 +635,21 @@ find_deployment () {

# Ignore: See if you can use ${variable//search/replace} instead.
# shellcheck disable=SC2001
file="$dir/kubernetes-$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/\([0-9]*\)\.\([0-9]*\).*/\1.\2/')/deploy-hostpath.sh"
file="$dir/kubernetes-$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/\([0-9]*\)\.\([0-9]*\).*/\1.\2/')/deploy.sh"
if ! [ -e "$file" ]; then
file="$dir/kubernetes-latest/deploy-hostpath.sh"
file="$dir/kubernetes-latest/deploy.sh"
if ! [ -e "$file" ]; then
return 1
fi
fi
echo "$file"
}

# This installs the hostpath driver example. CSI_PROW_HOSTPATH_CANARY overrides all
# image versions with that canary version. The parameters of install_hostpath can be
# used to override registry and/or tag of individual images (CSI_PROVISIONER_REGISTRY=localhost:9000
# CSI_PROVISIONER_TAG=latest).
install_hostpath () {
local images deploy_hostpath
# This installs the CSI driver. It's called with a list of env variables
# that override the default images. CSI_PROW_DRIVER_CANARY overrides all
# image versions with that canary version.
install_csi_driver () {
local images deploy_driver
images="$*"

if [ "${CSI_PROW_DEPLOYMENT}" = "none" ]; then
Expand All @@ -654,31 +665,31 @@ install_hostpath () {
done
fi

if deploy_hostpath="$(find_deployment "$(pwd)/deploy")"; then
if deploy_driver="$(find_deployment "$(pwd)/deploy")"; then
:
elif [ "${CSI_PROW_HOSTPATH_REPO}" = "none" ]; then
elif [ "${CSI_PROW_DRIVER_REPO}" = "none" ]; then
return 1
else
git_checkout "${CSI_PROW_HOSTPATH_REPO}" "${CSI_PROW_WORK}/hostpath" "${CSI_PROW_HOSTPATH_VERSION}" --depth=1 || die "checking out hostpath repo failed"
if deploy_hostpath="$(find_deployment "${CSI_PROW_WORK}/hostpath/deploy")"; then
git_checkout "${CSI_PROW_DRIVER_REPO}" "${CSI_PROW_WORK}/csi-driver" "${CSI_PROW_DRIVER_VERSION}" --depth=1 || die "checking out CSI driver repo failed"
if deploy_driver="$(find_deployment "${CSI_PROW_WORK}/csi-driver/deploy")"; then
:
else
die "deploy-hostpath.sh not found in ${CSI_PROW_HOSTPATH_REPO} ${CSI_PROW_HOSTPATH_VERSION}. To disable E2E testing, set CSI_PROW_HOSTPATH_REPO=none"
die "deploy.sh not found in ${CSI_PROW_DRIVER_REPO} ${CSI_PROW_DRIVER_VERSION}. To disable E2E testing, set CSI_PROW_DRIVER_REPO=none"
fi
fi

if [ "${CSI_PROW_HOSTPATH_CANARY}" != "stable" ]; then
images="$images IMAGE_TAG=${CSI_PROW_HOSTPATH_CANARY}"
if [ "${CSI_PROW_DRIVER_CANARY}" != "stable" ]; then
images="$images IMAGE_TAG=${CSI_PROW_DRIVER_CANARY}"
fi
# Ignore: Double quote to prevent globbing and word splitting.
# It's intentional here for $images.
# shellcheck disable=SC2086
if ! run env $images "${deploy_hostpath}"; then
if ! run env "CSI_PROW_TEST_DRIVER=${CSI_PROW_WORK}/test-driver.yaml" $images "${deploy_driver}"; then
# Collect information about failed deployment before failing.
collect_cluster_info
(start_loggers >/dev/null; wait)
info "For container output see job artifacts."
die "deploying the hostpath driver with ${deploy_hostpath} failed"
die "deploying the CSI driver with ${deploy_driver} failed"
fi
}

Expand Down Expand Up @@ -804,33 +815,6 @@ install_sanity () (
run_with_go "${CSI_PROW_GO_VERSION_SANITY}" go test -c -o "${CSI_PROW_WORK}/csi-sanity" "${CSI_PROW_SANITY_IMPORT_PATH}/cmd/csi-sanity" || die "building csi-sanity failed"
)

# The default implementation of this function generates a external
# driver test configuration for the hostpath driver.
#
# The content depends on both what the E2E suite expects and what the
# installed hostpath driver supports. Generating it here seems prone
# to breakage, but it is uncertain where a better place might be.
generate_test_driver () {
cat <<EOF
ShortName: csiprow
StorageClass:
FromName: true
SnapshotClass:
FromName: true
DriverInfo:
Name: ${CSI_PROW_HOSTPATH_DRIVER_NAME}
Capabilities:
block: true
persistence: true
dataSource: true
multipods: true
nodeExpansion: true
controllerExpansion: true
snapshotDataSource: true
singleNodeVolume: true
EOF
}

# Captures pod output while running some other command.
run_with_loggers () (
loggers=$(start_loggers -f)
Expand All @@ -852,8 +836,6 @@ run_e2e () (
install_e2e || die "building e2e.test failed"
install_ginkgo || die "installing ginkgo failed"

generate_test_driver >"${CSI_PROW_WORK}/test-driver.yaml" || die "generating test-driver.yaml failed"

# Rename, merge and filter JUnit files. Necessary in case that we run the E2E suite again
# and to avoid the large number of "skipped" tests that we get from using
# the full Kubernetes E2E testsuite while only running a few tests.
Expand Down Expand Up @@ -1063,7 +1045,7 @@ main () {
cmds="$(grep '^\s*CMDS\s*=' Makefile | sed -e 's/\s*CMDS\s*=//')"
# Get the image that was just built (if any) from the
# top-level Makefile CMDS variable and set the
# deploy-hostpath.sh env variables for it. We also need to
# deploy.sh env variables for it. We also need to
# side-load those images into the cluster.
for i in $cmds; do
e=$(echo "$i" | tr '[:lower:]' '[:upper:]' | tr - _)
Expand Down Expand Up @@ -1101,7 +1083,7 @@ main () {
fi

# Installing the driver might be disabled.
if install_hostpath "$images"; then
if ${CSI_PROW_DRIVER_INSTALL} "$images"; then
collect_cluster_info

if sanity_enabled; then
Expand Down Expand Up @@ -1158,7 +1140,7 @@ main () {
fi

# Installing the driver might be disabled.
if install_hostpath "$images"; then
if ${CSI_PROW_DRIVER_INSTALL} "$images"; then
collect_cluster_info

if tests_enabled "parallel-alpha"; then
Expand Down

0 comments on commit 84f78b1

Please sign in to comment.